Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd5056d4c9 | ||
|
|
39fb821481 | ||
|
|
beb87a8c43 | ||
|
|
4327fc939c | ||
|
|
ef1046c6f5 | ||
|
|
ef8cf719f2 | ||
|
|
6aa6b32a6c | ||
|
|
33dff04d47 | ||
|
|
65d697b7be | ||
|
|
06fc42ed37 | ||
|
|
3c5c567077 | ||
|
|
8f65e550c8 | ||
|
|
6b83879741 | ||
|
|
be265e9cc0 | ||
|
|
680c36ab59 | ||
|
|
96b4f65cd1 | ||
|
|
b1a897e51c | ||
|
|
e5a34efee9 | ||
|
|
f9ac4e4dbf | ||
|
|
1b40e29f40 | ||
|
|
7eba1fb487 | ||
|
|
838083b909 | ||
|
|
8f5eb36b5f | ||
|
|
b7d1bcce3d | ||
|
|
03d3173ca6 | ||
|
|
38a61d7b50 | ||
|
|
0a429e1f7b | ||
|
|
857ba953e3 | ||
|
|
e180018c99 | ||
|
|
ac9956bf00 | ||
|
|
62b5a8bf65 | ||
|
|
303efefcb7 | ||
|
|
feeb7c2d92 | ||
|
|
ea9a54421a | ||
|
|
fdf99b2bb0 | ||
|
|
c7cd641f89 | ||
|
|
18b90c8df3 | ||
|
|
8d3bc1c2e2 | ||
|
|
079d988034 | ||
|
|
aa9d388337 | ||
|
|
92bd3d9a47 | ||
|
|
53058d1504 | ||
|
|
3fe8463a03 | ||
|
|
5c4ce5d727 | ||
|
|
459193e7b1 | ||
|
|
98f21323fb | ||
|
|
515248d438 | ||
|
|
b0c69ad3e0 | ||
|
|
c5f29ab4ae | ||
|
|
876ee898d8 | ||
|
|
e93bab6ea7 |
No files matched your search
@@ -1,6 +1,8 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_parser.py
|
||||
DESCRIPTION: Zerlegt Markdown in Blöcke und extrahiert Kanten-Strings.
|
||||
DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
|
||||
Hält alle Überschriftenebenen (H1-H6) im Stream.
|
||||
Stellt die Funktion parse_edges_robust zur Verfügung.
|
||||
"""
|
||||
import re
|
||||
from typing import List, Tuple, Set
|
||||
@@ -11,69 +13,86 @@ _WS = re.compile(r'\s+')
|
||||
_SENT_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ0-9„(])')
|
||||
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
"""Teilt Text in Sätze auf."""
|
||||
"""Teilt Text in Sätze auf unter Berücksichtigung deutscher Interpunktion."""
|
||||
text = _WS.sub(' ', text.strip())
|
||||
if not text: return []
|
||||
# Splittet bei Punkt, Ausrufezeichen oder Fragezeichen, gefolgt von Leerzeichen und Großbuchstabe
|
||||
return [p.strip() for p in _SENT_SPLIT.split(text) if p.strip()]
|
||||
|
||||
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""Zerlegt Text in logische Einheiten."""
|
||||
"""Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6."""
|
||||
blocks = []
|
||||
h1_title = "Dokument"; section_path = "/"; current_h2 = None
|
||||
h1_title = "Dokument"
|
||||
section_path = "/"
|
||||
current_section_title = None
|
||||
|
||||
# Frontmatter entfernen
|
||||
fm, text_without_fm = extract_frontmatter_from_text(md_text)
|
||||
|
||||
# H1 für Note-Titel extrahieren (Metadaten-Zweck)
|
||||
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
|
||||
if h1_match: h1_title = h1_match.group(1).strip()
|
||||
if h1_match:
|
||||
h1_title = h1_match.group(1).strip()
|
||||
|
||||
lines = text_without_fm.split('\n')
|
||||
buffer = []
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
|
||||
# H1 ignorieren (ist Doc Title)
|
||||
if stripped.startswith('# '):
|
||||
continue
|
||||
|
||||
# Generische Heading-Erkennung (H2 bis H6) für flexible Split-Levels
|
||||
heading_match = re.match(r'^(#{2,6})\s+(.*)', stripped)
|
||||
# Heading-Erkennung (H1 bis H6)
|
||||
heading_match = re.match(r'^(#{1,6})\s+(.*)', stripped)
|
||||
if heading_match:
|
||||
# Buffer leeren (vorherigen Text abschließen)
|
||||
# Vorherigen Text-Block abschließen
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content: blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
buffer = []
|
||||
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
|
||||
# Pfad-Logik: H2 setzt den Haupt-Pfad
|
||||
if level == 2:
|
||||
current_h2 = title
|
||||
section_path = f"/{current_h2}"
|
||||
# Bei H3+ bleibt der section_path beim Parent, aber das Level wird korrekt gesetzt
|
||||
# Pfad- und Titel-Update für die Metadaten der folgenden Blöcke
|
||||
if level == 1:
|
||||
current_section_title = title; section_path = "/"
|
||||
elif level == 2:
|
||||
current_section_title = title; section_path = f"/{current_section_title}"
|
||||
|
||||
blocks.append(RawBlock("heading", stripped, level, section_path, current_h2))
|
||||
|
||||
elif not stripped:
|
||||
# Die Überschrift selbst als regulären Block hinzufügen
|
||||
blocks.append(RawBlock("heading", stripped, level, section_path, current_section_title))
|
||||
continue
|
||||
|
||||
# Trenner (---) oder Leerzeilen beenden Blöcke, außer innerhalb von Callouts
|
||||
if (not stripped or stripped == "---") and not line.startswith('>'):
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content: blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
buffer = []
|
||||
if stripped == "---":
|
||||
blocks.append(RawBlock("separator", "---", None, section_path, current_section_title))
|
||||
else:
|
||||
buffer.append(line)
|
||||
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content: blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
|
||||
return blocks, h1_title
|
||||
|
||||
def parse_edges_robust(text: str) -> Set[str]:
|
||||
"""Extrahiert Kanten-Kandidaten (Wikilinks, Callouts)."""
|
||||
"""Extrahiert Kanten-Kandidaten aus Wikilinks und Callouts."""
|
||||
found_edges = set()
|
||||
# 1. Wikilinks [[rel:kind|target]]
|
||||
inlines = re.findall(r'\[\[rel:([^\|\]]+)\|?([^\]]*)\]\]', text)
|
||||
for kind, target in inlines:
|
||||
k = kind.strip().lower()
|
||||
t = target.strip()
|
||||
if k and t: found_edges.add(f"{k}:{t}")
|
||||
|
||||
# 2. Callout Edges > [!edge] kind
|
||||
lines = text.split('\n')
|
||||
current_edge_type = None
|
||||
for line in lines:
|
||||
@@ -81,13 +100,16 @@ def parse_edges_robust(text: str) -> Set[str]:
|
||||
callout_match = re.match(r'>\s*\[!edge\]\s*([^:\s]+)', stripped)
|
||||
if callout_match:
|
||||
current_edge_type = callout_match.group(1).strip().lower()
|
||||
# Links in der gleichen Zeile des Callouts
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
continue
|
||||
# Links in Folgezeilen des Callouts
|
||||
if current_edge_type and stripped.startswith('>'):
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
elif not stripped.startswith('>'): current_edge_type = None
|
||||
elif not stripped.startswith('>'):
|
||||
current_edge_type = None
|
||||
return found_edges
|
||||
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_processor.py
|
||||
DESCRIPTION: Der zentrale Orchestrator für das Chunking-System.
|
||||
AUDIT v3.3.3: Wiederherstellung der "Gold-Standard" Qualität.
|
||||
AUDIT v3.3.4: Wiederherstellung der "Gold-Standard" Qualität.
|
||||
- Fix: Synchronisierung der Parameter (context_prefix) für alle Strategien.
|
||||
- Integriert physikalische Kanten-Injektion (Propagierung).
|
||||
- Stellt H1-Kontext-Fenster sicher.
|
||||
- Baut den Candidate-Pool für die WP-15b Ingestion auf.
|
||||
@@ -30,16 +31,19 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
fm, body_text = extract_frontmatter_from_text(md_text)
|
||||
blocks, doc_title = parse_blocks(md_text)
|
||||
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster (Breadcrumbs)
|
||||
h1_prefix = f"# {doc_title}" if doc_title else ""
|
||||
|
||||
# 2. Anwendung der Splitting-Strategie
|
||||
# Wir übergeben den Dokument-Titel/Präfix für die Window-Bildung.
|
||||
# Alle Strategien nutzen nun einheitlich context_prefix für die Window-Bildung.
|
||||
if config.get("strategy") == "by_heading":
|
||||
chunks = await asyncio.to_thread(strategy_by_heading, blocks, config, note_id, doc_title)
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_by_heading, blocks, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
else:
|
||||
# sliding_window nutzt nun den context_prefix für das Window-Feld.
|
||||
chunks = await asyncio.to_thread(strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix)
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
|
||||
if not chunks:
|
||||
return []
|
||||
@@ -52,6 +56,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
# Zuerst die explizit im Text vorhandenen Kanten sammeln.
|
||||
for ch in chunks:
|
||||
# Wir extrahieren aus dem bereits (durch Propagation) angereicherten Text.
|
||||
# ch.candidate_pool wird im Modell-Konstruktor als leere Liste initialisiert.
|
||||
for e_str in parse_edges_robust(ch.text):
|
||||
parts = e_str.split(':', 1)
|
||||
if len(parts) == 2:
|
||||
@@ -71,7 +76,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
parts = e_str.split(':', 1)
|
||||
if len(parts) == 2:
|
||||
k, t = parts
|
||||
# Diese Kanten werden als "Global Pool" markiert für die spätere KI-Prüfung.
|
||||
# Diese Kanten werden als "global_pool" markiert für die spätere KI-Prüfung.
|
||||
for ch in chunks:
|
||||
ch.candidate_pool.append({"kind": k, "to": t, "provenance": "global_pool"})
|
||||
|
||||
@@ -80,6 +85,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
seen = set()
|
||||
unique = []
|
||||
for c in ch.candidate_pool:
|
||||
# Eindeutigkeit über Typ, Ziel und Herkunft (Provenance)
|
||||
key = (c["kind"], c["to"], c["provenance"])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_propagation.py
|
||||
DESCRIPTION: Injiziert Sektions-Kanten physisch in den Text (Embedding-Enrichment).
|
||||
Stellt die "Gold-Standard"-Qualität von v3.1.0 wieder her.
|
||||
VERSION: 3.3.1
|
||||
STATUS: Active
|
||||
Fix v3.3.6: Nutzt robustes Parsing zur Erkennung vorhandener Kanten,
|
||||
um Dopplungen direkt hinter [!edge] Callouts format-agnostisch zu verhindern.
|
||||
"""
|
||||
from typing import List, Dict, Set
|
||||
from .chunking_models import Chunk
|
||||
@@ -12,7 +11,7 @@ from .chunking_parser import parse_edges_robust
|
||||
def propagate_section_edges(chunks: List[Chunk]) -> List[Chunk]:
|
||||
"""
|
||||
Sammelt Kanten pro Sektion und schreibt sie hart in den Text und das Window.
|
||||
Dies ist essenziell für die Vektorisierung der Beziehungen.
|
||||
Verhindert Dopplungen, wenn Kanten bereits via [!edge] Callout vorhanden sind.
|
||||
"""
|
||||
# 1. Sammeln: Alle expliziten Kanten pro Sektions-Pfad aggregieren
|
||||
section_map: Dict[str, Set[str]] = {} # path -> set(kind:target)
|
||||
@@ -36,21 +35,28 @@ def propagate_section_edges(chunks: List[Chunk]) -> List[Chunk]:
|
||||
if not edges_to_add:
|
||||
continue
|
||||
|
||||
# Vorhandene Kanten (Typ:Ziel) in DIESEM Chunk ermitteln,
|
||||
# um Dopplungen (z.B. durch Callouts) zu vermeiden.
|
||||
existing_edges = parse_edges_robust(ch.text)
|
||||
|
||||
injections = []
|
||||
for e_str in edges_to_add:
|
||||
# Sortierung für deterministische Ergebnisse
|
||||
for e_str in sorted(list(edges_to_add)):
|
||||
# Wenn die Kante (Typ + Ziel) bereits vorhanden ist (egal welches Format),
|
||||
# überspringen wir die Injektion für diesen Chunk.
|
||||
if e_str in existing_edges:
|
||||
continue
|
||||
|
||||
kind, target = e_str.split(':', 1)
|
||||
# Nur injizieren, wenn die Kante nicht bereits im Text steht
|
||||
token = f"[[rel:{kind}|{target}]]"
|
||||
if token not in ch.text:
|
||||
injections.append(token)
|
||||
injections.append(f"[[rel:{kind}|{target}]]")
|
||||
|
||||
if injections:
|
||||
# Physische Anreicherung (Der v3.1.0 Qualitäts-Fix)
|
||||
# Physische Anreicherung
|
||||
# Triple-Newline für saubere Trennung im Embedding-Fenster
|
||||
block = "\n\n\n" + " ".join(injections)
|
||||
ch.text += block
|
||||
|
||||
# ENTSCHEIDEND: Auch ins Window schreiben, da Qdrant hier sucht!
|
||||
# Auch ins Window schreiben, da Qdrant hier sucht!
|
||||
if ch.window:
|
||||
ch.window += block
|
||||
else:
|
||||
|
||||
@@ -1,142 +1,166 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_strategies.py
|
||||
DESCRIPTION: Mathematische Splitting-Strategien.
|
||||
AUDIT v3.3.2: 100% Konformität zur 'by_heading' Spezifikation.
|
||||
- Implementiert Hybrid-Safety-Net (Sliding Window für Übergrößen).
|
||||
- Breadcrumb-Kontext im Window (H1 > H2).
|
||||
- Sliding Window mit H1-Kontext (Gold-Standard v3.1.0).
|
||||
DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.9.
|
||||
Implementiert das 'Pack-and-Carry-Over' Verfahren nach Regel 1-3.
|
||||
- Keine redundante Kanten-Injektion.
|
||||
- Strikte Einhaltung von Sektionsgrenzen via Look-Ahead.
|
||||
- Fix: Synchronisierung der Parameter mit dem Orchestrator (context_prefix).
|
||||
"""
|
||||
from typing import List, Dict, Any, Optional
|
||||
from .chunking_models import RawBlock, Chunk
|
||||
from .chunking_utils import estimate_tokens
|
||||
from .chunking_parser import split_sentences
|
||||
|
||||
def _create_context_win(doc_title: str, sec_title: Optional[str], text: str) -> str:
|
||||
def _create_win(context_prefix: str, sec_title: Optional[str], text: str) -> str:
|
||||
"""Baut den Breadcrumb-Kontext für das Embedding-Fenster."""
|
||||
parts = []
|
||||
if doc_title: parts.append(doc_title)
|
||||
if sec_title and sec_title != doc_title: parts.append(sec_title)
|
||||
parts = [context_prefix] if context_prefix else []
|
||||
# Verhindert Dopplung, falls der Context-Prefix (H1) bereits den Sektionsnamen enthält
|
||||
if sec_title and f"# {sec_title}" != context_prefix and sec_title not in (context_prefix or ""):
|
||||
parts.append(sec_title)
|
||||
prefix = " > ".join(parts)
|
||||
return f"{prefix}\n{text}".strip() if prefix else text
|
||||
|
||||
def strategy_sliding_window(blocks: List[RawBlock],
|
||||
config: Dict[str, Any],
|
||||
note_id: str,
|
||||
context_prefix: str = "") -> List[Chunk]:
|
||||
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
|
||||
"""
|
||||
Fasst Blöcke zusammen und schneidet bei 'target' Tokens.
|
||||
Ignoriert H2-Überschriften beim Splitting, um Kontext zu wahren.
|
||||
"""
|
||||
target = config.get("target", 400)
|
||||
max_tokens = config.get("max", 600)
|
||||
overlap_val = config.get("overlap", (50, 80))
|
||||
overlap = sum(overlap_val) // 2 if isinstance(overlap_val, tuple) else overlap_val
|
||||
|
||||
chunks: List[Chunk] = []
|
||||
buf: List[RawBlock] = []
|
||||
|
||||
def _add(txt, sec, path):
|
||||
idx = len(chunks)
|
||||
# H1-Kontext Präfix für das Window-Feld
|
||||
win = f"{context_prefix}\n{txt}".strip() if context_prefix else txt
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
section_title=sec, section_path=path,
|
||||
neighbors_prev=None, neighbors_next=None
|
||||
))
|
||||
|
||||
def flush():
|
||||
nonlocal buf
|
||||
if not buf: return
|
||||
text_body = "\n\n".join([b.text for b in buf])
|
||||
sec_title = buf[-1].section_title; sec_path = buf[-1].section_path
|
||||
|
||||
if estimate_tokens(text_body) <= max_tokens:
|
||||
_add(text_body, sec_title, sec_path)
|
||||
else:
|
||||
sents = split_sentences(text_body); cur_sents = []; cur_len = 0
|
||||
for s in sents:
|
||||
slen = estimate_tokens(s)
|
||||
if cur_len + slen > target and cur_sents:
|
||||
_add(" ".join(cur_sents), sec_title, sec_path)
|
||||
ov_s = []; ov_l = 0
|
||||
for os in reversed(cur_sents):
|
||||
if ov_l + estimate_tokens(os) < overlap:
|
||||
ov_s.insert(0, os); ov_l += estimate_tokens(os)
|
||||
else: break
|
||||
cur_sents = list(ov_s); cur_sents.append(s); cur_len = ov_l + slen
|
||||
else:
|
||||
cur_sents.append(s); cur_len += slen
|
||||
if cur_sents:
|
||||
_add(" ".join(cur_sents), sec_title, sec_path)
|
||||
buf = []
|
||||
|
||||
for b in blocks:
|
||||
# H2-Überschriften werden ignoriert, um den Zusammenhang zu wahren
|
||||
if b.kind == "heading": continue
|
||||
if estimate_tokens("\n\n".join([x.text for x in buf])) + estimate_tokens(b.text) >= target:
|
||||
flush()
|
||||
buf.append(b)
|
||||
flush()
|
||||
return chunks
|
||||
|
||||
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
|
||||
"""
|
||||
Splittet Text basierend auf Markdown-Überschriften mit Hybrid-Safety-Net.
|
||||
Universelle Heading-Strategie mit Carry-Over Logik.
|
||||
Synchronisiert auf context_prefix für Kompatibilität mit dem Orchestrator.
|
||||
"""
|
||||
smart_edge = config.get("enable_smart_edge_allocation", True)
|
||||
strict = config.get("strict_heading_split", False)
|
||||
target = config.get("target", 400)
|
||||
max_tokens = config.get("max", 600)
|
||||
split_level = config.get("split_level", 2)
|
||||
overlap = sum(config.get("overlap", (50, 80))) // 2
|
||||
overlap_cfg = config.get("overlap", (50, 80))
|
||||
overlap = sum(overlap_cfg) // 2 if isinstance(overlap_cfg, (list, tuple)) else overlap_cfg
|
||||
|
||||
chunks: List[Chunk] = []
|
||||
buf: List[str] = []
|
||||
cur_tokens = 0
|
||||
|
||||
def _add_to_chunks(txt, title, path):
|
||||
def _emit(txt, title, path):
|
||||
"""Schreibt den finalen Chunk ohne Text-Modifikationen."""
|
||||
idx = len(chunks)
|
||||
win = _create_context_win(doc_title, title, txt)
|
||||
win = _create_win(context_prefix, title, txt)
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
section_title=title, section_path=path,
|
||||
neighbors_prev=None, neighbors_next=None
|
||||
section_title=title, section_path=path, neighbors_prev=None, neighbors_next=None
|
||||
))
|
||||
|
||||
def _flush(title, path):
|
||||
nonlocal buf, cur_tokens
|
||||
if not buf: return
|
||||
full_text = "\n\n".join(buf)
|
||||
if estimate_tokens(full_text) <= max_tokens:
|
||||
_add_to_chunks(full_text, title, path)
|
||||
else:
|
||||
sents = split_sentences(full_text); cur_sents = []; sub_len = 0
|
||||
for s in sents:
|
||||
slen = estimate_tokens(s)
|
||||
if sub_len + slen > target and cur_sents:
|
||||
_add_to_chunks(" ".join(cur_sents), title, path)
|
||||
ov_s = []; ov_l = 0
|
||||
for os in reversed(cur_sents):
|
||||
if ov_l + estimate_tokens(os) < overlap:
|
||||
ov_s.insert(0, os); ov_l += estimate_tokens(os)
|
||||
else: break
|
||||
cur_sents = list(ov_s); cur_sents.append(s); sub_len = ov_l + slen
|
||||
else: cur_sents.append(s); sub_len += slen
|
||||
if cur_sents: _add_to_chunks(" ".join(cur_sents), title, path)
|
||||
buf = []; cur_tokens = 0
|
||||
|
||||
# --- SCHRITT 1: Gruppierung in atomare Sektions-Einheiten ---
|
||||
sections: List[Dict[str, Any]] = []
|
||||
curr_blocks = []
|
||||
for b in blocks:
|
||||
if b.kind == "heading":
|
||||
if b.level < split_level: _flush(b.section_title, b.section_path)
|
||||
elif b.level == split_level:
|
||||
if strict or cur_tokens >= target: _flush(b.section_title, b.section_path)
|
||||
if b.kind == "heading" and b.level <= split_level:
|
||||
if curr_blocks:
|
||||
sections.append({
|
||||
"text": "\n\n".join([x.text for x in curr_blocks]),
|
||||
"meta": curr_blocks[0],
|
||||
"is_empty": len(curr_blocks) == 1 and curr_blocks[0].kind == "heading"
|
||||
})
|
||||
curr_blocks = [b]
|
||||
else:
|
||||
curr_blocks.append(b)
|
||||
if curr_blocks:
|
||||
sections.append({
|
||||
"text": "\n\n".join([x.text for x in curr_blocks]),
|
||||
"meta": curr_blocks[0],
|
||||
"is_empty": len(curr_blocks) == 1 and curr_blocks[0].kind == "heading"
|
||||
})
|
||||
|
||||
# --- SCHRITT 2: Verarbeitung der Queue ---
|
||||
queue = list(sections)
|
||||
current_chunk_text = ""
|
||||
current_meta = {"title": None, "path": "/"}
|
||||
|
||||
# Bestimmung des Modus: Hard-Split wenn smart_edge=False ODER strict=True
|
||||
is_hard_split_mode = (not smart_edge) or (strict)
|
||||
|
||||
while queue:
|
||||
item = queue.pop(0)
|
||||
item_text = item["text"]
|
||||
|
||||
# Initialisierung für neuen Chunk
|
||||
if not current_chunk_text:
|
||||
current_meta["title"] = item["meta"].section_title
|
||||
current_meta["path"] = item["meta"].section_path
|
||||
|
||||
# FALL A: HARD SPLIT MODUS
|
||||
if is_hard_split_mode:
|
||||
# Leere Überschriften (z.B. H1 direkt vor H2) verbleiben am nächsten Chunk
|
||||
if item.get("is_empty", False) and queue:
|
||||
current_chunk_text = (current_chunk_text + "\n\n" + item_text).strip()
|
||||
continue
|
||||
|
||||
combined = (current_chunk_text + "\n\n" + item_text).strip()
|
||||
# Wenn durch Verschmelzung das Limit gesprengt würde, vorher flashen
|
||||
if estimate_tokens(combined) > max_tokens and current_chunk_text:
|
||||
_emit(current_chunk_text, current_meta["title"], current_meta["path"])
|
||||
current_chunk_text = item_text
|
||||
else:
|
||||
current_chunk_text = combined
|
||||
|
||||
# Im Hard-Split wird nach jeder Sektion geflasht
|
||||
_emit(current_chunk_text, current_meta["title"], current_meta["path"])
|
||||
current_chunk_text = ""
|
||||
continue
|
||||
bt = estimate_tokens(b.text)
|
||||
if cur_tokens + bt > max_tokens and buf: _flush(b.section_title, b.section_path)
|
||||
buf.append(b.text); cur_tokens += bt
|
||||
|
||||
# FALL B: SMART MODE (Regel 1-3)
|
||||
combined_text = (current_chunk_text + "\n\n" + item_text).strip() if current_chunk_text else item_text
|
||||
combined_est = estimate_tokens(combined_text)
|
||||
|
||||
if combined_est <= max_tokens:
|
||||
# Regel 1 & 2: Passt rein laut Schätzung -> Aufnehmen
|
||||
current_chunk_text = combined_text
|
||||
else:
|
||||
if current_chunk_text:
|
||||
# Regel 2: Flashen an Sektionsgrenze, Item zurücklegen
|
||||
_emit(current_chunk_text, current_meta["title"], current_meta["path"])
|
||||
current_chunk_text = ""
|
||||
queue.insert(0, item)
|
||||
else:
|
||||
# Regel 3: Einzelne Sektion zu groß -> Smart Zerlegung
|
||||
sents = split_sentences(item_text)
|
||||
header_prefix = item["meta"].text if item["meta"].kind == "heading" else ""
|
||||
|
||||
take_sents = []; take_len = 0
|
||||
while sents:
|
||||
s = sents.pop(0); slen = estimate_tokens(s)
|
||||
if take_len + slen > target and take_sents:
|
||||
sents.insert(0, s); break
|
||||
take_sents.append(s); take_len += slen
|
||||
|
||||
_emit(" ".join(take_sents), current_meta["title"], current_meta["path"])
|
||||
|
||||
if sents:
|
||||
remainder = " ".join(sents)
|
||||
# Kontext-Erhalt: Überschrift für den Rest wiederholen
|
||||
if header_prefix and not remainder.startswith(header_prefix):
|
||||
remainder = header_prefix + "\n\n" + remainder
|
||||
# Carry-Over: Rest wird vorne in die Queue geschoben
|
||||
queue.insert(0, {"text": remainder, "meta": item["meta"], "is_split": True})
|
||||
|
||||
if current_chunk_text:
|
||||
_emit(current_chunk_text, current_meta["title"], current_meta["path"])
|
||||
|
||||
return chunks
|
||||
|
||||
def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
|
||||
"""Standard-Sliding-Window für flache Texte ohne Sektionsfokus."""
|
||||
target = config.get("target", 400); max_tokens = config.get("max", 600)
|
||||
chunks: List[Chunk] = []; buf: List[RawBlock] = []
|
||||
|
||||
for b in blocks:
|
||||
b_tokens = estimate_tokens(b.text)
|
||||
curr_tokens = sum(estimate_tokens(x.text) for x in buf) if buf else 0
|
||||
if curr_tokens + b_tokens > max_tokens and buf:
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=curr_tokens, section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
|
||||
buf = []
|
||||
buf.append(b)
|
||||
|
||||
if buf:
|
||||
last_b = blocks[-1] if blocks else None
|
||||
_flush(last_b.section_title if last_b else None, last_b.section_path if last_b else "/")
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=estimate_tokens(txt), section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
|
||||
|
||||
return chunks
|
||||
@@ -3,7 +3,7 @@ FILE: app/core/database/qdrant.py
|
||||
DESCRIPTION: Qdrant-Client Factory und Schema-Management.
|
||||
Erstellt Collections und Payload-Indizes.
|
||||
MODULARISIERUNG: Verschoben in das database-Paket für WP-14.
|
||||
VERSION: 2.2.1
|
||||
VERSION: 2.2.2 (WP-Fix: Index für target_section)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, dataclasses, os
|
||||
"""
|
||||
@@ -124,7 +124,7 @@ def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||
Stellt sicher, dass alle benötigten Payload-Indizes für die Suche existieren.
|
||||
- notes: note_id, type, title, updated, tags
|
||||
- chunks: note_id, chunk_id, index, type, tags
|
||||
- edges: note_id, kind, scope, source_id, target_id, chunk_id
|
||||
- edges: note_id, kind, scope, source_id, target_id, chunk_id, target_section
|
||||
"""
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
|
||||
@@ -156,6 +156,8 @@ def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||
("source_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("target_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||
# NEU: Index für Section-Links (WP-15b)
|
||||
("target_section", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, edges, field, schema)
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
FILE: app/core/database/qdrant_points.py
|
||||
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges) in PointStructs und generiert deterministische UUIDs.
|
||||
VERSION: 1.5.0
|
||||
VERSION: 1.5.1 (WP-Fix: Explicit Target Section Support)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, uuid, os
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
LAST_ANALYSIS: 2025-12-29
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
@@ -46,16 +46,25 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
|
||||
return chunks_col, points
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
"""Normalisiert Edge-Felder und sichert Schema-Konformität."""
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||
|
||||
# WP-Fix: target_section explizit durchreichen
|
||||
target_section = pl.get("target_section")
|
||||
|
||||
pl.setdefault("kind", kind)
|
||||
pl.setdefault("source_id", source_id)
|
||||
pl.setdefault("target_id", target_id)
|
||||
|
||||
if seq is not None and "seq" not in pl:
|
||||
pl["seq"] = seq
|
||||
|
||||
if target_section is not None:
|
||||
pl["target_section"] = target_section
|
||||
|
||||
return pl
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/derive_edges.py
|
||||
DESCRIPTION: Facade für das neue graph Package.
|
||||
WP-14: Modularisierung abgeschlossen.
|
||||
VERSION: 2.2.0
|
||||
"""
|
||||
from .graph.graph_derive_edges import build_edges_for_note
|
||||
from .graph.graph_utils import PROVENANCE_PRIORITY
|
||||
|
||||
__all__ = ["build_edges_for_note", "PROVENANCE_PRIORITY"]
|
||||
@@ -1,10 +1,14 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_derive_edges.py
|
||||
DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
|
||||
AUDIT:
|
||||
- Nutzt parse_link_target
|
||||
- Übergibt Section als 'variant' an ID-Gen
|
||||
- Dedup basiert jetzt auf Edge-ID (erlaubt Multigraph für Sections)
|
||||
"""
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
from .graph_utils import (
|
||||
_get, _edge, _mk_edge_id, _dedupe_seq,
|
||||
_get, _edge, _mk_edge_id, _dedupe_seq, parse_link_target,
|
||||
PROVENANCE_PRIORITY, load_types_registry, get_edge_defaults_for
|
||||
)
|
||||
from .graph_extractors import (
|
||||
@@ -53,47 +57,85 @@ def build_edges_for_note(
|
||||
|
||||
# Typed & Candidate Pool (WP-15b Integration)
|
||||
typed, rem = extract_typed_relations(raw)
|
||||
for k, t in typed:
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id(k, cid, t, "chunk", "inline:rel"),
|
||||
for k, raw_t in typed:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
if not t: continue
|
||||
|
||||
payload = {
|
||||
"chunk_id": cid,
|
||||
# Variant=sec sorgt für eindeutige ID pro Abschnitt
|
||||
"edge_id": _mk_edge_id(k, cid, t, "chunk", "inline:rel", variant=sec),
|
||||
"provenance": "explicit", "rule_id": "inline:rel", "confidence": PROVENANCE_PRIORITY["inline:rel"]
|
||||
}))
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
pool = ch.get("candidate_pool") or ch.get("candidate_edges") or []
|
||||
for cand in pool:
|
||||
t, k, p = cand.get("to"), cand.get("kind", "related_to"), cand.get("provenance", "semantic_ai")
|
||||
raw_t, k, p = cand.get("to"), cand.get("kind", "related_to"), cand.get("provenance", "semantic_ai")
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
if t:
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id(k, cid, t, "chunk", f"candidate:{p}"),
|
||||
payload = {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, cid, t, "chunk", f"candidate:{p}", variant=sec),
|
||||
"provenance": p, "rule_id": f"candidate:{p}", "confidence": PROVENANCE_PRIORITY.get(p, 0.90)
|
||||
}))
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
# Callouts & Wikilinks
|
||||
call_pairs, rem2 = extract_callout_relations(rem)
|
||||
for k, t in call_pairs:
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id(k, cid, t, "chunk", "callout:edge"),
|
||||
for k, raw_t in call_pairs:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
if not t: continue
|
||||
|
||||
payload = {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, cid, t, "chunk", "callout:edge", variant=sec),
|
||||
"provenance": "explicit", "rule_id": "callout:edge", "confidence": PROVENANCE_PRIORITY["callout:edge"]
|
||||
}))
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
refs = extract_wikilinks(rem2)
|
||||
for r in refs:
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, {
|
||||
"chunk_id": cid, "ref_text": r, "edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink"),
|
||||
for raw_r in refs:
|
||||
r, sec = parse_link_target(raw_r, note_id)
|
||||
if not r: continue
|
||||
|
||||
payload = {
|
||||
"chunk_id": cid, "ref_text": raw_r,
|
||||
"edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink", variant=sec),
|
||||
"provenance": "explicit", "rule_id": "explicit:wikilink", "confidence": PROVENANCE_PRIORITY["explicit:wikilink"]
|
||||
}))
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, payload))
|
||||
|
||||
for rel in defaults:
|
||||
if rel != "references":
|
||||
edges.append(_edge(rel, "chunk", cid, r, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{rel}"),
|
||||
def_payload = {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{rel}", variant=sec),
|
||||
"provenance": "rule", "rule_id": f"edge_defaults:{rel}", "confidence": PROVENANCE_PRIORITY["edge_defaults"]
|
||||
}))
|
||||
refs_all.extend(refs)
|
||||
}
|
||||
if sec: def_payload["target_section"] = sec
|
||||
edges.append(_edge(rel, "chunk", cid, r, note_id, def_payload))
|
||||
|
||||
# Für Note-Scope Sammlung nutzen wir den Original-String zur Dedup, aber gesäubert
|
||||
refs_all.extend([parse_link_target(r, note_id)[0] for r in refs])
|
||||
|
||||
# 3) Note-Scope & De-Duplizierung
|
||||
if include_note_scope_refs:
|
||||
refs_note = _dedupe_seq((refs_all or []) + (note_level_references or []))
|
||||
# refs_all ist jetzt schon gesäubert (nur Targets)
|
||||
# note_level_references müssen auch gesäubert werden
|
||||
cleaned_note_refs = [parse_link_target(r, note_id)[0] for r in (note_level_references or [])]
|
||||
refs_note = _dedupe_seq((refs_all or []) + cleaned_note_refs)
|
||||
|
||||
for r in refs_note:
|
||||
if not r: continue
|
||||
edges.append(_edge("references", "note", note_id, r, note_id, {
|
||||
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||
"provenance": "explicit", "confidence": PROVENANCE_PRIORITY["explicit:note_scope"]
|
||||
@@ -103,10 +145,13 @@ def build_edges_for_note(
|
||||
"provenance": "rule", "confidence": PROVENANCE_PRIORITY["derived:backlink"]
|
||||
}))
|
||||
|
||||
unique_map: Dict[Tuple[str, str, str], dict] = {}
|
||||
# Deduplizierung: Wir nutzen jetzt die EDGE-ID als Schlüssel.
|
||||
# Da die Edge-ID nun 'variant' (Section) enthält, bleiben unterschiedliche Sections erhalten.
|
||||
unique_map: Dict[str, dict] = {}
|
||||
for e in edges:
|
||||
key = (str(e.get("source_id")), str(e.get("target_id")), str(e.get("kind")))
|
||||
if key not in unique_map or e.get("confidence", 0) > unique_map[key].get("confidence", 0):
|
||||
unique_map[key] = e
|
||||
eid = e["edge_id"]
|
||||
# Bei Konflikt (gleiche ID = exakt gleiche Kante und Section) gewinnt die höhere Confidence
|
||||
if eid not in unique_map or e.get("confidence", 0) > unique_map[eid].get("confidence", 0):
|
||||
unique_map[eid] = e
|
||||
|
||||
return list(unique_map.values())
|
||||
@@ -1,25 +1,36 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_extractors.py
|
||||
DESCRIPTION: Regex-basierte Extraktion von Relationen aus Text.
|
||||
AUDIT:
|
||||
- Regex für Wikilinks liberalisiert (Umlaute, Sonderzeichen).
|
||||
- Callout-Parser erweitert für Multi-Line-Listen und Header-Typen.
|
||||
"""
|
||||
import re
|
||||
from typing import List, Tuple
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||
# Erlaube alle Zeichen außer ']' im Target (fängt Umlaute, Emojis, '&', '#' ab)
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([^\]]+)\]\]")
|
||||
|
||||
_REL_PIPE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s*\|\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
_REL_SPACE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s+(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
_REL_TEXT = re.compile(r"rel\s*:\s*(?P<kind>[a-z_]+)\s*\[\[\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
|
||||
_CALLOUT_START = re.compile(r"^\s*>\s*\[!edge\]\s*(.*)$", re.IGNORECASE)
|
||||
# Erkennt "kind: targets..."
|
||||
_REL_LINE = re.compile(r"^(?P<kind>[a-z_]+)\s*:\s*(?P<targets>.+?)\s*$", re.IGNORECASE)
|
||||
_WIKILINKS_IN_LINE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||
# Erkennt reine Typen (z.B. "depends_on" im Header)
|
||||
_SIMPLE_KIND = re.compile(r"^[a-z_]+$", re.IGNORECASE)
|
||||
|
||||
def extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""Extrahiert [[rel:KIND|Target]]."""
|
||||
def extract_typed_relations(text: str) -> Tuple[List[Tuple[str, str]], str]:
|
||||
"""
|
||||
Findet Inline-Relationen wie [[rel:depends_on Target]].
|
||||
Gibt (Liste[(kind, target)], bereinigter_text) zurück.
|
||||
"""
|
||||
if not text: return [], ""
|
||||
pairs = []
|
||||
def _collect(m):
|
||||
k, t = (m.group("kind") or "").strip().lower(), (m.group("target") or "").strip()
|
||||
if k and t: pairs.append((k, t))
|
||||
k, t = m.group("kind").strip().lower(), m.group("target").strip()
|
||||
pairs.append((k, t))
|
||||
return ""
|
||||
text = _REL_PIPE.sub(_collect, text)
|
||||
text = _REL_SPACE.sub(_collect, text)
|
||||
@@ -27,29 +38,90 @@ def extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
return pairs, text
|
||||
|
||||
def extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""Verarbeitet Obsidian [!edge]-Callouts."""
|
||||
"""
|
||||
Verarbeitet Obsidian [!edge]-Callouts.
|
||||
Unterstützt zwei Formate:
|
||||
1. Explizit: "kind: [[Target]]"
|
||||
2. Implizit (Header): "> [!edge] kind" gefolgt von "[[Target]]" Zeilen
|
||||
"""
|
||||
if not text: return [], text
|
||||
lines = text.splitlines(); out_pairs, keep_lines, i = [], [], 0
|
||||
lines = text.splitlines()
|
||||
out_pairs = []
|
||||
keep_lines = []
|
||||
i = 0
|
||||
|
||||
while i < len(lines):
|
||||
m = _CALLOUT_START.match(lines[i])
|
||||
line = lines[i]
|
||||
m = _CALLOUT_START.match(line)
|
||||
if not m:
|
||||
keep_lines.append(lines[i]); i += 1; continue
|
||||
block_lines = [m.group(1)] if m.group(1).strip() else []
|
||||
keep_lines.append(line)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Callout-Block gefunden. Wir sammeln alle relevanten Zeilen.
|
||||
block_lines = []
|
||||
|
||||
# Header Content prüfen (z.B. "type" aus "> [!edge] type")
|
||||
header_raw = m.group(1).strip()
|
||||
if header_raw:
|
||||
block_lines.append(header_raw)
|
||||
|
||||
i += 1
|
||||
while i < len(lines) and lines[i].lstrip().startswith('>'):
|
||||
block_lines.append(lines[i].lstrip()[1:].lstrip()); i += 1
|
||||
# Entferne '>' und führende Leerzeichen
|
||||
content = lines[i].lstrip()[1:].lstrip()
|
||||
if content:
|
||||
block_lines.append(content)
|
||||
i += 1
|
||||
|
||||
# Verarbeitung des Blocks
|
||||
current_kind = None
|
||||
|
||||
# Heuristik: Ist die allererste Zeile (meist aus dem Header) ein reiner Typ?
|
||||
# Dann setzen wir diesen als Default für den Block.
|
||||
if block_lines:
|
||||
first = block_lines[0]
|
||||
# Wenn es NICHT wie "Key: Value" aussieht, aber wie ein Wort:
|
||||
if not _REL_LINE.match(first) and _SIMPLE_KIND.match(first):
|
||||
current_kind = first.lower()
|
||||
|
||||
for bl in block_lines:
|
||||
# 1. Prüfen auf explizites "Kind: Targets" (überschreibt Header-Typ für diese Zeile)
|
||||
mrel = _REL_LINE.match(bl)
|
||||
if not mrel: continue
|
||||
kind, targets = mrel.group("kind").strip().lower(), mrel.group("targets") or ""
|
||||
found = _WIKILINKS_IN_LINE.findall(targets)
|
||||
if mrel:
|
||||
line_kind = mrel.group("kind").strip().lower()
|
||||
targets = mrel.group("targets")
|
||||
|
||||
# Links extrahieren
|
||||
found = _WIKILINK_RE.findall(targets)
|
||||
if found:
|
||||
for t in found: out_pairs.append((line_kind, t.strip()))
|
||||
else:
|
||||
# Fallback für kommagetrennten Plaintext
|
||||
for raw in re.split(r"[,;]", targets):
|
||||
if raw.strip(): out_pairs.append((line_kind, raw.strip()))
|
||||
|
||||
# Wenn wir eine explizite Zeile gefunden haben, aktualisieren wir NICHT
|
||||
# den current_kind für nachfolgende Zeilen (Design-Entscheidung: lokal scope),
|
||||
# oder wir machen es doch?
|
||||
# Üblicher ist: Header setzt Default, Zeile überschreibt lokal.
|
||||
# Wir lassen current_kind also unangetastet.
|
||||
continue
|
||||
|
||||
# 2. Kein Key:Value Muster -> Prüfen auf Links, die den current_kind nutzen
|
||||
found = _WIKILINK_RE.findall(bl)
|
||||
if found:
|
||||
for t in found: out_pairs.append((kind, t.strip()))
|
||||
else:
|
||||
for raw in re.split(r"[,;]", targets):
|
||||
if raw.strip(): out_pairs.append((kind, raw.strip()))
|
||||
if current_kind:
|
||||
for t in found: out_pairs.append((current_kind, t.strip()))
|
||||
else:
|
||||
# Link ohne Typ und ohne Header-Typ.
|
||||
# Wird ignoriert oder könnte als 'related_to' fallback dienen.
|
||||
# Aktuell: Ignorieren, um False Positives zu vermeiden.
|
||||
pass
|
||||
|
||||
return out_pairs, "\n".join(keep_lines)
|
||||
|
||||
def extract_wikilinks(text: str) -> List[str]:
|
||||
"""Extrahiert Standard-Wikilinks."""
|
||||
return [m.group(1).strip() for m in _WIKILINK_RE.finditer(text or "")]
|
||||
"""Findet Standard-Wikilinks [[Target]] oder [[Alias|Target]]."""
|
||||
if not text: return []
|
||||
return [m.strip() for m in _WIKILINK_RE.findall(text) if m.strip()]
|
||||
@@ -1,10 +1,11 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_utils.py
|
||||
DESCRIPTION: Basale Werkzeuge, ID-Generierung und Provenance-Konfiguration für den Graphen.
|
||||
AUDIT: Erweitert um parse_link_target für sauberes Section-Splitting (WP-Fix).
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
from typing import Iterable, List, Optional, Set, Any
|
||||
from typing import Iterable, List, Optional, Set, Any, Tuple
|
||||
|
||||
try:
|
||||
import yaml
|
||||
@@ -40,10 +41,19 @@ def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||
seen.add(s); out.append(s)
|
||||
return out
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||
"""Erzeugt eine deterministische 12-Byte ID mittels BLAKE2s."""
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None, variant: Optional[str] = None) -> str:
|
||||
"""
|
||||
Erzeugt eine deterministische 12-Byte ID mittels BLAKE2s.
|
||||
|
||||
WP-Fix: 'variant' (z.B. Section) fließt in den Hash ein, um mehrere Kanten
|
||||
zum gleichen Target-Node (aber unterschiedlichen Abschnitten) zu unterscheiden.
|
||||
"""
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id: base += f"|{rule_id}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
if variant:
|
||||
base += f"|{variant}" # <--- Hier entsteht die Eindeutigkeit für verschiedene Sections
|
||||
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||
@@ -59,6 +69,27 @@ def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, e
|
||||
if extra: pl.update(extra)
|
||||
return pl
|
||||
|
||||
def parse_link_target(raw: str, current_note_id: Optional[str] = None) -> Tuple[str, Optional[str]]:
|
||||
"""
|
||||
Zerlegt einen Link (z.B. 'Note#Section') in Target-ID und Section.
|
||||
Behandelt Self-Links ('#Section'), indem current_note_id eingesetzt wird.
|
||||
|
||||
Returns:
|
||||
(target_id, target_section)
|
||||
"""
|
||||
if not raw:
|
||||
return "", None
|
||||
|
||||
parts = raw.split("#", 1)
|
||||
target = parts[0].strip()
|
||||
section = parts[1].strip() if len(parts) > 1 else None
|
||||
|
||||
# Handle Self-Link [[#Section]] -> target wird zu current_note_id
|
||||
if not target and section and current_note_id:
|
||||
target = current_note_id
|
||||
|
||||
return target, section
|
||||
|
||||
def load_types_registry() -> dict:
|
||||
"""Lädt die YAML-Registry."""
|
||||
p = os.getenv("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/graph_adapter.py
|
||||
DESCRIPTION: Facade für das neue graph Package (Adapter-Teil).
|
||||
WP-14: Modularisierung abgeschlossen.
|
||||
VERSION: 0.5.0
|
||||
"""
|
||||
from .graph.graph_subgraph import Subgraph, expand
|
||||
from .graph.graph_weights import EDGE_BASE_WEIGHTS
|
||||
|
||||
__all__ = ["Subgraph", "expand", "EDGE_BASE_WEIGHTS"]
|
||||
@@ -3,9 +3,8 @@ FILE: app/core/ingestion/ingestion_note_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für mindnet_notes.
|
||||
FEATURES:
|
||||
- Multi-Hash (body/full) für flexible Change Detection.
|
||||
- Fix v2.4.4: Integration der zentralen Registry (WP-14) für konsistente Defaults.
|
||||
VERSION: 2.4.4
|
||||
STATUS: Active
|
||||
- Fix v2.4.5: Präzise Hash-Logik für Profil-Änderungen.
|
||||
- Integration der zentralen Registry (WP-14).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, Tuple, Optional
|
||||
@@ -45,14 +44,23 @@ def _compute_hash(content: str) -> str:
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
|
||||
"""Generiert den Hash-Input-String basierend auf Body oder Metadaten."""
|
||||
body = str(n.get("body") or "")
|
||||
"""
|
||||
Generiert den Hash-Input-String basierend auf Body oder Metadaten.
|
||||
Fix: Inkludiert nun alle entscheidungsrelevanten Profil-Parameter.
|
||||
"""
|
||||
body = str(n.get("body") or "").strip()
|
||||
if mode == "body": return body
|
||||
if mode == "full":
|
||||
fm = n.get("frontmatter") or {}
|
||||
meta_parts = []
|
||||
# Sortierte Liste für deterministische Hashes
|
||||
for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]):
|
||||
# Wir inkludieren alle Felder, die das Chunking oder Retrieval beeinflussen
|
||||
# Jede Änderung hier führt nun zwingend zu einem neuen Full-Hash
|
||||
keys = [
|
||||
"title", "type", "status", "tags",
|
||||
"chunking_profile", "chunk_profile",
|
||||
"retriever_weight", "split_level", "strict_heading_split"
|
||||
]
|
||||
for k in sorted(keys):
|
||||
val = fm.get(k)
|
||||
if val is not None: meta_parts.append(f"{k}:{val}")
|
||||
return f"{'|'.join(meta_parts)}||{body}"
|
||||
@@ -79,11 +87,11 @@ def _cfg_defaults(reg: dict) -> dict:
|
||||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Baut das Note-Payload inklusive Multi-Hash und Audit-Validierung.
|
||||
WP-14: Nutzt nun die zentrale Registry für alle Fallbacks.
|
||||
WP-14: Nutzt die zentrale Registry für alle Fallbacks.
|
||||
"""
|
||||
n = _as_dict(note)
|
||||
|
||||
# Nutzt übergebene Registry oder lädt sie global
|
||||
# Registry & Context Settings
|
||||
reg = kwargs.get("types_cfg") or load_type_registry()
|
||||
hash_source = kwargs.get("hash_source", "parsed")
|
||||
hash_normalize = kwargs.get("hash_normalize", "canonical")
|
||||
@@ -96,7 +104,6 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
ingest_cfg = reg.get("ingestion_settings", {})
|
||||
|
||||
# --- retriever_weight Audit ---
|
||||
# Priorität: Frontmatter -> Typ-Config -> globale Config -> Env-Var
|
||||
default_rw = float(os.environ.get("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0))
|
||||
retriever_weight = fm.get("retriever_weight")
|
||||
if retriever_weight is None:
|
||||
@@ -107,14 +114,13 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
retriever_weight = default_rw
|
||||
|
||||
# --- chunk_profile Audit ---
|
||||
# Nutzt nun primär die ingestion_settings aus der Registry
|
||||
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||
if chunk_profile is None:
|
||||
chunk_profile = cfg_type.get("chunking_profile") or cfg_type.get("chunk_profile")
|
||||
if chunk_profile is None:
|
||||
chunk_profile = ingest_cfg.get("default_chunk_profile", cfg_def.get("chunking_profile", "sliding_standard"))
|
||||
|
||||
# --- edge_defaults ---
|
||||
# --- edge_defaults Audit ---
|
||||
edge_defaults = fm.get("edge_defaults")
|
||||
if edge_defaults is None:
|
||||
edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", []))
|
||||
@@ -138,21 +144,24 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
# --- MULTI-HASH ---
|
||||
# Generiert Hashes für Change Detection
|
||||
# Generiert Hashes für Change Detection (WP-15b)
|
||||
for mode in ["body", "full"]:
|
||||
content = _get_hash_source_content(n, mode)
|
||||
payload["hashes"][f"{mode}:{hash_source}:{hash_normalize}"] = _compute_hash(content)
|
||||
|
||||
# Metadaten Anreicherung
|
||||
# Metadaten Anreicherung (Tags, Aliases, Zeitstempel)
|
||||
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||
if tags: payload["tags"] = _ensure_list(tags)
|
||||
if fm.get("aliases"): payload["aliases"] = _ensure_list(fm.get("aliases"))
|
||||
|
||||
aliases = fm.get("aliases")
|
||||
if aliases: payload["aliases"] = _ensure_list(aliases)
|
||||
|
||||
for k in ("created", "modified", "date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v: payload[k] = str(v)
|
||||
|
||||
if n.get("body"): payload["fulltext"] = str(n["body"])
|
||||
if n.get("body"):
|
||||
payload["fulltext"] = str(n["body"])
|
||||
|
||||
# Final JSON Validation Audit
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
@@ -4,8 +4,8 @@ DESCRIPTION: Der zentrale IngestionService (Orchestrator).
|
||||
WP-14: Modularisierung der Datenbank-Ebene (app.core.database).
|
||||
WP-15b: Two-Pass Workflow mit globalem Kontext-Cache.
|
||||
WP-20/22: Cloud-Resilienz und Content-Lifecycle integriert.
|
||||
AUDIT v2.13.10: Umstellung auf app.core.database Infrastruktur.
|
||||
VERSION: 2.13.10
|
||||
AUDIT v2.13.12: Synchronisierung der Profil-Auflösung mit Registry-Defaults.
|
||||
VERSION: 2.13.12
|
||||
STATUS: Active
|
||||
"""
|
||||
import logging
|
||||
@@ -38,7 +38,7 @@ from .ingestion_chunk_payload import make_chunk_payloads
|
||||
|
||||
# Fallback für Edges (Struktur-Verknüpfung)
|
||||
try:
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
from app.core.graph.graph_derive_edges import build_edges_for_note
|
||||
except ImportError:
|
||||
def build_edges_for_note(*args, **kwargs): return []
|
||||
|
||||
@@ -60,6 +60,7 @@ class IngestionService:
|
||||
self.embedder = EmbeddingsClient()
|
||||
self.llm = LLMService()
|
||||
|
||||
# Festlegen, welcher Hash für die Change-Detection maßgeblich ist
|
||||
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||
self.batch_cache: Dict[str, NoteContext] = {} # WP-15b LocalBatchCache
|
||||
|
||||
@@ -130,12 +131,18 @@ class IngestionService:
|
||||
)
|
||||
note_id = note_pl["note_id"]
|
||||
|
||||
# Abgleich mit der Datenbank (Qdrant)
|
||||
old_payload = None if force_replace else fetch_note_payload(self.client, self.prefix, note_id)
|
||||
|
||||
# Prüfung gegen den konfigurierten Hash-Modus (body vs. full)
|
||||
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
||||
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
||||
new_hash = note_pl.get("hashes", {}).get(check_key)
|
||||
|
||||
# Check ob Chunks oder Kanten in der DB fehlen (Reparatur-Modus)
|
||||
c_miss, e_miss = artifacts_missing(self.client, self.prefix, note_id)
|
||||
|
||||
# Wenn Hash identisch und Artefakte vorhanden -> Skip
|
||||
if not (force_replace or not old_payload or old_hash != new_hash or c_miss or e_miss):
|
||||
return {**result, "status": "unchanged", "note_id": note_id}
|
||||
|
||||
@@ -146,36 +153,49 @@ class IngestionService:
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
edge_registry.ensure_latest()
|
||||
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
|
||||
|
||||
# Profil-Auflösung via Registry
|
||||
# FIX: Wir nutzen das Profil, das bereits in make_note_payload unter
|
||||
# Berücksichtigung der types.yaml (Registry) ermittelt wurde.
|
||||
profile = note_pl.get("chunk_profile", "sliding_standard")
|
||||
|
||||
chunk_cfg = get_chunk_config_by_profile(self.registry, profile, note_type)
|
||||
enable_smart = chunk_cfg.get("enable_smart_edge_allocation", False)
|
||||
|
||||
# WP-15b: Chunker-Aufruf bereitet Candidate-Pool vor
|
||||
# WP-15b: Chunker-Aufruf bereitet den Candidate-Pool pro Chunk vor.
|
||||
# assemble_chunks führt intern auch die Propagierung durch.
|
||||
chunks = await assemble_chunks(note_id, body_text, note_type, config=chunk_cfg)
|
||||
|
||||
# Semantische Kanten-Validierung (Smart Edge Allocation)
|
||||
for ch in chunks:
|
||||
filtered = []
|
||||
for cand in getattr(ch, "candidate_pool", []):
|
||||
# WP-15b: Nur global_pool Kandidaten erfordern binäre Validierung
|
||||
# Nur global_pool Kandidaten (aus dem Pool am Ende) erfordern KI-Validierung
|
||||
if cand.get("provenance") == "global_pool" and enable_smart:
|
||||
if await validate_edge_candidate(ch.text, cand, self.batch_cache, self.llm, self.settings.MINDNET_LLM_PROVIDER):
|
||||
filtered.append(cand)
|
||||
else:
|
||||
# Explizite Kanten (Wikilinks/Callouts) werden ungeprüft übernommen
|
||||
filtered.append(cand)
|
||||
ch.candidate_pool = filtered
|
||||
|
||||
# Payload-Erstellung via interne Module
|
||||
# Payload-Erstellung für die Chunks
|
||||
chunk_pls = make_chunk_payloads(
|
||||
fm, note_pl["path"], chunks, file_path=file_path,
|
||||
types_cfg=self.registry
|
||||
)
|
||||
|
||||
# Vektorisierung der Fenster-Texte
|
||||
vecs = await self.embedder.embed_documents([c.get("window") or "" for c in chunk_pls]) if chunk_pls else []
|
||||
|
||||
# Kanten-Aggregation
|
||||
# Aggregation aller finalen Kanten (Edges)
|
||||
edges = build_edges_for_note(
|
||||
note_id, chunk_pls,
|
||||
note_level_references=note_pl.get("references", []),
|
||||
include_note_scope_refs=note_scope_refs
|
||||
)
|
||||
|
||||
# Kanten-Typen via Registry validieren/auflösen
|
||||
for e in edges:
|
||||
e["kind"] = edge_registry.resolve(
|
||||
e.get("kind", "related_to"),
|
||||
@@ -184,16 +204,20 @@ class IngestionService:
|
||||
)
|
||||
|
||||
# 4. DB Upsert via modularisierter Points-Logik
|
||||
# WICHTIG: Wenn sich der Inhalt geändert hat, löschen wir erst alle alten Fragmente.
|
||||
if purge_before and old_payload:
|
||||
purge_artifacts(self.client, self.prefix, note_id)
|
||||
|
||||
# Speichern der Haupt-Note
|
||||
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
||||
upsert_batch(self.client, n_name, n_pts)
|
||||
|
||||
# Speichern der Chunks
|
||||
if chunk_pls and vecs:
|
||||
c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_chunks", c_pts)
|
||||
|
||||
# Speichern der Kanten
|
||||
if edges:
|
||||
e_pts = points_for_edges(self.prefix, edges)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_edges", e_pts)
|
||||
@@ -217,4 +241,5 @@ class IngestionService:
|
||||
with open(target_path, "w", encoding="utf-8") as f:
|
||||
f.write(markdown_content)
|
||||
await asyncio.sleep(0.1)
|
||||
# Triggert sofortigen Import mit force_replace/purge_before
|
||||
return await self.process_file(file_path=target_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)
|
||||
@@ -1,22 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/qdrant.py
|
||||
DESCRIPTION: Proxy-Modul zur Aufrechterhaltung der Abwärtskompatibilität (WP-14).
|
||||
Leitet alle Aufrufe an das neue database-Paket weiter.
|
||||
STATUS: Proxy (Legacy-Support)
|
||||
"""
|
||||
from .database.qdrant import (
|
||||
QdrantConfig,
|
||||
get_client,
|
||||
ensure_collections,
|
||||
ensure_payload_indexes,
|
||||
collection_names
|
||||
)
|
||||
|
||||
# Re-Export für 100% Kompatibilität
|
||||
__all__ = [
|
||||
"QdrantConfig",
|
||||
"get_client",
|
||||
"ensure_collections",
|
||||
"ensure_payload_indexes",
|
||||
"collection_names",
|
||||
]
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/qdrant_points.py
|
||||
DESCRIPTION: Proxy-Modul zur Aufrechterhaltung der Abwärtskompatibilität (WP-14).
|
||||
Leitet Point-Operationen an das neue database-Paket weiter.
|
||||
STATUS: Proxy (Legacy-Support)
|
||||
"""
|
||||
from .database.qdrant_points import (
|
||||
points_for_note,
|
||||
points_for_chunks,
|
||||
points_for_edges,
|
||||
upsert_batch,
|
||||
get_edges_for_sources,
|
||||
search_chunks_by_vector
|
||||
)
|
||||
|
||||
# Re-Export für 100% Kompatibilität
|
||||
__all__ = [
|
||||
"points_for_note",
|
||||
"points_for_chunks",
|
||||
"points_for_edges",
|
||||
"upsert_batch",
|
||||
"get_edges_for_sources",
|
||||
"search_chunks_by_vector"
|
||||
]
|
||||
@@ -25,10 +25,10 @@ import app.core.database.qdrant as qdr
|
||||
import app.core.database.qdrant_points as qp
|
||||
|
||||
import app.services.embeddings_client as ec
|
||||
import app.core.graph_adapter as ga
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
|
||||
# Mathematische Engine importieren (Bleibt vorerst in app.core)
|
||||
from app.core.retriever_scoring import get_weights, compute_wp22_score
|
||||
# Mathematische Engine importieren
|
||||
from app.core.retrieval.retriever_scoring import get_weights, compute_wp22_score
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/retriever.py
|
||||
DESCRIPTION: Proxy-Modul zur Aufrechterhaltung der Abwärtskompatibilität (WP-14).
|
||||
Leitet Retrieval-Anfragen an das neue retrieval-Paket weiter.
|
||||
STATUS: Proxy (Legacy-Support)
|
||||
"""
|
||||
from .retrieval.retriever import (
|
||||
Retriever,
|
||||
hybrid_retrieve,
|
||||
semantic_retrieve
|
||||
)
|
||||
|
||||
# Re-Export für 100% Kompatibilität
|
||||
__all__ = ["Retriever", "hybrid_retrieve", "semantic_retrieve"]
|
||||
@@ -1,18 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/retriever_scoring.py
|
||||
DESCRIPTION: Proxy-Modul zur Aufrechterhaltung der Abwärtskompatibilität (WP-14).
|
||||
Leitet Scoring-Berechnungen an das neue retrieval-Paket weiter.
|
||||
STATUS: Proxy (Legacy-Support)
|
||||
"""
|
||||
from .retrieval.retriever_scoring import (
|
||||
get_weights,
|
||||
compute_wp22_score,
|
||||
get_status_multiplier
|
||||
)
|
||||
|
||||
# Re-Export für 100% Kompatibilität
|
||||
__all__ = [
|
||||
"get_weights",
|
||||
"compute_wp22_score",
|
||||
"get_status_multiplier"
|
||||
]
|
||||
@@ -69,20 +69,35 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
search_term = st.text_input("Suche Notiz", placeholder="Titel eingeben...", key="cy_search")
|
||||
|
||||
if search_term:
|
||||
hits, _ = graph_service.client.scroll(
|
||||
collection_name=f"{COLLECTION_PREFIX}_notes",
|
||||
limit=10,
|
||||
scroll_filter=models.Filter(must=[models.FieldCondition(key="title", match=models.MatchText(text=search_term))])
|
||||
)
|
||||
options = {h.payload['title']: h.payload['note_id'] for h in hits}
|
||||
|
||||
if options:
|
||||
selected_title = st.selectbox("Ergebnisse:", list(options.keys()), key="cy_select")
|
||||
if st.button("Laden", use_container_width=True, key="cy_load"):
|
||||
new_id = options[selected_title]
|
||||
st.session_state.graph_center_id = new_id
|
||||
st.session_state.graph_inspected_id = new_id
|
||||
st.rerun()
|
||||
try:
|
||||
hits, _ = graph_service.client.scroll(
|
||||
collection_name=f"{COLLECTION_PREFIX}_notes",
|
||||
limit=10,
|
||||
scroll_filter=models.Filter(must=[models.FieldCondition(key="title", match=models.MatchText(text=search_term))])
|
||||
)
|
||||
options = {}
|
||||
for h in hits:
|
||||
if h.payload and 'title' in h.payload and 'note_id' in h.payload:
|
||||
title = h.payload['title']
|
||||
note_id = h.payload['note_id']
|
||||
# Vermeide Duplikate (falls mehrere Chunks/Notes denselben Titel haben)
|
||||
if title not in options:
|
||||
options[title] = note_id
|
||||
|
||||
if options:
|
||||
selected_title = st.selectbox("Ergebnisse:", list(options.keys()), key="cy_select")
|
||||
if st.button("Laden", use_container_width=True, key="cy_load"):
|
||||
new_id = options[selected_title]
|
||||
st.session_state.graph_center_id = new_id
|
||||
st.session_state.graph_inspected_id = new_id
|
||||
st.rerun()
|
||||
else:
|
||||
# Zeige Info, wenn keine Ergebnisse gefunden wurden
|
||||
st.info(f"Keine Notizen mit '{search_term}' im Titel gefunden.")
|
||||
except Exception as e:
|
||||
st.error(f"Fehler bei der Suche: {e}")
|
||||
import traceback
|
||||
st.code(traceback.format_exc())
|
||||
|
||||
st.divider()
|
||||
|
||||
@@ -138,6 +153,26 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
)
|
||||
# 2. Detail Daten (Inspector)
|
||||
inspected_data = graph_service.get_note_with_full_content(inspected_id)
|
||||
|
||||
# DEBUG: Zeige Debug-Informationen
|
||||
with st.expander("🔍 Debug-Informationen", expanded=False):
|
||||
st.write(f"**Gefundene Knoten:** {len(nodes_data) if nodes_data else 0}")
|
||||
st.write(f"**Gefundene Kanten:** {len(edges_data) if edges_data else 0}")
|
||||
if nodes_data:
|
||||
st.write("**Knoten-IDs:**")
|
||||
for n in nodes_data[:10]:
|
||||
nid = getattr(n, 'id', 'N/A')
|
||||
st.write(f" - {nid}")
|
||||
if len(nodes_data) > 10:
|
||||
st.write(f" ... und {len(nodes_data) - 10} weitere")
|
||||
if edges_data:
|
||||
st.write("**Kanten:**")
|
||||
for e in edges_data[:10]:
|
||||
src = getattr(e, "source", "N/A")
|
||||
tgt = getattr(e, "to", getattr(e, "target", "N/A"))
|
||||
st.write(f" - {src} -> {tgt}")
|
||||
if len(edges_data) > 10:
|
||||
st.write(f" ... und {len(edges_data) - 10} weitere")
|
||||
|
||||
# --- ACTION BAR ---
|
||||
action_container = st.container()
|
||||
@@ -174,7 +209,12 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
st.markdown(f"**ID:** `{inspected_data.get('note_id')}`")
|
||||
st.markdown(f"**Typ:** `{inspected_data.get('type')}`")
|
||||
with col_i2:
|
||||
st.markdown(f"**Tags:** {', '.join(inspected_data.get('tags', []))}")
|
||||
tags = inspected_data.get('tags', [])
|
||||
if isinstance(tags, list):
|
||||
tags_str = ', '.join(tags) if tags else "Keine"
|
||||
else:
|
||||
tags_str = str(tags) if tags else "Keine"
|
||||
st.markdown(f"**Tags:** {tags_str}")
|
||||
path_check = "✅" if inspected_data.get('path') else "❌"
|
||||
st.markdown(f"**Pfad:** {path_check}")
|
||||
|
||||
@@ -189,12 +229,27 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
# --- GRAPH ELEMENTS ---
|
||||
cy_elements = []
|
||||
|
||||
# Validierung: Prüfe ob nodes_data vorhanden ist
|
||||
if not nodes_data:
|
||||
st.warning("⚠️ Keine Knoten gefunden. Bitte wähle eine andere Notiz.")
|
||||
# Zeige trotzdem den Inspector, falls Daten vorhanden
|
||||
if inspected_data:
|
||||
st.info(f"**Hinweis:** Die Notiz '{inspected_data.get('title', inspected_id)}' wurde gefunden, hat aber keine Verbindungen im Graphen.")
|
||||
return
|
||||
|
||||
# Erstelle Set aller Node-IDs für schnelle Validierung
|
||||
node_ids = {n.id for n in nodes_data if hasattr(n, 'id') and n.id}
|
||||
|
||||
# Nodes hinzufügen
|
||||
for n in nodes_data:
|
||||
if not hasattr(n, 'id') or not n.id:
|
||||
continue
|
||||
|
||||
is_center = (n.id == center_id)
|
||||
is_inspected = (n.id == inspected_id)
|
||||
|
||||
tooltip_text = n.title if n.title else n.label
|
||||
display_label = n.label
|
||||
tooltip_text = getattr(n, 'title', None) or getattr(n, 'label', '')
|
||||
display_label = getattr(n, 'label', str(n.id))
|
||||
if len(display_label) > 15 and " " in display_label:
|
||||
display_label = display_label.replace(" ", "\n", 1)
|
||||
|
||||
@@ -202,7 +257,7 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
"data": {
|
||||
"id": n.id,
|
||||
"label": display_label,
|
||||
"bg_color": n.color,
|
||||
"bg_color": getattr(n, 'color', '#8395a7'),
|
||||
"tooltip": tooltip_text
|
||||
},
|
||||
# Wir steuern das Aussehen rein über Klassen (.inspected / .center)
|
||||
@@ -211,18 +266,22 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
}
|
||||
cy_elements.append(cy_node)
|
||||
|
||||
for e in edges_data:
|
||||
target_id = getattr(e, "to", getattr(e, "target", None))
|
||||
if target_id:
|
||||
cy_edge = {
|
||||
"data": {
|
||||
"source": e.source,
|
||||
"target": target_id,
|
||||
"label": e.label,
|
||||
"line_color": e.color
|
||||
# Edges hinzufügen - nur wenn beide Nodes im Graph vorhanden sind
|
||||
if edges_data:
|
||||
for e in edges_data:
|
||||
source_id = getattr(e, "source", None)
|
||||
target_id = getattr(e, "to", getattr(e, "target", None))
|
||||
# Nur hinzufügen, wenn beide IDs vorhanden UND beide Nodes im Graph sind
|
||||
if source_id and target_id and source_id in node_ids and target_id in node_ids:
|
||||
cy_edge = {
|
||||
"data": {
|
||||
"source": source_id,
|
||||
"target": target_id,
|
||||
"label": getattr(e, "label", ""),
|
||||
"line_color": getattr(e, "color", "#bdc3c7")
|
||||
}
|
||||
}
|
||||
}
|
||||
cy_elements.append(cy_edge)
|
||||
cy_elements.append(cy_edge)
|
||||
|
||||
# --- STYLESHEET ---
|
||||
stylesheet = [
|
||||
@@ -292,43 +351,47 @@ def render_graph_explorer_cytoscape(graph_service):
|
||||
]
|
||||
|
||||
# --- RENDER ---
|
||||
graph_key = f"cy_{center_id}_{st.session_state.cy_depth}_{st.session_state.cy_ideal_edge_len}"
|
||||
# Nur rendern, wenn Elemente vorhanden sind
|
||||
if not cy_elements:
|
||||
st.warning("⚠️ Keine Graph-Elemente zum Anzeigen gefunden.")
|
||||
else:
|
||||
graph_key = f"cy_{center_id}_{st.session_state.cy_depth}_{st.session_state.cy_ideal_edge_len}"
|
||||
|
||||
clicked_elements = cytoscape(
|
||||
elements=cy_elements,
|
||||
stylesheet=stylesheet,
|
||||
layout={
|
||||
"name": "cose",
|
||||
"idealEdgeLength": st.session_state.cy_ideal_edge_len,
|
||||
"nodeOverlap": 20,
|
||||
"refresh": 20,
|
||||
"fit": True,
|
||||
"padding": 50,
|
||||
"randomize": False,
|
||||
"componentSpacing": 100,
|
||||
"nodeRepulsion": st.session_state.cy_node_repulsion,
|
||||
"edgeElasticity": 100,
|
||||
"nestingFactor": 5,
|
||||
"gravity": 80,
|
||||
"numIter": 1000,
|
||||
"initialTemp": 200,
|
||||
"coolingFactor": 0.95,
|
||||
"minTemp": 1.0,
|
||||
"animate": False
|
||||
},
|
||||
key=graph_key,
|
||||
height="700px"
|
||||
)
|
||||
clicked_elements = cytoscape(
|
||||
elements=cy_elements,
|
||||
stylesheet=stylesheet,
|
||||
layout={
|
||||
"name": "cose",
|
||||
"idealEdgeLength": st.session_state.cy_ideal_edge_len,
|
||||
"nodeOverlap": 20,
|
||||
"refresh": 20,
|
||||
"fit": True,
|
||||
"padding": 50,
|
||||
"randomize": False,
|
||||
"componentSpacing": 100,
|
||||
"nodeRepulsion": st.session_state.cy_node_repulsion,
|
||||
"edgeElasticity": 100,
|
||||
"nestingFactor": 5,
|
||||
"gravity": 80,
|
||||
"numIter": 1000,
|
||||
"initialTemp": 200,
|
||||
"coolingFactor": 0.95,
|
||||
"minTemp": 1.0,
|
||||
"animate": False
|
||||
},
|
||||
key=graph_key,
|
||||
height="700px"
|
||||
)
|
||||
|
||||
# --- EVENT HANDLING ---
|
||||
if clicked_elements:
|
||||
clicked_nodes = clicked_elements.get("nodes", [])
|
||||
if clicked_nodes:
|
||||
clicked_id = clicked_nodes[0]
|
||||
|
||||
if clicked_id != st.session_state.graph_inspected_id:
|
||||
st.session_state.graph_inspected_id = clicked_id
|
||||
st.rerun()
|
||||
# --- EVENT HANDLING ---
|
||||
if clicked_elements:
|
||||
clicked_nodes = clicked_elements.get("nodes", [])
|
||||
if clicked_nodes:
|
||||
clicked_id = clicked_nodes[0]
|
||||
|
||||
if clicked_id != st.session_state.graph_inspected_id:
|
||||
st.session_state.graph_inspected_id = clicked_id
|
||||
st.rerun()
|
||||
|
||||
else:
|
||||
st.info("👈 Bitte wähle links eine Notiz aus.")
|
||||
@@ -10,15 +10,19 @@ LAST_ANALYSIS: 2025-12-15
|
||||
import re
|
||||
from qdrant_client import QdrantClient, models
|
||||
from streamlit_agraph import Node, Edge
|
||||
from ui_config import GRAPH_COLORS, get_edge_color, SYSTEM_EDGES
|
||||
from ui_config import COLLECTION_PREFIX, GRAPH_COLORS, get_edge_color, SYSTEM_EDGES
|
||||
|
||||
class GraphExplorerService:
|
||||
def __init__(self, url, api_key=None, prefix="mindnet"):
|
||||
def __init__(self, url, api_key=None, prefix=None):
|
||||
"""
|
||||
Initialisiert den Service. Nutzt COLLECTION_PREFIX aus der Config,
|
||||
sofern kein spezifischer Prefix übergeben wurde.
|
||||
"""
|
||||
self.client = QdrantClient(url=url, api_key=api_key)
|
||||
self.prefix = prefix
|
||||
self.notes_col = f"{prefix}_notes"
|
||||
self.chunks_col = f"{prefix}_chunks"
|
||||
self.edges_col = f"{prefix}_edges"
|
||||
self.prefix = prefix if prefix else COLLECTION_PREFIX
|
||||
self.notes_col = f"{self.prefix}_notes"
|
||||
self.chunks_col = f"{self.prefix}_chunks"
|
||||
self.edges_col = f"{self.prefix}_edges"
|
||||
self._note_cache = {}
|
||||
|
||||
def get_note_with_full_content(self, note_id):
|
||||
@@ -163,31 +167,33 @@ class GraphExplorerService:
|
||||
return previews
|
||||
|
||||
def _find_connected_edges(self, note_ids, note_title=None):
|
||||
"""Findet eingehende und ausgehende Kanten."""
|
||||
"""
|
||||
Findet eingehende und ausgehende Kanten.
|
||||
|
||||
WICHTIG: target_id enthält nur den Titel (ohne #Abschnitt).
|
||||
target_section ist ein separates Feld für Abschnitt-Informationen.
|
||||
"""
|
||||
results = []
|
||||
if not note_ids:
|
||||
return results
|
||||
|
||||
# 1. OUTGOING EDGES (Der "Owner"-Fix)
|
||||
# Wir suchen Kanten, die im Feld 'note_id' (Owner) eine unserer Notizen haben.
|
||||
# Das findet ALLE ausgehenden Kanten, egal ob sie an einem Chunk oder der Note hängen.
|
||||
if note_ids:
|
||||
out_filter = models.Filter(must=[
|
||||
models.FieldCondition(key="note_id", match=models.MatchAny(any=note_ids)),
|
||||
models.FieldCondition(key="kind", match=models.MatchExcept(**{"except": SYSTEM_EDGES}))
|
||||
])
|
||||
# Limit hoch, um alles zu finden
|
||||
res_out, _ = self.client.scroll(self.edges_col, scroll_filter=out_filter, limit=500, with_payload=True)
|
||||
results.extend(res_out)
|
||||
out_filter = models.Filter(must=[
|
||||
models.FieldCondition(key="note_id", match=models.MatchAny(any=note_ids)),
|
||||
models.FieldCondition(key="kind", match=models.MatchExcept(**{"except": SYSTEM_EDGES}))
|
||||
])
|
||||
res_out, _ = self.client.scroll(self.edges_col, scroll_filter=out_filter, limit=2000, with_payload=True)
|
||||
results.extend(res_out)
|
||||
|
||||
# 2. INCOMING EDGES (Ziel = Chunk ID oder Titel oder Note ID)
|
||||
# Hier müssen wir Chunks auflösen, um Treffer auf Chunks zu finden.
|
||||
# 2. INCOMING EDGES (Ziel = Chunk ID, Note ID oder Titel)
|
||||
# WICHTIG: target_id enthält nur den Titel, target_section ist separat
|
||||
|
||||
# Chunk IDs der aktuellen Notes holen
|
||||
chunk_ids = []
|
||||
if note_ids:
|
||||
c_filter = models.Filter(must=[models.FieldCondition(key="note_id", match=models.MatchAny(any=note_ids))])
|
||||
chunks, _ = self.client.scroll(self.chunks_col, scroll_filter=c_filter, limit=300)
|
||||
chunk_ids = [c.id for c in chunks]
|
||||
c_filter = models.Filter(must=[models.FieldCondition(key="note_id", match=models.MatchAny(any=note_ids))])
|
||||
chunks, _ = self.client.scroll(self.chunks_col, scroll_filter=c_filter, limit=1000, with_payload=False)
|
||||
chunk_ids = [c.id for c in chunks]
|
||||
|
||||
shoulds = []
|
||||
# Case A: Edge zeigt auf einen unserer Chunks
|
||||
@@ -195,42 +201,92 @@ class GraphExplorerService:
|
||||
shoulds.append(models.FieldCondition(key="target_id", match=models.MatchAny(any=chunk_ids)))
|
||||
|
||||
# Case B: Edge zeigt direkt auf unsere Note ID
|
||||
if note_ids:
|
||||
shoulds.append(models.FieldCondition(key="target_id", match=models.MatchAny(any=note_ids)))
|
||||
|
||||
# Case C: Edge zeigt auf unseren Titel (Wikilinks)
|
||||
if note_title:
|
||||
shoulds.append(models.FieldCondition(key="target_id", match=models.MatchValue(value=note_title)))
|
||||
shoulds.append(models.FieldCondition(key="target_id", match=models.MatchAny(any=note_ids)))
|
||||
|
||||
# Case C: Edge zeigt auf unseren Titel
|
||||
# WICHTIG: target_id enthält nur den Titel (z.B. "Meine Prinzipien 2025")
|
||||
# target_section enthält die Abschnitt-Information (z.B. "P3 – Disziplin"), wenn gesetzt
|
||||
|
||||
# Sammle alle relevanten Titel (inkl. Aliase)
|
||||
titles_to_search = []
|
||||
if note_title:
|
||||
titles_to_search.append(note_title)
|
||||
|
||||
# Lade auch Titel aus den Notes selbst (falls note_title nicht übergeben wurde)
|
||||
for nid in note_ids:
|
||||
note = self._fetch_note_cached(nid)
|
||||
if note:
|
||||
note_title_from_db = note.get("title")
|
||||
if note_title_from_db and note_title_from_db not in titles_to_search:
|
||||
titles_to_search.append(note_title_from_db)
|
||||
# Aliase hinzufügen
|
||||
aliases = note.get("aliases", [])
|
||||
if isinstance(aliases, str):
|
||||
aliases = [aliases]
|
||||
for alias in aliases:
|
||||
if alias and alias not in titles_to_search:
|
||||
titles_to_search.append(alias)
|
||||
|
||||
# Für jeden Titel: Suche nach exaktem Match
|
||||
# target_id enthält nur den Titel, daher reicht MatchValue
|
||||
for title in titles_to_search:
|
||||
shoulds.append(models.FieldCondition(key="target_id", match=models.MatchValue(value=title)))
|
||||
|
||||
if shoulds:
|
||||
in_filter = models.Filter(
|
||||
must=[models.FieldCondition(key="kind", match=models.MatchExcept(**{"except": SYSTEM_EDGES}))],
|
||||
should=shoulds
|
||||
)
|
||||
res_in, _ = self.client.scroll(self.edges_col, scroll_filter=in_filter, limit=500, with_payload=True)
|
||||
res_in, _ = self.client.scroll(self.edges_col, scroll_filter=in_filter, limit=2000, with_payload=True)
|
||||
results.extend(res_in)
|
||||
|
||||
return results
|
||||
|
||||
def _find_connected_edges_batch(self, note_ids):
|
||||
# Wrapper für Level 2 Suche
|
||||
return self._find_connected_edges(note_ids)
|
||||
"""
|
||||
Wrapper für Level 2 Suche.
|
||||
Lädt Titel der ersten Note für Titel-basierte Suche.
|
||||
"""
|
||||
if not note_ids:
|
||||
return []
|
||||
first_note = self._fetch_note_cached(note_ids[0])
|
||||
note_title = first_note.get("title") if first_note else None
|
||||
return self._find_connected_edges(note_ids, note_title=note_title)
|
||||
|
||||
def _process_edge(self, record, nodes_dict, unique_edges, current_depth):
|
||||
"""Verarbeitet eine rohe Edge, löst IDs auf und fügt sie den Dictionaries hinzu."""
|
||||
"""
|
||||
Verarbeitet eine rohe Edge, löst IDs auf und fügt sie den Dictionaries hinzu.
|
||||
|
||||
WICHTIG: Beide Richtungen werden unterstützt:
|
||||
- Ausgehende Kanten: source_id gehört zu unserer Note (via note_id Owner)
|
||||
- Eingehende Kanten: target_id zeigt auf unsere Note (via target_id Match)
|
||||
"""
|
||||
if not record or not record.payload:
|
||||
return None, None
|
||||
|
||||
payload = record.payload
|
||||
src_ref = payload.get("source_id")
|
||||
tgt_ref = payload.get("target_id")
|
||||
kind = payload.get("kind")
|
||||
provenance = payload.get("provenance", "explicit")
|
||||
|
||||
# Prüfe, ob beide Referenzen vorhanden sind
|
||||
if not src_ref or not tgt_ref:
|
||||
return None, None
|
||||
|
||||
# IDs zu Notes auflösen
|
||||
# WICHTIG: source_id kann Chunk-ID (note_id#c01), Note-ID oder Titel sein
|
||||
# WICHTIG: target_id kann Chunk-ID, Note-ID oder Titel sein (ohne #Abschnitt)
|
||||
src_note = self._resolve_note_from_ref(src_ref)
|
||||
tgt_note = self._resolve_note_from_ref(tgt_ref)
|
||||
|
||||
if src_note and tgt_note:
|
||||
src_id = src_note['note_id']
|
||||
tgt_id = tgt_note['note_id']
|
||||
src_id = src_note.get('note_id')
|
||||
tgt_id = tgt_note.get('note_id')
|
||||
|
||||
# Prüfe, ob beide IDs vorhanden sind
|
||||
if not src_id or not tgt_id:
|
||||
return None, None
|
||||
|
||||
if src_id != tgt_id:
|
||||
# Nodes hinzufügen
|
||||
@@ -245,7 +301,7 @@ class GraphExplorerService:
|
||||
# Bevorzuge explizite Kanten vor Smart Kanten
|
||||
is_current_explicit = (provenance in ["explicit", "rule"])
|
||||
if existing:
|
||||
is_existing_explicit = (existing['provenance'] in ["explicit", "rule"])
|
||||
is_existing_explicit = (existing.get('provenance', '') in ["explicit", "rule"])
|
||||
if is_existing_explicit and not is_current_explicit:
|
||||
should_update = False
|
||||
|
||||
@@ -267,38 +323,109 @@ class GraphExplorerService:
|
||||
return None
|
||||
|
||||
def _resolve_note_from_ref(self, ref_str):
|
||||
"""Löst eine ID (Chunk, Note oder Titel) zu einer Note Payload auf."""
|
||||
if not ref_str: return None
|
||||
"""
|
||||
Löst eine Referenz zu einer Note Payload auf.
|
||||
|
||||
# Fall A: Chunk ID (enthält #)
|
||||
WICHTIG: Wenn ref_str ein Titel#Abschnitt Format hat, wird nur der Titel-Teil verwendet.
|
||||
Unterstützt:
|
||||
- Note-ID: "20250101-meine-note"
|
||||
- Chunk-ID: "20250101-meine-note#c01"
|
||||
- Titel: "Meine Prinzipien 2025"
|
||||
- Titel#Abschnitt: "Meine Prinzipien 2025#P3 – Disziplin" (trennt Abschnitt ab, sucht nur nach Titel)
|
||||
"""
|
||||
if not ref_str:
|
||||
return None
|
||||
|
||||
# Fall A: Enthält # (kann Chunk-ID oder Titel#Abschnitt sein)
|
||||
if "#" in ref_str:
|
||||
try:
|
||||
# Versuch 1: Chunk ID direkt
|
||||
# Versuch 1: Chunk ID direkt (Format: note_id#c01)
|
||||
res = self.client.retrieve(self.chunks_col, ids=[ref_str], with_payload=True)
|
||||
if res: return self._fetch_note_cached(res[0].payload.get("note_id"))
|
||||
except: pass
|
||||
if res and res[0].payload:
|
||||
note_id = res[0].payload.get("note_id")
|
||||
if note_id:
|
||||
return self._fetch_note_cached(note_id)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Versuch 2: NoteID#Section (Hash abtrennen)
|
||||
possible_note_id = ref_str.split("#")[0]
|
||||
if self._fetch_note_cached(possible_note_id): return self._fetch_note_cached(possible_note_id)
|
||||
# Versuch 2: NoteID#Section (Hash abtrennen und als Note-ID versuchen)
|
||||
# z.B. "20250101-meine-note#Abschnitt" -> "20250101-meine-note"
|
||||
possible_note_id = ref_str.split("#")[0].strip()
|
||||
note = self._fetch_note_cached(possible_note_id)
|
||||
if note:
|
||||
return note
|
||||
|
||||
# Versuch 3: Titel#Abschnitt (Hash abtrennen und als Titel suchen)
|
||||
# z.B. "Meine Prinzipien 2025#P3 – Disziplin" -> "Meine Prinzipien 2025"
|
||||
# WICHTIG: target_id enthält nur den Titel, daher suchen wir nur nach dem Titel-Teil
|
||||
possible_title = ref_str.split("#")[0].strip()
|
||||
if possible_title:
|
||||
res, _ = self.client.scroll(
|
||||
collection_name=self.notes_col,
|
||||
scroll_filter=models.Filter(must=[
|
||||
models.FieldCondition(key="title", match=models.MatchValue(value=possible_title))
|
||||
]),
|
||||
limit=1, with_payload=True
|
||||
)
|
||||
if res and res[0].payload:
|
||||
payload = res[0].payload
|
||||
self._note_cache[payload['note_id']] = payload
|
||||
return payload
|
||||
|
||||
# Fallback: Text-Suche für Fuzzy-Matching
|
||||
res, _ = self.client.scroll(
|
||||
collection_name=self.notes_col,
|
||||
scroll_filter=models.Filter(must=[
|
||||
models.FieldCondition(key="title", match=models.MatchText(text=possible_title))
|
||||
]),
|
||||
limit=10, with_payload=True
|
||||
)
|
||||
if res:
|
||||
# Nimm das erste Ergebnis, das exakt oder beginnend mit possible_title übereinstimmt
|
||||
for r in res:
|
||||
if r.payload:
|
||||
note_title = r.payload.get("title", "")
|
||||
if note_title == possible_title or note_title.startswith(possible_title):
|
||||
payload = r.payload
|
||||
self._note_cache[payload['note_id']] = payload
|
||||
return payload
|
||||
|
||||
# Fall B: Note ID direkt
|
||||
if self._fetch_note_cached(ref_str): return self._fetch_note_cached(ref_str)
|
||||
note = self._fetch_note_cached(ref_str)
|
||||
if note:
|
||||
return note
|
||||
|
||||
# Fall C: Titel
|
||||
# Fall C: Titel (exakte Übereinstimmung)
|
||||
res, _ = self.client.scroll(
|
||||
collection_name=self.notes_col,
|
||||
scroll_filter=models.Filter(must=[models.FieldCondition(key="title", match=models.MatchValue(value=ref_str))]),
|
||||
scroll_filter=models.Filter(must=[
|
||||
models.FieldCondition(key="title", match=models.MatchValue(value=ref_str))
|
||||
]),
|
||||
limit=1, with_payload=True
|
||||
)
|
||||
if res:
|
||||
self._note_cache[res[0].payload['note_id']] = res[0].payload
|
||||
return res[0].payload
|
||||
if res and res[0].payload:
|
||||
payload = res[0].payload
|
||||
self._note_cache[payload['note_id']] = payload
|
||||
return payload
|
||||
|
||||
# Fall D: Titel (Text-Suche für Fuzzy-Matching)
|
||||
res, _ = self.client.scroll(
|
||||
collection_name=self.notes_col,
|
||||
scroll_filter=models.Filter(must=[
|
||||
models.FieldCondition(key="title", match=models.MatchText(text=ref_str))
|
||||
]),
|
||||
limit=1, with_payload=True
|
||||
)
|
||||
if res and res[0].payload:
|
||||
payload = res[0].payload
|
||||
self._note_cache[payload['note_id']] = payload
|
||||
return payload
|
||||
|
||||
return None
|
||||
|
||||
def _add_node_to_dict(self, node_dict, note_payload, level=1):
|
||||
nid = note_payload.get("note_id")
|
||||
if nid in node_dict: return
|
||||
if not nid or nid in node_dict: return
|
||||
|
||||
ntype = note_payload.get("type", "default")
|
||||
color = GRAPH_COLORS.get(ntype, GRAPH_COLORS["default"])
|
||||
|
||||
+3
-2
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
FILE: app/models/dto.py
|
||||
DESCRIPTION: Pydantic-Modelle (DTOs) für Request/Response Bodies. Definiert das API-Schema.
|
||||
VERSION: 0.6.6 (WP-22 Debug & Stability Update)
|
||||
VERSION: 0.6.7 (WP-Fix: Target Section Support)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: pydantic, typing, uuid
|
||||
LAST_ANALYSIS: 2025-12-18
|
||||
LAST_ANALYSIS: 2025-12-29
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,6 +43,7 @@ class EdgeDTO(BaseModel):
|
||||
direction: Literal["out", "in", "undirected"] = "out"
|
||||
provenance: Optional[Literal["explicit", "rule", "smart", "structure"]] = "explicit"
|
||||
confidence: float = 1.0
|
||||
target_section: Optional[str] = None # Neu: Speichert den Anker (z.B. #Abschnitt)
|
||||
|
||||
|
||||
# --- Request Models ---
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ from pathlib import Path
|
||||
from app.config import get_settings
|
||||
from app.models.dto import ChatRequest, ChatResponse, QueryRequest, QueryHit
|
||||
from app.services.llm_service import LLMService
|
||||
from app.core.retriever import Retriever
|
||||
from app.core.retrieval.retriever import Retriever
|
||||
from app.services.feedback_service import log_search
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -12,7 +12,7 @@ from typing import List, Optional
|
||||
from fastapi import APIRouter, Query
|
||||
from qdrant_client import QdrantClient
|
||||
from app.models.dto import GraphResponse, NodeDTO, EdgeDTO
|
||||
from app.core.graph_adapter import expand
|
||||
from app.core.graph.graph_subgraph import expand
|
||||
from app.config import get_settings
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -10,7 +10,7 @@ LAST_ANALYSIS: 2025-12-15
|
||||
from __future__ import annotations
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||
from app.models.dto import QueryRequest, QueryResponse
|
||||
from app.core.retriever import hybrid_retrieve, semantic_retrieve
|
||||
from app.core.retrieval.retriever import hybrid_retrieve, semantic_retrieve
|
||||
# NEU:
|
||||
from app.services.feedback_service import log_search
|
||||
|
||||
|
||||
@@ -13,9 +13,9 @@ import os
|
||||
from typing import List, Dict, Any, Optional, Set
|
||||
import yaml
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
from app.models.dto import QueryRequest
|
||||
from app.core.retriever import hybrid_retrieve
|
||||
from app.core.retrieval.retriever import hybrid_retrieve
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
doc_type: glossary
|
||||
audience: all
|
||||
status: active
|
||||
version: 2.8.1
|
||||
version: 2.9.1
|
||||
context: "Zentrales Glossar für Mindnet v2.8. Enthält Definitionen zu Hybrid-Cloud Resilienz, WP-14 Modularisierung, WP-15b Two-Pass Ingestion und Mistral-safe Parsing."
|
||||
---
|
||||
|
||||
@@ -14,7 +14,7 @@ context: "Zentrales Glossar für Mindnet v2.8. Enthält Definitionen zu Hybrid-C
|
||||
|
||||
* **Note:** Repräsentiert eine Markdown-Datei. Die fachliche Haupteinheit. Verfügt über einen **Status** (stable, draft, system), der das Scoring beeinflusst.
|
||||
* **Chunk:** Ein Textabschnitt einer Note. Die technische Sucheinheit (Vektor).
|
||||
* **Edge:** Eine gerichtete Verbindung zwischen zwei Knoten. Wird in WP-22 durch die Registry validiert.
|
||||
* **Edge:** Eine gerichtete Verbindung zwischen zwei Knoten. Wird in WP-22 durch die Registry validiert. Seit v2.9.1 unterstützt Edges **Section-basierte Links** (`target_section`), sodass mehrere Kanten zwischen denselben Knoten existieren können, wenn sie auf verschiedene Abschnitte zeigen.
|
||||
* **Vault:** Der lokale Ordner mit den Markdown-Dateien (Source of Truth).
|
||||
* **Frontmatter:** Der YAML-Header am Anfang einer Notiz (enthält `id`, `type`, `title`, `status`).
|
||||
|
||||
@@ -47,4 +47,7 @@ context: "Zentrales Glossar für Mindnet v2.8. Enthält Definitionen zu Hybrid-C
|
||||
* **Two-Pass Workflow (WP-15b):** Optimiertes Ingestion-Verfahren:
|
||||
* **Pass 1 (Pre-Scan):** Schnelles Scannen aller Dateien zur Befüllung des LocalBatchCache.
|
||||
* **Pass 2 (Semantic Processing):** Tiefenverarbeitung (Chunking, Embedding, Validierung) nur für geänderte Dateien.
|
||||
* **Circular Import Registry (WP-14):** Entkopplung von Kern-Logik (wie Textbereinigung) in eine neutrale `registry.py`, um Abhängigkeitsschleifen zwischen Diensten und Ingestion-Utilities zu verhindern.
|
||||
* **Circular Import Registry (WP-14):** Entkopplung von Kern-Logik (wie Textbereinigung) in eine neutrale `registry.py`, um Abhängigkeitsschleifen zwischen Diensten und Ingestion-Utilities zu verhindern.
|
||||
* **Deep-Link / Section-basierter Link:** Ein Link wie `[[Note#Section]]`, der auf einen spezifischen Abschnitt innerhalb einer Note verweist. Seit v2.9.1 wird dieser in `target_id="Note"` und `target_section="Section"` aufgeteilt, um "Phantom-Knoten" zu vermeiden und Multigraph-Support zu ermöglichen.
|
||||
* **Atomic Section Logic (v3.9.9):** Chunking-Verfahren, das Sektions-Überschriften und deren Inhalte atomar in Chunks hält (Pack-and-Carry-Over). Verhindert, dass Überschriften über Chunk-Grenzen hinweg getrennt werden.
|
||||
* **Registry-First Profiling (v2.13.12):** Hierarchische Auflösung des Chunking-Profils: Frontmatter > types.yaml Typ-Config > Global Defaults. Stellt sicher, dass Note-Typen automatisch das korrekte Profil erhalten.
|
||||
@@ -3,7 +3,7 @@ doc_type: user_manual
|
||||
audience: user, author
|
||||
scope: vault, markdown, schema
|
||||
status: active
|
||||
version: 2.8.0
|
||||
version: 2.9.1
|
||||
context: "Regelwerk für das Erstellen von Notizen im Vault. Die 'Source of Truth' für Autoren."
|
||||
---
|
||||
|
||||
@@ -208,6 +208,12 @@ Dies ist die **mächtigste** Methode. Du sagst dem System explizit, **wie** Ding
|
||||
> "Daher [[rel:depends_on Qdrant]]."
|
||||
> "Dieses Konzept ist [[rel:similar_to Pinecone]]."
|
||||
|
||||
**Deep-Links zu Abschnitten (v2.9.1):**
|
||||
Du kannst auch auf spezifische Abschnitte innerhalb einer Note verlinken:
|
||||
> "Siehe [[rel:based_on Mein Leitbild#P3 – Disziplin]]."
|
||||
|
||||
Das System trennt automatisch den Note-Namen (`Mein Leitbild`) vom Abschnitts-Namen (`P3 – Disziplin`), sodass mehrere Links zur gleichen Note möglich sind, wenn sie auf verschiedene Abschnitte zeigen.
|
||||
|
||||
**Gültige Relationen:**
|
||||
* `depends_on`: Hängt ab von / Benötigt.
|
||||
* `blocks`: Blockiert oder gefährdet (z.B. Risiko -> Projekt).
|
||||
@@ -226,6 +232,12 @@ Für Zusammenfassungen am Ende einer Notiz, oder eines Absatzes:
|
||||
> [[AI Agents]]
|
||||
```
|
||||
|
||||
**Multi-Line Support (v2.9.1):**
|
||||
Callout-Blocks mit mehreren Zeilen werden korrekt verarbeitet. Das System erkennt automatisch, wenn mehrere Links im gleichen Callout-Block stehen, und erstellt für jeden Link eine separate Kante (auch bei Deep-Links zu verschiedenen Sections).
|
||||
|
||||
**Format-agnostische De-Duplizierung:**
|
||||
Wenn Kanten bereits via `[!edge]` Callout vorhanden sind, werden sie nicht mehrfach injiziert. Das System erkennt vorhandene Kanten unabhängig vom Format (Inline, Callout, Wikilink).
|
||||
|
||||
### 4.3 Implizite Bidirektionalität (Edger-Logik) [NEU] [PRÜFEN!]
|
||||
In Mindnet musst du Kanten **nicht** manuell in beide Richtungen pflegen. Der **Edger** übernimmt die Paarbildung automatisch im Hintergrund.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ doc_type: concept
|
||||
audience: architect, product_owner
|
||||
scope: graph, logic, provenance
|
||||
status: active
|
||||
version: 2.7.0
|
||||
version: 2.9.1
|
||||
context: "Fachliche Beschreibung des Wissensgraphen: Knoten, Kanten, Provenance, Matrix-Logik und WP-22 Scoring-Prinzipien."
|
||||
---
|
||||
|
||||
@@ -118,8 +118,30 @@ Der Intent-Router injiziert spezifische Multiplikatoren für kanonische Typen:
|
||||
|
||||
---
|
||||
|
||||
## 6. Idempotenz & Konsistenz
|
||||
## 6. Section-basierte Links & Multigraph-Support
|
||||
|
||||
Seit v2.9.1 unterstützt Mindnet **Deep-Links** zu spezifischen Abschnitten innerhalb einer Note.
|
||||
|
||||
### 6.1 Link-Parsing
|
||||
Links wie `[[Note#Section]]` werden in zwei Komponenten aufgeteilt:
|
||||
* **`target_id`:** Enthält nur den Note-Namen (z.B. "Mein Leitbild")
|
||||
* **`target_section`:** Enthält den Abschnitts-Namen (z.B. "P3 – Disziplin")
|
||||
|
||||
**Vorteil:** Verhindert "Phantom-Knoten", die durch das Einbeziehen des Anchors in die `target_id` entstanden wären.
|
||||
|
||||
### 6.2 Multigraph-Support
|
||||
Die Edge-ID enthält nun einen `variant`-Parameter (die Section), sodass mehrere Kanten zwischen denselben Knoten existieren können, wenn sie auf verschiedene Sections zeigen:
|
||||
* `[[Note#Section1]]` → Edge-ID: `src->tgt:kind@Section1`
|
||||
* `[[Note#Section2]]` → Edge-ID: `src->tgt:kind@Section2`
|
||||
|
||||
### 6.3 Semantische Deduplizierung
|
||||
Die Deduplizierung basiert auf dem `src->tgt:kind@sec` Key, um sicherzustellen, dass identische Links (gleiche Quelle, Ziel, Typ und Section) nicht mehrfach erstellt werden.
|
||||
|
||||
---
|
||||
|
||||
## 7. Idempotenz & Konsistenz
|
||||
|
||||
Das System garantiert fachliche Konsistenz auch bei mehrfachen Importen.
|
||||
* **Stabile IDs:** Deterministische IDs verhindern Duplikate bei Re-Imports.
|
||||
* **Deduplizierung:** Kanten werden anhand ihrer Identität erkannt. Die "stärkere" Provenance gewinnt.
|
||||
* **Deduplizierung:** Kanten werden anhand ihrer Identität (inkl. Section) erkannt. Die "stärkere" Provenance gewinnt.
|
||||
* **Format-agnostische Erkennung:** Kanten werden unabhängig vom Format (Inline, Callout, Wikilink) erkannt, um Dopplungen zu vermeiden.
|
||||
@@ -144,8 +144,11 @@ Lädt den Subgraphen um eine Note herum.
|
||||
"kind": "depends_on",
|
||||
"source": "uuid",
|
||||
"target": "uuid",
|
||||
"target_section": "P3 – Disziplin", // Optional: Abschnitts-Name bei Deep-Links
|
||||
"weight": 1.4,
|
||||
"direction": "out"
|
||||
"direction": "out",
|
||||
"provenance": "explicit",
|
||||
"confidence": 1.0
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
|
||||
@@ -3,7 +3,7 @@ doc_type: technical_reference
|
||||
audience: developer, architect
|
||||
scope: database, qdrant, schema
|
||||
status: active
|
||||
version: 2.8.0
|
||||
version: 2.9.1
|
||||
context: "Exakte Definition der Datenmodelle (Payloads) in Qdrant und Index-Anforderungen. Berücksichtigt WP-14 Modularisierung und WP-15b Multi-Hashes."
|
||||
---
|
||||
|
||||
@@ -96,15 +96,19 @@ Es müssen Payload-Indizes für folgende Felder existieren:
|
||||
|
||||
## 4. Edge Payload (`mindnet_edges`)
|
||||
|
||||
Gerichtete Kanten zwischen Knoten. Stark erweitert in v2.6 für Provenienz-Tracking.
|
||||
Gerichtete Kanten zwischen Knoten. Stark erweitert in v2.6 für Provenienz-Tracking. Seit v2.9.1 unterstützt das System **Section-basierte Links** (`[[Note#Section]]`), die in `target_id` und `target_section` aufgeteilt werden.
|
||||
|
||||
**JSON-Schema:**
|
||||
|
||||
```json
|
||||
{
|
||||
"edge_id": "string (keyword)", // Deterministischer Hash aus (src, dst, kind)
|
||||
"edge_id": "string (keyword)", // Deterministischer Hash aus (src, dst, kind, variant)
|
||||
// variant = target_section (erlaubt Multigraph für Sections)
|
||||
"source_id": "string (keyword)", // Chunk-ID (Start)
|
||||
"target_id": "string (keyword)", // Chunk-ID oder Note-Titel (bei Unresolved)
|
||||
// WICHTIG: Enthält NUR den Note-Namen, KEINE Section-Info
|
||||
"target_section": "string (keyword)", // Optional: Abschnitts-Name (z.B. "P3 – Disziplin")
|
||||
// Wird aus [[Note#Section]] extrahiert
|
||||
"kind": "string (keyword)", // Beziehungsart (z.B. 'depends_on')
|
||||
"scope": "string (keyword)", // Immer 'chunk' (Legacy-Support: 'note')
|
||||
"note_id": "string (keyword)", // Owner Note ID (Ursprung der Kante)
|
||||
@@ -116,10 +120,16 @@ Gerichtete Kanten zwischen Knoten. Stark erweitert in v2.6 für Provenienz-Track
|
||||
}
|
||||
```
|
||||
|
||||
**Section-Support:**
|
||||
* Links wie `[[Note#Section]]` werden in `target_id="Note"` und `target_section="Section"` aufgeteilt.
|
||||
* Die Edge-ID enthält die Section als `variant`, sodass mehrere Kanten zwischen denselben Knoten existieren können, wenn sie auf verschiedene Sections zeigen.
|
||||
* Semantische Deduplizierung basiert auf `src->tgt:kind@sec` Key, um "Phantom-Knoten" zu vermeiden.
|
||||
|
||||
**Erforderliche Indizes:**
|
||||
Es müssen Payload-Indizes für folgende Felder existieren:
|
||||
* `source_id`
|
||||
* `target_id`
|
||||
* `target_section` (neu: Keyword-Index für Section-basierte Filterung)
|
||||
* `kind`
|
||||
* `scope`
|
||||
* `note_id`
|
||||
|
||||
@@ -3,7 +3,7 @@ doc_type: technical_reference
|
||||
audience: developer, frontend_architect
|
||||
scope: architecture, graph_viz, state_management
|
||||
status: active
|
||||
version: 2.7.0
|
||||
version: 2.9.1
|
||||
context: "Technische Dokumentation des modularen Streamlit-Frontends, der Graph-Engines und des Editors."
|
||||
---
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ doc_type: technical_reference
|
||||
audience: developer, devops
|
||||
scope: backend, ingestion, smart_edges, edge_registry, modularization
|
||||
status: active
|
||||
version: 2.9.0
|
||||
version: 2.13.12
|
||||
context: "Detaillierte technische Beschreibung der Import-Pipeline, Two-Pass-Workflow (WP-15b) und modularer Datenbank-Architektur (WP-14). Integriert Mistral-safe Parsing und Deep Fallback."
|
||||
---
|
||||
|
||||
@@ -31,9 +31,10 @@ Der Prozess ist **asynchron**, **idempotent** und wird nun in zwei logische Durc
|
||||
4. **Edge Registry Initialisierung (WP-22):**
|
||||
* Laden der Singleton-Instanz der `EdgeRegistry`.
|
||||
* Validierung der Vokabular-Datei unter `MINDNET_VOCAB_PATH`.
|
||||
5. **Config Resolution (WP-14):**
|
||||
5. **Config Resolution (WP-14 / v2.13.12):**
|
||||
* Bestimmung von `chunking_profile` und `retriever_weight` via zentraler `TypeRegistry`.
|
||||
* **Priorität:** 1. Frontmatter (Override) -> 2. `types.yaml` (Type) -> 3. Global Default.
|
||||
* **Registry-First Profiling:** Automatische Anwendung der korrekten Profile basierend auf dem Note-Typ (z.B. `value` nutzt automatisch `structured_smart_edges_strict`).
|
||||
6. **LocalBatchCache & Summary Generation (WP-15b):**
|
||||
* Erstellung von Kurz-Zusammenfassungen für jede Note.
|
||||
* Speicherung im `batch_cache` als Referenzrahmen für die spätere Kantenvalidierung.
|
||||
@@ -126,19 +127,44 @@ Das Chunking ist profilbasiert und bezieht seine Konfiguration dynamisch aus der
|
||||
| `sliding_smart_edges`| `sliding_window` | Max: 600, Target: 400 | Fließtexte (Projekte). |
|
||||
| `structured_smart_edges` | `by_heading` | `strict: false` | Strukturierte Texte. |
|
||||
|
||||
### 3.2 Die `by_heading` Logik (v2.9 Hybrid)
|
||||
### 3.2 Die `by_heading` Logik (v3.9.9 Atomic Section Logic)
|
||||
|
||||
Die Strategie `by_heading` zerlegt Texte anhand ihrer Struktur (Überschriften). Sie unterstützt ein "Safety Net" gegen zu große Chunks.
|
||||
Die Strategie `by_heading` implementiert seit v3.9.9 das **"Pack-and-Carry-Over"** Verfahren (Regel 1-3), um Sektions-Überschriften und deren Inhalte atomar in Chunks zu halten.
|
||||
|
||||
* **Split Level:** Definiert die Tiefe (z.B. `2` = H1 & H2 triggern Split).
|
||||
* **Modus "Strict" (`strict_heading_split: true`):**
|
||||
* Jede Überschrift (`<= split_level`) erzwingt einen neuen Chunk.
|
||||
* *Merge-Check:* Wenn der vorherige Chunk leer war (nur Überschriften), wird gemergt.
|
||||
* *Safety Net:* Wird ein Abschnitt zu lang (> `max` Token), wird auch ohne Überschrift getrennt.
|
||||
* **Modus "Soft" (`strict_heading_split: false`):**
|
||||
* **Hierarchie-Check:** Überschriften *oberhalb* des Split-Levels erzwingen **immer** einen Split.
|
||||
* **Füll-Logik:** Überschriften *auf* dem Split-Level lösen nur dann einen neuen Chunk aus, wenn der aktuelle Chunk die `target`-Größe erreicht hat.
|
||||
* *Safety Net:* Auch hier greift das `max` Token Limit.
|
||||
**Kernprinzipien:**
|
||||
* **Atomic Section Logic:** Überschriften und deren Inhalte werden als atomare Einheiten behandelt und nicht über Chunk-Grenzen hinweg getrennt.
|
||||
* **H1-Context Preservation:** Der Dokumenttitel (H1) wird zuverlässig als Breadcrumb in das Embedding-Fenster (`window`) aller Chunks injiziert.
|
||||
* **Signature Alignment:** Parameter-Synchronisierung zwischen Orchestrator und Strategien (`context_prefix` statt `doc_title`).
|
||||
|
||||
**Split Level:** Definiert die Tiefe (z.B. `2` = H1 & H2 triggern Split).
|
||||
|
||||
**Modus "Strict" (`strict_heading_split: true`):**
|
||||
* Jede Überschrift (`<= split_level`) erzwingt einen neuen Chunk.
|
||||
* *Merge-Check:* Wenn der vorherige Chunk leer war (nur Überschriften), wird gemergt.
|
||||
* *Safety Net:* Wird ein Abschnitt zu lang (> `max` Token), wird auch ohne Überschrift getrennt.
|
||||
|
||||
**Modus "Soft" (`strict_heading_split: false`):**
|
||||
* **Hierarchie-Check:** Überschriften *oberhalb* des Split-Levels erzwingen **immer** einen Split.
|
||||
* **Füll-Logik:** Überschriften *auf* dem Split-Level lösen nur dann einen neuen Chunk aus, wenn der aktuelle Chunk die `target`-Größe erreicht hat.
|
||||
* **Pack-and-Carry-Over:** Wenn ein Abschnitt zu groß ist, wird er intelligent zerlegt, wobei der Rest (mit Überschrift) zurück in die Queue gelegt wird.
|
||||
* *Safety Net:* Auch hier greift das `max` Token Limit.
|
||||
|
||||
### 3.3 Registry-First Profiling (v2.13.12)
|
||||
|
||||
Seit v2.13.12 nutzt der `IngestionService` die korrekte Hierarchie zur Ermittlung des Chunking-Profils:
|
||||
|
||||
**Priorität:**
|
||||
1. **Frontmatter** (Override) - Explizite `chunking_profile` Angabe
|
||||
2. **`types.yaml` Typ-Config** - Profil basierend auf `type`
|
||||
3. **Global Defaults** - Fallback auf `sliding_standard`
|
||||
|
||||
**Wichtig:** Ein Hard-Fallback auf `sliding_standard` erfolgt nur noch, wenn keine Konfiguration existiert. Dies stellt sicher, dass Note-Typen wie `value` automatisch das korrekte Profil (z.B. `structured_smart_edges_strict`) erhalten.
|
||||
|
||||
### 3.4 Deterministic Hashing (v2.13.12)
|
||||
|
||||
Der `full`-Hash inkludiert nun alle strategischen Parameter (z.B. `split_level`, `strict_heading_split`), sodass Konfigurationsänderungen im Frontmatter zwingend einen Re-Import auslösen.
|
||||
|
||||
**Impact:** Änderungen an Chunking-Parametern werden zuverlässig erkannt, auch wenn der Text unverändert bleibt.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -279,4 +279,11 @@ python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet" --yes
|
||||
|
||||
# 2. Neu importieren (Force Hash recalculation)
|
||||
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply --force
|
||||
```
|
||||
```
|
||||
|
||||
**Wichtig (v2.9.1 Migration):**
|
||||
Nach dem Update auf v2.9.1 (Section-basierte Links, Multigraph-Support) ist ein vollständiger Re-Import erforderlich, um "Phantom-Knoten" zu beheben und die neue Edge-Struktur zu konsolidieren:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply --force
|
||||
```
|
||||
Dies stellt sicher, dass alle bestehenden Links korrekt in `target_id` und `target_section` aufgeteilt werden.
|
||||
@@ -0,0 +1,99 @@
|
||||
# Branch Merge Commit Message: WP4d
|
||||
|
||||
```
|
||||
feat: Section-basierte Links, Atomic Section Chunking & Registry-First Profiling (v2.9.1)
|
||||
|
||||
## Graph Topology & Edge Management
|
||||
|
||||
### Section-basierte Links (Multigraph-Support)
|
||||
- Split `[[Note#Section]]` Links in `target_id="Note"` und `target_section="Section"`
|
||||
- Edge-ID enthält nun `variant` (Section), ermöglicht mehrere Kanten zwischen denselben Knoten
|
||||
- Semantische Deduplizierung basiert auf `src->tgt:kind@sec` Key
|
||||
- Behebt "Phantom-Knoten" durch korrekte Trennung von Note-Name und Abschnitt
|
||||
|
||||
**Geänderte Dateien:**
|
||||
- `app/core/graph/graph_utils.py`: `parse_link_target()` für Section-Extraktion
|
||||
- `app/core/graph/graph_derive_edges.py`: `target_section` in Edge-Payload
|
||||
- `app/core/database/qdrant.py`: Keyword-Index für `target_section`
|
||||
- `app/core/database/qdrant_points.py`: Explizites Durchreichen von `target_section`
|
||||
- `app/models/dto.py`: `EdgeDTO` mit `target_section` Feld
|
||||
|
||||
### Extraction & Parsing Verbesserungen
|
||||
- Multi-line Callout-Blocks korrekt verarbeitet (stop-check logic)
|
||||
- Robuster Fallback für "headless" Blocks (split chunks)
|
||||
- Liberalisierte Regex für Umlaute und Sonderzeichen in Targets
|
||||
|
||||
**Geänderte Dateien:**
|
||||
- `app/core/graph/graph_extractors.py`: Multi-line Callout-Parser, erweiterte Regex
|
||||
|
||||
## Chunking & Ingestion (v3.9.9 / v2.13.12)
|
||||
|
||||
### Atomic Section Logic (v3.9.9)
|
||||
- Vollständige Implementierung des "Pack-and-Carry-Over" Verfahrens (Regel 1-3)
|
||||
- Sektions-Überschriften und Inhalte bleiben atomar in Chunks
|
||||
- H1-Context Preservation: Dokumenttitel als Breadcrumb in Embedding-Fenster
|
||||
- Signature Alignment: Parameter-Synchronisierung (`context_prefix` statt `doc_title`)
|
||||
|
||||
**Geänderte Dateien:**
|
||||
- `app/core/chunking/chunking_strategies.py`: Atomic Section Logic implementiert
|
||||
|
||||
### Format-agnostische De-Duplizierung
|
||||
- Prüfung auf vorhandene Kanten basiert auf Ziel (`target`), nicht String-Match
|
||||
- Verhindert Dopplung von Kanten, die bereits via `[!edge]` Callout vorhanden sind
|
||||
- Global Pool Integration für unzugeordnete Kanten
|
||||
|
||||
**Geänderte Dateien:**
|
||||
- `app/core/chunking/chunking_propagation.py`: Ziel-basierte Prüfung
|
||||
|
||||
### Registry-First Profiling (v2.13.12)
|
||||
- Korrekte Hierarchie: Frontmatter > types.yaml Typ-Config > Global Defaults
|
||||
- Hard-Fallback auf `sliding_standard` nur wenn keine Konfiguration existiert
|
||||
- Automatische Anwendung korrekter Profile basierend auf Note-Typ
|
||||
|
||||
### Deterministic Hashing
|
||||
- `full`-Hash inkludiert strategische Parameter (`split_level`, `strict_heading_split`)
|
||||
- Konfigurationsänderungen im Frontmatter lösen zwingend Re-Import aus
|
||||
|
||||
**Geänderte Dateien:**
|
||||
- `app/core/ingestion/ingestion_processor.py`: Registry-First Profiling, Deterministic Hashing
|
||||
|
||||
## Impact & Breaking Changes
|
||||
|
||||
### Migration erforderlich
|
||||
**WICHTIG:** Vollständiger Re-Import erforderlich für bestehende Vaults:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply --force
|
||||
```
|
||||
|
||||
**Grund:**
|
||||
- Behebt "Phantom-Knoten" durch korrekte Aufteilung von `[[Note#Section]]` Links
|
||||
- Konsolidiert Edge-Struktur mit `target_section` Feld
|
||||
- Aktualisiert Chunking basierend auf neuen Strategien
|
||||
|
||||
### Fixes
|
||||
- ✅ Resolves: Mehrere Links zur gleichen Note in einem Callout-Block wurden zu einer Kante gemergt
|
||||
- ✅ Resolves: "Phantom-Knoten" durch Einbeziehung des Anchors in `target_id`
|
||||
- ✅ Resolves: Redundante `[[rel:...]]` Links in Chunks
|
||||
- ✅ Resolves: Inkonsistente Metadaten in Qdrant durch Registry-First Profiling
|
||||
|
||||
## Dokumentation
|
||||
|
||||
Alle relevanten Dokumente aktualisiert:
|
||||
- `03_tech_data_model.md`: Edge Payload Schema mit `target_section`
|
||||
- `02_concept_graph_logic.md`: Section-basierte Links & Multigraph-Support
|
||||
- `03_tech_ingestion_pipeline.md`: Chunking-Strategien, Registry-First Profiling
|
||||
- `03_tech_api_reference.md`: EdgeDTO mit `target_section`
|
||||
- `01_knowledge_design.md`: Deep-Links dokumentiert
|
||||
- `00_glossary.md`: Neue Begriffe ergänzt
|
||||
- `04_admin_operations.md`: Migration-Hinweis
|
||||
|
||||
## Versionen
|
||||
|
||||
- Graph Topology: v2.9.1
|
||||
- Chunking Strategies: v3.9.9
|
||||
- Ingestion Processor: v2.13.12
|
||||
- API DTO: v0.6.7
|
||||
|
||||
Closes #[issue-number]
|
||||
```
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
# Release Notes: Mindnet v2.9.1 (WP4d)
|
||||
|
||||
**Release Date:** 2025-01-XX
|
||||
**Type:** Feature Release mit Breaking Changes
|
||||
**Branch:** WP4d
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Übersicht
|
||||
|
||||
Diese Version führt **Section-basierte Links** ein, verbessert das Chunking durch **Atomic Section Logic** und implementiert **Registry-First Profiling** für konsistentere Konfigurationsauflösung. Die Änderungen erfordern einen **vollständigen Re-Import** bestehender Vaults.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Neue Features
|
||||
|
||||
### Section-basierte Links (Deep-Links)
|
||||
|
||||
Mindnet unterstützt nun **Deep-Links** zu spezifischen Abschnitten innerhalb einer Note:
|
||||
|
||||
```markdown
|
||||
[[rel:based_on Mein Leitbild#P3 – Disziplin]]
|
||||
```
|
||||
|
||||
**Vorteile:**
|
||||
- Mehrere Links zur gleichen Note möglich (verschiedene Sections)
|
||||
- Präzise Kontext-Ladung (nur relevanter Abschnitt)
|
||||
- Keine "Phantom-Knoten" mehr durch korrekte Trennung von Note-Name und Abschnitt
|
||||
|
||||
**Technische Details:**
|
||||
- Links werden in `target_id="Note"` und `target_section="Section"` aufgeteilt
|
||||
- Edge-ID enthält `variant` (Section) für Multigraph-Support
|
||||
- Semantische Deduplizierung basiert auf `src->tgt:kind@sec` Key
|
||||
|
||||
### Atomic Section Logic (Chunking v3.9.9)
|
||||
|
||||
Das Chunking hält nun Sektions-Überschriften und deren Inhalte **atomar** zusammen:
|
||||
|
||||
**"Pack-and-Carry-Over" Verfahren:**
|
||||
- Regel 1 & 2: Sektionen werden zusammengepackt, wenn sie in den Token-Limit passen
|
||||
- Regel 3: Zu große Sektionen werden intelligent zerlegt, Rest wird zurück in Queue gelegt
|
||||
- H1-Context Preservation: Dokumenttitel wird als Breadcrumb in alle Chunks injiziert
|
||||
|
||||
**Vorteile:**
|
||||
- Keine getrennten Überschriften mehr
|
||||
- Bessere semantische Kohärenz in Chunks
|
||||
- Verbesserte Retrieval-Qualität durch vollständigen Kontext
|
||||
|
||||
### Registry-First Profiling (v2.13.12)
|
||||
|
||||
Die Konfigurationsauflösung folgt nun einer klaren Hierarchie:
|
||||
|
||||
1. **Frontmatter** (höchste Priorität)
|
||||
2. **types.yaml Typ-Config**
|
||||
3. **Global Defaults**
|
||||
|
||||
**Impact:**
|
||||
- Note-Typen wie `value` erhalten automatisch das korrekte Profil (`structured_smart_edges_strict`)
|
||||
- Keine manuellen Overrides mehr nötig für Standard-Typen
|
||||
- Konsistente Metadaten in Qdrant
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Verbesserungen
|
||||
|
||||
### Extraction & Parsing
|
||||
|
||||
- **Multi-line Callout-Blocks:** Korrekte Verarbeitung von mehrzeiligen `[!edge]` Callouts
|
||||
- **Robuste Fallbacks:** "Headless" Blocks werden korrekt behandelt
|
||||
- **Liberalisierte Regex:** Unterstützung für Umlaute und Sonderzeichen in Link-Targets
|
||||
|
||||
### Format-agnostische De-Duplizierung
|
||||
|
||||
- Kanten werden unabhängig vom Format (Inline, Callout, Wikilink) erkannt
|
||||
- Verhindert Dopplungen, wenn Kanten bereits via `[!edge]` Callout vorhanden sind
|
||||
- Ziel-basierte Prüfung statt String-Match
|
||||
|
||||
### Deterministic Hashing
|
||||
|
||||
- `full`-Hash inkludiert strategische Parameter (`split_level`, `strict_heading_split`)
|
||||
- Konfigurationsänderungen im Frontmatter lösen zwingend Re-Import aus
|
||||
- Zuverlässigere Change Detection
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Bugfixes
|
||||
|
||||
- ✅ **Behoben:** Mehrere Links zur gleichen Note in einem Callout-Block wurden zu einer Kante gemergt
|
||||
- ✅ **Behoben:** "Phantom-Knoten" durch Einbeziehung des Anchors in `target_id`
|
||||
- ✅ **Behoben:** Redundante `[[rel:...]]` Links in Chunks
|
||||
- ✅ **Behoben:** Inkonsistente Metadaten in Qdrant durch fehlerhafte Profil-Auflösung
|
||||
- ✅ **Behoben:** `TypeError` durch Parameter-Mismatch zwischen Orchestrator und Strategien
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Breaking Changes & Migration
|
||||
|
||||
### Migration erforderlich
|
||||
|
||||
**WICHTIG:** Nach dem Update auf v2.9.1 ist ein **vollständiger Re-Import** erforderlich:
|
||||
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply --force
|
||||
```
|
||||
|
||||
**Warum?**
|
||||
- Behebt "Phantom-Knoten" durch korrekte Aufteilung von `[[Note#Section]]` Links
|
||||
- Konsolidiert Edge-Struktur mit neuem `target_section` Feld
|
||||
- Aktualisiert Chunking basierend auf Atomic Section Logic
|
||||
|
||||
**Was passiert beim Re-Import?**
|
||||
- Alle bestehenden Links werden neu geparst und in `target_id` + `target_section` aufgeteilt
|
||||
- Chunks werden mit neuer Atomic Section Logic neu generiert
|
||||
- Edge-Struktur wird konsolidiert (Multigraph-Support)
|
||||
|
||||
**Dauer:** Abhängig von Vault-Größe (typischerweise 5-30 Minuten)
|
||||
|
||||
---
|
||||
|
||||
## 📚 API-Änderungen
|
||||
|
||||
### EdgeDTO erweitert
|
||||
|
||||
```python
|
||||
class EdgeDTO(BaseModel):
|
||||
# ... bestehende Felder ...
|
||||
target_section: Optional[str] = None # Neu: Abschnitts-Name
|
||||
```
|
||||
|
||||
**Impact für API-Consumer:**
|
||||
- Graph-Endpunkte (`/graph/{note_id}`) enthalten nun `target_section` in Edge-Objekten
|
||||
- Frontend kann Section-Informationen für präzisere Visualisierung nutzen
|
||||
|
||||
---
|
||||
|
||||
## 📖 Dokumentation
|
||||
|
||||
Alle relevanten Dokumente wurden aktualisiert:
|
||||
|
||||
- ✅ `03_tech_data_model.md`: Edge Payload Schema mit `target_section`
|
||||
- ✅ `02_concept_graph_logic.md`: Section-basierte Links & Multigraph-Support
|
||||
- ✅ `03_tech_ingestion_pipeline.md`: Chunking-Strategien, Registry-First Profiling
|
||||
- ✅ `03_tech_api_reference.md`: EdgeDTO mit `target_section`
|
||||
- ✅ `01_knowledge_design.md`: Deep-Links dokumentiert
|
||||
- ✅ `00_glossary.md`: Neue Begriffe ergänzt
|
||||
- ✅ `04_admin_operations.md`: Migration-Hinweis
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Technische Details
|
||||
|
||||
### Geänderte Module
|
||||
|
||||
**Graph Topology:**
|
||||
- `app/core/graph/graph_utils.py`: `parse_link_target()` für Section-Extraktion
|
||||
- `app/core/graph/graph_derive_edges.py`: `target_section` in Edge-Payload
|
||||
- `app/core/graph/graph_extractors.py`: Multi-line Callout-Parser
|
||||
|
||||
**Chunking:**
|
||||
- `app/core/chunking/chunking_strategies.py`: Atomic Section Logic (v3.9.9)
|
||||
- `app/core/chunking/chunking_propagation.py`: Format-agnostische De-Duplizierung
|
||||
|
||||
**Ingestion:**
|
||||
- `app/core/ingestion/ingestion_processor.py`: Registry-First Profiling (v2.13.12), Deterministic Hashing
|
||||
|
||||
**Database:**
|
||||
- `app/core/database/qdrant.py`: Keyword-Index für `target_section`
|
||||
- `app/core/database/qdrant_points.py`: Explizites Durchreichen von `target_section`
|
||||
|
||||
**API:**
|
||||
- `app/models/dto.py`: `EdgeDTO` mit `target_section` Feld (v0.6.7)
|
||||
|
||||
### Versionsnummern
|
||||
|
||||
- Graph Topology: **v2.9.1**
|
||||
- Chunking Strategies: **v3.9.9**
|
||||
- Ingestion Processor: **v2.13.12**
|
||||
- API DTO: **v0.6.7**
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Upgrade-Pfad
|
||||
|
||||
### Für Administratoren
|
||||
|
||||
1. **Backup erstellen:**
|
||||
```bash
|
||||
docker stop qdrant
|
||||
tar -czf qdrant_backup_$(date +%F).tar.gz ./qdrant_data
|
||||
docker start qdrant
|
||||
```
|
||||
|
||||
2. **Code aktualisieren:**
|
||||
```bash
|
||||
git pull origin main
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
3. **Re-Import durchführen:**
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply --force
|
||||
```
|
||||
|
||||
4. **Services neu starten:**
|
||||
```bash
|
||||
sudo systemctl restart mindnet-prod
|
||||
sudo systemctl restart mindnet-ui-prod
|
||||
```
|
||||
|
||||
### Für Entwickler
|
||||
|
||||
- Keine Code-Änderungen erforderlich, wenn nur API genutzt wird
|
||||
- Frontend kann `target_section` Feld in Edge-Objekten nutzen (optional)
|
||||
|
||||
---
|
||||
|
||||
## 📝 Bekannte Einschränkungen
|
||||
|
||||
- **Migration-Dauer:** Große Vaults (>10.000 Notizen) können 30+ Minuten benötigen
|
||||
- **Temporärer Speicher:** Während des Re-Imports kann Qdrant-Speicher temporär ansteigen
|
||||
|
||||
---
|
||||
|
||||
## 🙏 Danksagungen
|
||||
|
||||
Diese Version wurde durch umfangreiche Code-Analyse und Dokumentationsprüfung ermöglicht. Besonderer Fokus lag auf:
|
||||
- Konsistenz zwischen Code und Dokumentation
|
||||
- Vollständige Abdeckung aller Rollen (Entwickler, Administratoren, Anwender, Tester, Deployment)
|
||||
- Klare Migration-Pfade
|
||||
|
||||
---
|
||||
|
||||
**Vollständige Changelog:** Siehe Git-Commits für detaillierte Änderungen
|
||||
**Support:** Bei Fragen zur Migration siehe [Admin Operations Guide](../04_Operations/04_admin_operations.md)
|
||||
|
||||
@@ -64,7 +64,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
|
||||
@@ -61,9 +61,10 @@ import argparse, glob, json, os
|
||||
from typing import List, Tuple
|
||||
|
||||
from app.core.parser import read_markdown # gibt je nach Implementierung ein Objekt ODER ein (fm, body)-Tuple
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
from app.core.qdrant_points import points_for_edges, upsert_batch
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
from app.core.database.qdrant_points import points_for_edges, upsert_batch
|
||||
# TODO: build_note_index und derive_wikilink_edges müssen noch implementiert werden
|
||||
# from app.core.graph.graph_derive_edges import build_note_index, derive_wikilink_edges
|
||||
|
||||
|
||||
def _coerce_parsed(note_or_tuple):
|
||||
|
||||
@@ -71,7 +71,7 @@ from pathlib import Path
|
||||
sys.path.insert(0, os.path.abspath("."))
|
||||
|
||||
from app.core.chunking import assemble_chunks
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
from app.core.graph.graph_derive_edges import build_edges_for_note
|
||||
from app.core.parser import extract_wikilinks
|
||||
from app.core.graph.graph_extractors import extract_typed_relations, extract_callout_relations
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ v1.0.0: Initial Release
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, os, json
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
def count_points(client, collection: str) -> int:
|
||||
try:
|
||||
|
||||
@@ -60,7 +60,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
|
||||
@@ -76,7 +76,7 @@ from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
from app.core.graph.graph_derive_edges import build_edges_for_note
|
||||
|
||||
def _iter_markdown(vault: str):
|
||||
for p in Path(vault).rglob("*.md"):
|
||||
|
||||
@@ -67,7 +67,7 @@ import json
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
|
||||
@@ -80,8 +80,7 @@ from typing import Dict, List, Optional, Tuple, Any
|
||||
import yaml
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.qdrant import ensure_collections # safety
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helpers
|
||||
|
||||
@@ -77,7 +77,7 @@ from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
|
||||
|
||||
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
|
||||
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
|
||||
|
||||
@@ -73,7 +73,7 @@ from app.core.parser import read_markdown, normalize_frontmatter, validate_requi
|
||||
from app.core.chunking import assemble_chunks
|
||||
from app.core.ingestion.ingestion_note_payload import make_note_payload
|
||||
from app.core.ingestion.ingestion_chunk_payload import make_chunk_payloads
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
from app.core.graph.graph_derive_edges import build_edges_for_note
|
||||
|
||||
async def process_file(path: str, root: str, args):
|
||||
"""Verarbeitet eine einzelne Datei asynchron."""
|
||||
|
||||
@@ -85,7 +85,7 @@ from typing import Dict, List, Set, Tuple
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.parser import read_markdown, normalize_frontmatter
|
||||
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ import os
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
def collections(prefix: str):
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
@@ -83,7 +83,7 @@ from dotenv import load_dotenv
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import (
|
||||
from app.core.database.qdrant import (
|
||||
QdrantConfig,
|
||||
get_client,
|
||||
ensure_collections,
|
||||
|
||||
@@ -70,8 +70,8 @@ import uuid
|
||||
from typing import List, Dict, Any, Iterable
|
||||
|
||||
from qdrant_client import models
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.qdrant_points import points_for_edges
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant_points import points_for_edges
|
||||
|
||||
# Logging Setup
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
|
||||
@@ -67,7 +67,7 @@ from typing import Dict, List, Tuple, Optional
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
def _chunk_sort_key(p: Dict, pid: str) -> Tuple[int,int,str]:
|
||||
# Primär: payload.chunk_index, sekundär: Nummer am Ende der ID (#cNN oder #NN), sonst 0
|
||||
|
||||
@@ -23,7 +23,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
|
||||
@@ -9,7 +9,7 @@ if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
def scroll_all(c, col, flt=None):
|
||||
out, nextp = [], None
|
||||
|
||||
@@ -18,7 +18,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
|
||||
@@ -18,7 +18,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Diagnose-Script: Prüft welche Kanten für eine Note gefunden werden.
|
||||
Hilft beim Debugging der Graph-Anzeige.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Projekt-Root zum sys.path hinzufügen
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def find_edges_for_note(note_id: str, prefix: str = "mindnet"):
|
||||
"""Findet alle Kanten für eine Note und zeigt sie an."""
|
||||
cfg = QdrantConfig.from_env()
|
||||
if prefix:
|
||||
cfg.prefix = prefix
|
||||
|
||||
client = get_client(cfg)
|
||||
_, _, edges_col = collection_names(cfg.prefix)
|
||||
notes_col = f"{cfg.prefix}_notes"
|
||||
|
||||
# 1. Lade Note, um Titel zu bekommen
|
||||
note_res, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]),
|
||||
limit=1, with_payload=True
|
||||
)
|
||||
|
||||
if not note_res:
|
||||
print(f"❌ Note '{note_id}' nicht gefunden!")
|
||||
return
|
||||
|
||||
note = note_res[0].payload
|
||||
note_title = note.get("title", "")
|
||||
print(f"📄 Note: {note_id}")
|
||||
print(f" Titel: {note_title}\n")
|
||||
|
||||
# 2. Ausgehende Kanten (note_id = unsere Note)
|
||||
print("🔍 AUSGEHENDE KANTEN (note_id = unsere Note):")
|
||||
out_filter = rest.Filter(must=[
|
||||
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id)),
|
||||
rest.FieldCondition(key="kind", match=rest.MatchExcept(**{"except": ["prev", "next", "belongs_to"]}))
|
||||
])
|
||||
out_edges, _ = client.scroll(edges_col, scroll_filter=out_filter, limit=2000, with_payload=True)
|
||||
print(f" Gefunden: {len(out_edges)} Kanten")
|
||||
for i, e in enumerate(out_edges[:10], 1): # Zeige erste 10
|
||||
pl = e.payload
|
||||
print(f" {i}. {pl.get('kind')}: {pl.get('source_id')} -> {pl.get('target_id')}")
|
||||
if len(out_edges) > 10:
|
||||
print(f" ... und {len(out_edges) - 10} weitere\n")
|
||||
else:
|
||||
print()
|
||||
|
||||
# 3. Eingehende Kanten - exakte Titel-Suche
|
||||
print("🔍 EINGEHENDE KANTEN (target_id = exakter Titel):")
|
||||
if note_title:
|
||||
in_filter_exact = rest.Filter(
|
||||
must=[rest.FieldCondition(key="kind", match=rest.MatchExcept(**{"except": ["prev", "next", "belongs_to"]}))],
|
||||
should=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_title))]
|
||||
)
|
||||
in_edges_exact, _ = client.scroll(edges_col, scroll_filter=in_filter_exact, limit=2000, with_payload=True)
|
||||
print(f" Gefunden: {len(in_edges_exact)} Kanten (exakter Titel)")
|
||||
for i, e in enumerate(in_edges_exact[:10], 1):
|
||||
pl = e.payload
|
||||
print(f" {i}. {pl.get('kind')}: {pl.get('source_id')} -> {pl.get('target_id')}")
|
||||
if len(in_edges_exact) > 10:
|
||||
print(f" ... und {len(in_edges_exact) - 10} weitere\n")
|
||||
else:
|
||||
print()
|
||||
|
||||
# 4. Eingehende Kanten - Titel#Abschnitt Varianten mit Fuzzy-Matching
|
||||
print("🔍 EINGEHENDE KANTEN (target_id beginnt mit 'Titel#' + Fuzzy-Matching):")
|
||||
if note_title:
|
||||
import re
|
||||
# Lade alle Kanten und filtere clientseitig (wie Case D in ui_graph_service.py)
|
||||
all_filter = rest.Filter(
|
||||
must=[rest.FieldCondition(key="kind", match=rest.MatchExcept(**{"except": ["prev", "next", "belongs_to"]}))]
|
||||
)
|
||||
all_edges, _ = client.scroll(edges_col, scroll_filter=all_filter, limit=10000, with_payload=True)
|
||||
print(f" Gesamt geladen: {len(all_edges)} Kanten aus der Datenbank")
|
||||
|
||||
# Normalisierungs-Funktion (wie in ui_graph_service.py)
|
||||
def normalize_title(t):
|
||||
if not t:
|
||||
return ""
|
||||
t = re.sub(r'\s*\([^)]*\)', '', t)
|
||||
t = re.sub(r'\s*\d{4}[\s–\-]*\d{0,4}', '', t)
|
||||
t = re.sub(r'^(Mein|Meine)\s+', '', t, flags=re.IGNORECASE)
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
return t.lower()
|
||||
|
||||
note_title_norm = normalize_title(note_title)
|
||||
print(f" Normalisierter Note-Titel: '{note_title_norm}'")
|
||||
|
||||
# Clientseitige Filterung: Exakte Matches
|
||||
matched_exact = []
|
||||
for e in all_edges:
|
||||
tgt_id = e.payload.get("target_id", "")
|
||||
if tgt_id and (tgt_id == note_title or tgt_id.startswith(note_title + "#")):
|
||||
matched_exact.append(e)
|
||||
|
||||
# Clientseitige Filterung: Fuzzy-Matches
|
||||
matched_fuzzy = []
|
||||
for e in all_edges:
|
||||
tgt_id = e.payload.get("target_id", "")
|
||||
if not tgt_id or e in matched_exact:
|
||||
continue
|
||||
tgt_base = tgt_id.split("#")[0].strip()
|
||||
tgt_norm = normalize_title(tgt_base)
|
||||
if tgt_norm and note_title_norm and len(note_title_norm) > 5:
|
||||
if (tgt_norm == note_title_norm or
|
||||
tgt_norm.startswith(note_title_norm) or
|
||||
note_title_norm.startswith(tgt_norm)):
|
||||
matched_fuzzy.append((e, tgt_norm))
|
||||
|
||||
matched = matched_exact + [e for e, _ in matched_fuzzy]
|
||||
print(f" Gefunden: {len(matched_exact)} exakte Matches, {len(matched_fuzzy)} Fuzzy-Matches")
|
||||
print(f" Gesamt: {len(matched)} Kanten")
|
||||
for i, e in enumerate(matched[:10], 1):
|
||||
pl = e.payload
|
||||
match_type = "EXAKT" if e in matched_exact else "FUZZY"
|
||||
print(f" {i}. [{match_type}] {pl.get('kind')}: {pl.get('source_id')} -> {pl.get('target_id')}")
|
||||
if len(matched) > 10:
|
||||
print(f" ... und {len(matched) - 10} weitere\n")
|
||||
else:
|
||||
print()
|
||||
|
||||
# Zeige auch einige Beispiele von target_ids, die NICHT matchen
|
||||
print(" 🔍 DEBUG: Beispiel target_ids die NICHT matchen (erste 10):")
|
||||
non_matched = []
|
||||
for e in all_edges[:200]: # Erste 200 prüfen
|
||||
tgt_id = e.payload.get("target_id", "")
|
||||
if not tgt_id or e in matched:
|
||||
continue
|
||||
tgt_base = tgt_id.split("#")[0].strip()
|
||||
tgt_norm = normalize_title(tgt_base)
|
||||
non_matched.append((tgt_id, tgt_norm))
|
||||
if len(non_matched) >= 10:
|
||||
break
|
||||
for i, (tgt, tgt_norm) in enumerate(non_matched, 1):
|
||||
print(f" {i}. '{tgt}' (normalisiert: '{tgt_norm}')")
|
||||
print()
|
||||
|
||||
# 5. Zeige Beispiel target_ids aus der Datenbank
|
||||
print("📊 BEISPIEL target_ids aus der Datenbank (erste 20 mit #):")
|
||||
sample_filter = rest.Filter(
|
||||
must=[rest.FieldCondition(key="kind", match=rest.MatchExcept(**{"except": ["prev", "next", "belongs_to"]}))]
|
||||
)
|
||||
sample_edges, _ = client.scroll(edges_col, scroll_filter=sample_filter, limit=100, with_payload=True)
|
||||
hash_edges = [e for e in sample_edges if "#" in str(e.payload.get("target_id", ""))]
|
||||
for i, e in enumerate(hash_edges[:20], 1):
|
||||
pl = e.payload
|
||||
print(f" {i}. target_id: '{pl.get('target_id')}' (kind: {pl.get('kind')})")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--note-id", required=True, help="Note-ID zum Testen")
|
||||
parser.add_argument("--prefix", default="mindnet", help="Collection-Prefix")
|
||||
args = parser.parse_args()
|
||||
|
||||
find_edges_for_note(args.note_id, args.prefix)
|
||||
|
||||
@@ -37,7 +37,7 @@ if PROJECT_ROOT not in sys.path:
|
||||
|
||||
from qdrant_client.http import models as rest # type: ignore
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
|
||||
def fetch_edges_for_note(note_id: str, prefix: str | None, limit: int = 512) -> List[dict]:
|
||||
|
||||
@@ -10,7 +10,7 @@ WICHTIG: Einige Qdrant-Versionen liefern payload_schema nur, wenn
|
||||
from __future__ import annotations
|
||||
import argparse, json
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
def compact_schema(ps: dict | None) -> dict:
|
||||
if not isinstance(ps, dict):
|
||||
|
||||
@@ -33,8 +33,8 @@ PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client # type: ignore
|
||||
import app.core.graph_adapter as ga # type: ignore
|
||||
from app.core.database.qdrant import QdrantConfig, get_client # type: ignore
|
||||
import app.core.graph.graph_subgraph as ga # type: ignore
|
||||
|
||||
|
||||
def build_subgraph_for_note(
|
||||
|
||||
@@ -14,7 +14,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
|
||||
@@ -17,7 +17,7 @@ from app.core.parser import read_markdown, normalize_frontmatter, validate_requi
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
try:
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
from app.core.graph.graph_derive_edges import build_edges_for_note
|
||||
except Exception:
|
||||
from app.core.edges import build_edges_for_note # type: ignore
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import argparse, json, os, sys
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
@@ -14,7 +14,7 @@ from unittest.mock import MagicMock, patch, AsyncMock
|
||||
import app.routers.chat
|
||||
from app.models.dto import ChatRequest, QueryHit, QueryRequest
|
||||
from app.services.edge_registry import EdgeRegistry
|
||||
from app.core.retriever import _compute_total_score, _get_status_multiplier
|
||||
from app.core.retrieval.retriever_scoring import compute_wp22_score, get_status_multiplier
|
||||
from app.routers.chat import _classify_intent, get_decision_strategy, chat_endpoint
|
||||
|
||||
class TestWP22Integration(unittest.IsolatedAsyncioTestCase):
|
||||
@@ -93,19 +93,19 @@ class TestWP22Integration(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
def test_scoring_math(self):
|
||||
print("\n🔵 TEST 2: Scoring Math (Lifecycle)")
|
||||
with patch("app.core.retriever._get_scoring_weights", return_value=(1.0, 1.0, 0.0)):
|
||||
with patch("app.core.retrieval.retriever_scoring.get_weights", return_value=(1.0, 1.0, 0.0)):
|
||||
# Stable (1.2)
|
||||
self.assertEqual(_get_status_multiplier({"status": "stable"}), 1.2)
|
||||
self.assertEqual(get_status_multiplier({"status": "stable"}), 1.2)
|
||||
# Draft (0.5)
|
||||
self.assertEqual(_get_status_multiplier({"status": "draft"}), 0.5)
|
||||
self.assertEqual(get_status_multiplier({"status": "draft"}), 0.5)
|
||||
|
||||
# Scoring Formel Test: BaseScore * (1 + ConfigWeight + DynamicBoost)
|
||||
# BaseScore = 0.5 (sem) * 1.2 (stable) = 0.6
|
||||
# ConfigWeight = 1.0 (neutral) - 1.0 = 0.0
|
||||
# DynamicBoost = (1.0 * 0.5) = 0.5
|
||||
# Total = 0.6 * (1 + 0 + 0.5) = 0.9
|
||||
total, _, _ = _compute_total_score(0.5, {"status": "stable", "retriever_weight": 1.0}, edge_bonus_raw=0.5)
|
||||
self.assertAlmostEqual(total, 0.9)
|
||||
result = compute_wp22_score(0.5, {"status": "stable", "retriever_weight": 1.0}, edge_bonus_raw=0.5)
|
||||
self.assertAlmostEqual(result["total"], 0.9)
|
||||
print("✅ Scoring OK.")
|
||||
|
||||
async def test_router_intent(self):
|
||||
|
||||
@@ -16,7 +16,7 @@ import json
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
|
||||
def _scroll_all(client, collection: str):
|
||||
|
||||
@@ -8,7 +8,7 @@ Zeigt, ob typenbasierte edge_defaults-Relationen (z. B. depends_on/related_to) a
|
||||
from __future__ import annotations
|
||||
from collections import Counter
|
||||
import json
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
|
||||
@@ -7,7 +7,7 @@ import os
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
|
||||
def _rel(pl: dict) -> str:
|
||||
|
||||
@@ -13,7 +13,7 @@ import argparse, json
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
KINDS = ["belongs_to", "next", "prev", "references", "backlink"]
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import create_app
|
||||
import app.core.graph_adapter as ga
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
|
||||
def _fake_get_edges_for_sources(client, prefix, source_ids, edge_types=None, limit=2048):
|
||||
note_id = source_ids[0]
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import create_app
|
||||
import app.services.embeddings_client as ec
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.graph_adapter as ga
|
||||
import app.core.database.qdrant_points as qp
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
|
||||
def _fake_embed_text(text: str):
|
||||
# Liefert stabilen 384-d Vektor ohne echtes Modell
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import create_app
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.graph_adapter as ga
|
||||
import app.core.database.qdrant_points as qp
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
|
||||
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||
return [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from app.core import retriever as r
|
||||
from app.core.retrieval import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.database.qdrant as qdr
|
||||
import app.core.database.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from app.core import retriever as r
|
||||
from app.core.retrieval import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.graph_adapter as ga
|
||||
import app.core.database.qdrant as qdr
|
||||
import app.core.database.qdrant_points as qp
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from app.core import retriever as r
|
||||
from app.core.retrieval import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.database.qdrant as qdr
|
||||
import app.core.database.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
# Import der Kernkomponenten
|
||||
from app.core import chunker
|
||||
from app.core import derive_edges
|
||||
from app.core.graph import graph_derive_edges as derive_edges
|
||||
from app.services.semantic_analyzer import SemanticAnalyzer
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ if PROJECT_ROOT not in sys.path:
|
||||
import yaml
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from app.core.database.qdrant import QdrantConfig, get_client
|
||||
|
||||
# --------------------------- Helpers ---------------------------
|
||||
|
||||
|
||||
Reference in new issue
Block a user