app/core/derive_edges.py aktualisiert
Some checks failed
Deploy mindnet to llm-node / deploy (push) Failing after 2s

This commit is contained in:
Lars 2025-09-04 09:44:19 +02:00
parent a9d4c2ec0e
commit d4a2d9c0e1

View File

@ -4,6 +4,7 @@ import re
import unicodedata import unicodedata
from typing import Dict, List, Tuple from typing import Dict, List, Tuple
# [[Ziel]], [[Ziel|Alias]], [[Ziel#Heading]], [[Ziel#Heading|Alias]]
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]") WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]")
def _slug(s: str) -> str: def _slug(s: str) -> str:
@ -13,9 +14,8 @@ def _slug(s: str) -> str:
s = unicodedata.normalize("NFKD", s) s = unicodedata.normalize("NFKD", s)
s = "".join(ch for ch in s if not unicodedata.combining(ch)) s = "".join(ch for ch in s if not unicodedata.combining(ch))
s = s.replace("\\", "/") s = s.replace("\\", "/")
s = s.split("/")[-1] # nur letzter Pfadteil s = s.split("/")[-1]
s = s.lower() s = s.lower().replace(" ", "-")
s = s.replace(" ", "-")
s = re.sub(r"[^a-z0-9\-]+", "", s) s = re.sub(r"[^a-z0-9\-]+", "", s)
s = re.sub(r"-{2,}", "-", s).strip("-") s = re.sub(r"-{2,}", "-", s).strip("-")
return s return s
@ -30,7 +30,7 @@ def build_note_index(notes_payloads: List[dict]) -> Tuple[Dict[str, dict], Dict[
continue continue
by_id[nid] = n by_id[nid] = n
title = n.get("title", "") title = n.get("title", "")
path = n.get("path", "") # z.B. "Ordner/Datei.md" path = n.get("path", "")
file_slug = _slug(path.split("/")[-1]) if path else "" file_slug = _slug(path.split("/")[-1]) if path else ""
if title: if title:
by_slug[_slug(title)] = n by_slug[_slug(title)] = n
@ -38,14 +38,11 @@ def build_note_index(notes_payloads: List[dict]) -> Tuple[Dict[str, dict], Dict[
by_file_slug[file_slug] = n by_file_slug[file_slug] = n
return by_id, by_slug, by_file_slug return by_id, by_slug, by_file_slug
def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict[str,dict]]) -> Tuple[str|None, str]: def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict[str,dict]]):
"""
Liefert (target_note_id | None, resolution_hint)
"""
by_id, by_slug, by_file_slug = idx by_id, by_slug, by_file_slug = idx
key = note_like.strip() key = note_like.strip()
if key in by_id: if key in by_id:
return key, "by_id" return by_id[key]["note_id"], "by_id"
s = _slug(key) s = _slug(key)
if s in by_slug: if s in by_slug:
return by_slug[s]["note_id"], "by_slug" return by_slug[s]["note_id"], "by_slug"
@ -54,29 +51,18 @@ def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict
return None, "unresolved" return None, "unresolved"
def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_index) -> List[dict]: def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_index) -> List[dict]:
"""
Erzeugt Edges aus [[...]] im Note-Text und in Chunk-Text:
- references: source_note -> target_note
- backlink: target_note -> source_note
- optional: references_at: source_chunk -> target_note (wo steht der Link?)
"""
edges: List[dict] = [] edges: List[dict] = []
source_note_id = note_payload["note_id"] source_note_id = note_payload["note_id"]
def _make_edge(kind: str, src: str, tgt: str, seq: int|None=None, extra: dict|None=None) -> dict: def _make_edge(kind: str, src: str, tgt: str, seq=None, extra: dict|None=None):
e = { e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
"edge_id": None, # später vergeben (qdrant_points macht robusten Fallback + UUID)
"kind": kind,
"source_id": src,
"target_id": tgt,
}
if seq is not None: if seq is not None:
e["seq"] = seq e["seq"] = seq
if extra: if extra:
e.update(extra) e.update(extra)
return e return e
# 1) Links im Note-Gesamttext (falls vorhanden) # Links im Volltext (falls vorhanden)
fulltext = note_payload.get("fulltext") or note_payload.get("body") or "" fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
if fulltext: if fulltext:
for m in WIKILINK_RE.finditer(fulltext): for m in WIKILINK_RE.finditer(fulltext):
@ -85,14 +71,13 @@ def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how} extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id: if target_id:
edges.append(_make_edge("references", source_note_id, target_id, extra=extra)) edges.append(_make_edge("references", source_note_id, target_id, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, extra=extra)) edges.append(_make_edge("backlink", target_id, source_note_id, extra=extra))
else: else:
# Unresolved: referenziere Slug/Raw, markiere Status
extra["status"] = "unresolved" extra["status"] = "unresolved"
extra["target_label"] = raw_target extra["target_label"] = raw_target
edges.append(_make_edge("references", source_note_id, raw_target, extra=extra)) edges.append(_make_edge("references", source_note_id, raw_target, extra=extra))
# 2) Links in den einzelnen Chunks # Links in Chunks (wenn übergeben)
for i, ch in enumerate(chunks_payloads, start=1): for i, ch in enumerate(chunks_payloads, start=1):
txt = ch.get("text") or ch.get("content") or "" txt = ch.get("text") or ch.get("content") or ""
if not txt: if not txt:
@ -101,13 +86,10 @@ def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_
raw_target, heading, alias = m.groups() raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index) target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how} extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
# optional: chunk→note Referenz (wo steht der Link?)
if target_id: if target_id:
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, extra=extra)) edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, extra=extra))
# die Note→Note-Kanten sind oben schon generiert; doppelt vermeiden:
else: else:
extra["status"] = "unresolved" extra["status"] = "unresolved"
extra["target_label"] = raw_target extra["target_label"] = raw_target
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, extra=extra)) edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, extra=extra))
return edges return edges