Enhance chunking strategies and graph utilities for section-type transitions
- Implemented WP-26 v1.1: Section-Type-Wechsel erzwingt immer einen neuen Chunk, um konsistente Chunking-Verhalten bei unterschiedlichen section_types zu gewährleisten. - Introduced automatic Intra-Note-Edges zwischen Sektionen mit unterschiedlichen Typen, um semantische Beziehungen zu erfassen. - Updated graph utilities to support automatic edge type derivation based on section transitions. - Added unit tests for section-type changes and automatic edge generation to ensure functionality and reliability.
This commit is contained in:
1 parent
cc258008dc
commit
af3cc0a254
4 files changed
+505
-50
No files matched your search
@@ -32,7 +32,8 @@ import logging
|
||||
from typing import List, Optional, Dict, Tuple, Set
|
||||
from .graph_utils import (
|
||||
_get, _edge, _mk_edge_id, _dedupe_seq, parse_link_target,
|
||||
PROVENANCE_PRIORITY, load_types_registry, get_edge_defaults_for
|
||||
PROVENANCE_PRIORITY, load_types_registry, get_edge_defaults_for,
|
||||
get_typical_edge_for # WP-26 v1.1: Für automatische Intra-Note-Edges
|
||||
)
|
||||
from .graph_extractors import (
|
||||
extract_typed_relations, extract_callout_relations, extract_wikilinks
|
||||
@@ -645,6 +646,52 @@ def build_edges_for_note(
|
||||
"edge_id": _mk_edge_id("prev", next_id, cid, "chunk"),
|
||||
"provenance": "structure", "rule_id": "structure:order", "confidence": PROVENANCE_PRIORITY["structure:order"]
|
||||
}))
|
||||
|
||||
# 1b) WP-26 v1.1: Automatische Intra-Note-Edges zwischen Sektionen mit unterschiedlichen Typen
|
||||
# Wenn sich der section_type zwischen aufeinanderfolgenden Chunks ändert,
|
||||
# wird eine semantische Kante basierend auf graph_schema.md erstellt.
|
||||
for idx, ch in enumerate(chunks):
|
||||
if idx >= len(chunks) - 1:
|
||||
continue # Kein nächster Chunk
|
||||
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
next_ch = chunks[idx + 1]
|
||||
next_id = _get(next_ch, "chunk_id", "id")
|
||||
|
||||
if not cid or not next_id:
|
||||
continue
|
||||
|
||||
# Hole die effective_types der Chunks
|
||||
# WP-26 v1.1: section_type oder note_type (effective_type)
|
||||
current_section_type = ch.get("section_type")
|
||||
next_section_type = next_ch.get("section_type")
|
||||
current_type = current_section_type or ch.get("type") or note_type
|
||||
next_type = next_section_type or next_ch.get("type") or note_type
|
||||
|
||||
# Prüfe, ob es einen Section-Type-Wechsel gibt
|
||||
# Nur wenn beide einen expliziten section_type haben oder sich die effective_types unterscheiden
|
||||
is_section_change = (
|
||||
(current_section_type is not None or next_section_type is not None) and
|
||||
current_type != next_type
|
||||
)
|
||||
|
||||
if is_section_change:
|
||||
# Ermittle den passenden Edge-Typ aus graph_schema.md
|
||||
edge_kind = get_typical_edge_for(current_type, next_type)
|
||||
|
||||
logger.debug(f"WP-26 Intra-Note-Edge: {current_type} -> {next_type} = {edge_kind}")
|
||||
|
||||
# Erstelle die automatische Edge (Forward-Richtung)
|
||||
edges.append(_edge(edge_kind, "chunk", cid, next_id, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(edge_kind, cid, next_id, "chunk"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "inferred:section_transition",
|
||||
"source_hint": "schema_default",
|
||||
"confidence": PROVENANCE_PRIORITY.get("schema_default", 0.85),
|
||||
"is_internal": True, # Explizit als Intra-Note-Edge markieren
|
||||
"section_transition": f"{current_type}->{next_type}" # Debug-Info
|
||||
}))
|
||||
|
||||
# 2) Inhaltliche Kanten (Explicit & Candidate Pool)
|
||||
reg = load_types_registry()
|
||||
|
||||
@@ -33,6 +33,7 @@ PROVENANCE_PRIORITY = {
|
||||
"derived:backlink": 0.90,
|
||||
"edge_defaults": 0.70,
|
||||
"schema_default": 0.85,
|
||||
"inferred:section_transition": 0.85, # WP-26 v1.1: Automatische Section-Übergänge
|
||||
# Struktur-Kanten (provenance: "structure")
|
||||
"structure:belongs_to": 1.00,
|
||||
"structure:order": 0.95,
|
||||
@@ -56,6 +57,7 @@ PROVENANCE_TO_DTO = {
|
||||
"edge_defaults": ("rule", "edge_defaults"),
|
||||
"schema_default": ("rule", "schema_default"),
|
||||
"inferred:schema": ("rule", "schema_default"),
|
||||
"inferred:section_transition": ("rule", "schema_default"), # WP-26 v1.1
|
||||
"rule": ("rule", None),
|
||||
# structure
|
||||
"structure:belongs_to": ("structure", "belongs_to"),
|
||||
@@ -255,4 +257,157 @@ def get_edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||
|
||||
return []
|
||||
return []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WP-26 v1.1: Graph-Schema Parser für automatische Edge-Typ-Ableitung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cache für geladenes Schema (vermeidet mehrfaches Parsen)
|
||||
_GRAPH_SCHEMA_CACHE: Optional[Dict[str, Dict[str, List[str]]]] = None
|
||||
|
||||
def load_graph_schema() -> Dict[str, Dict[str, List[str]]]:
|
||||
"""
|
||||
WP-26 v1.1: Parst das graph_schema.md und extrahiert Typical Edge-Types.
|
||||
|
||||
Das Schema hat folgendes Format:
|
||||
## Source: `experience`
|
||||
| Target-Note-type | Typical Edge-Types | Prohibited Edge-Types |
|
||||
| :--- | :--- | :--- |
|
||||
| `event` | `caused_by` | `consists_of` |
|
||||
|
||||
Returns:
|
||||
Dict[source_type, Dict[target_type, List[typical_edges]]]
|
||||
Beispiel: {"experience": {"event": ["caused_by"], "insight": ["resulted_in"]}}
|
||||
"""
|
||||
global _GRAPH_SCHEMA_CACHE
|
||||
if _GRAPH_SCHEMA_CACHE is not None:
|
||||
return _GRAPH_SCHEMA_CACHE
|
||||
|
||||
import re
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
schema_path = get_schema_path()
|
||||
|
||||
# Versuche verschiedene Pfade
|
||||
paths_to_try = [
|
||||
schema_path,
|
||||
os.path.join(os.getcwd(), "config", "graph_schema.md"),
|
||||
os.path.join(os.path.dirname(__file__), "..", "..", "..", "config", "graph_schema.md"),
|
||||
]
|
||||
|
||||
# Falls MINDNET_OBSIDIAN_DICTIONARY gesetzt ist, nutze diesen Pfad
|
||||
obsidian_dict = os.getenv("MINDNET_OBSIDIAN_DICTIONARY")
|
||||
if obsidian_dict:
|
||||
paths_to_try.insert(0, os.path.join(obsidian_dict, "graph_schema.md"))
|
||||
|
||||
content = None
|
||||
for path in paths_to_try:
|
||||
if os.path.isfile(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
logger.debug(f"Graph-Schema geladen von: {path}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Fehler beim Laden von {path}: {e}")
|
||||
|
||||
if not content:
|
||||
logger.warning("Graph-Schema nicht gefunden. Fallback auf leeres Schema.")
|
||||
_GRAPH_SCHEMA_CACHE = {}
|
||||
return _GRAPH_SCHEMA_CACHE
|
||||
|
||||
schema: Dict[str, Dict[str, List[str]]] = {}
|
||||
current_source = None
|
||||
|
||||
# Regex für Source-Header: ## Source: `experience`
|
||||
source_pattern = re.compile(r'^##\s+Source:\s*`(\w+)`', re.IGNORECASE)
|
||||
|
||||
# Regex für Tabellen-Zeile: | `event` | `caused_by` | `consists_of` |
|
||||
# oder | `event` | `caused_by`, `resulted_in` | - |
|
||||
table_row_pattern = re.compile(
|
||||
r'^\|\s*`(\w+)`\s*\|\s*([^|]+)\s*\|\s*([^|]*)\s*\|'
|
||||
)
|
||||
|
||||
for line in content.split('\n'):
|
||||
stripped = line.strip()
|
||||
|
||||
# Prüfe auf Source-Header
|
||||
source_match = source_pattern.match(stripped)
|
||||
if source_match:
|
||||
current_source = source_match.group(1).lower()
|
||||
if current_source not in schema:
|
||||
schema[current_source] = {}
|
||||
continue
|
||||
|
||||
# Prüfe auf Tabellen-Zeile (nur wenn wir einen Source haben)
|
||||
if current_source:
|
||||
row_match = table_row_pattern.match(stripped)
|
||||
if row_match:
|
||||
target_type = row_match.group(1).lower()
|
||||
typical_edges_raw = row_match.group(2).strip()
|
||||
|
||||
# Parse die Edge-Types (können mit Backticks und Kommas getrennt sein)
|
||||
# Format: `caused_by`, `resulted_in` oder `caused_by`
|
||||
edge_pattern = re.compile(r'`(\w+)`')
|
||||
typical_edges = edge_pattern.findall(typical_edges_raw)
|
||||
|
||||
if typical_edges:
|
||||
schema[current_source][target_type] = typical_edges
|
||||
|
||||
logger.info(f"Graph-Schema geladen: {len(schema)} Source-Types")
|
||||
_GRAPH_SCHEMA_CACHE = schema
|
||||
return schema
|
||||
|
||||
def get_typical_edge_for(source_type: str, target_type: str) -> Optional[str]:
|
||||
"""
|
||||
WP-26 v1.1: Ermittelt den ersten "Typical Edge-Type" für ein Typ-Paar.
|
||||
|
||||
Args:
|
||||
source_type: Typ der Quell-Sektion (z.B. "experience")
|
||||
target_type: Typ der Ziel-Sektion (z.B. "insight")
|
||||
|
||||
Returns:
|
||||
Der erste Typical Edge-Type (z.B. "resulted_in") oder None
|
||||
"""
|
||||
schema = load_graph_schema()
|
||||
|
||||
source_lower = source_type.lower() if source_type else "default"
|
||||
target_lower = target_type.lower() if target_type else "any"
|
||||
|
||||
# 1. Exakter Match
|
||||
if source_lower in schema:
|
||||
source_rules = schema[source_lower]
|
||||
if target_lower in source_rules:
|
||||
edges = source_rules[target_lower]
|
||||
if edges:
|
||||
return edges[0]
|
||||
# 2. Fallback auf "any" Target
|
||||
if "any" in source_rules:
|
||||
edges = source_rules["any"]
|
||||
if edges:
|
||||
return edges[0]
|
||||
|
||||
# 3. Fallback auf "default" Source
|
||||
if "default" in schema:
|
||||
default_rules = schema["default"]
|
||||
if target_lower in default_rules:
|
||||
edges = default_rules[target_lower]
|
||||
if edges:
|
||||
return edges[0]
|
||||
if "any" in default_rules:
|
||||
edges = default_rules["any"]
|
||||
if edges:
|
||||
return edges[0]
|
||||
|
||||
# 4. Absoluter Fallback
|
||||
return "related_to"
|
||||
|
||||
def clear_graph_schema_cache():
|
||||
"""
|
||||
WP-26 v1.1: Löscht den Cache für das Graph-Schema.
|
||||
Nützlich für Tests oder wenn das Schema neu geladen werden soll.
|
||||
"""
|
||||
global _GRAPH_SCHEMA_CACHE
|
||||
_GRAPH_SCHEMA_CACHE = None
|
||||
Reference in new issue
Block a user