neue parser logik

This commit is contained in:
Lars committed 2025-12-23 07:31:33 +01:00
1 parent 2d43e0596c
commit 99482ad65a
3 files changed
+107 -48

No files matched your search

+39 -13
View File
@@ -4,8 +4,9 @@ DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen (Notes
FIX: Korrekte Priorisierung von Frontmatter für chunk_profile und retriever_weight.
Lade Chunk-Config basierend auf dem effektiven Profil, nicht nur dem Notiz-Typ.
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
WP-22: Multi-Hash Refresh für konsistente Change Detection.
VERSION: 2.8.6 (WP-22 Lifecycle & Registry)
VERSION: 2.9.0 (WP-22 Full Integration: Context-Aware Registry)
STATUS: Active
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client, app.services.edge_registry
EXTERNAL_CONFIG: config/types.yaml
@@ -21,6 +22,7 @@ from app.core.parser import (
read_markdown,
normalize_frontmatter,
validate_required_frontmatter,
extract_edges_with_context, # WP-22: Neue Funktion für Zeilennummern
)
from app.core.note_payload import make_note_payload
from app.core.chunker import assemble_chunks, get_chunk_config
@@ -158,7 +160,7 @@ class IngestionService:
logger.error(f"Validation failed for {file_path}: {e}")
return {**result, "error": f"Validation failed: {str(e)}"}
# --- WP-22: Content Lifecycle Gate (Teil A) ---
# --- WP-22: Content Lifecycle Gate ---
status = fm.get("status", "draft").lower().strip()
# Hard Skip für System- oder Archiv-Dateien
@@ -229,6 +231,9 @@ class IngestionService:
try:
body_text = getattr(parsed, "body", "") or ""
# WP-22: Sicherstellen, dass die Registry aktuell ist (Lazy Reload)
edge_registry.ensure_latest()
# Konfiguration für das spezifische Profil laden
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
@@ -251,26 +256,47 @@ class IngestionService:
logger.error(f"Embedding failed: {e}")
raise RuntimeError(f"Embedding failed: {e}")
# Kanten generieren
# --- WP-22: Kanten-Extraktion & Validierung ---
# A. Explizite User-Kanten mit Zeilennummern extrahieren
explicit_edges = extract_edges_with_context(parsed)
# B. System-Kanten generieren (Struktur: belongs_to, next, prev)
try:
raw_edges = build_edges_for_note(
raw_system_edges = build_edges_for_note(
note_id,
chunk_pls,
note_level_references=note_pl.get("references", []),
include_note_scope_refs=note_scope_refs
)
except TypeError:
raw_edges = build_edges_for_note(note_id, chunk_pls)
raw_system_edges = build_edges_for_note(note_id, chunk_pls)
# --- WP-22: Edge Registry Validation (Teil B) ---
# C. Alle Kanten validieren und über die Registry mappen
edges = []
if raw_edges:
for edge in raw_edges:
original_kind = edge.get("kind", "related_to")
# Normalisierung über die Registry (Alias-Auflösung)
canonical_kind = edge_registry.resolve(original_kind)
edge["kind"] = canonical_kind
edges.append(edge)
context = {"file": file_path, "note_id": note_id}
# Zuerst User-Kanten (provenance="explicit")
for e in explicit_edges:
valid_kind = edge_registry.resolve(
edge_type=e["kind"],
provenance="explicit",
context={**context, "line": e.get("line")}
)
e["kind"] = valid_kind
edges.append(e)
# Dann System-Kanten (provenance="structure")
for e in raw_system_edges:
# Sicherstellen, dass System-Kanten korrekt markiert sind
valid_kind = edge_registry.resolve(
edge_type=e.get("kind", "belongs_to"),
provenance="structure",
context={**context, "line": "system"}
)
e["kind"] = valid_kind
# Nur hinzufügen, wenn die Registry einen validen Typ zurückgibt
if valid_kind:
edges.append(e)
except Exception as e:
logger.error(f"Processing failed: {e}", exc_info=True)
+17
View File
@@ -231,3 +231,20 @@ def extract_wikilinks(text: str) -> List[str]:
if raw:
out.append(raw)
return out
def extract_edges_with_context(note: ParsedNote) -> List[Dict[str, Any]]:
"""Extrahiert Kanten-Typen, Ziele und Zeilennummern."""
edges = []
lines = note.body.splitlines()
# Erkennt [[rel:typ Ziel]]
rel_pattern = re.compile(r"\[\[rel:([a-zA-Z0-9_-]+)\s+([^\]|#]+)\]\]")
for i, line in enumerate(lines):
for match in rel_pattern.finditer(line):
edges.append({
"kind": match.group(1).strip(),
"target": match.group(2).strip(),
"line": i + 1,
"provenance": "explicit"
})
return edges