neue parser logik
This commit is contained in:
1 parent
2d43e0596c
commit
99482ad65a
3 files changed
+107
-48
No files matched your search
+39
-13
@@ -4,8 +4,9 @@ DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen (Notes
|
||||
FIX: Korrekte Priorisierung von Frontmatter für chunk_profile und retriever_weight.
|
||||
Lade Chunk-Config basierend auf dem effektiven Profil, nicht nur dem Notiz-Typ.
|
||||
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
|
||||
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
|
||||
WP-22: Multi-Hash Refresh für konsistente Change Detection.
|
||||
VERSION: 2.8.6 (WP-22 Lifecycle & Registry)
|
||||
VERSION: 2.9.0 (WP-22 Full Integration: Context-Aware Registry)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client, app.services.edge_registry
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
@@ -21,6 +22,7 @@ from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
extract_edges_with_context, # WP-22: Neue Funktion für Zeilennummern
|
||||
)
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks, get_chunk_config
|
||||
@@ -158,7 +160,7 @@ class IngestionService:
|
||||
logger.error(f"Validation failed for {file_path}: {e}")
|
||||
return {**result, "error": f"Validation failed: {str(e)}"}
|
||||
|
||||
# --- WP-22: Content Lifecycle Gate (Teil A) ---
|
||||
# --- WP-22: Content Lifecycle Gate ---
|
||||
status = fm.get("status", "draft").lower().strip()
|
||||
|
||||
# Hard Skip für System- oder Archiv-Dateien
|
||||
@@ -229,6 +231,9 @@ class IngestionService:
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
|
||||
# WP-22: Sicherstellen, dass die Registry aktuell ist (Lazy Reload)
|
||||
edge_registry.ensure_latest()
|
||||
|
||||
# Konfiguration für das spezifische Profil laden
|
||||
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
|
||||
|
||||
@@ -251,26 +256,47 @@ class IngestionService:
|
||||
logger.error(f"Embedding failed: {e}")
|
||||
raise RuntimeError(f"Embedding failed: {e}")
|
||||
|
||||
# Kanten generieren
|
||||
# --- WP-22: Kanten-Extraktion & Validierung ---
|
||||
# A. Explizite User-Kanten mit Zeilennummern extrahieren
|
||||
explicit_edges = extract_edges_with_context(parsed)
|
||||
|
||||
# B. System-Kanten generieren (Struktur: belongs_to, next, prev)
|
||||
try:
|
||||
raw_edges = build_edges_for_note(
|
||||
raw_system_edges = build_edges_for_note(
|
||||
note_id,
|
||||
chunk_pls,
|
||||
note_level_references=note_pl.get("references", []),
|
||||
include_note_scope_refs=note_scope_refs
|
||||
)
|
||||
except TypeError:
|
||||
raw_edges = build_edges_for_note(note_id, chunk_pls)
|
||||
raw_system_edges = build_edges_for_note(note_id, chunk_pls)
|
||||
|
||||
# --- WP-22: Edge Registry Validation (Teil B) ---
|
||||
# C. Alle Kanten validieren und über die Registry mappen
|
||||
edges = []
|
||||
if raw_edges:
|
||||
for edge in raw_edges:
|
||||
original_kind = edge.get("kind", "related_to")
|
||||
# Normalisierung über die Registry (Alias-Auflösung)
|
||||
canonical_kind = edge_registry.resolve(original_kind)
|
||||
edge["kind"] = canonical_kind
|
||||
edges.append(edge)
|
||||
context = {"file": file_path, "note_id": note_id}
|
||||
|
||||
# Zuerst User-Kanten (provenance="explicit")
|
||||
for e in explicit_edges:
|
||||
valid_kind = edge_registry.resolve(
|
||||
edge_type=e["kind"],
|
||||
provenance="explicit",
|
||||
context={**context, "line": e.get("line")}
|
||||
)
|
||||
e["kind"] = valid_kind
|
||||
edges.append(e)
|
||||
|
||||
# Dann System-Kanten (provenance="structure")
|
||||
for e in raw_system_edges:
|
||||
# Sicherstellen, dass System-Kanten korrekt markiert sind
|
||||
valid_kind = edge_registry.resolve(
|
||||
edge_type=e.get("kind", "belongs_to"),
|
||||
provenance="structure",
|
||||
context={**context, "line": "system"}
|
||||
)
|
||||
e["kind"] = valid_kind
|
||||
# Nur hinzufügen, wenn die Registry einen validen Typ zurückgibt
|
||||
if valid_kind:
|
||||
edges.append(e)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Processing failed: {e}", exc_info=True)
|
||||
|
||||
@@ -231,3 +231,20 @@ def extract_wikilinks(text: str) -> List[str]:
|
||||
if raw:
|
||||
out.append(raw)
|
||||
return out
|
||||
|
||||
def extract_edges_with_context(note: ParsedNote) -> List[Dict[str, Any]]:
|
||||
"""Extrahiert Kanten-Typen, Ziele und Zeilennummern."""
|
||||
edges = []
|
||||
lines = note.body.splitlines()
|
||||
# Erkennt [[rel:typ Ziel]]
|
||||
rel_pattern = re.compile(r"\[\[rel:([a-zA-Z0-9_-]+)\s+([^\]|#]+)\]\]")
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
for match in rel_pattern.finditer(line):
|
||||
edges.append({
|
||||
"kind": match.group(1).strip(),
|
||||
"target": match.group(2).strip(),
|
||||
"line": i + 1,
|
||||
"provenance": "explicit"
|
||||
})
|
||||
return edges
|
||||
Reference in new issue
Block a user