erster Aufschlag WP22

This commit is contained in:
Lars committed 2025-12-18 11:50:52 +01:00
1 parent 77a6db7e92
commit e2ee5df815
7 files changed
+313 -106

No files matched your search

+27 -7
View File
@@ -3,9 +3,10 @@ FILE: app/core/ingestion.py
DESCRIPTION: Haupt-Ingestion-Logik.
FIX: Korrekte Priorisierung von Frontmatter für chunk_profile und retriever_weight.
Lade Chunk-Config basierend auf dem effektiven Profil, nicht nur dem Notiz-Typ.
VERSION: 2.7.0 (Fix: Frontmatter Overrides & Config Loading)
WP-22: Integration von Content Lifecycle (Status) und Edge Registry.
VERSION: 2.8.0 (WP-22 Lifecycle & Registry)
STATUS: Active
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client, app.services.edge_registry
EXTERNAL_CONFIG: config/types.yaml
"""
import os
@@ -39,6 +40,7 @@ from app.core.qdrant_points import (
)
from app.services.embeddings_client import EmbeddingsClient
from app.services.edge_registry import registry as edge_registry
logger = logging.getLogger(__name__)
@@ -157,13 +159,20 @@ class IngestionService:
logger.error(f"Validation failed for {file_path}: {e}")
return {**result, "error": f"Validation failed: {str(e)}"}
# --- WP-22: Content Lifecycle Gate ---
status = fm.get("status", "draft").lower().strip()
# Hard Skip für System-Dateien
if status in ["system", "template", "archive", "hidden"]:
logger.info(f"Skipping file {file_path} (Status: {status})")
return {**result, "status": "skipped", "reason": f"lifecycle_status_{status}"}
# 2. Type & Config Resolution (FIXED)
# Wir ermitteln erst den Typ
note_type = resolve_note_type(fm.get("type"), self.registry)
fm["type"] = note_type
# Dann ermitteln wir die effektiven Werte unter Berücksichtigung des Frontmatters!
# Hier lag der Fehler: Vorher wurde einfach überschrieben.
effective_profile = effective_chunk_profile_name(fm, note_type, self.registry)
effective_weight = effective_retriever_weight(fm, note_type, self.registry)
@@ -186,6 +195,8 @@ class IngestionService:
# Update Payload with explicit effective values (Sicherheit)
note_pl["retriever_weight"] = effective_weight
note_pl["chunk_profile"] = effective_profile
# WP-22: Status speichern für Dynamic Scoring
note_pl["status"] = status
note_id = note_pl["note_id"]
except Exception as e:
@@ -222,7 +233,6 @@ class IngestionService:
body_text = getattr(parsed, "body", "") or ""
# FIX: Wir laden jetzt die Config für das SPEZIFISCHE Profil
# (z.B. wenn User "sliding_short" wollte, laden wir dessen Params)
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config)
@@ -244,15 +254,26 @@ class IngestionService:
logger.error(f"Embedding failed: {e}")
raise RuntimeError(f"Embedding failed: {e}")
# Raw Edges generieren
try:
edges = build_edges_for_note(
raw_edges = build_edges_for_note(
note_id,
chunk_pls,
note_level_references=note_pl.get("references", []),
include_note_scope_refs=note_scope_refs
)
except TypeError:
edges = build_edges_for_note(note_id, chunk_pls)
raw_edges = build_edges_for_note(note_id, chunk_pls)
# --- WP-22: Edge Registry Validation ---
edges = []
if raw_edges:
for edge in raw_edges:
original_kind = edge.get("kind", "related_to")
# Resolve via Registry (Canonical mapping + Unknown Logging)
canonical_kind = edge_registry.resolve(original_kind)
edge["kind"] = canonical_kind
edges.append(edge)
except Exception as e:
logger.error(f"Processing failed: {e}", exc_info=True)
@@ -286,7 +307,6 @@ class IngestionService:
logger.error(f"Upsert failed: {e}", exc_info=True)
return {**result, "error": f"DB Upsert failed: {e}"}
# ... (Restliche Methoden wie _fetch_note_payload bleiben unverändert) ...
def _fetch_note_payload(self, note_id: str) -> Optional[dict]:
from qdrant_client.http import models as rest
col = f"{self.prefix}_notes"
+66 -8
View File
@@ -1,10 +1,11 @@
"""
FILE: app/core/retriever.py
DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability).
VERSION: 0.5.3
WP-22 Update: Dynamic Edge Boosting & Lifecycle Scoring.
VERSION: 0.6.0 (WP-22 Dynamic Scoring)
STATUS: Active
DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter
LAST_ANALYSIS: 2025-12-15
LAST_ANALYSIS: 2025-12-18
"""
from __future__ import annotations
@@ -97,14 +98,29 @@ def _semantic_hits(
results.append((str(pid), float(score), dict(payload or {})))
return results
# --- WP-22 Helper: Lifecycle Multipliers ---
def _get_status_multiplier(payload: Dict[str, Any]) -> float:
"""
WP-22: Drafts werden bestraft, Stable Notes belohnt.
"""
status = str(payload.get("status", "draft")).lower()
if status == "stable": return 1.2
if status == "active": return 1.0
if status == "draft": return 0.8 # Malus für Entwürfe
# Fallback für andere oder leere Status
return 1.0
def _compute_total_score(
semantic_score: float,
payload: Dict[str, Any],
edge_bonus: float = 0.0,
cent_bonus: float = 0.0,
dynamic_edge_boosts: Dict[str, float] = None
) -> Tuple[float, float, float]:
"""Berechnet total_score."""
"""
Berechnet total_score.
WP-22 Update: Integration von Status-Bonus und Dynamic Edge Boosts.
"""
raw_weight = payload.get("retriever_weight", 1.0)
try:
weight = float(raw_weight)
@@ -114,7 +130,17 @@ def _compute_total_score(
weight = 0.0
sem_w, edge_w, cent_w = _get_scoring_weights()
total = (sem_w * float(semantic_score) * weight) + (edge_w * edge_bonus) + (cent_w * cent_bonus)
status_mult = _get_status_multiplier(payload)
# Dynamic Edge Boosting
# Wenn dynamische Boosts aktiv sind, erhöhen wir den Einfluss des Graphen
# Dies ist eine Vereinfachung, da der echte Boost im Subgraph passiert sein sollte.
final_edge_score = edge_w * edge_bonus
if dynamic_edge_boosts and edge_bonus > 0:
# Globaler Boost für Graph-Signale bei spezifischen Intents
final_edge_score *= 1.2
total = (sem_w * float(semantic_score) * weight * status_mult) + final_edge_score + (cent_w * cent_bonus)
return float(total), float(edge_bonus), float(cent_bonus)
@@ -138,10 +164,11 @@ def _build_explanation(
except (TypeError, ValueError):
type_weight = 1.0
status_mult = _get_status_multiplier(payload)
note_type = payload.get("type", "unknown")
breakdown = ScoreBreakdown(
semantic_contribution=(sem_w * semantic_score * type_weight),
semantic_contribution=(sem_w * semantic_score * type_weight * status_mult),
edge_contribution=(edge_w_cfg * edge_bonus),
centrality_contribution=(cent_w_cfg * cent_bonus),
raw_semantic=semantic_score,
@@ -162,6 +189,10 @@ def _build_explanation(
msg = "Bevorzugt" if type_weight > 1.0 else "Leicht abgewertet"
reasons.append(Reason(kind="type", message=f"{msg} aufgrund des Typs '{note_type}'.", score_impact=(sem_w * semantic_score * (type_weight - 1.0))))
if status_mult != 1.0:
msg = "Status-Bonus" if status_mult > 1.0 else "Status-Malus"
reasons.append(Reason(kind="lifecycle", message=f"{msg} ({payload.get('status')}).", score_impact=0.0))
if subgraph and node_key and edge_bonus > 0:
if hasattr(subgraph, "get_outgoing_edges"):
outgoing = subgraph.get_outgoing_edges(node_key)
@@ -226,6 +257,7 @@ def _build_hits_from_semantic(
used_mode: str,
subgraph: ga.Subgraph | None = None,
explain: bool = False,
dynamic_edge_boosts: Dict[str, float] = None
) -> QueryResponse:
"""Baut strukturierte QueryHits."""
t0 = time.time()
@@ -246,7 +278,13 @@ def _build_hits_from_semantic(
except Exception:
cent_bonus = 0.0
total, edge_bonus, cent_bonus = _compute_total_score(semantic_score, payload, edge_bonus=edge_bonus, cent_bonus=cent_bonus)
total, edge_bonus, cent_bonus = _compute_total_score(
semantic_score,
payload,
edge_bonus=edge_bonus,
cent_bonus=cent_bonus,
dynamic_edge_boosts=dynamic_edge_boosts
)
enriched.append((pid, float(semantic_score), payload, total, edge_bonus, cent_bonus))
enriched_sorted = sorted(enriched, key=lambda h: h[3], reverse=True)
@@ -280,7 +318,6 @@ def _build_hits_from_semantic(
"section": payload.get("section") or payload.get("section_title"),
"text": text_content
},
# --- FIX: Wir füllen das payload-Feld explizit ---
payload=payload,
explanation=explanation_obj
))
@@ -311,6 +348,10 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
depth, edge_types = _extract_expand_options(req)
# WP-22: Dynamic Boosts aus dem Request (vom Router)
boost_edges = getattr(req, "boost_edges", {})
subgraph: ga.Subgraph | None = None
if depth and depth > 0:
seed_ids: List[str] = []
@@ -320,11 +361,28 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
seed_ids.append(key)
if seed_ids:
try:
# Hier könnten wir boost_edges auch an expand übergeben, wenn ga.expand es unterstützt
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
# Manuelles Boosten der Kantengewichte im Graphen falls aktiv
if boost_edges and subgraph and hasattr(subgraph, "graph"):
for u, v, data in subgraph.graph.edges(data=True):
k = data.get("kind")
if k in boost_edges:
# Gewicht erhöhen für diesen Query-Kontext
data["weight"] = data.get("weight", 1.0) * boost_edges[k]
except Exception:
subgraph = None
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="hybrid", subgraph=subgraph, explain=req.explain)
return _build_hits_from_semantic(
hits,
top_k=top_k,
used_mode="hybrid",
subgraph=subgraph,
explain=req.explain,
dynamic_edge_boosts=boost_edges
)
class Retriever:
+109
View File
@@ -0,0 +1,109 @@
"""
FILE: app/services/edge_registry.py
DESCRIPTION: Single Source of Truth für Kanten-Typen. Parst '01_edge_vocabulary.md'.
Implementiert WP-22 Teil B (Registry & Validation).
"""
import re
import os
import json
import logging
from typing import Dict, Optional, Set
logger = logging.getLogger(__name__)
class EdgeRegistry:
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = super(EdgeRegistry, cls).__new__(cls)
cls._instance.initialized = False
return cls._instance
def __init__(self):
if self.initialized: return
# Pfad korrespondiert mit dem Frontmatter Pfad in 01_edge_vocabulary.md
self.vocab_path = "01_User_Manual/01_edge_vocabulary.md"
self.unknown_log_path = "data/logs/unknown_edges.jsonl"
self.canonical_map: Dict[str, str] = {} # alias -> canonical
self.valid_types: Set[str] = set()
self._load_vocabulary()
self.initialized = True
def _load_vocabulary(self):
"""Parst die Markdown-Tabelle in 01_edge_vocabulary.md"""
# Fallback Suche, falls das Skript aus Root oder app ausgeführt wird
candidates = [
self.vocab_path,
os.path.join("..", self.vocab_path),
"vault/01_User_Manual/01_edge_vocabulary.md"
]
found_path = None
for p in candidates:
if os.path.exists(p):
found_path = p
break
if not found_path:
logger.warning(f"Edge Vocabulary not found (checked: {candidates}). Registry empty.")
return
# Regex für Tabellenzeilen: | **canonical** | alias, alias | ...
# Matcht: | **caused_by** | ausgelöst_durch, wegen |
pattern = re.compile(r"\|\s*\*\*([a-z_]+)\*\*\s*\|\s*([^|]+)\|")
try:
with open(found_path, "r", encoding="utf-8") as f:
for line in f:
match = pattern.search(line)
if match:
canonical = match.group(1).strip()
aliases_str = match.group(2).strip()
self.valid_types.add(canonical)
self.canonical_map[canonical] = canonical # Self-ref
# Aliases parsen
if aliases_str and "Kein Alias" not in aliases_str:
aliases = [a.strip() for a in aliases_str.split(",") if a.strip()]
for alias in aliases:
# Clean up user inputs (e.g. remove backticks if present)
clean_alias = alias.replace("`", "")
self.canonical_map[clean_alias] = canonical
logger.info(f"EdgeRegistry loaded: {len(self.valid_types)} canonical types from {found_path}.")
except Exception as e:
logger.error(f"Failed to parse Edge Vocabulary: {e}")
def resolve(self, edge_type: str) -> str:
"""
Normalisiert Kanten-Typen. Loggt unbekannte Typen, verwirft sie aber nicht (Learning System).
"""
if not edge_type:
return "related_to"
clean_type = edge_type.lower().strip().replace(" ", "_")
# 1. Lookup
if clean_type in self.canonical_map:
return self.canonical_map[clean_type]
# 2. Unknown Handling
self._log_unknown(clean_type)
return clean_type # Pass-through (nicht verwerfen, aber loggen)
def _log_unknown(self, edge_type: str):
"""Schreibt unbekannte Typen in ein Append-Only Log für Review."""
try:
os.makedirs(os.path.dirname(self.unknown_log_path), exist_ok=True)
# Einfaches JSONL Format
entry = {"unknown_type": edge_type, "status": "new"}
with open(self.unknown_log_path, "a", encoding="utf-8") as f:
f.write(json.dumps(entry) + "\n")
except Exception:
pass # Silent fail bei Logging, darf Ingestion nicht stoppen
# Singleton Accessor
registry = EdgeRegistry()