Enhance Slot Evaluation and Scoring Mechanisms
Deploy Development / deploy (push) Successful in 46s
Test Suite / pytest-backend (push) Successful in 44s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m26s

- Introduced new functions `_off_topic_semantic_scores_by_slot` and `_score_exercise_stage_fit_for_spec` to improve the evaluation of off-topic steps and exercise stage fit, enhancing the quality assessment process.
- Updated `_run_unified_slot_improvement_review` to incorporate off-topic scores and exercise stage fit scoring, refining the decision-making process for slot suggestions.
- Enhanced existing logic to streamline the handling of slot scores and improve the overall robustness of slot management in path evaluations.
This commit is contained in:
Lars committed 2026-06-13 12:33:16 +02:00
1 parent e9bf5bd1a5
commit cd457e3ea0
5 files changed
+654 -437

No files matched your search

+284 -199
View File
@@ -35,6 +35,7 @@ from planning_path_rematch import (
from planning_path_refine_stage import apply_stage_spec_refinements, collect_refine_stage_targets
from planning_stage_context import build_contextualized_stage_goal, resolve_path_start_target
from planning_exercise_path_qa import (
_load_exercise_text_bundle,
apply_llm_path_reorder,
build_path_qa_summary,
compute_deterministic_path_quality_score,
@@ -66,6 +67,7 @@ from planning_exercise_semantics import (
exercise_passes_stage_fit,
exercise_title_matches_peer_stage_goal,
pick_best_path_hit,
score_exercise_stage_fit,
resolve_semantic_skill_weights,
step_phase_for_index,
step_retrieval_query,
@@ -3019,6 +3021,92 @@ def _suggestion_as_slot_diff(entry: Mapping[str, Any]) -> Dict[str, Any]:
}
_SLOT_FIT_POOR_THRESHOLD = 0.30
def _off_topic_semantic_scores_by_slot(
off_topic_steps: Sequence[Mapping[str, Any]],
) -> Dict[int, float]:
scores: Dict[int, float] = {}
for item in off_topic_steps or []:
if not isinstance(item, dict):
continue
midx = item.get("roadmap_major_step_index")
if midx is None:
continue
try:
key = int(midx)
raw = item.get("semantic_score")
if raw is not None:
scores[key] = round(float(raw), 4)
except (TypeError, ValueError):
continue
return scores
def _score_exercise_stage_fit_for_spec(
cur,
*,
exercise_id: int,
step: Mapping[str, Any],
stage_spec: StageSpecArtifact,
semantic_brief: PlanningSemanticBrief,
step_index: int,
stage_count: int,
) -> Optional[float]:
try:
eid = int(exercise_id)
except (TypeError, ValueError):
return None
if eid < 1:
return None
bundle = _load_exercise_text_bundle(cur, eid)
stage_goal = (stage_spec.learning_goal or step.get("roadmap_learning_goal") or "").strip()
phase = (
(step.get("roadmap_phase") or "").strip().lower()
or step_phase_for_index(semantic_brief, step_index, stage_count)
)
stage_anti = list(stage_spec.anti_patterns or step.get("roadmap_anti_patterns") or [])
stage_match_brief = (
build_stage_match_brief(
learning_goal=stage_goal,
anti_patterns=stage_anti or None,
phase=phase or None,
)
if stage_goal
else None
)
if not stage_match_brief:
return None
score, _ = score_exercise_stage_fit(
title=bundle["title"],
summary=bundle["summary"],
goal=bundle["goal"],
variant_names=bundle["variant_names"],
stage_brief=stage_match_brief,
step_phase=phase,
)
return round(float(score), 4)
def _slot_auto_select_library(
*,
baseline_slot_score: Optional[float],
proposed_slot_score: Optional[float],
baseline_exercise_id: Optional[int],
proposed_exercise_id: Optional[int],
) -> bool:
if proposed_exercise_id is None:
return False
if baseline_exercise_id is not None and int(baseline_exercise_id) == int(proposed_exercise_id):
return False
if proposed_slot_score is None:
return False
if baseline_slot_score is None:
return True
return float(proposed_slot_score) > float(baseline_slot_score) + 0.001
def _run_unified_slot_improvement_review(
cur,
*,
@@ -3109,6 +3197,10 @@ def _run_unified_slot_improvement_review(
spec_by_major = {int(s.major_step_index): s for s in roadmap_ctx.stage_specs}
stage_count = len(roadmap_ctx.stage_specs)
off_topic_scores = _off_topic_semantic_scores_by_slot(
baseline_qa.get("off_topic_steps") or [],
)
slot_reviews: List[Dict[str, Any]] = []
suggestions: List[Dict[str, Any]] = []
rejected: List[Dict[str, Any]] = []
@@ -3116,6 +3208,7 @@ def _run_unified_slot_improvement_review(
major_idx = int(stage_spec.major_step_index)
current = dict(steps_by_major.get(major_idx, {}))
current.setdefault("roadmap_major_step_index", major_idx)
current.setdefault("roadmap_learning_goal", stage_spec.learning_goal)
current_id = current.get("exercise_id")
slot_problem = major_idx in problem_slots
off_topic = slot_problem or major_idx in off_topic_map or bool(
@@ -3123,6 +3216,18 @@ def _run_unified_slot_improvement_review(
)
off_reasons = list(problem_slots.get(major_idx, [])) + off_topic_map.get(major_idx, [])
baseline_slot_score: Optional[float] = off_topic_scores.get(major_idx)
if baseline_slot_score is None and current_id is not None and not current.get("is_ai_proposal"):
baseline_slot_score = _score_exercise_stage_fit_for_spec(
cur,
exercise_id=int(current_id),
step=current,
stage_spec=stage_spec,
semantic_brief=semantic_brief,
step_index=step_index,
stage_count=stage_count,
)
planned_ids = [
int(s["exercise_id"])
for midx, s in sorted(steps_by_major.items())
@@ -3141,19 +3246,6 @@ def _run_unified_slot_improvement_review(
vid = step.get("variant_id")
anchor_variant_id = int(vid) if vid is not None else None
exclude_id: Optional[int] = None
if current_id is not None and not (off_topic or slot_problem):
try:
exclude_id = int(current_id)
except (TypeError, ValueError):
exclude_id = None
elif current_id is not None and (off_topic or slot_problem):
try:
exclude_id = int(current_id)
except (TypeError, ValueError):
exclude_id = None
relax_match_gate = bool(off_topic or slot_problem)
candidates = _roadmap_slot_library_candidates(
cur,
tenant=tenant,
@@ -3171,208 +3263,199 @@ def _run_unified_slot_improvement_review(
anchor_id=anchor_id,
anchor_variant_id=anchor_variant_id,
used=used_other,
exclude_exercise_id=int(current_id) if current_id is not None else exclude_id,
max_candidates=5 if relax_match_gate else 3,
skip_post_match_gate=relax_match_gate,
exclude_exercise_id=int(current_id) if current_id is not None else None,
max_candidates=5,
skip_post_match_gate=True,
)
accepted_for_slot = False
best_candidate: Optional[Dict[str, Any]] = None
for candidate in candidates:
try:
cand_id = int(candidate.get("exercise_id"))
except (TypeError, ValueError):
continue
if (
current_id is not None
and not (off_topic or slot_problem)
and int(current_id) == cand_id
):
if current_id is not None and int(current_id) == cand_id:
continue
diff_stub = {
best_candidate = candidate
break
proposed_slot_score: Optional[float] = None
quality_delta: Optional[float] = None
projected_qa: Optional[Dict[str, Any]] = None
library_alt: Optional[Dict[str, Any]] = None
if best_candidate is not None:
try:
cand_id = int(best_candidate.get("exercise_id"))
except (TypeError, ValueError):
cand_id = None
if cand_id is not None:
proposed_slot_score = _score_exercise_stage_fit_for_spec(
cur,
exercise_id=cand_id,
step={**current, **best_candidate, "roadmap_major_step_index": major_idx},
stage_spec=stage_spec,
semantic_brief=semantic_brief,
step_index=step_index,
stage_count=stage_count,
)
diff_stub = {
"roadmap_major_step_index": major_idx,
"baseline_exercise_id": int(current_id) if current_id is not None else None,
"baseline_title": (current.get("title") or "").strip() or None,
"proposed_exercise_id": cand_id,
"proposed_title": (best_candidate.get("title") or "").strip() or None,
}
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, baseline_steps)
for i, raw in enumerate(merged_steps):
if int(raw.get("roadmap_major_step_index", -1)) == major_idx:
merged_steps[i] = {
**raw,
**best_candidate,
"roadmap_major_step_index": major_idx,
}
break
projected_qa = _quick_evaluate_steps_qa(
cur,
goal_query=goal_query,
semantic_brief=semantic_brief,
steps=merged_steps,
roadmap_ctx=roadmap_ctx,
)
quality_delta = _quality_delta(
baseline_score,
_path_qa_quality_score(projected_qa),
)
suggestion_type = (
"remove_and_replace"
if (off_topic or slot_problem) and current_id is not None
else ("library_fill" if current_id is None else "library_improvement")
)
auto_select = _slot_auto_select_library(
baseline_slot_score=baseline_slot_score,
proposed_slot_score=proposed_slot_score,
baseline_exercise_id=int(current_id) if current_id is not None else None,
proposed_exercise_id=cand_id,
)
library_alt = {
"exercise_id": cand_id,
"title": (best_candidate.get("title") or "").strip() or None,
"slot_score": proposed_slot_score,
"slot_score_delta": (
round(float(proposed_slot_score) - float(baseline_slot_score), 4)
if proposed_slot_score is not None and baseline_slot_score is not None
else None
),
"quality_delta": quality_delta,
"auto_select": auto_select,
"suggestion_type": suggestion_type,
"reasons": list(best_candidate.get("reasons") or [])[:4],
"pro_contra": _build_slot_pro_contra(
current_step=current,
proposed_step=best_candidate,
suggestion_type=suggestion_type,
baseline_qa=baseline_qa,
projected_qa=projected_qa,
quality_delta=quality_delta,
off_topic_reasons=off_reasons,
candidate_reasons=best_candidate.get("reasons") or [],
),
}
lib_entry = {
"roadmap_major_step_index": major_idx,
"baseline_exercise_id": int(current_id) if current_id is not None else None,
"baseline_title": (current.get("title") or "").strip() or None,
"proposed_exercise_id": cand_id,
"proposed_title": library_alt["title"],
"baseline_slot_status": current.get("slot_status"),
"proposed_slot_status": best_candidate.get("slot_status") or "matched",
"suggestion_type": suggestion_type,
"quality_delta": quality_delta,
"baseline_slot_score": baseline_slot_score,
"proposed_slot_score": proposed_slot_score,
"slot_score_delta": library_alt["slot_score_delta"],
"auto_select": auto_select,
"baseline_quality_score": baseline_score,
"projected_quality_score": _path_qa_quality_score(projected_qa),
"projected_path_qa": projected_qa,
"improves_path": auto_select,
"off_topic": off_topic,
"slot_problem": slot_problem,
"problem_reasons": off_reasons[:6],
"proposed_is_ai_proposal": False,
"pro_contra": library_alt["pro_contra"],
}
if auto_select:
suggestions.append(lib_entry)
elif cand_id is not None:
rejected.append(lib_entry)
show_ai_option = bool(
body.include_ai_gap_fill
and (
current_id is None
or off_topic
or slot_problem
or bool(current.get("is_ai_proposal"))
or (
baseline_slot_score is not None
and baseline_slot_score < _SLOT_FIT_POOR_THRESHOLD
)
)
)
ai_alt: Optional[Dict[str, Any]] = None
if show_ai_option:
slot_offer = next(
(
o
for o in gap_fill_offers
if isinstance(o, dict)
and int(o.get("roadmap_major_step_index", -1)) == major_idx
),
None,
)
if not slot_offer:
empty_specs = _build_evaluate_empty_slot_gap_specs(
[current],
goal_query=goal_query,
)
if empty_specs:
slot_offer = build_gap_fill_offer(
spec=empty_specs[0],
steps=baseline_steps,
goal_query=goal_query,
brief=semantic_brief,
proposal=None,
roadmap_snapshot=_roadmap_gap_snapshot_for_spec(
cur,
roadmap_ctx,
empty_specs[0],
goal_query=goal_query,
semantic_brief=semantic_brief,
),
)
gap_fill_offers.append(slot_offer)
if slot_offer:
ai_alt = {
"title_hint": slot_offer.get("title_hint") or f"Slot {major_idx + 1}",
"gap_offer": slot_offer,
"auto_select": False,
}
slot_reviews.append(
{
"roadmap_major_step_index": major_idx,
"roadmap_learning_goal": (stage_spec.learning_goal or "").strip() or None,
"baseline_exercise_id": int(current_id) if current_id is not None else None,
"baseline_title": (current.get("title") or "").strip() or None,
"proposed_exercise_id": cand_id,
"proposed_title": (candidate.get("title") or "").strip() or None,
}
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, baseline_steps)
for i, raw in enumerate(merged_steps):
if int(raw.get("roadmap_major_step_index", -1)) == major_idx:
merged_steps[i] = {**raw, **candidate, "roadmap_major_step_index": major_idx}
break
eval_res = _quick_evaluate_steps_qa(
cur,
goal_query=goal_query,
semantic_brief=semantic_brief,
steps=merged_steps,
roadmap_ctx=roadmap_ctx,
)
projected_qa = eval_res if isinstance(eval_res, dict) else None
projected_score = _path_qa_quality_score(projected_qa)
delta = _quality_delta(baseline_score, projected_score)
improves = _slot_suggestion_accepted(
baseline_qa=baseline_qa,
projected_qa=projected_qa,
baseline_score=baseline_score,
projected_score=projected_score,
diff=diff_stub,
off_topic=off_topic,
major_idx=major_idx,
slot_problem=slot_problem,
stage_specs=roadmap_ctx.stage_specs,
baseline_steps=baseline_steps,
projected_steps=merged_steps,
)
suggestion_type = (
"remove_and_replace"
if (off_topic or slot_problem) and current_id is not None
else ("library_fill" if current_id is None else "library_improvement")
)
entry = {
**diff_stub,
"baseline_slot_score": baseline_slot_score,
"baseline_slot_status": current.get("slot_status"),
"proposed_slot_status": candidate.get("slot_status") or "matched",
"suggestion_type": suggestion_type,
"quality_delta": delta,
"projected_quality_score": projected_score,
"baseline_quality_score": baseline_score,
"projected_path_qa": projected_qa,
"improves_path": improves,
"off_topic": off_topic,
"slot_problem": slot_problem,
"off_topic": off_topic,
"problem_reasons": off_reasons[:6],
"proposed_is_ai_proposal": False,
"pro_contra": _build_slot_pro_contra(
current_step=current,
proposed_step=candidate,
suggestion_type=suggestion_type,
baseline_qa=baseline_qa,
projected_qa=projected_qa,
quality_delta=delta,
off_topic_reasons=off_reasons,
candidate_reasons=candidate.get("reasons") or [],
),
"library_alternative": library_alt,
"ai_alternative": ai_alt,
}
if improves:
suggestions.append(entry)
accepted_for_slot = True
break
rejected.append(entry)
if accepted_for_slot:
continue
# Kein Bibliotheks-Treffer oder keine Verbesserung → KI-Angebot wenn Slot leer/off-topic/KI
needs_ai = (
current_id is None
or off_topic
or slot_problem
or bool(current.get("is_ai_proposal"))
)
if not needs_ai or not body.include_ai_gap_fill:
continue
slot_offer = next(
(
o
for o in gap_fill_offers
if isinstance(o, dict)
and int(o.get("roadmap_major_step_index", -1)) == major_idx
),
None,
)
if not slot_offer:
empty_specs = _build_evaluate_empty_slot_gap_specs(
[current],
goal_query=goal_query,
)
if empty_specs:
slot_offer = build_gap_fill_offer(
spec=empty_specs[0],
steps=baseline_steps,
goal_query=goal_query,
brief=semantic_brief,
proposal=None,
roadmap_snapshot=_roadmap_gap_snapshot_for_spec(
cur,
roadmap_ctx,
empty_specs[0],
goal_query=goal_query,
semantic_brief=semantic_brief,
),
)
gap_fill_offers.append(slot_offer)
ai_step = {
**current,
"exercise_id": None,
"is_ai_proposal": True,
"title": slot_offer.get("title_hint") or current.get("title") or f"Slot {major_idx + 1}",
"roadmap_major_step_index": major_idx,
"gap_offer": slot_offer,
}
diff_stub = {
"roadmap_major_step_index": major_idx,
"baseline_exercise_id": int(current_id) if current_id is not None else None,
"baseline_title": (current.get("title") or "").strip() or None,
"proposed_exercise_id": None,
"proposed_title": ai_step.get("title"),
}
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, [ai_step])
eval_res = _quick_evaluate_steps_qa(
cur,
goal_query=goal_query,
semantic_brief=semantic_brief,
steps=merged_steps,
roadmap_ctx=roadmap_ctx,
)
projected_qa = eval_res if isinstance(eval_res, dict) else None
projected_score = _path_qa_quality_score(projected_qa)
delta = _quality_delta(baseline_score, projected_score)
improves = _slot_suggestion_accepted(
baseline_qa=baseline_qa,
projected_qa=projected_qa,
baseline_score=baseline_score,
projected_score=projected_score,
diff=diff_stub,
off_topic=off_topic,
major_idx=major_idx,
slot_problem=slot_problem,
stage_specs=roadmap_ctx.stage_specs,
baseline_steps=baseline_steps,
projected_steps=merged_steps,
)
entry = {
**diff_stub,
"baseline_slot_status": current.get("slot_status"),
"proposed_slot_status": "ai_proposal",
"suggestion_type": "ai_gap",
"quality_delta": delta,
"projected_quality_score": projected_score,
"baseline_quality_score": baseline_score,
"projected_path_qa": projected_qa,
"improves_path": improves or slot_problem,
"off_topic": off_topic,
"slot_problem": slot_problem,
"problem_reasons": off_reasons[:6],
"proposed_is_ai_proposal": True,
"gap_offer": slot_offer,
"pro_contra": _build_slot_pro_contra(
current_step=current,
proposed_step=None,
suggestion_type="ai_gap",
baseline_qa=baseline_qa,
projected_qa=projected_qa,
quality_delta=delta,
off_topic_reasons=off_reasons,
candidate_reasons=[],
gap_offer=slot_offer,
),
}
if improves or slot_problem:
entry["improves_path"] = True
suggestions.append(entry)
else:
rejected.append(entry)
improvement_diffs = [_suggestion_as_slot_diff(s) for s in suggestions]
problem_slot_payload = {
@@ -3412,9 +3495,11 @@ def _run_unified_slot_improvement_review(
"suggestion_count": len(suggestions),
"rejected_count": len(rejected),
"problem_slot_count": len(problem_slots),
"slot_review_count": len(slot_reviews),
},
"retrieval_phase": "unified_slot_review",
"unified_slot_review": True,
"slot_reviews": slot_reviews,
"problem_slots": problem_slot_payload,
"slot_suggestions": suggestions,
"slot_diff_scoring": slot_diff_scoring,
@@ -2,6 +2,7 @@
from planning_exercise_path_builder import (
_parse_slot_refs_from_text,
_problematic_slots_from_path_qa,
_slot_auto_select_library,
_slot_suggestion_accepted,
)
from planning_progression_roadmap import StageSpecArtifact
@@ -95,3 +96,18 @@ def test_problematic_slots_from_llm_schritt_text():
specs = [_spec(7)]
problems = _problematic_slots_from_path_qa(qa, steps, specs)
assert 7 in problems
def test_slot_auto_select_requires_higher_score():
assert _slot_auto_select_library(
baseline_slot_score=0.5,
proposed_slot_score=0.51,
baseline_exercise_id=1,
proposed_exercise_id=2,
)
assert not _slot_auto_select_library(
baseline_slot_score=0.5,
proposed_slot_score=0.5,
baseline_exercise_id=1,
proposed_exercise_id=2,
)