Enhance Slot Evaluation and Scoring Mechanisms
Deploy Development / deploy (push) Successful in 46s
Test Suite / pytest-backend (push) Successful in 44s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m26s
Deploy Development / deploy (push) Successful in 46s
Test Suite / pytest-backend (push) Successful in 44s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m26s
- Introduced new functions `_off_topic_semantic_scores_by_slot` and `_score_exercise_stage_fit_for_spec` to improve the evaluation of off-topic steps and exercise stage fit, enhancing the quality assessment process. - Updated `_run_unified_slot_improvement_review` to incorporate off-topic scores and exercise stage fit scoring, refining the decision-making process for slot suggestions. - Enhanced existing logic to streamline the handling of slot scores and improve the overall robustness of slot management in path evaluations.
This commit is contained in:
1 parent
e9bf5bd1a5
commit
cd457e3ea0
5 files changed
+654
-437
No files matched your search
@@ -35,6 +35,7 @@ from planning_path_rematch import (
|
||||
from planning_path_refine_stage import apply_stage_spec_refinements, collect_refine_stage_targets
|
||||
from planning_stage_context import build_contextualized_stage_goal, resolve_path_start_target
|
||||
from planning_exercise_path_qa import (
|
||||
_load_exercise_text_bundle,
|
||||
apply_llm_path_reorder,
|
||||
build_path_qa_summary,
|
||||
compute_deterministic_path_quality_score,
|
||||
@@ -66,6 +67,7 @@ from planning_exercise_semantics import (
|
||||
exercise_passes_stage_fit,
|
||||
exercise_title_matches_peer_stage_goal,
|
||||
pick_best_path_hit,
|
||||
score_exercise_stage_fit,
|
||||
resolve_semantic_skill_weights,
|
||||
step_phase_for_index,
|
||||
step_retrieval_query,
|
||||
@@ -3019,6 +3021,92 @@ def _suggestion_as_slot_diff(entry: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
_SLOT_FIT_POOR_THRESHOLD = 0.30
|
||||
|
||||
|
||||
def _off_topic_semantic_scores_by_slot(
|
||||
off_topic_steps: Sequence[Mapping[str, Any]],
|
||||
) -> Dict[int, float]:
|
||||
scores: Dict[int, float] = {}
|
||||
for item in off_topic_steps or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
midx = item.get("roadmap_major_step_index")
|
||||
if midx is None:
|
||||
continue
|
||||
try:
|
||||
key = int(midx)
|
||||
raw = item.get("semantic_score")
|
||||
if raw is not None:
|
||||
scores[key] = round(float(raw), 4)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return scores
|
||||
|
||||
|
||||
def _score_exercise_stage_fit_for_spec(
|
||||
cur,
|
||||
*,
|
||||
exercise_id: int,
|
||||
step: Mapping[str, Any],
|
||||
stage_spec: StageSpecArtifact,
|
||||
semantic_brief: PlanningSemanticBrief,
|
||||
step_index: int,
|
||||
stage_count: int,
|
||||
) -> Optional[float]:
|
||||
try:
|
||||
eid = int(exercise_id)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if eid < 1:
|
||||
return None
|
||||
bundle = _load_exercise_text_bundle(cur, eid)
|
||||
stage_goal = (stage_spec.learning_goal or step.get("roadmap_learning_goal") or "").strip()
|
||||
phase = (
|
||||
(step.get("roadmap_phase") or "").strip().lower()
|
||||
or step_phase_for_index(semantic_brief, step_index, stage_count)
|
||||
)
|
||||
stage_anti = list(stage_spec.anti_patterns or step.get("roadmap_anti_patterns") or [])
|
||||
stage_match_brief = (
|
||||
build_stage_match_brief(
|
||||
learning_goal=stage_goal,
|
||||
anti_patterns=stage_anti or None,
|
||||
phase=phase or None,
|
||||
)
|
||||
if stage_goal
|
||||
else None
|
||||
)
|
||||
if not stage_match_brief:
|
||||
return None
|
||||
score, _ = score_exercise_stage_fit(
|
||||
title=bundle["title"],
|
||||
summary=bundle["summary"],
|
||||
goal=bundle["goal"],
|
||||
variant_names=bundle["variant_names"],
|
||||
stage_brief=stage_match_brief,
|
||||
step_phase=phase,
|
||||
)
|
||||
return round(float(score), 4)
|
||||
|
||||
|
||||
def _slot_auto_select_library(
|
||||
*,
|
||||
baseline_slot_score: Optional[float],
|
||||
proposed_slot_score: Optional[float],
|
||||
baseline_exercise_id: Optional[int],
|
||||
proposed_exercise_id: Optional[int],
|
||||
) -> bool:
|
||||
if proposed_exercise_id is None:
|
||||
return False
|
||||
if baseline_exercise_id is not None and int(baseline_exercise_id) == int(proposed_exercise_id):
|
||||
return False
|
||||
if proposed_slot_score is None:
|
||||
return False
|
||||
if baseline_slot_score is None:
|
||||
return True
|
||||
return float(proposed_slot_score) > float(baseline_slot_score) + 0.001
|
||||
|
||||
|
||||
def _run_unified_slot_improvement_review(
|
||||
cur,
|
||||
*,
|
||||
@@ -3109,6 +3197,10 @@ def _run_unified_slot_improvement_review(
|
||||
spec_by_major = {int(s.major_step_index): s for s in roadmap_ctx.stage_specs}
|
||||
stage_count = len(roadmap_ctx.stage_specs)
|
||||
|
||||
off_topic_scores = _off_topic_semantic_scores_by_slot(
|
||||
baseline_qa.get("off_topic_steps") or [],
|
||||
)
|
||||
slot_reviews: List[Dict[str, Any]] = []
|
||||
suggestions: List[Dict[str, Any]] = []
|
||||
rejected: List[Dict[str, Any]] = []
|
||||
|
||||
@@ -3116,6 +3208,7 @@ def _run_unified_slot_improvement_review(
|
||||
major_idx = int(stage_spec.major_step_index)
|
||||
current = dict(steps_by_major.get(major_idx, {}))
|
||||
current.setdefault("roadmap_major_step_index", major_idx)
|
||||
current.setdefault("roadmap_learning_goal", stage_spec.learning_goal)
|
||||
current_id = current.get("exercise_id")
|
||||
slot_problem = major_idx in problem_slots
|
||||
off_topic = slot_problem or major_idx in off_topic_map or bool(
|
||||
@@ -3123,6 +3216,18 @@ def _run_unified_slot_improvement_review(
|
||||
)
|
||||
off_reasons = list(problem_slots.get(major_idx, [])) + off_topic_map.get(major_idx, [])
|
||||
|
||||
baseline_slot_score: Optional[float] = off_topic_scores.get(major_idx)
|
||||
if baseline_slot_score is None and current_id is not None and not current.get("is_ai_proposal"):
|
||||
baseline_slot_score = _score_exercise_stage_fit_for_spec(
|
||||
cur,
|
||||
exercise_id=int(current_id),
|
||||
step=current,
|
||||
stage_spec=stage_spec,
|
||||
semantic_brief=semantic_brief,
|
||||
step_index=step_index,
|
||||
stage_count=stage_count,
|
||||
)
|
||||
|
||||
planned_ids = [
|
||||
int(s["exercise_id"])
|
||||
for midx, s in sorted(steps_by_major.items())
|
||||
@@ -3141,19 +3246,6 @@ def _run_unified_slot_improvement_review(
|
||||
vid = step.get("variant_id")
|
||||
anchor_variant_id = int(vid) if vid is not None else None
|
||||
|
||||
exclude_id: Optional[int] = None
|
||||
if current_id is not None and not (off_topic or slot_problem):
|
||||
try:
|
||||
exclude_id = int(current_id)
|
||||
except (TypeError, ValueError):
|
||||
exclude_id = None
|
||||
elif current_id is not None and (off_topic or slot_problem):
|
||||
try:
|
||||
exclude_id = int(current_id)
|
||||
except (TypeError, ValueError):
|
||||
exclude_id = None
|
||||
|
||||
relax_match_gate = bool(off_topic or slot_problem)
|
||||
candidates = _roadmap_slot_library_candidates(
|
||||
cur,
|
||||
tenant=tenant,
|
||||
@@ -3171,208 +3263,199 @@ def _run_unified_slot_improvement_review(
|
||||
anchor_id=anchor_id,
|
||||
anchor_variant_id=anchor_variant_id,
|
||||
used=used_other,
|
||||
exclude_exercise_id=int(current_id) if current_id is not None else exclude_id,
|
||||
max_candidates=5 if relax_match_gate else 3,
|
||||
skip_post_match_gate=relax_match_gate,
|
||||
exclude_exercise_id=int(current_id) if current_id is not None else None,
|
||||
max_candidates=5,
|
||||
skip_post_match_gate=True,
|
||||
)
|
||||
|
||||
accepted_for_slot = False
|
||||
best_candidate: Optional[Dict[str, Any]] = None
|
||||
for candidate in candidates:
|
||||
try:
|
||||
cand_id = int(candidate.get("exercise_id"))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if (
|
||||
current_id is not None
|
||||
and not (off_topic or slot_problem)
|
||||
and int(current_id) == cand_id
|
||||
):
|
||||
if current_id is not None and int(current_id) == cand_id:
|
||||
continue
|
||||
diff_stub = {
|
||||
best_candidate = candidate
|
||||
break
|
||||
|
||||
proposed_slot_score: Optional[float] = None
|
||||
quality_delta: Optional[float] = None
|
||||
projected_qa: Optional[Dict[str, Any]] = None
|
||||
library_alt: Optional[Dict[str, Any]] = None
|
||||
if best_candidate is not None:
|
||||
try:
|
||||
cand_id = int(best_candidate.get("exercise_id"))
|
||||
except (TypeError, ValueError):
|
||||
cand_id = None
|
||||
if cand_id is not None:
|
||||
proposed_slot_score = _score_exercise_stage_fit_for_spec(
|
||||
cur,
|
||||
exercise_id=cand_id,
|
||||
step={**current, **best_candidate, "roadmap_major_step_index": major_idx},
|
||||
stage_spec=stage_spec,
|
||||
semantic_brief=semantic_brief,
|
||||
step_index=step_index,
|
||||
stage_count=stage_count,
|
||||
)
|
||||
diff_stub = {
|
||||
"roadmap_major_step_index": major_idx,
|
||||
"baseline_exercise_id": int(current_id) if current_id is not None else None,
|
||||
"baseline_title": (current.get("title") or "").strip() or None,
|
||||
"proposed_exercise_id": cand_id,
|
||||
"proposed_title": (best_candidate.get("title") or "").strip() or None,
|
||||
}
|
||||
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, baseline_steps)
|
||||
for i, raw in enumerate(merged_steps):
|
||||
if int(raw.get("roadmap_major_step_index", -1)) == major_idx:
|
||||
merged_steps[i] = {
|
||||
**raw,
|
||||
**best_candidate,
|
||||
"roadmap_major_step_index": major_idx,
|
||||
}
|
||||
break
|
||||
projected_qa = _quick_evaluate_steps_qa(
|
||||
cur,
|
||||
goal_query=goal_query,
|
||||
semantic_brief=semantic_brief,
|
||||
steps=merged_steps,
|
||||
roadmap_ctx=roadmap_ctx,
|
||||
)
|
||||
quality_delta = _quality_delta(
|
||||
baseline_score,
|
||||
_path_qa_quality_score(projected_qa),
|
||||
)
|
||||
suggestion_type = (
|
||||
"remove_and_replace"
|
||||
if (off_topic or slot_problem) and current_id is not None
|
||||
else ("library_fill" if current_id is None else "library_improvement")
|
||||
)
|
||||
auto_select = _slot_auto_select_library(
|
||||
baseline_slot_score=baseline_slot_score,
|
||||
proposed_slot_score=proposed_slot_score,
|
||||
baseline_exercise_id=int(current_id) if current_id is not None else None,
|
||||
proposed_exercise_id=cand_id,
|
||||
)
|
||||
library_alt = {
|
||||
"exercise_id": cand_id,
|
||||
"title": (best_candidate.get("title") or "").strip() or None,
|
||||
"slot_score": proposed_slot_score,
|
||||
"slot_score_delta": (
|
||||
round(float(proposed_slot_score) - float(baseline_slot_score), 4)
|
||||
if proposed_slot_score is not None and baseline_slot_score is not None
|
||||
else None
|
||||
),
|
||||
"quality_delta": quality_delta,
|
||||
"auto_select": auto_select,
|
||||
"suggestion_type": suggestion_type,
|
||||
"reasons": list(best_candidate.get("reasons") or [])[:4],
|
||||
"pro_contra": _build_slot_pro_contra(
|
||||
current_step=current,
|
||||
proposed_step=best_candidate,
|
||||
suggestion_type=suggestion_type,
|
||||
baseline_qa=baseline_qa,
|
||||
projected_qa=projected_qa,
|
||||
quality_delta=quality_delta,
|
||||
off_topic_reasons=off_reasons,
|
||||
candidate_reasons=best_candidate.get("reasons") or [],
|
||||
),
|
||||
}
|
||||
lib_entry = {
|
||||
"roadmap_major_step_index": major_idx,
|
||||
"baseline_exercise_id": int(current_id) if current_id is not None else None,
|
||||
"baseline_title": (current.get("title") or "").strip() or None,
|
||||
"proposed_exercise_id": cand_id,
|
||||
"proposed_title": library_alt["title"],
|
||||
"baseline_slot_status": current.get("slot_status"),
|
||||
"proposed_slot_status": best_candidate.get("slot_status") or "matched",
|
||||
"suggestion_type": suggestion_type,
|
||||
"quality_delta": quality_delta,
|
||||
"baseline_slot_score": baseline_slot_score,
|
||||
"proposed_slot_score": proposed_slot_score,
|
||||
"slot_score_delta": library_alt["slot_score_delta"],
|
||||
"auto_select": auto_select,
|
||||
"baseline_quality_score": baseline_score,
|
||||
"projected_quality_score": _path_qa_quality_score(projected_qa),
|
||||
"projected_path_qa": projected_qa,
|
||||
"improves_path": auto_select,
|
||||
"off_topic": off_topic,
|
||||
"slot_problem": slot_problem,
|
||||
"problem_reasons": off_reasons[:6],
|
||||
"proposed_is_ai_proposal": False,
|
||||
"pro_contra": library_alt["pro_contra"],
|
||||
}
|
||||
if auto_select:
|
||||
suggestions.append(lib_entry)
|
||||
elif cand_id is not None:
|
||||
rejected.append(lib_entry)
|
||||
|
||||
show_ai_option = bool(
|
||||
body.include_ai_gap_fill
|
||||
and (
|
||||
current_id is None
|
||||
or off_topic
|
||||
or slot_problem
|
||||
or bool(current.get("is_ai_proposal"))
|
||||
or (
|
||||
baseline_slot_score is not None
|
||||
and baseline_slot_score < _SLOT_FIT_POOR_THRESHOLD
|
||||
)
|
||||
)
|
||||
)
|
||||
ai_alt: Optional[Dict[str, Any]] = None
|
||||
if show_ai_option:
|
||||
slot_offer = next(
|
||||
(
|
||||
o
|
||||
for o in gap_fill_offers
|
||||
if isinstance(o, dict)
|
||||
and int(o.get("roadmap_major_step_index", -1)) == major_idx
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not slot_offer:
|
||||
empty_specs = _build_evaluate_empty_slot_gap_specs(
|
||||
[current],
|
||||
goal_query=goal_query,
|
||||
)
|
||||
if empty_specs:
|
||||
slot_offer = build_gap_fill_offer(
|
||||
spec=empty_specs[0],
|
||||
steps=baseline_steps,
|
||||
goal_query=goal_query,
|
||||
brief=semantic_brief,
|
||||
proposal=None,
|
||||
roadmap_snapshot=_roadmap_gap_snapshot_for_spec(
|
||||
cur,
|
||||
roadmap_ctx,
|
||||
empty_specs[0],
|
||||
goal_query=goal_query,
|
||||
semantic_brief=semantic_brief,
|
||||
),
|
||||
)
|
||||
gap_fill_offers.append(slot_offer)
|
||||
if slot_offer:
|
||||
ai_alt = {
|
||||
"title_hint": slot_offer.get("title_hint") or f"Slot {major_idx + 1}",
|
||||
"gap_offer": slot_offer,
|
||||
"auto_select": False,
|
||||
}
|
||||
|
||||
slot_reviews.append(
|
||||
{
|
||||
"roadmap_major_step_index": major_idx,
|
||||
"roadmap_learning_goal": (stage_spec.learning_goal or "").strip() or None,
|
||||
"baseline_exercise_id": int(current_id) if current_id is not None else None,
|
||||
"baseline_title": (current.get("title") or "").strip() or None,
|
||||
"proposed_exercise_id": cand_id,
|
||||
"proposed_title": (candidate.get("title") or "").strip() or None,
|
||||
}
|
||||
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, baseline_steps)
|
||||
for i, raw in enumerate(merged_steps):
|
||||
if int(raw.get("roadmap_major_step_index", -1)) == major_idx:
|
||||
merged_steps[i] = {**raw, **candidate, "roadmap_major_step_index": major_idx}
|
||||
break
|
||||
eval_res = _quick_evaluate_steps_qa(
|
||||
cur,
|
||||
goal_query=goal_query,
|
||||
semantic_brief=semantic_brief,
|
||||
steps=merged_steps,
|
||||
roadmap_ctx=roadmap_ctx,
|
||||
)
|
||||
projected_qa = eval_res if isinstance(eval_res, dict) else None
|
||||
projected_score = _path_qa_quality_score(projected_qa)
|
||||
delta = _quality_delta(baseline_score, projected_score)
|
||||
improves = _slot_suggestion_accepted(
|
||||
baseline_qa=baseline_qa,
|
||||
projected_qa=projected_qa,
|
||||
baseline_score=baseline_score,
|
||||
projected_score=projected_score,
|
||||
diff=diff_stub,
|
||||
off_topic=off_topic,
|
||||
major_idx=major_idx,
|
||||
slot_problem=slot_problem,
|
||||
stage_specs=roadmap_ctx.stage_specs,
|
||||
baseline_steps=baseline_steps,
|
||||
projected_steps=merged_steps,
|
||||
)
|
||||
suggestion_type = (
|
||||
"remove_and_replace"
|
||||
if (off_topic or slot_problem) and current_id is not None
|
||||
else ("library_fill" if current_id is None else "library_improvement")
|
||||
)
|
||||
entry = {
|
||||
**diff_stub,
|
||||
"baseline_slot_score": baseline_slot_score,
|
||||
"baseline_slot_status": current.get("slot_status"),
|
||||
"proposed_slot_status": candidate.get("slot_status") or "matched",
|
||||
"suggestion_type": suggestion_type,
|
||||
"quality_delta": delta,
|
||||
"projected_quality_score": projected_score,
|
||||
"baseline_quality_score": baseline_score,
|
||||
"projected_path_qa": projected_qa,
|
||||
"improves_path": improves,
|
||||
"off_topic": off_topic,
|
||||
"slot_problem": slot_problem,
|
||||
"off_topic": off_topic,
|
||||
"problem_reasons": off_reasons[:6],
|
||||
"proposed_is_ai_proposal": False,
|
||||
"pro_contra": _build_slot_pro_contra(
|
||||
current_step=current,
|
||||
proposed_step=candidate,
|
||||
suggestion_type=suggestion_type,
|
||||
baseline_qa=baseline_qa,
|
||||
projected_qa=projected_qa,
|
||||
quality_delta=delta,
|
||||
off_topic_reasons=off_reasons,
|
||||
candidate_reasons=candidate.get("reasons") or [],
|
||||
),
|
||||
"library_alternative": library_alt,
|
||||
"ai_alternative": ai_alt,
|
||||
}
|
||||
if improves:
|
||||
suggestions.append(entry)
|
||||
accepted_for_slot = True
|
||||
break
|
||||
rejected.append(entry)
|
||||
|
||||
if accepted_for_slot:
|
||||
continue
|
||||
|
||||
# Kein Bibliotheks-Treffer oder keine Verbesserung → KI-Angebot wenn Slot leer/off-topic/KI
|
||||
needs_ai = (
|
||||
current_id is None
|
||||
or off_topic
|
||||
or slot_problem
|
||||
or bool(current.get("is_ai_proposal"))
|
||||
)
|
||||
if not needs_ai or not body.include_ai_gap_fill:
|
||||
continue
|
||||
slot_offer = next(
|
||||
(
|
||||
o
|
||||
for o in gap_fill_offers
|
||||
if isinstance(o, dict)
|
||||
and int(o.get("roadmap_major_step_index", -1)) == major_idx
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not slot_offer:
|
||||
empty_specs = _build_evaluate_empty_slot_gap_specs(
|
||||
[current],
|
||||
goal_query=goal_query,
|
||||
)
|
||||
if empty_specs:
|
||||
slot_offer = build_gap_fill_offer(
|
||||
spec=empty_specs[0],
|
||||
steps=baseline_steps,
|
||||
goal_query=goal_query,
|
||||
brief=semantic_brief,
|
||||
proposal=None,
|
||||
roadmap_snapshot=_roadmap_gap_snapshot_for_spec(
|
||||
cur,
|
||||
roadmap_ctx,
|
||||
empty_specs[0],
|
||||
goal_query=goal_query,
|
||||
semantic_brief=semantic_brief,
|
||||
),
|
||||
)
|
||||
gap_fill_offers.append(slot_offer)
|
||||
|
||||
ai_step = {
|
||||
**current,
|
||||
"exercise_id": None,
|
||||
"is_ai_proposal": True,
|
||||
"title": slot_offer.get("title_hint") or current.get("title") or f"Slot {major_idx + 1}",
|
||||
"roadmap_major_step_index": major_idx,
|
||||
"gap_offer": slot_offer,
|
||||
}
|
||||
diff_stub = {
|
||||
"roadmap_major_step_index": major_idx,
|
||||
"baseline_exercise_id": int(current_id) if current_id is not None else None,
|
||||
"baseline_title": (current.get("title") or "").strip() or None,
|
||||
"proposed_exercise_id": None,
|
||||
"proposed_title": ai_step.get("title"),
|
||||
}
|
||||
merged_steps = _apply_slot_diff_to_steps(baseline_steps, diff_stub, [ai_step])
|
||||
eval_res = _quick_evaluate_steps_qa(
|
||||
cur,
|
||||
goal_query=goal_query,
|
||||
semantic_brief=semantic_brief,
|
||||
steps=merged_steps,
|
||||
roadmap_ctx=roadmap_ctx,
|
||||
)
|
||||
projected_qa = eval_res if isinstance(eval_res, dict) else None
|
||||
projected_score = _path_qa_quality_score(projected_qa)
|
||||
delta = _quality_delta(baseline_score, projected_score)
|
||||
improves = _slot_suggestion_accepted(
|
||||
baseline_qa=baseline_qa,
|
||||
projected_qa=projected_qa,
|
||||
baseline_score=baseline_score,
|
||||
projected_score=projected_score,
|
||||
diff=diff_stub,
|
||||
off_topic=off_topic,
|
||||
major_idx=major_idx,
|
||||
slot_problem=slot_problem,
|
||||
stage_specs=roadmap_ctx.stage_specs,
|
||||
baseline_steps=baseline_steps,
|
||||
projected_steps=merged_steps,
|
||||
)
|
||||
entry = {
|
||||
**diff_stub,
|
||||
"baseline_slot_status": current.get("slot_status"),
|
||||
"proposed_slot_status": "ai_proposal",
|
||||
"suggestion_type": "ai_gap",
|
||||
"quality_delta": delta,
|
||||
"projected_quality_score": projected_score,
|
||||
"baseline_quality_score": baseline_score,
|
||||
"projected_path_qa": projected_qa,
|
||||
"improves_path": improves or slot_problem,
|
||||
"off_topic": off_topic,
|
||||
"slot_problem": slot_problem,
|
||||
"problem_reasons": off_reasons[:6],
|
||||
"proposed_is_ai_proposal": True,
|
||||
"gap_offer": slot_offer,
|
||||
"pro_contra": _build_slot_pro_contra(
|
||||
current_step=current,
|
||||
proposed_step=None,
|
||||
suggestion_type="ai_gap",
|
||||
baseline_qa=baseline_qa,
|
||||
projected_qa=projected_qa,
|
||||
quality_delta=delta,
|
||||
off_topic_reasons=off_reasons,
|
||||
candidate_reasons=[],
|
||||
gap_offer=slot_offer,
|
||||
),
|
||||
}
|
||||
if improves or slot_problem:
|
||||
entry["improves_path"] = True
|
||||
suggestions.append(entry)
|
||||
else:
|
||||
rejected.append(entry)
|
||||
|
||||
improvement_diffs = [_suggestion_as_slot_diff(s) for s in suggestions]
|
||||
problem_slot_payload = {
|
||||
@@ -3412,9 +3495,11 @@ def _run_unified_slot_improvement_review(
|
||||
"suggestion_count": len(suggestions),
|
||||
"rejected_count": len(rejected),
|
||||
"problem_slot_count": len(problem_slots),
|
||||
"slot_review_count": len(slot_reviews),
|
||||
},
|
||||
"retrieval_phase": "unified_slot_review",
|
||||
"unified_slot_review": True,
|
||||
"slot_reviews": slot_reviews,
|
||||
"problem_slots": problem_slot_payload,
|
||||
"slot_suggestions": suggestions,
|
||||
"slot_diff_scoring": slot_diff_scoring,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from planning_exercise_path_builder import (
|
||||
_parse_slot_refs_from_text,
|
||||
_problematic_slots_from_path_qa,
|
||||
_slot_auto_select_library,
|
||||
_slot_suggestion_accepted,
|
||||
)
|
||||
from planning_progression_roadmap import StageSpecArtifact
|
||||
@@ -95,3 +96,18 @@ def test_problematic_slots_from_llm_schritt_text():
|
||||
specs = [_spec(7)]
|
||||
problems = _problematic_slots_from_path_qa(qa, steps, specs)
|
||||
assert 7 in problems
|
||||
|
||||
|
||||
def test_slot_auto_select_requires_higher_score():
|
||||
assert _slot_auto_select_library(
|
||||
baseline_slot_score=0.5,
|
||||
proposed_slot_score=0.51,
|
||||
baseline_exercise_id=1,
|
||||
proposed_exercise_id=2,
|
||||
)
|
||||
assert not _slot_auto_select_library(
|
||||
baseline_slot_score=0.5,
|
||||
proposed_slot_score=0.5,
|
||||
baseline_exercise_id=1,
|
||||
proposed_exercise_id=2,
|
||||
)
|
||||
Reference in new issue
Block a user