Enhance Progression Path Evaluation and Comparison Logic
Deploy Development / deploy (push) Successful in 45s
Test Suite / pytest-backend (push) Successful in 45s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 15s
Test Suite / k6 /health Baseline (push) Successful in 38s
Test Suite / playwright-tests (push) Successful in 1m23s
Deploy Development / deploy (push) Successful in 45s
Test Suite / pytest-backend (push) Successful in 45s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 15s
Test Suite / k6 /health Baseline (push) Successful in 38s
Test Suite / playwright-tests (push) Successful in 1m23s
- Introduced `_steps_to_evaluate_payloads` to convert path steps into evaluation payloads for improved quality assessments. - Updated `_build_progression_compare_response` to include a new `proposed_eval` parameter, allowing for fair quality assessment comparisons. - Enhanced `ProgressionGraphEditor` to utilize the new pipeline quality assessment data. - Modified `ProgressionOptimizeCompareModal` to display detailed comparison results, including handling of trivial slot differences and optimization hints. - Bumped version to reflect the new features and improvements.
This commit is contained in:
1 parent
5bca5ef9eb
commit
e828a5da32
4 files changed
+175
-16
No files matched your search
@@ -2176,6 +2176,52 @@ def _steps_by_major_index(steps: Sequence[Mapping[str, Any]]) -> Dict[int, Dict[
|
||||
return out
|
||||
|
||||
|
||||
def _steps_to_evaluate_payloads(steps: Sequence[Mapping[str, Any]]) -> List[EvaluateStepPayload]:
|
||||
"""Pfad-Schritte → evaluate_steps (für faire QS auf dem End-Stand)."""
|
||||
payloads: List[EvaluateStepPayload] = []
|
||||
for step in steps or []:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
midx = step.get("roadmap_major_step_index")
|
||||
if midx is None:
|
||||
continue
|
||||
eid = step.get("exercise_id")
|
||||
is_proposal = bool(step.get("is_ai_proposal")) or eid is None
|
||||
payloads.append(
|
||||
EvaluateStepPayload(
|
||||
exercise_id=int(eid) if eid is not None and not is_proposal else None,
|
||||
variant_id=step.get("variant_id"),
|
||||
title=step.get("title"),
|
||||
is_ai_proposal=is_proposal,
|
||||
ai_suggestion=step.get("ai_suggestion") if isinstance(step.get("ai_suggestion"), dict) else None,
|
||||
proposal_key=step.get("proposal_key"),
|
||||
roadmap_major_step_index=int(midx),
|
||||
roadmap_phase=step.get("roadmap_phase"),
|
||||
roadmap_learning_goal=step.get("roadmap_learning_goal"),
|
||||
)
|
||||
)
|
||||
payloads.sort(key=lambda p: int(p.roadmap_major_step_index or 0))
|
||||
return payloads
|
||||
|
||||
|
||||
def _normalize_slot_title(title: Optional[str]) -> str:
|
||||
return (title or "").strip().casefold()
|
||||
|
||||
|
||||
def _filter_trivial_slot_diffs(diffs: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Gleicher sichtbarer Titel = kein inhaltlicher Wechsel (nur ID-Doppel in der Bibliothek)."""
|
||||
out: List[Dict[str, Any]] = []
|
||||
for raw in diffs or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
bt = _normalize_slot_title(raw.get("baseline_title"))
|
||||
pt = _normalize_slot_title(raw.get("proposed_title"))
|
||||
if bt and pt and bt == pt:
|
||||
continue
|
||||
out.append(dict(raw))
|
||||
return out
|
||||
|
||||
|
||||
def _build_progression_slot_diffs(
|
||||
baseline_steps: Sequence[Mapping[str, Any]],
|
||||
proposed_steps: Sequence[Mapping[str, Any]],
|
||||
@@ -2211,24 +2257,35 @@ def _build_progression_slot_diffs(
|
||||
def _build_progression_compare_response(
|
||||
baseline: Mapping[str, Any],
|
||||
proposed: Mapping[str, Any],
|
||||
*,
|
||||
proposed_eval: Optional[Mapping[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
baseline_steps = list(baseline.get("steps") or [])
|
||||
proposed_steps = list(proposed.get("steps") or [])
|
||||
baseline_qa = baseline.get("path_qa") if isinstance(baseline.get("path_qa"), dict) else {}
|
||||
proposed_qa = proposed.get("path_qa") if isinstance(proposed.get("path_qa"), dict) else {}
|
||||
slot_diffs = _build_progression_slot_diffs(baseline_steps, proposed_steps)
|
||||
pipeline_qa = proposed.get("path_qa") if isinstance(proposed.get("path_qa"), dict) else {}
|
||||
fair_qa = (
|
||||
proposed_eval.get("path_qa")
|
||||
if isinstance(proposed_eval, dict) and isinstance(proposed_eval.get("path_qa"), dict)
|
||||
else pipeline_qa
|
||||
)
|
||||
slot_diffs = _filter_trivial_slot_diffs(
|
||||
_build_progression_slot_diffs(baseline_steps, proposed_steps),
|
||||
)
|
||||
return {
|
||||
**dict(proposed),
|
||||
"comparison_mode": True,
|
||||
"baseline_steps": baseline_steps,
|
||||
"baseline_path_qa": baseline_qa,
|
||||
"proposed_steps": proposed_steps,
|
||||
"proposed_path_qa": proposed_qa,
|
||||
"proposed_path_qa": fair_qa,
|
||||
"proposed_path_qa_pipeline": pipeline_qa,
|
||||
"slot_diffs": slot_diffs,
|
||||
"slot_diff_count": len(slot_diffs),
|
||||
"baseline_quality_score": _path_qa_quality_score(baseline_qa),
|
||||
"proposed_quality_score": _path_qa_quality_score(proposed_qa),
|
||||
"path_qa": proposed_qa,
|
||||
"proposed_quality_score": _path_qa_quality_score(fair_qa),
|
||||
"proposed_pipeline_quality_score": _path_qa_quality_score(pipeline_qa),
|
||||
"path_qa": fair_qa,
|
||||
"steps": proposed_steps,
|
||||
}
|
||||
|
||||
@@ -2273,7 +2330,21 @@ def suggest_progression_path(
|
||||
}
|
||||
)
|
||||
proposed = suggest_progression_path(cur, tenant=tenant, body=proposed_body)
|
||||
return _build_progression_compare_response(baseline, proposed)
|
||||
proposed_eval_payloads = _steps_to_evaluate_payloads(proposed.get("steps") or [])
|
||||
proposed_eval: Optional[Dict[str, Any]] = None
|
||||
if proposed_eval_payloads:
|
||||
proposed_eval_body = body.model_copy(
|
||||
update={
|
||||
"evaluate_only": True,
|
||||
"evaluate_steps": proposed_eval_payloads,
|
||||
"compare_with_assignments": False,
|
||||
"include_llm_intent": False,
|
||||
"auto_rematch_after_qa": False,
|
||||
"include_roadmap_preview": False,
|
||||
}
|
||||
)
|
||||
proposed_eval = suggest_progression_path(cur, tenant=tenant, body=proposed_eval_body)
|
||||
return _build_progression_compare_response(baseline, proposed, proposed_eval=proposed_eval)
|
||||
|
||||
goal_query = _normalize_query(body.query)
|
||||
if len(goal_query) < 3:
|
||||
|
||||
Reference in new issue
Block a user