Enhance Rematch Suggestion Logic and Progression Path Evaluation
All checks were successful
Deploy Development / deploy (push) Successful in 41s
Test Suite / pytest-backend (push) Successful in 45s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m14s
All checks were successful
Deploy Development / deploy (push) Successful in 41s
Test Suite / pytest-backend (push) Successful in 45s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m14s
- Introduced `_baseline_slot_accepts_rematch_suggestion` to filter out filled or invalid slots from rematch suggestions, improving the accuracy of rematch logic. - Updated `_build_rematch_suggestion_diffs` to skip non-eligible baseline slots, streamlining the rematch suggestion process. - Added `_evaluate_steps_for_compare_qa` to evaluate steps against the current state, enhancing the quality assessment during progression path suggestions. - Modified `_build_progression_compare_response` to ensure proper handling of slot differences and quality scores, improving response clarity. - Updated frontend components to reflect changes in rematch handling and evaluation logic. - Bumped version to reflect the new features and improvements.
This commit is contained in:
parent
dccb065181
commit
69ce3f6975
|
|
@ -2247,6 +2247,19 @@ def _last_rematch_replacements_by_slot(
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _baseline_slot_accepts_rematch_suggestion(base: Mapping[str, Any]) -> bool:
|
||||||
|
"""Rematch-Protokoll nur für leere oder explizit ungültige Slots — nicht kuratierte Zuordnungen ersetzen."""
|
||||||
|
if not base:
|
||||||
|
return True
|
||||||
|
base_id = base.get("exercise_id")
|
||||||
|
status = str(base.get("slot_status") or "").strip().lower()
|
||||||
|
if base_id is None:
|
||||||
|
return True
|
||||||
|
if status in {"unfilled", "stripped", "gap", "off_topic"}:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _build_rematch_suggestion_diffs(
|
def _build_rematch_suggestion_diffs(
|
||||||
baseline_steps: Sequence[Mapping[str, Any]],
|
baseline_steps: Sequence[Mapping[str, Any]],
|
||||||
rematch_log: Sequence[Mapping[str, Any]],
|
rematch_log: Sequence[Mapping[str, Any]],
|
||||||
|
|
@ -2257,6 +2270,8 @@ def _build_rematch_suggestion_diffs(
|
||||||
diffs: List[Dict[str, Any]] = []
|
diffs: List[Dict[str, Any]] = []
|
||||||
for midx, entry in sorted(replacements.items()):
|
for midx, entry in sorted(replacements.items()):
|
||||||
base = base_by.get(midx, {})
|
base = base_by.get(midx, {})
|
||||||
|
if not _baseline_slot_accepts_rematch_suggestion(base):
|
||||||
|
continue
|
||||||
base_id = base.get("exercise_id")
|
base_id = base.get("exercise_id")
|
||||||
new_id = entry.get("new_exercise_id")
|
new_id = entry.get("new_exercise_id")
|
||||||
base_title = (base.get("title") or "").strip() or None
|
base_title = (base.get("title") or "").strip() or None
|
||||||
|
|
@ -2354,6 +2369,31 @@ def _build_progression_slot_diffs(
|
||||||
return diffs
|
return diffs
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate_steps_for_compare_qa(
|
||||||
|
cur,
|
||||||
|
*,
|
||||||
|
tenant: TenantContext,
|
||||||
|
body: ProgressionPathSuggestRequest,
|
||||||
|
steps: Sequence[Mapping[str, Any]],
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Evaluate-only auf konkretem Schritt-Stand (gleiche Pipeline wie Graph bewerten)."""
|
||||||
|
payloads = _steps_to_evaluate_payloads(steps)
|
||||||
|
if not payloads:
|
||||||
|
return None
|
||||||
|
eval_body = body.model_copy(
|
||||||
|
update={
|
||||||
|
"evaluate_only": True,
|
||||||
|
"evaluate_steps": payloads,
|
||||||
|
"compare_with_assignments": False,
|
||||||
|
"preserve_slot_assignments": False,
|
||||||
|
"include_llm_intent": False,
|
||||||
|
"auto_rematch_after_qa": False,
|
||||||
|
"include_roadmap_preview": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return suggest_progression_path(cur, tenant=tenant, body=eval_body)
|
||||||
|
|
||||||
|
|
||||||
def _build_progression_compare_response(
|
def _build_progression_compare_response(
|
||||||
baseline: Mapping[str, Any],
|
baseline: Mapping[str, Any],
|
||||||
proposed: Mapping[str, Any],
|
proposed: Mapping[str, Any],
|
||||||
|
|
@ -2373,22 +2413,7 @@ def _build_progression_compare_response(
|
||||||
_build_progression_slot_diffs(baseline_steps, proposed_steps),
|
_build_progression_slot_diffs(baseline_steps, proposed_steps),
|
||||||
)
|
)
|
||||||
actionable_diffs = _actionable_slot_diffs(slot_diffs)
|
actionable_diffs = _actionable_slot_diffs(slot_diffs)
|
||||||
slot_diffs_source = "steps"
|
|
||||||
rematch_log = (
|
|
||||||
pipeline_qa.get("rematch_log")
|
|
||||||
if isinstance(pipeline_qa.get("rematch_log"), list)
|
|
||||||
else []
|
|
||||||
)
|
|
||||||
apply_steps = list(proposed_steps)
|
apply_steps = list(proposed_steps)
|
||||||
if not actionable_diffs and rematch_log:
|
|
||||||
rematch_raw = _build_rematch_suggestion_diffs(baseline_steps, rematch_log)
|
|
||||||
rematch_diffs = _annotate_slot_diffs(rematch_raw)
|
|
||||||
rematch_actionable = _actionable_slot_diffs(rematch_diffs)
|
|
||||||
if rematch_actionable:
|
|
||||||
actionable_diffs = rematch_actionable
|
|
||||||
slot_diffs = rematch_diffs
|
|
||||||
slot_diffs_source = "rematch_log"
|
|
||||||
apply_steps = _overlay_rematch_suggestions_on_steps(proposed_steps, rematch_actionable)
|
|
||||||
return {
|
return {
|
||||||
**dict(proposed),
|
**dict(proposed),
|
||||||
"comparison_mode": True,
|
"comparison_mode": True,
|
||||||
|
|
@ -2402,7 +2427,7 @@ def _build_progression_compare_response(
|
||||||
"slot_diffs_actionable": actionable_diffs,
|
"slot_diffs_actionable": actionable_diffs,
|
||||||
"slot_diff_count": len(actionable_diffs),
|
"slot_diff_count": len(actionable_diffs),
|
||||||
"slot_diff_count_including_trivial": len(slot_diffs),
|
"slot_diff_count_including_trivial": len(slot_diffs),
|
||||||
"slot_diffs_source": slot_diffs_source,
|
"slot_diffs_source": "steps",
|
||||||
"optimization_actionable": len(actionable_diffs) > 0,
|
"optimization_actionable": len(actionable_diffs) > 0,
|
||||||
"baseline_quality_score": _path_qa_quality_score(baseline_qa),
|
"baseline_quality_score": _path_qa_quality_score(baseline_qa),
|
||||||
"proposed_quality_score": _path_qa_quality_score(fair_qa),
|
"proposed_quality_score": _path_qa_quality_score(fair_qa),
|
||||||
|
|
@ -2447,26 +2472,31 @@ def suggest_progression_path(
|
||||||
proposed_body = body.model_copy(
|
proposed_body = body.model_copy(
|
||||||
update={
|
update={
|
||||||
"compare_with_assignments": False,
|
"compare_with_assignments": False,
|
||||||
"preserve_slot_assignments": False,
|
"preserve_slot_assignments": True,
|
||||||
"evaluate_only": False,
|
"evaluate_only": False,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
proposed = suggest_progression_path(cur, tenant=tenant, body=proposed_body)
|
proposed = suggest_progression_path(cur, tenant=tenant, body=proposed_body)
|
||||||
proposed_eval_payloads = _steps_to_evaluate_payloads(proposed.get("steps") or [])
|
result = _build_progression_compare_response(baseline, proposed, proposed_eval=None)
|
||||||
proposed_eval: Optional[Dict[str, Any]] = None
|
if result.get("slot_diff_count", 0) > 0:
|
||||||
if proposed_eval_payloads:
|
apply_eval = _evaluate_steps_for_compare_qa(
|
||||||
proposed_eval_body = body.model_copy(
|
cur,
|
||||||
update={
|
tenant=tenant,
|
||||||
"evaluate_only": True,
|
body=body,
|
||||||
"evaluate_steps": proposed_eval_payloads,
|
steps=result.get("proposed_steps") or [],
|
||||||
"compare_with_assignments": False,
|
|
||||||
"include_llm_intent": False,
|
|
||||||
"auto_rematch_after_qa": False,
|
|
||||||
"include_roadmap_preview": False,
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
proposed_eval = suggest_progression_path(cur, tenant=tenant, body=proposed_eval_body)
|
if isinstance(apply_eval, dict) and isinstance(apply_eval.get("path_qa"), dict):
|
||||||
return _build_progression_compare_response(baseline, proposed, proposed_eval=proposed_eval)
|
fair = apply_eval["path_qa"]
|
||||||
|
result["proposed_path_qa"] = fair
|
||||||
|
result["path_qa"] = fair
|
||||||
|
result["proposed_quality_score"] = _path_qa_quality_score(fair)
|
||||||
|
elif isinstance(baseline.get("path_qa"), dict):
|
||||||
|
# Kein übernehmbarer Unterschied — Vorschlag-QS = Baseline (kein Pipeline-Artefakt)
|
||||||
|
fair = baseline["path_qa"]
|
||||||
|
result["proposed_path_qa"] = fair
|
||||||
|
result["path_qa"] = fair
|
||||||
|
result["proposed_quality_score"] = _path_qa_quality_score(fair)
|
||||||
|
return result
|
||||||
|
|
||||||
goal_query = _normalize_query(body.query)
|
goal_query = _normalize_query(body.query)
|
||||||
if len(goal_query) < 3:
|
if len(goal_query) < 3:
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Tests Vergleichs-Diffs (triviale ID-Tausche markieren, Rematch-Vorschläge)."""
|
"""Tests Vergleichs-Diffs (triviale ID-Tausche markieren, Rematch-Filter)."""
|
||||||
from planning_exercise_path_builder import (
|
from planning_exercise_path_builder import (
|
||||||
_actionable_slot_diffs,
|
_actionable_slot_diffs,
|
||||||
_annotate_slot_diffs,
|
_annotate_slot_diffs,
|
||||||
|
|
@ -55,12 +55,32 @@ def test_build_slot_diffs_then_annotate():
|
||||||
assert _actionable_slot_diffs(annotated) == []
|
assert _actionable_slot_diffs(annotated) == []
|
||||||
|
|
||||||
|
|
||||||
def test_rematch_suggestion_diffs_when_end_path_matches_baseline():
|
def test_rematch_suggestion_skips_filled_baseline_slot():
|
||||||
|
baseline = [
|
||||||
|
{
|
||||||
|
"roadmap_major_step_index": 1,
|
||||||
|
"exercise_id": 5727,
|
||||||
|
"title": "Einführung von Richtungswechseln",
|
||||||
|
"slot_status": "preserved",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
rematch_log = [
|
||||||
|
{
|
||||||
|
"roadmap_major_step_index": 1,
|
||||||
|
"action": "replaced",
|
||||||
|
"round": 3,
|
||||||
|
"new_exercise_id": 5594,
|
||||||
|
"new_title": "Kumite Beinarbeit — vertiefung",
|
||||||
|
"replaced_exercise_id": 5727,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
assert _build_rematch_suggestion_diffs(baseline, rematch_log) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_rematch_suggestion_keeps_empty_baseline_slot():
|
||||||
baseline = [
|
baseline = [
|
||||||
{"roadmap_major_step_index": 1, "exercise_id": None, "title": "Lernziel Slot 2"},
|
{"roadmap_major_step_index": 1, "exercise_id": None, "title": "Lernziel Slot 2"},
|
||||||
{"roadmap_major_step_index": 4, "exercise_id": 50, "title": "Bestehend"},
|
|
||||||
]
|
]
|
||||||
proposed = list(baseline)
|
|
||||||
rematch_log = [
|
rematch_log = [
|
||||||
{
|
{
|
||||||
"roadmap_major_step_index": 1,
|
"roadmap_major_step_index": 1,
|
||||||
|
|
@ -68,35 +88,23 @@ def test_rematch_suggestion_diffs_when_end_path_matches_baseline():
|
||||||
"round": 1,
|
"round": 1,
|
||||||
"new_exercise_id": 101,
|
"new_exercise_id": 101,
|
||||||
"new_title": "Rhythmuswechsel in der Kumite-Beinarbeit",
|
"new_title": "Rhythmuswechsel in der Kumite-Beinarbeit",
|
||||||
"replaced_exercise_id": None,
|
|
||||||
"replaced_title": None,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"roadmap_major_step_index": 1,
|
|
||||||
"action": "replaced",
|
|
||||||
"round": 3,
|
|
||||||
"new_exercise_id": 102,
|
|
||||||
"new_title": "Kumite Beinarbeit — vertiefung",
|
|
||||||
"replaced_exercise_id": 101,
|
|
||||||
"replaced_title": "Rhythmuswechsel in der Kumite-Beinarbeit",
|
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
diffs = _build_rematch_suggestion_diffs(baseline, rematch_log)
|
diffs = _build_rematch_suggestion_diffs(baseline, rematch_log)
|
||||||
assert len(diffs) == 1
|
assert len(diffs) == 1
|
||||||
assert diffs[0]["proposed_exercise_id"] == 102
|
assert diffs[0]["proposed_exercise_id"] == 101
|
||||||
assert diffs[0]["from_rematch_log"] is True
|
|
||||||
|
|
||||||
compare = _build_progression_compare_response(
|
|
||||||
{"steps": baseline, "path_qa": {"overall_ok": True, "quality_score": 0.88}},
|
def test_compare_response_no_step_diffs_uses_baseline_qa_not_pipeline():
|
||||||
{
|
baseline = {
|
||||||
"steps": proposed,
|
"steps": [{"roadmap_major_step_index": 0, "exercise_id": 1, "title": "A"}],
|
||||||
"path_qa": {
|
"path_qa": {"overall_ok": True, "quality_score": 0.88},
|
||||||
"overall_ok": False,
|
}
|
||||||
"quality_score": 0.65,
|
proposed = {
|
||||||
"rematch_log": rematch_log,
|
"steps": [{"roadmap_major_step_index": 0, "exercise_id": 1, "title": "A"}],
|
||||||
},
|
"path_qa": {"overall_ok": False, "quality_score": 0.65, "rematch_log": [{"action": "replaced"}]},
|
||||||
},
|
}
|
||||||
)
|
compare = _build_progression_compare_response(baseline, proposed, proposed_eval=None)
|
||||||
assert compare["slot_diffs_source"] == "rematch_log"
|
assert compare["slot_diff_count"] == 0
|
||||||
assert compare["slot_diff_count"] == 1
|
assert compare["slot_diffs_source"] == "steps"
|
||||||
assert compare["proposed_steps"][0]["exercise_id"] == 102
|
assert compare["proposed_path_qa"]["quality_score"] == 0.65
|
||||||
|
|
|
||||||
|
|
@ -491,7 +491,7 @@ export default function ProgressionGraphEditor({ graphId, embedded = false, onSa
|
||||||
const res = await api.suggestProgressionPath({
|
const res = await api.suggestProgressionPath({
|
||||||
...buildMatchRequestBase(synced),
|
...buildMatchRequestBase(synced),
|
||||||
evaluate_steps: slotsToEvaluateSteps(synced),
|
evaluate_steps: slotsToEvaluateSteps(synced),
|
||||||
preserve_slot_assignments: false,
|
preserve_slot_assignments: true,
|
||||||
compare_with_assignments: true,
|
compare_with_assignments: true,
|
||||||
})
|
})
|
||||||
if (!res?.comparison_mode) {
|
if (!res?.comparison_mode) {
|
||||||
|
|
@ -1217,8 +1217,6 @@ export default function ProgressionGraphEditor({ graphId, embedded = false, onSa
|
||||||
onGenerateGapAi={openGapFillPrep}
|
onGenerateGapAi={openGapFillPrep}
|
||||||
onRematchSlots={runMatch}
|
onRematchSlots={runMatch}
|
||||||
onOptimizeCompare={runOptimizeCompare}
|
onOptimizeCompare={runOptimizeCompare}
|
||||||
optimizationPreviewQa={compareOpen ? proposedPathQa : null}
|
|
||||||
optimizationPreviewFairQa={compareOpen ? comparePayload?.proposed_path_qa : null}
|
|
||||||
canOptimizeCompare={draftHasLibrarySlotAssignments(draft)}
|
canOptimizeCompare={draftHasLibrarySlotAssignments(draft)}
|
||||||
optimizeCompareBusy={comparing}
|
optimizeCompareBusy={comparing}
|
||||||
rematchBusy={matching}
|
rematchBusy={matching}
|
||||||
|
|
|
||||||
|
|
@ -82,8 +82,9 @@ export default function ProgressionOptimizeCompareModal({
|
||||||
Optimierung vergleichen
|
Optimierung vergleichen
|
||||||
</h3>
|
</h3>
|
||||||
<p style={{ fontSize: '12px', color: 'var(--text3)', marginTop: 0, lineHeight: 1.45 }}>
|
<p style={{ fontSize: '12px', color: 'var(--text3)', marginTop: 0, lineHeight: 1.45 }}>
|
||||||
Vergleicht deinen Pfad mit dem End-Stand nach Match — beide Seiten mit derselben Bewertungslogik
|
Vergleicht deinen Pfad mit dem Match-Vorschlag — beide Seiten mit derselben Bewertungslogik
|
||||||
wie „Graph bewerten“. Auto-Rematch-Details stehen im Panel, nicht in der Prozentzahl.
|
wie „Graph bewerten“. Prozentwerte beziehen sich auf den übernehmbaren End-Stand, nicht auf
|
||||||
|
das Auto-Rematch-Protokoll.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
{noMeaningfulDiffs || proposedNotBetter ? (
|
{noMeaningfulDiffs || proposedNotBetter ? (
|
||||||
|
|
@ -173,14 +174,7 @@ export default function ProgressionOptimizeCompareModal({
|
||||||
: ''}
|
: ''}
|
||||||
{refineCount > 0 ? ` · ${refineCount} Stufen-Spec verfeinert` : ''}
|
{refineCount > 0 ? ` · ${refineCount} Stufen-Spec verfeinert` : ''}
|
||||||
{hintCount > 0 ? ` · ${hintCount} Handlungshinweis(e)` : ''}
|
{hintCount > 0 ? ` · ${hintCount} Handlungshinweis(e)` : ''}
|
||||||
. Details im Panel „Graph-Bewertung“.
|
. Nur Prozessinfo — nicht für die Prozentzahl links/rechts.
|
||||||
</p>
|
|
||||||
) : null}
|
|
||||||
|
|
||||||
{comparison?.slot_diffs_source === 'rematch_log' ? (
|
|
||||||
<p style={{ fontSize: '11px', color: 'var(--text3)', margin: '0 0 14px', lineHeight: 1.45 }}>
|
|
||||||
Vorschläge stammen aus dem Auto-Rematch-Protokoll (letzte Runde je Slot), weil der sichtbare
|
|
||||||
End-Pfad deinem aktuellen Stand entspricht.
|
|
||||||
</p>
|
</p>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user