Enhance Slot Evaluation and Scoring Mechanisms
Deploy Development / deploy (push) Successful in 46s
Test Suite / pytest-backend (push) Successful in 44s
Test Suite / lint-backend (push) Successful in 0s
Test Suite / build-frontend (push) Successful in 14s
Test Suite / k6 /health Baseline (push) Successful in 33s
Test Suite / playwright-tests (push) Successful in 1m26s

- Introduced new functions `_off_topic_semantic_scores_by_slot` and `_score_exercise_stage_fit_for_spec` to improve the evaluation of off-topic steps and exercise stage fit, enhancing the quality assessment process.
- Updated `_run_unified_slot_improvement_review` to incorporate off-topic scores and exercise stage fit scoring, refining the decision-making process for slot suggestions.
- Enhanced existing logic to streamline the handling of slot scores and improve the overall robustness of slot management in path evaluations.
This commit is contained in:
Lars committed 2026-06-13 12:33:16 +02:00
1 parent e9bf5bd1a5
commit cd457e3ea0
5 files changed
+654 -437

No files matched your search

+89 -15
View File
@@ -1010,8 +1010,39 @@ export function annotateCompareDiffKinds(diffs) {
}))
}
export function slotFitScorePercent(score) {
if (score == null || !Number.isFinite(Number(score))) return null
return Math.round(Number(score) * 100)
}
export function slotReviewSelectionKey(midx, kind = 'library') {
return `${Number(midx)}:${kind}`
}
export function parseSlotReviewSelection(raw) {
if (raw == null) return null
const text = String(raw)
if (text.includes(':')) {
const [midxRaw, kind] = text.split(':')
const midx = Number(midxRaw)
if (!Number.isFinite(midx)) return null
return { midx, kind: kind === 'ai' ? 'ai' : 'library' }
}
const midx = Number(text)
if (!Number.isFinite(midx)) return null
return { midx, kind: 'library' }
}
/** Alle Slot-Reviews aus Match-Antwort (je Slot eine Zeile). */
export function compareSlotReviews(comparison) {
return Array.isArray(comparison?.slot_reviews) ? comparison.slot_reviews : []
}
/** Nur übernehmbare Verbesserungsvorschläge (Bibliothek oder KI-Angebot). */
export function compareDiffsForDialog(comparison) {
const reviews = compareSlotReviews(comparison)
if (reviews.length > 0) return reviews
const fromSuggestions = (comparison?.slot_suggestions || []).filter((s) => s?.improves_path)
if (fromSuggestions.length > 0) {
return fromSuggestions
@@ -1037,6 +1068,16 @@ export function compareDiffsForDialog(comparison) {
)
}
export function defaultSelectedCompareDiffs(comparison) {
const reviews = compareSlotReviews(comparison)
if (reviews.length > 0) {
return reviews
.filter((review) => review?.library_alternative?.auto_select)
.map((review) => slotReviewSelectionKey(review.roadmap_major_step_index, 'library'))
}
return compareDiffsForDialog(comparison).map((d) => Number(d.roadmap_major_step_index))
}
export function suggestionDiffKind(suggestion) {
if (!suggestion) return 'skip'
if (suggestion.suggestion_type === 'ai_gap') return 'ai_gap'
@@ -1069,10 +1110,6 @@ export function gapOnlyCompareDiffs(comparison) {
).filter((d) => d.diff_kind === 'gap_only')
}
export function defaultSelectedCompareDiffs(comparison) {
return compareDiffsForDialog(comparison).map((d) => Number(d.roadmap_major_step_index))
}
function mergeGapFillOffersFromSteps(steps, offers) {
const merged = (offers || []).map((o) => ({ ...o }))
const seen = new Set(merged.map((o) => o.offer_id).filter(Boolean))
@@ -1160,11 +1197,13 @@ export function buildUnifiedSlotReviewComparePayload(res, baselineRes = null) {
: (Array.isArray(res?.baseline_steps) ? res.baseline_steps : (res?.steps || []))
const baselineQa = baselineRes?.path_qa || res?.baseline_path_qa || res?.path_qa || null
const scoring = res?.slot_diff_scoring
const slotReviews = compareSlotReviews(res)
const suggestions = Array.isArray(res?.slot_suggestions) ? res.slot_suggestions : []
const improving = suggestions.filter((s) => s?.improves_path)
const improving = suggestions.filter((s) => s?.improves_path || s?.auto_select)
const rejected = Array.isArray(scoring?.rejected_diffs) ? scoring.rejected_diffs : []
const proposedSteps = improving.map(suggestionToApplyStep).filter(Boolean)
const gapFillOffers = Array.isArray(res?.gap_fill_offers) ? res.gap_fill_offers : []
const autoSelectCount = slotReviews.filter((r) => r?.library_alternative?.auto_select).length
return {
...res,
@@ -1177,14 +1216,15 @@ export function buildUnifiedSlotReviewComparePayload(res, baselineRes = null) {
proposed_path_qa: baselineQa,
proposed_path_qa_pipeline: null,
gap_fill_offers: gapFillOffers,
slot_reviews: slotReviews,
slot_suggestions: suggestions,
slot_diffs: improving,
slot_diffs_improving: improving,
slot_diffs_rejected: rejected,
slot_diffs_dialog: improving,
slot_diffs_dialog: slotReviews.length > 0 ? slotReviews : improving,
slot_diffs_recommended: improving,
slot_diff_count: improving.length,
slot_diff_count_recommended: improving.length,
slot_diff_count: autoSelectCount || improving.length,
slot_diff_count_recommended: autoSelectCount || improving.length,
slot_diff_count_rejected: rejected.length,
slot_diffs_source: 'unified_slot_review',
slot_diff_scoring: scoring,
@@ -1220,25 +1260,59 @@ function suggestionToApplyStep(suggestion) {
}
/** Ausgewählte Slot-Vorschläge aus unified review übernehmen. */
export function applySelectedSlotSuggestions(draft, comparison, selectedMajorIndices) {
export function applySelectedSlotSuggestions(draft, comparison, selectedKeys) {
const reviews = compareSlotReviews(comparison)
if (reviews.length > 0) {
const selected = new Set((selectedKeys || []).map((x) => String(x)))
const steps = []
for (const review of reviews) {
const midx = Number(review.roadmap_major_step_index)
const libKey = slotReviewSelectionKey(midx, 'library')
const aiKey = slotReviewSelectionKey(midx, 'ai')
if (selected.has(aiKey) && review.ai_alternative?.gap_offer) {
const offer = review.ai_alternative.gap_offer
steps.push({
roadmap_major_step_index: midx,
exercise_id: null,
title: offer.title_hint || review.ai_alternative.title_hint || `Slot ${midx + 1}`,
is_ai_proposal: true,
proposal_key: offer.offer_id || `roadmap-unfilled-${midx}`,
gap_offer: offer,
slot_status: 'ai_proposal',
})
continue
}
if (selected.has(libKey) && review.library_alternative?.exercise_id != null) {
steps.push({
roadmap_major_step_index: midx,
exercise_id: review.library_alternative.exercise_id,
title: review.library_alternative.title,
slot_status: 'matched',
is_ai_proposal: false,
})
}
}
if (steps.length) return applyMatchStepsToSlots(draft, steps)
}
const selected = new Set(
(selectedMajorIndices || [])
.map((x) => Number(x))
(selectedKeys || [])
.map((x) => parseSlotReviewSelection(x)?.midx ?? Number(x))
.filter((x) => Number.isFinite(x)),
)
if (!selected.size) return draft
const steps = (comparison?.slot_suggestions || [])
const legacySteps = (comparison?.slot_suggestions || [])
.filter((s) => selected.has(Number(s.roadmap_major_step_index)))
.map(suggestionToApplyStep)
.filter(Boolean)
if (!steps.length) {
if (!legacySteps.length) {
return applySelectedCompareSteps(
draft,
comparison?.proposed_steps || comparison?.steps,
selectedMajorIndices,
selectedKeys,
)
}
return applyMatchStepsToSlots(draft, steps)
return applyMatchStepsToSlots(draft, legacySteps)
}
/** Alle Slot-Diffs inkl. reiner ID-Tausche (gleicher Titel). */