refactor(csv-import): Simplify test execution and enhance custom equivalence handling

- Updated the test execution command in the CI workflow to run all tests excluding slow ones, improving efficiency. - Enhanced the AdminCsvTemplateEditorPage to support custom equivalence for unit conversions, allowing for more flexible data handling. - Added markers in pytest configuration for categorizing tests, facilitating better test management.
2026-04-10 11:38:54 +02:00 · 2026-04-10 11:38:54 +02:00 · 8b67f7ab55
commit 8b67f7ab55
parent 8ee9fb84ba
3 changed files with 228 additions and 79 deletions
--- a/.gitea/workflows/test.yml
+++ b/.gitea/workflows/test.yml
@ -34,13 +34,7 @@ jobs:
          docker compose -f "$COMPOSE_FILE" exec -T backend sh -lc "
            pip install -r /app/requirements-dev.txt &&
            cd /app &&
-            python -m pytest \
-              tests/test_csv_parser_core.py \
-              tests/test_csv_import_executor.py \
-              tests/test_mapping_suggest.py \
-              tests/test_placeholder_metadata.py \
-              tests/test_placeholder_metadata_v2.py \
-              -q --tb=short
+            python -m pytest tests -m 'not slow' -q --tb=short
          "

  lint-backend:
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@ -3,3 +3,7 @@ testpaths = tests
 python_files = test_*.py
 python_functions = test_*
 addopts = -q --tb=short
+markers =
+    smoke: Fast smoke tests for core regression checks.
+    integration: Integration tests across modules/db/api behavior.
+    slow: Long-running or heavy tests.
--- a/frontend/src/pages/AdminCsvTemplateEditorPage.jsx
+++ b/frontend/src/pages/AdminCsvTemplateEditorPage.jsx
@ -13,6 +13,19 @@ const MODULE_LABEL = {
  vitals_baseline: 'Vitalwerte (Baseline)',
 }

+/** Vorschläge für Freitext „Quelleinheit“ (Volumen, Stück …); Ziel kommt aus dem Datenmodell. */
+const CUSTOM_SOURCE_UNIT_HINTS = [
+  'ml',
+  'l',
+  'cl',
+  'dl',
+  'µl',
+  'mg',
+  'µg',
+  'Stück',
+  'Portion',
+]
+
 /** Erlaubt Eingaben wie 1,03 oder 1.03 während des Tippens; Finale normalisiert bei Blur/Speichern. */
 function normalizeDecimalInputString(raw) {
  let s = String(raw).trim().replace(/\s/g, '')
@ -31,21 +44,70 @@ function normalizeDecimalInputString(raw) {
  return s
 }

-function applyCustomFactorToTcObject(tc, fieldKey, raw) {
+/**
+ * Semantik: „source_amount [source_unit_label] entspricht target_amount [canonical storage unit]“
+ * → CSV-Zahl (in der linken Einheit) × (target_amount / source_amount) → Speicherwert.
+ * custom_equivalence dient der Dokumentation im JSON; der Importer nutzt nur conversion_factor.
+ */
+function loadEquivalenceFromTc(tc, fieldKey) {
+  const row = tc[fieldKey]
+  if (!row || typeof row !== 'object') {
+    return { srcAmt: '1', srcUnit: '', tgtAmt: '' }
+  }
+  const eq = row.custom_equivalence
+  if (eq && typeof eq === 'object') {
+    const sa = eq.source_amount
+    const ta = eq.target_amount
+    return {
+      srcAmt: sa != null && sa !== '' ? String(sa) : '',
+      srcUnit: eq.source_unit_label != null ? String(eq.source_unit_label) : '',
+      tgtAmt: ta != null && ta !== '' ? String(ta) : '',
+    }
+  }
+  const f = row.conversion_factor
+  if (f != null && f !== '') {
+    return { srcAmt: '1', srcUnit: '', tgtAmt: String(f) }
+  }
+  return { srcAmt: '1', srcUnit: '', tgtAmt: '' }
+}
+
+function applyCustomEquivalenceToTcObject(tc, fieldKey, draft, canonicalUnitLabel) {
  const base =
    tc[fieldKey] && typeof tc[fieldKey] === 'object'
      ? { ...tc[fieldKey] }
      : { type: 'float', decimal_separator: 'auto', flexible: true }
-  const normalized = normalizeDecimalInputString(raw)
-  if (normalized === '') {
+  const sa = normalizeDecimalInputString(draft.srcAmt ?? '')
+  const ta = normalizeDecimalInputString(draft.tgtAmt ?? '')
+  const unitLbl = String(draft.srcUnit ?? '').trim()
+  const bothEmpty = !sa && !ta
+
+  if (bothEmpty) {
    delete base.conversion_factor
-  } else {
-    const num = Number(normalized)
-    if (Number.isNaN(num)) {
-      return { ok: false, message: `Konvertierungsfaktor (${fieldKey}): keine gültige Zahl.` }
-    }
-    base.conversion_factor = num
+    delete base.custom_equivalence
    base.source_unit = 'custom'
+    delete base.target_unit
+    tc[fieldKey] = base
+    return { ok: true }
+  }
+  if (!sa || !ta) {
+    return { ok: true, partial: true }
+  }
+  const srcNum = Number(sa)
+  const tgtNum = Number(ta)
+  if (Number.isNaN(srcNum) || Number.isNaN(tgtNum)) {
+    return { ok: false, message: `Umrechnung (${fieldKey}): keine gültigen Zahlen.` }
+  }
+  if (srcNum === 0) {
+    return { ok: false, message: `Umrechnung (${fieldKey}): die linke Menge darf nicht 0 sein.` }
+  }
+  const factor = tgtNum / srcNum
+  base.conversion_factor = factor
+  base.source_unit = 'custom'
+  base.custom_equivalence = {
+    source_amount: srcNum,
+    source_unit_label: unitLbl || '(Quelleinheit)',
+    target_amount: tgtNum,
+    target_unit_label: canonicalUnitLabel || '(Ziel)',
  }
  delete base.target_unit
  tc[fieldKey] = base
@ -121,8 +183,8 @@ export default function AdminCsvTemplateEditorPage() {
  const [analyzing, setAnalyzing] = useState(false)
  const [saving, setSaving] = useState(false)
  const [error, setError] = useState(null)
-  /** Lokaler Text für Konvertierungsfaktor (nur bei source_unit custom), Commit bei Blur/Speichern — verhindert 1. → 1 beim Tippen. */
-  const [customFactorDraftByField, setCustomFactorDraftByField] = useState({})
+  /** Entwurf für „Quelle entspricht Ziel“ (nur source_unit custom); Commit bei Blur/Speichern. */
+  const [customEquivalenceDraftByField, setCustomEquivalenceDraftByField] = useState({})

  const modMeta = useMemo(() => modules.find((m) => m.id === module), [modules, module])
  const aggregateSleepImport = modMeta?.import_mode === 'apple_sleep_aggregate'
@ -176,7 +238,7 @@ export default function AdminCsvTemplateEditorPage() {
        setFieldMappings(fm)
        setColumns(Object.keys(fm))
        setTypeConversionsText(JSON.stringify(t.type_conversions || {}, null, 2))
-        setCustomFactorDraftByField({})
+        setCustomEquivalenceDraftByField({})
        setSampleRows([])
        setSeedHint(null)
      })
@ -229,6 +291,10 @@ export default function AdminCsvTemplateEditorPage() {
      const hit = opts.find((o) => o.id === sid)
      if (hit) return hit.id
    }
+    const ce = tc[fieldKey]?.custom_equivalence
+    if (ce && typeof ce === 'object' && opts.some((o) => o.id === 'custom')) {
+      return 'custom'
+    }
    if (tc[fieldKey]?.conversion_factor != null && tc[fieldKey]?.conversion_factor !== '') {
      const hasCustomOpt = opts.some((o) => o.id === 'custom')
      if (hasCustomOpt) return 'custom'
@ -254,16 +320,17 @@ export default function AdminCsvTemplateEditorPage() {
    if (sourceUnitId === canonical || sourceUnitId === '' || sourceUnitId == null) {
      delete base.source_unit
      delete base.conversion_factor
+      delete base.custom_equivalence
    } else if (sourceUnitId === 'custom') {
      base.source_unit = 'custom'
-      // conversion_factor nur per Eingabefeld setzen/löschen
    } else {
      base.source_unit = sourceUnitId
      delete base.conversion_factor
+      delete base.custom_equivalence
    }
    tc[fieldKey] = base
    setTypeConversionsText(JSON.stringify(tc, null, 2))
-    setCustomFactorDraftByField((prev) => {
+    setCustomEquivalenceDraftByField((prev) => {
      const next = { ...prev }
      delete next[fieldKey]
      return next
@ -271,28 +338,40 @@ export default function AdminCsvTemplateEditorPage() {
    setError(null)
  }

-  const getCustomConversionFactorInputValue = (fieldKey) => {
-    let tc
+  const getCanonicalStorageUnitLabel = (fieldKey) => {
+    const opts = modMeta?.fields?.[fieldKey]?.source_unit_options || []
+    const c = opts.find((o) => o.is_canonical)
+    return c?.canonical_unit || c?.id || '—'
+  }
+
+  const getEquivalenceDisplay = (fieldKey) => {
+    if (customEquivalenceDraftByField[fieldKey]) {
+      return customEquivalenceDraftByField[fieldKey]
+    }
    try {
-      tc = JSON.parse(typeConversionsText || '{}')
+      const tc = JSON.parse(typeConversionsText || '{}')
+      return loadEquivalenceFromTc(tc, fieldKey)
    } catch {
-      return ''
+      return { srcAmt: '1', srcUnit: '', tgtAmt: '' }
    }
-    const v = tc[fieldKey]?.conversion_factor
-    if (v == null || v === '') return ''
-    return String(v)
  }

-  const getCustomFactorFieldDisplay = (fieldKey) => {
-    if (Object.prototype.hasOwnProperty.call(customFactorDraftByField, fieldKey)) {
-      return customFactorDraftByField[fieldKey]
+  const mergeEquivalenceDraft = (fieldKey, patch) => {
+    setCustomEquivalenceDraftByField((prev) => {
+      let baseTc
+      try {
+        baseTc = JSON.parse(typeConversionsText || '{}')
+      } catch {
+        baseTc = {}
      }
-    return getCustomConversionFactorInputValue(fieldKey)
+      const cur = prev[fieldKey] || loadEquivalenceFromTc(baseTc, fieldKey)
+      return { ...prev, [fieldKey]: { ...cur, ...patch } }
+    })
  }

-  const commitCustomFactorOnBlur = (fieldKey, raw) => {
+  const commitCustomEquivalenceOnBlur = (fieldKey) => {
    if (getSourceUnitSelectValue(fieldKey) !== 'custom') {
-      setCustomFactorDraftByField((prev) => {
+      setCustomEquivalenceDraftByField((prev) => {
        const next = { ...prev }
        delete next[fieldKey]
        return next
@ -303,16 +382,23 @@ export default function AdminCsvTemplateEditorPage() {
    try {
      tc = JSON.parse(typeConversionsText || '{}')
    } catch {
-      setError('type_conversions: ungültiges JSON (Faktor kann nicht gesetzt werden).')
+      setError('type_conversions: ungültiges JSON (Umrechnung kann nicht gespeichert werden).')
+      return
+    }
+    const draft =
+      customEquivalenceDraftByField[fieldKey] || loadEquivalenceFromTc(tc, fieldKey)
+    const canon = getCanonicalStorageUnitLabel(fieldKey)
+    const result = applyCustomEquivalenceToTcObject(tc, fieldKey, draft, canon)
+    if (result.partial) {
+      setError(null)
      return
    }
-    const result = applyCustomFactorToTcObject(tc, fieldKey, raw)
    if (!result.ok) {
-      setError(result.message || 'Konvertierungsfaktor ungültig.')
+      setError(result.message || 'Ungültige Umrechnung.')
      return
    }
    setTypeConversionsText(JSON.stringify(tc, null, 2))
-    setCustomFactorDraftByField((prev) => {
+    setCustomEquivalenceDraftByField((prev) => {
      const next = { ...prev }
      delete next[fieldKey]
      return next
@ -320,6 +406,20 @@ export default function AdminCsvTemplateEditorPage() {
    setError(null)
  }

+  const derivedFactorHintLine = (fieldKey) => {
+    const d = getEquivalenceDisplay(fieldKey)
+    const sa = normalizeDecimalInputString(d.srcAmt ?? '')
+    const ta = normalizeDecimalInputString(d.tgtAmt ?? '')
+    if (!sa || !ta) return null
+    const a = Number(sa)
+    const b = Number(ta)
+    if (Number.isNaN(a) || Number.isNaN(b) || a === 0) return null
+    const f = b / a
+    const fDisp = Number.isFinite(f) ? Math.round(f * 1e9) / 1e9 : f
+    const canon = getCanonicalStorageUnitLabel(fieldKey)
+    return `Abgeleitet: CSV-Zahl × ${fDisp} → Wert in ${canon} (Speicher)`
+  }
+
  const handleAnalyze = async () => {
    if (!file) {
      setError('Bitte eine CSV-Datei wählen.')
@ -339,7 +439,7 @@ export default function AdminCsvTemplateEditorPage() {
      setDelimiter(res.delimiter || ';')
      setEncoding(res.encoding || 'utf-8')
      setSeedHint(res.seed_template || null)
-      setCustomFactorDraftByField({})
+      setCustomEquivalenceDraftByField({})
    } catch (e) {
      setError(e.message || 'Analyse fehlgeschlagen')
    } finally {
@ -354,22 +454,32 @@ export default function AdminCsvTemplateEditorPage() {
  const handleSave = async () => {
    setError(null)
    let textForTc = typeConversionsText
-    const pendingFactorDrafts = { ...customFactorDraftByField }
-    if (Object.keys(pendingFactorDrafts).length > 0) {
+    const pendingEquivalenceDrafts = { ...customEquivalenceDraftByField }
+    if (Object.keys(pendingEquivalenceDrafts).length > 0) {
      try {
        const tco = JSON.parse(textForTc || '{}')
-        for (const [fk, raw] of Object.entries(pendingFactorDrafts)) {
+        for (const fk of Object.keys(pendingEquivalenceDrafts)) {
          const su = String(tco[fk]?.source_unit || '').toLowerCase()
          if (su !== 'custom') continue
-          const result = applyCustomFactorToTcObject(tco, fk, raw)
+          const draft = pendingEquivalenceDrafts[fk]
+          const opts = modMeta?.fields?.[fk]?.source_unit_options || []
+          const c = opts.find((o) => o.is_canonical)
+          const canon = c?.canonical_unit || c?.id || '—'
+          const result = applyCustomEquivalenceToTcObject(tco, fk, draft, canon)
+          if (result.partial) {
+            setError(
+              `Benutzerdefinierte Umrechnung (${fk}): beide Mengen ausfüllen oder Entwurf verwerfen (Tab durch alle Felder).`,
+            )
+            return
+          }
          if (!result.ok) {
-            setError(result.message || 'Konvertierungsfaktor ungültig.')
+            setError(result.message || 'Benutzerdefinierte Umrechnung ungültig.')
            return
          }
        }
        textForTc = JSON.stringify(tco, null, 2)
        setTypeConversionsText(textForTc)
-        setCustomFactorDraftByField({})
+        setCustomEquivalenceDraftByField({})
      } catch {
        setError('type_conversions: ungültiges JSON.')
        return
@ -718,11 +828,10 @@ export default function AdminCsvTemplateEditorPage() {
        <div className="card" style={{ padding: 16, marginBottom: 16 }}>
          <div className="form-label">3b. Quelleinheit (optional)</div>
          <p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 8, lineHeight: 1.55 }}>
-            Ziel-Einheit kommt aus dem Datenmodell (z. B. kcal, g, kg, km). Standard-Umrechnungen wählen; für
-            abweichende Skalen (z. B. Volumen→Masse mit variabler Dichte) &quot;Benutzerdefiniert&quot; und den
-            Faktor eintragen (CSV-Wert × Faktor → Speicher-Einheit). Bei Registry-Optionen werden{' '}
-            <code>target_unit</code> und ein alter <code>conversion_factor</code> entfernt; bei
-            Benutzerdefiniert bleibt der Faktor erhalten, bis du ihn leerst.
+            Ziel-Einheit kommt aus dem Datenmodell. Standard-Umrechnungen im Dropdown; bei &quot;Benutzerdefiniert&quot;
+            die Bezugsgröße eintragen: <strong>Menge [Quelleinheit] entspricht Menge [Zieleinheit]</strong> (Ziel ist die
+            Speicher-Einheit des Feldes). Im JSON siehst du weiterhin <code>conversion_factor</code> und{' '}
+            <code>custom_equivalence</code> zur Dokumentation.
          </p>
          <div style={{ marginTop: 14, display: 'flex', flexDirection: 'column', gap: 16 }}>
            {unitTargets.map(({ field: fkey, options }) => (
@ -751,30 +860,74 @@ export default function AdminCsvTemplateEditorPage() {
                  </select>
                </label>
                {getSourceUnitSelectValue(fkey) === 'custom' && (
-                  <div style={{ marginTop: 10 }}>
-                    <span className="form-label" style={{ display: 'block', marginBottom: 6 }}>
-                      Konvertierungsfaktor (× CSV-Wert → Wert in Speicher-Einheit)
+                  <div style={{ marginTop: 12 }}>
+                    <span className="form-label" style={{ display: 'block', marginBottom: 8 }}>
+                      Bezugsgröße (CSV steht in der linken Einheit)
                    </span>
+                    <datalist id={`csv-custom-units-${fkey}`}>
+                      {CUSTOM_SOURCE_UNIT_HINTS.map((h) => (
+                        <option key={h} value={h} />
+                      ))}
+                    </datalist>
+                    <div
+                      style={{
+                        display: 'flex',
+                        flexWrap: 'wrap',
+                        alignItems: 'center',
+                        gap: 10,
+                        rowGap: 12,
+                      }}
+                    >
                      <input
                        type="text"
                        inputMode="decimal"
                        className="form-input"
-                      placeholder="z. B. 1,03 oder 1.03 (ml→g nach Dichte)"
-                      value={getCustomFactorFieldDisplay(fkey)}
-                      onChange={(e) =>
-                        setCustomFactorDraftByField((prev) => ({ ...prev, [fkey]: e.target.value }))
-                      }
-                      onBlur={(e) => commitCustomFactorOnBlur(fkey, e.target.value)}
-                      style={{ width: '100%', textAlign: 'left' }}
+                        placeholder="Menge"
+                        aria-label="Menge in Quelleinheit"
+                        value={getEquivalenceDisplay(fkey).srcAmt}
+                        onChange={(e) => mergeEquivalenceDraft(fkey, { srcAmt: e.target.value })}
+                        onBlur={() => commitCustomEquivalenceOnBlur(fkey)}
+                        style={{ width: 96, textAlign: 'left', minHeight: 46, padding: '11px 12px' }}
                      />
-                    <p style={{ fontSize: 12, color: 'var(--text3)', marginTop: 6 }}>
-                      Dezimalkomma oder -punkt; mit Tab oder Klick außerhalb übernehmen (oder direkt Speichern).
+                      <input
+                        type="text"
+                        className="form-input"
+                        placeholder="Quelleinheit"
+                        list={`csv-custom-units-${fkey}`}
+                        aria-label="Quelleinheit (z. B. ml)"
+                        value={getEquivalenceDisplay(fkey).srcUnit}
+                        onChange={(e) => mergeEquivalenceDraft(fkey, { srcUnit: e.target.value })}
+                        onBlur={() => commitCustomEquivalenceOnBlur(fkey)}
+                        style={{ width: 120, textAlign: 'left', minHeight: 46, padding: '11px 12px' }}
+                      />
+                      <span style={{ fontSize: 14, color: 'var(--text2)' }}>entspricht</span>
+                      <input
+                        type="text"
+                        inputMode="decimal"
+                        className="form-input"
+                        placeholder="Menge"
+                        aria-label="Menge in Speicher-Einheit"
+                        value={getEquivalenceDisplay(fkey).tgtAmt}
+                        onChange={(e) => mergeEquivalenceDraft(fkey, { tgtAmt: e.target.value })}
+                        onBlur={() => commitCustomEquivalenceOnBlur(fkey)}
+                        style={{ width: 96, textAlign: 'left', minHeight: 46, padding: '11px 12px' }}
+                      />
+                      <span style={{ fontSize: 15, fontWeight: 600, color: 'var(--text1)' }}>
+                        {getCanonicalStorageUnitLabel(fkey)}
+                      </span>
+                      <span style={{ fontSize: 13, color: 'var(--text3)' }}>(Ziel / Speicher)</span>
+                    </div>
+                    {derivedFactorHintLine(fkey) ? (
+                      <p style={{ fontSize: 12, color: 'var(--text2)', marginTop: 10 }}>
+                        {derivedFactorHintLine(fkey)}
                      </p>
-                    {getCustomFactorFieldDisplay(fkey) === '' && (
-                      <p style={{ fontSize: 12, color: 'var(--text3)', marginTop: 6 }}>
-                        Ohne Faktor gilt keine zusätzliche Skalierung (wie 1:1 nach dem Parsen).
+                    ) : null}
+                    <p style={{ fontSize: 12, color: 'var(--text3)', marginTop: 8, lineHeight: 1.5 }}>
+                      Beispiel: <strong>1</strong> <strong>ml</strong> entspricht <strong>1,03</strong>{' '}
+                      <strong>{getCanonicalStorageUnitLabel(fkey)}</strong>, wenn die CSV Milliliter liefert und die
+                      Dichte etwa 1,03&nbsp;g/ml ist. Alle drei Felder leer lassen = keine Zusatz-Umrechnung. Tab /
+                      Fokus weg oder Speichern übernimmt.
                    </p>
-                    )}
                  </div>
                )}
              </div>
@ -786,11 +939,9 @@ export default function AdminCsvTemplateEditorPage() {
      <div className="card" style={{ padding: 16, marginBottom: 16 }}>
        <div className="form-label">4. type_conversions (JSON)</div>
        <p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 8, lineHeight: 1.55 }}>
-          Vom Vorschlag übernommen; bei Dropdowns 3b wird <code>source_unit</code> ergänzt. Zusätzlich manuell
-          z. B. Datumsformat. <strong>Freie Umrechnung (nicht in der Liste 3b):</strong>{' '}
-          <code>conversion_factor</code> als Multiplikator nach dem Parsen (und nach Registry-
-          <code>source_unit</code>). Optional im JSON <code>source_unit: &apos;custom&apos;</code>, wenn nur{' '}
-          <code>conversion_factor</code> gelten soll.
+          Vom Vorschlag übernommen; bei Dropdowns 3b werden <code>source_unit</code>, ggf.{' '}
+          <code>conversion_factor</code> und <code>custom_equivalence</code> gesetzt. Zusätzlich manuell z. B.
+          Datumsformat.
        </p>
        <textarea
          className="form-input"
@ -805,7 +956,7 @@ export default function AdminCsvTemplateEditorPage() {
          value={typeConversionsText}
          onChange={(e) => {
            setTypeConversionsText(e.target.value)
-            setCustomFactorDraftByField({})
+            setCustomEquivalenceDraftByField({})
          }}
        />
      </div>