feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
Deploy Development / deploy (push) Successful in 52s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates.
- Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching.
- Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure.
- Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
Lars committed 2026-04-10 07:08:21 +02:00
1 parent c10da55ec6
commit b4cc3cb934
7 files changed
+223 -60

No files matched your search

@@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from 'react'
import { Link, useNavigate, useParams } from 'react-router-dom'
import { ArrowLeft, FileSpreadsheet, Loader2, Save, Trash2 } from 'lucide-react'
import { api } from '../utils/api'
import { csvPreviewTdStyle } from '../utils/csvPreviewCells'
const MODULE_LABEL = {
nutrition: 'Ernährung',
@@ -38,16 +39,7 @@ function SampleTable({ sampleRows, columns }) {
{sampleRows.slice(0, 5).map((row, i) => (
<tr key={i}>
{showCols.map((c) => (
<td
key={c}
style={{
padding: '6px',
borderBottom: '1px solid var(--border)',
maxWidth: 140,
overflow: 'hidden',
textOverflow: 'ellipsis',
}}
>
<td key={c} style={csvPreviewTdStyle(row[c] ?? '—')}>
{row[c] ?? '—'}
</td>
))}
@@ -298,7 +290,7 @@ export default function AdminCsvTemplateEditorPage() {
value={module}
disabled={!isNew}
onChange={(e) => setModule(e.target.value)}
style={{ width: '100%', marginTop: 8 }}
style={{ width: '100%', marginTop: 8, textAlign: 'left', minHeight: 46, padding: '11px 14px' }}
>
{modules.map((m) => (
<option key={m.id} value={m.id}>
@@ -327,7 +319,7 @@ export default function AdminCsvTemplateEditorPage() {
Trennzeichen (optional, sonst automatisch):
<select
className="form-input"
style={{ width: '100%', marginTop: 6 }}
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
value={delimiterOverride}
onChange={(e) => setDelimiterOverride(e.target.value)}
>
@@ -341,11 +333,11 @@ export default function AdminCsvTemplateEditorPage() {
Optional: feste Seed-Vorlage für Vorschläge:
<select
className="form-input"
style={{ width: '100%', marginTop: 6 }}
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
value={seedTemplateId}
onChange={(e) => setSeedTemplateId(e.target.value)}
>
<option value="">Beste passende System-Vorlage (Jaccard)</option>
<option value="">Beste passende System-Vorlage (Abdeckung der Vorlagen-Spalten)</option>
{seedOptions.map((s) => (
<option key={s.id} value={String(s.id)}>
{s.mapping_name}
@@ -371,9 +363,18 @@ export default function AdminCsvTemplateEditorPage() {
)}
</button>
{seedHint && (
<p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 12 }}>
Seed: <strong>{seedHint.mapping_name}</strong> · Übereinstimmung ca.{' '}
{Math.round((seedHint.confidence || 0) * 100)} %
<p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 12, lineHeight: 1.5 }}>
Seed: <strong>{seedHint.mapping_name}</strong> · Vorlage abgedeckt{' '}
<strong>{Math.round((seedHint.confidence || 0) * 100)} %</strong>
{seedHint.columns_matched != null && seedHint.columns_in_template != null
? ` (${seedHint.columns_matched}/${seedHint.columns_in_template} Spalten)`
: ''}
{seedHint.jaccard != null && (
<>
{' '}
· Jaccard <strong>{Math.round(seedHint.jaccard * 100)} %</strong>
</>
)}
</p>
)}
{sampleRows.length > 0 && <SampleTable sampleRows={sampleRows} columns={columns} />}
@@ -386,7 +387,7 @@ export default function AdminCsvTemplateEditorPage() {
</label>
<input
className="form-input"
style={{ width: '100%' }}
style={{ width: '100%', textAlign: 'left' }}
value={mappingName}
onChange={(e) => setMappingName(e.target.value)}
placeholder="z. B. FDDB Export 2026"
@@ -396,14 +397,19 @@ export default function AdminCsvTemplateEditorPage() {
</label>
<textarea
className="form-input"
style={{ width: '100%', minHeight: 64 }}
style={{ width: '100%', minHeight: 64, textAlign: 'left' }}
value={description}
onChange={(e) => setDescription(e.target.value)}
/>
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12, marginTop: 12 }}>
<label>
<span className="form-label">Trennzeichen (gespeichert)</span>
<select className="form-input" style={{ width: '100%', marginTop: 6 }} value={delimiter} onChange={(e) => setDelimiter(e.target.value)}>
<select
className="form-input"
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
value={delimiter}
onChange={(e) => setDelimiter(e.target.value)}
>
<option value=";">Semikolon</option>
<option value=",">Komma</option>
<option value="\t">Tab</option>
@@ -413,7 +419,7 @@ export default function AdminCsvTemplateEditorPage() {
<span className="form-label">Kopfzeile</span>
<select
className="form-input"
style={{ width: '100%', marginTop: 6 }}
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
value={hasHeader ? 'yes' : 'no'}
onChange={(e) => setHasHeader(e.target.value === 'yes')}
>
@@ -435,16 +441,25 @@ export default function AdminCsvTemplateEditorPage() {
key={col}
style={{
display: 'grid',
gridTemplateColumns: 'minmax(0, 1fr) minmax(140px, 200px)',
gap: 10,
gridTemplateColumns: 'minmax(0, 1fr) minmax(280px, min(52vw, 440px))',
gap: '10px 16px',
alignItems: 'center',
}}
>
<code style={{ fontSize: 12, wordBreak: 'break-word', color: 'var(--text2)' }}>{col}</code>
<code style={{ fontSize: 12, wordBreak: 'break-word', color: 'var(--text2)', textAlign: 'left' }}>
{col}
</code>
<select
className="form-input"
value={fieldMappings[col] || '-'}
onChange={(e) => updateMapping(col, e.target.value)}
style={{
width: '100%',
minHeight: 46,
textAlign: 'left',
padding: '11px 14px',
fontSize: 15,
}}
>
<option value="-">— ignorieren</option>
{targetOptions.map((o) => (
@@ -471,7 +486,14 @@ export default function AdminCsvTemplateEditorPage() {
</p>
<textarea
className="form-input"
style={{ width: '100%', minHeight: 200, marginTop: 8, fontFamily: 'monospace', fontSize: 12 }}
style={{
width: '100%',
minHeight: 200,
marginTop: 8,
fontFamily: 'monospace',
fontSize: 12,
textAlign: 'left',
}}
value={typeConversionsText}
onChange={(e) => setTypeConversionsText(e.target.value)}
/>
+40 -15
View File
@@ -2,6 +2,7 @@ import { useState, useMemo } from 'react'
import { useNavigate } from 'react-router-dom'
import { ArrowLeft, FileSpreadsheet, Loader2 } from 'lucide-react'
import { api } from '../utils/api'
import { csvPreviewTdStyle } from '../utils/csvPreviewCells'
/** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */
const EXECUTOR_READY = new Set(['nutrition', 'weight', 'blood_pressure'])
@@ -32,6 +33,11 @@ function mergeMappingChoices(detected, mapData) {
name: row.name,
is_system: row.is_system,
confidence: d.confidence ?? 0,
jaccard: d.jaccard,
template_recall: d.template_recall,
columns_matched: d.columns_matched,
columns_in_template: d.columns_in_template,
columns_in_csv: d.columns_in_csv,
})
seen.add(row.id)
}
@@ -78,16 +84,7 @@ function SampleTable({ sampleRows, columns }) {
{sampleRows.slice(0, 5).map((row, i) => (
<tr key={i}>
{showCols.map((c) => (
<td
key={c}
style={{
padding: '6px',
borderBottom: '1px solid var(--border)',
maxWidth: 140,
overflow: 'hidden',
textOverflow: 'ellipsis',
}}
>
<td key={c} style={csvPreviewTdStyle(row[c] ?? '—')}>
{row[c] ?? '—'}
</td>
))}
@@ -296,8 +293,24 @@ export default function UniversalCsvImportPage() {
>
<strong>Vorschlag:</strong>{' '}
{MODULE_LABEL[analyzeResult.recommended.module] || analyzeResult.recommended.module} —{' '}
{analyzeResult.recommended.mapping_name} (
{Math.round((analyzeResult.recommended.confidence || 0) * 100)} % Übereinstimmung der Spalten)
{analyzeResult.recommended.mapping_name}.
<br />
<span style={{ fontSize: 13, color: 'var(--text2)', fontWeight: 500 }}>
Vorlage abgedeckt:{' '}
<strong>{Math.round((analyzeResult.recommended.confidence || 0) * 100)} %</strong>
{analyzeResult.recommended.columns_matched != null &&
analyzeResult.recommended.columns_in_template != null
? ` (${analyzeResult.recommended.columns_matched}/${analyzeResult.recommended.columns_in_template} erwartete Spalten in der Datei)`
: ''}
.{' '}
{analyzeResult.recommended.jaccard != null && (
<>
Jaccard{' '}
<strong>{Math.round(analyzeResult.recommended.jaccard * 100)} %</strong> (gesamte
Spalten-Überlappung — niedriger, wenn die CSV viele Zusatzspalten hat).
</>
)}
</span>
</div>
)}
@@ -307,8 +320,9 @@ export default function UniversalCsvImportPage() {
<ul style={{ margin: '8px 0 0 18px', padding: 0 }}>
{analyzeResult.detected_mappings.slice(1, 8).map((d) => (
<li key={d.mapping_id}>
{MODULE_LABEL[d.module] || d.module}: {d.mapping_name} ·{' '}
{MODULE_LABEL[d.module] || d.module}: {d.mapping_name} · Vorlage{' '}
{Math.round((d.confidence || 0) * 100)} %
{d.jaccard != null ? ` · Jaccard ${Math.round(d.jaccard * 100)} %` : ''}
</li>
))}
</ul>
@@ -324,7 +338,14 @@ export default function UniversalCsvImportPage() {
className="form-input"
value={mappingId}
onChange={(e) => setMappingId(e.target.value)}
style={{ width: '100%', marginTop: 8 }}
style={{
width: '100%',
marginTop: 8,
minHeight: 48,
textAlign: 'left',
padding: '12px 14px',
fontSize: 15,
}}
>
{mappingChoices.length === 0 ? (
<option value="">Keine Vorlage geladen</option>
@@ -333,7 +354,11 @@ export default function UniversalCsvImportPage() {
<option key={o.id} value={o.id}>
{MODULE_LABEL[o.module] || o.module} — {o.name}
{o.is_system ? ' (System)' : ''}
{o.confidence > 0 ? ` · ${Math.round(o.confidence * 100)} %` : ''}
{o.confidence > 0
? ` · Vorlage ${Math.round(o.confidence * 100)} %${
o.jaccard != null ? ` · Jaccard ${Math.round(o.jaccard * 100)} %` : ''
}`
: ''}
{!EXECUTOR_READY.has(o.module) ? ' · Import: noch nicht hier' : ''}
</option>
))
+47
View File
@@ -0,0 +1,47 @@
/**
* Vorschau-Zellen: Zahlen rechts (tabular-nums), Text links — typisch für CSV-Tabellen.
*/
function stripForNumericTest(s) {
return String(s)
.trim()
.replace(/\u00a0/g, '')
.replace(/\s/g, '')
}
/**
* Einfache Heuristik: reine Zahl / Dezimal (ein Punkt oder ein Komma als Dezimaltrenner).
* Keine Datums-/Zeitstrings (mehrere Punkte ohne klares Muster werden nicht als Zahl gewertet).
*/
export function isCsvPreviewNumericCell(val) {
if (val == null || val === '—') return false
let s = stripForNumericTest(val)
if (!s) return false
if (/^\d{1,2}[./]\d{1,2}[./]\d{2,4}/.test(s)) return false
if (/^\d{4}-\d{2}-\d{2}/.test(s)) return false
if (/:/.test(s) && /\d/.test(s)) return false
s = s.replace(/^[-+]/, '')
const comma = (s.match(/,/g) || []).length
const dot = (s.match(/\./g) || []).length
if (comma > 1 || dot > 1 || (comma >= 1 && dot >= 1)) return false
s = s.replace(',', '.')
if (!/^\d*\.?\d+$/.test(s) && !/^\d+\.\d+$/.test(s)) return false
return Number.isFinite(Number(s))
}
/** td-Style für CSV-Vorschau (Import + Admin-Editor) */
export function csvPreviewTdStyle(val, extra = {}) {
const numeric = isCsvPreviewNumericCell(val)
return {
padding: '6px 8px',
borderBottom: '1px solid var(--border)',
maxWidth: 220,
overflow: 'hidden',
textOverflow: 'ellipsis',
textAlign: numeric ? 'right' : 'left',
fontVariantNumeric: numeric ? 'tabular-nums' : undefined,
...extra,
}
}