feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates. - Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching. - Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure. - Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
1 parent
c10da55ec6
commit
b4cc3cb934
7 files changed
+223
-60
No files matched your search
@@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from 'react'
|
||||
import { Link, useNavigate, useParams } from 'react-router-dom'
|
||||
import { ArrowLeft, FileSpreadsheet, Loader2, Save, Trash2 } from 'lucide-react'
|
||||
import { api } from '../utils/api'
|
||||
import { csvPreviewTdStyle } from '../utils/csvPreviewCells'
|
||||
|
||||
const MODULE_LABEL = {
|
||||
nutrition: 'Ernährung',
|
||||
@@ -38,16 +39,7 @@ function SampleTable({ sampleRows, columns }) {
|
||||
{sampleRows.slice(0, 5).map((row, i) => (
|
||||
<tr key={i}>
|
||||
{showCols.map((c) => (
|
||||
<td
|
||||
key={c}
|
||||
style={{
|
||||
padding: '6px',
|
||||
borderBottom: '1px solid var(--border)',
|
||||
maxWidth: 140,
|
||||
overflow: 'hidden',
|
||||
textOverflow: 'ellipsis',
|
||||
}}
|
||||
>
|
||||
<td key={c} style={csvPreviewTdStyle(row[c] ?? '—')}>
|
||||
{row[c] ?? '—'}
|
||||
</td>
|
||||
))}
|
||||
@@ -298,7 +290,7 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
value={module}
|
||||
disabled={!isNew}
|
||||
onChange={(e) => setModule(e.target.value)}
|
||||
style={{ width: '100%', marginTop: 8 }}
|
||||
style={{ width: '100%', marginTop: 8, textAlign: 'left', minHeight: 46, padding: '11px 14px' }}
|
||||
>
|
||||
{modules.map((m) => (
|
||||
<option key={m.id} value={m.id}>
|
||||
@@ -327,7 +319,7 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
Trennzeichen (optional, sonst automatisch):
|
||||
<select
|
||||
className="form-input"
|
||||
style={{ width: '100%', marginTop: 6 }}
|
||||
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
|
||||
value={delimiterOverride}
|
||||
onChange={(e) => setDelimiterOverride(e.target.value)}
|
||||
>
|
||||
@@ -341,11 +333,11 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
Optional: feste Seed-Vorlage für Vorschläge:
|
||||
<select
|
||||
className="form-input"
|
||||
style={{ width: '100%', marginTop: 6 }}
|
||||
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
|
||||
value={seedTemplateId}
|
||||
onChange={(e) => setSeedTemplateId(e.target.value)}
|
||||
>
|
||||
<option value="">Beste passende System-Vorlage (Jaccard)</option>
|
||||
<option value="">Beste passende System-Vorlage (Abdeckung der Vorlagen-Spalten)</option>
|
||||
{seedOptions.map((s) => (
|
||||
<option key={s.id} value={String(s.id)}>
|
||||
{s.mapping_name}
|
||||
@@ -371,9 +363,18 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
)}
|
||||
</button>
|
||||
{seedHint && (
|
||||
<p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 12 }}>
|
||||
Seed: <strong>{seedHint.mapping_name}</strong> · Übereinstimmung ca.{' '}
|
||||
{Math.round((seedHint.confidence || 0) * 100)} %
|
||||
<p style={{ fontSize: 13, color: 'var(--text2)', marginTop: 12, lineHeight: 1.5 }}>
|
||||
Seed: <strong>{seedHint.mapping_name}</strong> · Vorlage abgedeckt{' '}
|
||||
<strong>{Math.round((seedHint.confidence || 0) * 100)} %</strong>
|
||||
{seedHint.columns_matched != null && seedHint.columns_in_template != null
|
||||
? ` (${seedHint.columns_matched}/${seedHint.columns_in_template} Spalten)`
|
||||
: ''}
|
||||
{seedHint.jaccard != null && (
|
||||
<>
|
||||
{' '}
|
||||
· Jaccard <strong>{Math.round(seedHint.jaccard * 100)} %</strong>
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
{sampleRows.length > 0 && <SampleTable sampleRows={sampleRows} columns={columns} />}
|
||||
@@ -386,7 +387,7 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
</label>
|
||||
<input
|
||||
className="form-input"
|
||||
style={{ width: '100%' }}
|
||||
style={{ width: '100%', textAlign: 'left' }}
|
||||
value={mappingName}
|
||||
onChange={(e) => setMappingName(e.target.value)}
|
||||
placeholder="z. B. FDDB Export 2026"
|
||||
@@ -396,14 +397,19 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
</label>
|
||||
<textarea
|
||||
className="form-input"
|
||||
style={{ width: '100%', minHeight: 64 }}
|
||||
style={{ width: '100%', minHeight: 64, textAlign: 'left' }}
|
||||
value={description}
|
||||
onChange={(e) => setDescription(e.target.value)}
|
||||
/>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12, marginTop: 12 }}>
|
||||
<label>
|
||||
<span className="form-label">Trennzeichen (gespeichert)</span>
|
||||
<select className="form-input" style={{ width: '100%', marginTop: 6 }} value={delimiter} onChange={(e) => setDelimiter(e.target.value)}>
|
||||
<select
|
||||
className="form-input"
|
||||
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
|
||||
value={delimiter}
|
||||
onChange={(e) => setDelimiter(e.target.value)}
|
||||
>
|
||||
<option value=";">Semikolon</option>
|
||||
<option value=",">Komma</option>
|
||||
<option value="\t">Tab</option>
|
||||
@@ -413,7 +419,7 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
<span className="form-label">Kopfzeile</span>
|
||||
<select
|
||||
className="form-input"
|
||||
style={{ width: '100%', marginTop: 6 }}
|
||||
style={{ width: '100%', marginTop: 6, textAlign: 'left', minHeight: 44, padding: '10px 12px' }}
|
||||
value={hasHeader ? 'yes' : 'no'}
|
||||
onChange={(e) => setHasHeader(e.target.value === 'yes')}
|
||||
>
|
||||
@@ -435,16 +441,25 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
key={col}
|
||||
style={{
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'minmax(0, 1fr) minmax(140px, 200px)',
|
||||
gap: 10,
|
||||
gridTemplateColumns: 'minmax(0, 1fr) minmax(280px, min(52vw, 440px))',
|
||||
gap: '10px 16px',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
>
|
||||
<code style={{ fontSize: 12, wordBreak: 'break-word', color: 'var(--text2)' }}>{col}</code>
|
||||
<code style={{ fontSize: 12, wordBreak: 'break-word', color: 'var(--text2)', textAlign: 'left' }}>
|
||||
{col}
|
||||
</code>
|
||||
<select
|
||||
className="form-input"
|
||||
value={fieldMappings[col] || '-'}
|
||||
onChange={(e) => updateMapping(col, e.target.value)}
|
||||
style={{
|
||||
width: '100%',
|
||||
minHeight: 46,
|
||||
textAlign: 'left',
|
||||
padding: '11px 14px',
|
||||
fontSize: 15,
|
||||
}}
|
||||
>
|
||||
<option value="-">— ignorieren</option>
|
||||
{targetOptions.map((o) => (
|
||||
@@ -471,7 +486,14 @@ export default function AdminCsvTemplateEditorPage() {
|
||||
</p>
|
||||
<textarea
|
||||
className="form-input"
|
||||
style={{ width: '100%', minHeight: 200, marginTop: 8, fontFamily: 'monospace', fontSize: 12 }}
|
||||
style={{
|
||||
width: '100%',
|
||||
minHeight: 200,
|
||||
marginTop: 8,
|
||||
fontFamily: 'monospace',
|
||||
fontSize: 12,
|
||||
textAlign: 'left',
|
||||
}}
|
||||
value={typeConversionsText}
|
||||
onChange={(e) => setTypeConversionsText(e.target.value)}
|
||||
/>
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useState, useMemo } from 'react'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import { ArrowLeft, FileSpreadsheet, Loader2 } from 'lucide-react'
|
||||
import { api } from '../utils/api'
|
||||
import { csvPreviewTdStyle } from '../utils/csvPreviewCells'
|
||||
|
||||
/** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */
|
||||
const EXECUTOR_READY = new Set(['nutrition', 'weight', 'blood_pressure'])
|
||||
@@ -32,6 +33,11 @@ function mergeMappingChoices(detected, mapData) {
|
||||
name: row.name,
|
||||
is_system: row.is_system,
|
||||
confidence: d.confidence ?? 0,
|
||||
jaccard: d.jaccard,
|
||||
template_recall: d.template_recall,
|
||||
columns_matched: d.columns_matched,
|
||||
columns_in_template: d.columns_in_template,
|
||||
columns_in_csv: d.columns_in_csv,
|
||||
})
|
||||
seen.add(row.id)
|
||||
}
|
||||
@@ -78,16 +84,7 @@ function SampleTable({ sampleRows, columns }) {
|
||||
{sampleRows.slice(0, 5).map((row, i) => (
|
||||
<tr key={i}>
|
||||
{showCols.map((c) => (
|
||||
<td
|
||||
key={c}
|
||||
style={{
|
||||
padding: '6px',
|
||||
borderBottom: '1px solid var(--border)',
|
||||
maxWidth: 140,
|
||||
overflow: 'hidden',
|
||||
textOverflow: 'ellipsis',
|
||||
}}
|
||||
>
|
||||
<td key={c} style={csvPreviewTdStyle(row[c] ?? '—')}>
|
||||
{row[c] ?? '—'}
|
||||
</td>
|
||||
))}
|
||||
@@ -296,8 +293,24 @@ export default function UniversalCsvImportPage() {
|
||||
>
|
||||
<strong>Vorschlag:</strong>{' '}
|
||||
{MODULE_LABEL[analyzeResult.recommended.module] || analyzeResult.recommended.module} —{' '}
|
||||
{analyzeResult.recommended.mapping_name} (
|
||||
{Math.round((analyzeResult.recommended.confidence || 0) * 100)} % Übereinstimmung der Spalten)
|
||||
{analyzeResult.recommended.mapping_name}.
|
||||
<br />
|
||||
<span style={{ fontSize: 13, color: 'var(--text2)', fontWeight: 500 }}>
|
||||
Vorlage abgedeckt:{' '}
|
||||
<strong>{Math.round((analyzeResult.recommended.confidence || 0) * 100)} %</strong>
|
||||
{analyzeResult.recommended.columns_matched != null &&
|
||||
analyzeResult.recommended.columns_in_template != null
|
||||
? ` (${analyzeResult.recommended.columns_matched}/${analyzeResult.recommended.columns_in_template} erwartete Spalten in der Datei)`
|
||||
: ''}
|
||||
.{' '}
|
||||
{analyzeResult.recommended.jaccard != null && (
|
||||
<>
|
||||
Jaccard{' '}
|
||||
<strong>{Math.round(analyzeResult.recommended.jaccard * 100)} %</strong> (gesamte
|
||||
Spalten-Überlappung — niedriger, wenn die CSV viele Zusatzspalten hat).
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -307,8 +320,9 @@ export default function UniversalCsvImportPage() {
|
||||
<ul style={{ margin: '8px 0 0 18px', padding: 0 }}>
|
||||
{analyzeResult.detected_mappings.slice(1, 8).map((d) => (
|
||||
<li key={d.mapping_id}>
|
||||
{MODULE_LABEL[d.module] || d.module}: {d.mapping_name} ·{' '}
|
||||
{MODULE_LABEL[d.module] || d.module}: {d.mapping_name} · Vorlage{' '}
|
||||
{Math.round((d.confidence || 0) * 100)} %
|
||||
{d.jaccard != null ? ` · Jaccard ${Math.round(d.jaccard * 100)} %` : ''}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
@@ -324,7 +338,14 @@ export default function UniversalCsvImportPage() {
|
||||
className="form-input"
|
||||
value={mappingId}
|
||||
onChange={(e) => setMappingId(e.target.value)}
|
||||
style={{ width: '100%', marginTop: 8 }}
|
||||
style={{
|
||||
width: '100%',
|
||||
marginTop: 8,
|
||||
minHeight: 48,
|
||||
textAlign: 'left',
|
||||
padding: '12px 14px',
|
||||
fontSize: 15,
|
||||
}}
|
||||
>
|
||||
{mappingChoices.length === 0 ? (
|
||||
<option value="">Keine Vorlage geladen</option>
|
||||
@@ -333,7 +354,11 @@ export default function UniversalCsvImportPage() {
|
||||
<option key={o.id} value={o.id}>
|
||||
{MODULE_LABEL[o.module] || o.module} — {o.name}
|
||||
{o.is_system ? ' (System)' : ''}
|
||||
{o.confidence > 0 ? ` · ${Math.round(o.confidence * 100)} %` : ''}
|
||||
{o.confidence > 0
|
||||
? ` · Vorlage ${Math.round(o.confidence * 100)} %${
|
||||
o.jaccard != null ? ` · Jaccard ${Math.round(o.jaccard * 100)} %` : ''
|
||||
}`
|
||||
: ''}
|
||||
{!EXECUTOR_READY.has(o.module) ? ' · Import: noch nicht hier' : ''}
|
||||
</option>
|
||||
))
|
||||
|
||||
Reference in new issue
Block a user