From a51ee1d304ed27197cfcecf8e859cb8ae056ec65 Mon Sep 17 00:00:00 2001 From: Lars Date: Fri, 10 Apr 2026 15:22:31 +0200 Subject: [PATCH] feat(csv-import): Update versioning and enhance row processing features - Bumped version numbers for csv_import to 0.3.1 and admin_csv_templates to 0.2.0, reflecting recent enhancements. - Added support for import_row_processing_default in the CSV modules endpoint, improving data handling capabilities. - Introduced new row aggregation operations in the AdminCsvTemplateEditorPage, allowing for more flexible data processing options. - Implemented parsing and validation for custom row processing configurations, enhancing user experience in template management. --- backend/routers/csv_import.py | 1 + backend/version.py | 4 +- .../src/pages/AdminCsvTemplateEditorPage.jsx | 141 ++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/backend/routers/csv_import.py b/backend/routers/csv_import.py index 338db2e..8a7a590 100644 --- a/backend/routers/csv_import.py +++ b/backend/routers/csv_import.py @@ -74,6 +74,7 @@ def csv_modules(session: dict = Depends(require_auth)): "table": d["table"], "fields": fields_out, "import_mode": d.get("import_mode"), + "import_row_processing_default": d.get("import_row_processing_default"), } ) return {"modules": out} diff --git a/backend/version.py b/backend/version.py index 987a1a5..7148790 100644 --- a/backend/version.py +++ b/backend/version.py @@ -31,8 +31,8 @@ MODULE_VERSIONS = { "membership": "2.1.0", "workflow": "0.6.0", # Phase 4: End Node Template Engine "app_dashboard": "1.11.0", # Entitlements: DB-Override widget→features (AND), sonst Katalog - "csv_import": "0.3.0", # Analyse ohne Modul-Filter; Import nur mapping_id (Modul aus Vorlage) - "admin_csv_templates": "0.1.0", # Issue #21: System-Templates + Import-Limits (Admin) + "csv_import": "0.3.1", # GET /csv/modules: import_row_processing_default pro Modul + "admin_csv_templates": "0.2.0", # Admin-Editor: Zeilenaggregation (Schlüssel + gemeinsame Funktion) } CHANGELOG = [ diff --git a/frontend/src/pages/AdminCsvTemplateEditorPage.jsx b/frontend/src/pages/AdminCsvTemplateEditorPage.jsx index e8443bb..63ec53e 100644 --- a/frontend/src/pages/AdminCsvTemplateEditorPage.jsx +++ b/frontend/src/pages/AdminCsvTemplateEditorPage.jsx @@ -26,6 +26,69 @@ const CUSTOM_SOURCE_UNIT_HINTS = [ 'Portion', ] +/** Eine gemeinsame Funktion für alle nicht-Schlüssel-Zielfelder (pro-Feld-Mix = spätere Ausbaustufe). */ +const ROW_AGG_OPS = [ + { value: 'sum', label: 'Summe' }, + { value: 'mean', label: 'Mittelwert' }, + { value: 'min', label: 'Minimum' }, + { value: 'max', label: 'Maximum' }, + { value: 'median', label: 'Median' }, + { value: 'first', label: 'Erster Wert (Reihenfolge in der Datei)' }, + { value: 'last', label: 'Letzter Wert (Reihenfolge in der Datei)' }, +] + +const NUMERIC_ROW_AGG = new Set(['sum', 'mean', 'min', 'max', 'median']) + +function parseStoredImportRowProcessing(irp) { + if (!irp || typeof irp !== 'object' || Object.keys(irp).length === 0) { + return { + useCustom: false, + irregular: false, + groupBy: [], + mode: '', + } + } + const gb = irp.group_by + const agg = irp.aggregates + if (!Array.isArray(gb) || gb.length === 0 || agg == null || typeof agg !== 'object') { + return { + useCustom: true, + irregular: true, + groupBy: Array.isArray(gb) ? [...gb] : [], + mode: '', + } + } + const ops = [...new Set(Object.values(agg).map((x) => String(x)))] + if (ops.length !== 1) { + return { useCustom: true, irregular: true, groupBy: [...gb], mode: '' } + } + return { + useCustom: true, + irregular: false, + groupBy: [...gb], + mode: ops[0], + } +} + +function buildImportRowProcessingSimple(modFields, fm, groupBy, mode) { + const targets = new Set( + Object.values(fm).filter((v) => v && v !== '-' && v !== '_skip'), + ) + const aggregates = {} + const gbSet = new Set(groupBy) + for (const t of targets) { + if (gbSet.has(t)) continue + const typ = modFields?.[t]?.type + if (mode === 'first' || mode === 'last') { + aggregates[t] = mode + } else if (NUMERIC_ROW_AGG.has(mode)) { + if (typ === 'string') continue + aggregates[t] = mode + } + } + return { group_by: groupBy, aggregates } +} + /** Erlaubt Eingaben wie 1,03 oder 1.03 während des Tippens; Finale normalisiert bei Blur/Speichern. */ function normalizeDecimalInputString(raw) { let s = String(raw).trim().replace(/\s/g, '') @@ -186,6 +249,13 @@ export default function AdminCsvTemplateEditorPage() { /** Entwurf für „Quelle entspricht Ziel“ (nur source_unit custom); Commit bei Blur/Speichern. */ const [customEquivalenceDraftByField, setCustomEquivalenceDraftByField] = useState({}) + /** Zeilenaggregation: null in DB = Modul-Standard; sonst import_row_processing */ + const [rowAggUseCustom, setRowAggUseCustom] = useState(false) + const [rowAggGroupBy, setRowAggGroupBy] = useState([]) + const [rowAggMode, setRowAggMode] = useState('') + const [rowAggIrregular, setRowAggIrregular] = useState(false) + const [rowAggJsonText, setRowAggJsonText] = useState('{}') + const modMeta = useMemo(() => modules.find((m) => m.id === module), [modules, module]) const aggregateSleepImport = modMeta?.import_mode === 'apple_sleep_aggregate' const targetOptions = useMemo(() => { @@ -218,6 +288,15 @@ export default function AdminCsvTemplateEditorPage() { .catch(() => setSeedOptions([])) }, [module]) + useEffect(() => { + if (!isNew) return + setRowAggUseCustom(false) + setRowAggIrregular(false) + setRowAggGroupBy([]) + setRowAggMode('') + setRowAggJsonText('{}') + }, [module, isNew]) + useEffect(() => { if (isNew || !templateId) return let ok = true @@ -241,6 +320,12 @@ export default function AdminCsvTemplateEditorPage() { setCustomEquivalenceDraftByField({}) setSampleRows([]) setSeedHint(null) + const rp = parseStoredImportRowProcessing(t.import_row_processing) + setRowAggUseCustom(rp.useCustom) + setRowAggIrregular(rp.irregular) + setRowAggGroupBy(rp.groupBy) + setRowAggMode(rp.mode) + setRowAggJsonText(JSON.stringify(t.import_row_processing || {}, null, 2)) }) .catch((e) => { if (ok) setError(e.message) @@ -259,6 +344,17 @@ export default function AdminCsvTemplateEditorPage() { ) }, [fieldMappings]) + const rowAggGroupCandidates = useMemo(() => { + return [...assignedTargets].filter((t) => modMeta?.fields?.[t]).sort() + }, [assignedTargets, modMeta]) + + useEffect(() => { + const targets = new Set( + Object.values(fieldMappings).filter((v) => v && v !== '-' && v !== '_skip'), + ) + setRowAggGroupBy((prev) => prev.filter((g) => targets.has(g))) + }, [fieldMappings]) + const missingRequired = useMemo(() => { return requiredTargets.filter((r) => !assignedTargets.has(r)) }, [requiredTargets, assignedTargets]) @@ -440,6 +536,11 @@ export default function AdminCsvTemplateEditorPage() { setEncoding(res.encoding || 'utf-8') setSeedHint(res.seed_template || null) setCustomEquivalenceDraftByField({}) + setRowAggUseCustom(false) + setRowAggIrregular(false) + setRowAggGroupBy([]) + setRowAggMode('') + setRowAggJsonText('{}') } catch (e) { setError(e.message || 'Analyse fehlgeschlagen') } finally { @@ -506,6 +607,45 @@ export default function AdminCsvTemplateEditorPage() { return } + let import_row_processing = null + if (!aggregateSleepImport && rowAggUseCustom) { + if (rowAggIrregular) { + try { + import_row_processing = JSON.parse(rowAggJsonText || '{}') + if (!import_row_processing || typeof import_row_processing !== 'object') throw new Error('bad') + } catch { + setError('Zeilenaggregation: ungültiges JSON.') + return + } + const gb = import_row_processing.group_by + if (!Array.isArray(gb) || !gb.length) { + setError('Zeilenaggregation (JSON): „group_by“ muss eine nicht-leere Liste sein.') + return + } + } else { + if (!rowAggGroupBy.length) { + setError('Zeilenaggregation: mindestens ein Schlüsselfeld auswählen.') + return + } + if (!rowAggMode) { + setError('Zeilenaggregation: eine Funktion wählen (Summe, Mittelwert, …).') + return + } + for (const g of rowAggGroupBy) { + if (!assignedTargets.has(g)) { + setError(`Zeilenaggregation: Schlüsselfeld „${g}“ muss einer CSV-Spalte zugeordnet sein.`) + return + } + } + import_row_processing = buildImportRowProcessingSimple( + modMeta?.fields, + fieldMappings, + rowAggGroupBy, + rowAggMode, + ) + } + } + const payload = { module, mapping_name: mappingName.trim(), @@ -516,6 +656,7 @@ export default function AdminCsvTemplateEditorPage() { has_header: hasHeader, field_mappings: fieldMappings, type_conversions: tc, + import_row_processing, } if (!payload.column_signature?.length) {