feat(csv-import): Update versioning and enhance row processing features
All checks were successful
Deploy Development / deploy (push) Successful in 54s
Build Test / pytest-backend (push) Successful in 3s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Bumped version numbers for csv_import to 0.3.1 and admin_csv_templates to 0.2.0, reflecting recent enhancements.
- Added support for import_row_processing_default in the CSV modules endpoint, improving data handling capabilities.
- Introduced new row aggregation operations in the AdminCsvTemplateEditorPage, allowing for more flexible data processing options.
- Implemented parsing and validation for custom row processing configurations, enhancing user experience in template management.
This commit is contained in:
Lars 2026-04-10 15:22:31 +02:00
parent e35d167055
commit a51ee1d304
3 changed files with 144 additions and 2 deletions

View File

@ -74,6 +74,7 @@ def csv_modules(session: dict = Depends(require_auth)):
"table": d["table"],
"fields": fields_out,
"import_mode": d.get("import_mode"),
"import_row_processing_default": d.get("import_row_processing_default"),
}
)
return {"modules": out}

View File

@ -31,8 +31,8 @@ MODULE_VERSIONS = {
"membership": "2.1.0",
"workflow": "0.6.0", # Phase 4: End Node Template Engine
"app_dashboard": "1.11.0", # Entitlements: DB-Override widget→features (AND), sonst Katalog
"csv_import": "0.3.0", # Analyse ohne Modul-Filter; Import nur mapping_id (Modul aus Vorlage)
"admin_csv_templates": "0.1.0", # Issue #21: System-Templates + Import-Limits (Admin)
"csv_import": "0.3.1", # GET /csv/modules: import_row_processing_default pro Modul
"admin_csv_templates": "0.2.0", # Admin-Editor: Zeilenaggregation (Schlüssel + gemeinsame Funktion)
}
CHANGELOG = [

View File

@ -26,6 +26,69 @@ const CUSTOM_SOURCE_UNIT_HINTS = [
'Portion',
]
/** Eine gemeinsame Funktion für alle nicht-Schlüssel-Zielfelder (pro-Feld-Mix = spätere Ausbaustufe). */
const ROW_AGG_OPS = [
{ value: 'sum', label: 'Summe' },
{ value: 'mean', label: 'Mittelwert' },
{ value: 'min', label: 'Minimum' },
{ value: 'max', label: 'Maximum' },
{ value: 'median', label: 'Median' },
{ value: 'first', label: 'Erster Wert (Reihenfolge in der Datei)' },
{ value: 'last', label: 'Letzter Wert (Reihenfolge in der Datei)' },
]
const NUMERIC_ROW_AGG = new Set(['sum', 'mean', 'min', 'max', 'median'])
function parseStoredImportRowProcessing(irp) {
if (!irp || typeof irp !== 'object' || Object.keys(irp).length === 0) {
return {
useCustom: false,
irregular: false,
groupBy: [],
mode: '',
}
}
const gb = irp.group_by
const agg = irp.aggregates
if (!Array.isArray(gb) || gb.length === 0 || agg == null || typeof agg !== 'object') {
return {
useCustom: true,
irregular: true,
groupBy: Array.isArray(gb) ? [...gb] : [],
mode: '',
}
}
const ops = [...new Set(Object.values(agg).map((x) => String(x)))]
if (ops.length !== 1) {
return { useCustom: true, irregular: true, groupBy: [...gb], mode: '' }
}
return {
useCustom: true,
irregular: false,
groupBy: [...gb],
mode: ops[0],
}
}
function buildImportRowProcessingSimple(modFields, fm, groupBy, mode) {
const targets = new Set(
Object.values(fm).filter((v) => v && v !== '-' && v !== '_skip'),
)
const aggregates = {}
const gbSet = new Set(groupBy)
for (const t of targets) {
if (gbSet.has(t)) continue
const typ = modFields?.[t]?.type
if (mode === 'first' || mode === 'last') {
aggregates[t] = mode
} else if (NUMERIC_ROW_AGG.has(mode)) {
if (typ === 'string') continue
aggregates[t] = mode
}
}
return { group_by: groupBy, aggregates }
}
/** Erlaubt Eingaben wie 1,03 oder 1.03 während des Tippens; Finale normalisiert bei Blur/Speichern. */
function normalizeDecimalInputString(raw) {
let s = String(raw).trim().replace(/\s/g, '')
@ -186,6 +249,13 @@ export default function AdminCsvTemplateEditorPage() {
/** Entwurf für „Quelle entspricht Ziel“ (nur source_unit custom); Commit bei Blur/Speichern. */
const [customEquivalenceDraftByField, setCustomEquivalenceDraftByField] = useState({})
/** Zeilenaggregation: null in DB = Modul-Standard; sonst import_row_processing */
const [rowAggUseCustom, setRowAggUseCustom] = useState(false)
const [rowAggGroupBy, setRowAggGroupBy] = useState([])
const [rowAggMode, setRowAggMode] = useState('')
const [rowAggIrregular, setRowAggIrregular] = useState(false)
const [rowAggJsonText, setRowAggJsonText] = useState('{}')
const modMeta = useMemo(() => modules.find((m) => m.id === module), [modules, module])
const aggregateSleepImport = modMeta?.import_mode === 'apple_sleep_aggregate'
const targetOptions = useMemo(() => {
@ -218,6 +288,15 @@ export default function AdminCsvTemplateEditorPage() {
.catch(() => setSeedOptions([]))
}, [module])
useEffect(() => {
if (!isNew) return
setRowAggUseCustom(false)
setRowAggIrregular(false)
setRowAggGroupBy([])
setRowAggMode('')
setRowAggJsonText('{}')
}, [module, isNew])
useEffect(() => {
if (isNew || !templateId) return
let ok = true
@ -241,6 +320,12 @@ export default function AdminCsvTemplateEditorPage() {
setCustomEquivalenceDraftByField({})
setSampleRows([])
setSeedHint(null)
const rp = parseStoredImportRowProcessing(t.import_row_processing)
setRowAggUseCustom(rp.useCustom)
setRowAggIrregular(rp.irregular)
setRowAggGroupBy(rp.groupBy)
setRowAggMode(rp.mode)
setRowAggJsonText(JSON.stringify(t.import_row_processing || {}, null, 2))
})
.catch((e) => {
if (ok) setError(e.message)
@ -259,6 +344,17 @@ export default function AdminCsvTemplateEditorPage() {
)
}, [fieldMappings])
const rowAggGroupCandidates = useMemo(() => {
return [...assignedTargets].filter((t) => modMeta?.fields?.[t]).sort()
}, [assignedTargets, modMeta])
useEffect(() => {
const targets = new Set(
Object.values(fieldMappings).filter((v) => v && v !== '-' && v !== '_skip'),
)
setRowAggGroupBy((prev) => prev.filter((g) => targets.has(g)))
}, [fieldMappings])
const missingRequired = useMemo(() => {
return requiredTargets.filter((r) => !assignedTargets.has(r))
}, [requiredTargets, assignedTargets])
@ -440,6 +536,11 @@ export default function AdminCsvTemplateEditorPage() {
setEncoding(res.encoding || 'utf-8')
setSeedHint(res.seed_template || null)
setCustomEquivalenceDraftByField({})
setRowAggUseCustom(false)
setRowAggIrregular(false)
setRowAggGroupBy([])
setRowAggMode('')
setRowAggJsonText('{}')
} catch (e) {
setError(e.message || 'Analyse fehlgeschlagen')
} finally {
@ -506,6 +607,45 @@ export default function AdminCsvTemplateEditorPage() {
return
}
let import_row_processing = null
if (!aggregateSleepImport && rowAggUseCustom) {
if (rowAggIrregular) {
try {
import_row_processing = JSON.parse(rowAggJsonText || '{}')
if (!import_row_processing || typeof import_row_processing !== 'object') throw new Error('bad')
} catch {
setError('Zeilenaggregation: ungültiges JSON.')
return
}
const gb = import_row_processing.group_by
if (!Array.isArray(gb) || !gb.length) {
setError('Zeilenaggregation (JSON): „group_by“ muss eine nicht-leere Liste sein.')
return
}
} else {
if (!rowAggGroupBy.length) {
setError('Zeilenaggregation: mindestens ein Schlüsselfeld auswählen.')
return
}
if (!rowAggMode) {
setError('Zeilenaggregation: eine Funktion wählen (Summe, Mittelwert, …).')
return
}
for (const g of rowAggGroupBy) {
if (!assignedTargets.has(g)) {
setError(`Zeilenaggregation: Schlüsselfeld „${g}“ muss einer CSV-Spalte zugeordnet sein.`)
return
}
}
import_row_processing = buildImportRowProcessingSimple(
modMeta?.fields,
fieldMappings,
rowAggGroupBy,
rowAggMode,
)
}
}
const payload = {
module,
mapping_name: mappingName.trim(),
@ -516,6 +656,7 @@ export default function AdminCsvTemplateEditorPage() {
has_header: hasHeader,
field_mappings: fieldMappings,
type_conversions: tc,
import_row_processing,
}
if (!payload.column_signature?.length) {