feat(csv-parser): Implement CSV import functionality with mapping and type conversion
Deploy Development / deploy (push) Successful in 50s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 15s

- Added permissions for editing and deleting CSV field mappings.
- Created type converter for CSV cells to handle various data types.
- Implemented database migrations for CSV field mappings and import logs.
- Seeded initial system templates for nutrition and activity data imports.
- Developed admin endpoints for managing system CSV templates.
- Introduced user endpoints for CSV import analysis and mapping retrieval.
- Added tests for core CSV parser functionalities, including delimiter detection and value conversion.
This commit is contained in:
Lars committed 2026-04-09 21:37:19 +02:00
1 parent 73963e7140
commit 4a771f6a83
16 files changed
+3252 -1

No files matched your search

@@ -0,0 +1,75 @@
-- Migration 042: Universal CSV Parser – Mapping-Registry & Import-Log (Issue #21)
-- Tabellen für System-Templates (profile_id NULL, is_system true) und User-Mappings.
CREATE TABLE IF NOT EXISTS csv_field_mappings (
id SERIAL PRIMARY KEY,
profile_id UUID REFERENCES profiles(id) ON DELETE CASCADE,
is_system BOOLEAN NOT NULL DEFAULT false,
module VARCHAR(50) NOT NULL,
mapping_name VARCHAR(100) NOT NULL,
description TEXT,
column_signature TEXT[] NOT NULL DEFAULT '{}',
delimiter VARCHAR(10) NOT NULL DEFAULT ',',
encoding VARCHAR(20) NOT NULL DEFAULT 'utf-8',
has_header BOOLEAN NOT NULL DEFAULT true,
field_mappings JSONB NOT NULL DEFAULT '{}',
type_conversions JSONB,
usage_count INTEGER NOT NULL DEFAULT 0,
last_used_at TIMESTAMPTZ,
success_rate REAL NOT NULL DEFAULT 1.0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT csv_field_mappings_system_profile CHECK (
(is_system = true AND profile_id IS NULL)
OR (is_system = false AND profile_id IS NOT NULL)
)
);
COMMENT ON TABLE csv_field_mappings IS 'CSV Import: System-Templates + User-Mappings (Issue #21)';
COMMENT ON COLUMN csv_field_mappings.is_system IS 'true = globales Template (nur Admin pflegbar), false = User-Mapping';
CREATE UNIQUE INDEX IF NOT EXISTS idx_csv_field_mappings_system_module_name
ON csv_field_mappings (module, mapping_name)
WHERE is_system = true AND profile_id IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_csv_field_mappings_user_module_name
ON csv_field_mappings (profile_id, module, mapping_name)
WHERE is_system = false;
CREATE INDEX IF NOT EXISTS idx_csv_field_mappings_module_profile
ON csv_field_mappings (module, profile_id);
CREATE INDEX IF NOT EXISTS idx_csv_field_mappings_system_module
ON csv_field_mappings (module)
WHERE is_system = true;
CREATE TABLE IF NOT EXISTS csv_import_log (
id SERIAL PRIMARY KEY,
profile_id UUID NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
mapping_id INTEGER REFERENCES csv_field_mappings(id) ON DELETE SET NULL,
module VARCHAR(50) NOT NULL,
filename VARCHAR(255),
rows_total INTEGER,
rows_imported INTEGER,
rows_updated INTEGER,
rows_skipped INTEGER,
rows_errors INTEGER,
error_details JSONB,
started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
finished_at TIMESTAMPTZ,
status VARCHAR(20) NOT NULL DEFAULT 'running',
affected_ids JSONB
);
CREATE INDEX IF NOT EXISTS idx_csv_import_log_profile_module
ON csv_import_log (profile_id, module DESC, started_at DESC);
COMMENT ON COLUMN csv_import_log.affected_ids IS 'Pro Import gesammelte Primärschlüssel je Tabelle (Rollback / Bereinigung)';
INSERT INTO system_config (key, value, updated_at)
VALUES (
'csv_import',
'{"max_rows_per_file": 50000, "max_file_bytes": 52428800}'::jsonb,
CURRENT_TIMESTAMP
)
ON CONFLICT (key) DO NOTHING;