feat(csv-parser): Implement CSV import functionality with mapping and type conversion
- Added permissions for editing and deleting CSV field mappings. - Created type converter for CSV cells to handle various data types. - Implemented database migrations for CSV field mappings and import logs. - Seeded initial system templates for nutrition and activity data imports. - Developed admin endpoints for managing system CSV templates. - Introduced user endpoints for CSV import analysis and mapping retrieval. - Added tests for core CSV parser functionalities, including delimiter detection and value conversion.
This commit is contained in:
1 parent
73963e7140
commit
4a771f6a83
16 files changed
+3252
-1
No files matched your search
@@ -0,0 +1,75 @@
|
||||
-- Migration 042: Universal CSV Parser – Mapping-Registry & Import-Log (Issue #21)
|
||||
-- Tabellen für System-Templates (profile_id NULL, is_system true) und User-Mappings.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS csv_field_mappings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
profile_id UUID REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
is_system BOOLEAN NOT NULL DEFAULT false,
|
||||
module VARCHAR(50) NOT NULL,
|
||||
mapping_name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
column_signature TEXT[] NOT NULL DEFAULT '{}',
|
||||
delimiter VARCHAR(10) NOT NULL DEFAULT ',',
|
||||
encoding VARCHAR(20) NOT NULL DEFAULT 'utf-8',
|
||||
has_header BOOLEAN NOT NULL DEFAULT true,
|
||||
field_mappings JSONB NOT NULL DEFAULT '{}',
|
||||
type_conversions JSONB,
|
||||
usage_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_used_at TIMESTAMPTZ,
|
||||
success_rate REAL NOT NULL DEFAULT 1.0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT csv_field_mappings_system_profile CHECK (
|
||||
(is_system = true AND profile_id IS NULL)
|
||||
OR (is_system = false AND profile_id IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE csv_field_mappings IS 'CSV Import: System-Templates + User-Mappings (Issue #21)';
|
||||
COMMENT ON COLUMN csv_field_mappings.is_system IS 'true = globales Template (nur Admin pflegbar), false = User-Mapping';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_csv_field_mappings_system_module_name
|
||||
ON csv_field_mappings (module, mapping_name)
|
||||
WHERE is_system = true AND profile_id IS NULL;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_csv_field_mappings_user_module_name
|
||||
ON csv_field_mappings (profile_id, module, mapping_name)
|
||||
WHERE is_system = false;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_csv_field_mappings_module_profile
|
||||
ON csv_field_mappings (module, profile_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_csv_field_mappings_system_module
|
||||
ON csv_field_mappings (module)
|
||||
WHERE is_system = true;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS csv_import_log (
|
||||
id SERIAL PRIMARY KEY,
|
||||
profile_id UUID NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
mapping_id INTEGER REFERENCES csv_field_mappings(id) ON DELETE SET NULL,
|
||||
module VARCHAR(50) NOT NULL,
|
||||
filename VARCHAR(255),
|
||||
rows_total INTEGER,
|
||||
rows_imported INTEGER,
|
||||
rows_updated INTEGER,
|
||||
rows_skipped INTEGER,
|
||||
rows_errors INTEGER,
|
||||
error_details JSONB,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
finished_at TIMESTAMPTZ,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'running',
|
||||
affected_ids JSONB
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_csv_import_log_profile_module
|
||||
ON csv_import_log (profile_id, module DESC, started_at DESC);
|
||||
|
||||
COMMENT ON COLUMN csv_import_log.affected_ids IS 'Pro Import gesammelte Primärschlüssel je Tabelle (Rollback / Bereinigung)';
|
||||
|
||||
INSERT INTO system_config (key, value, updated_at)
|
||||
VALUES (
|
||||
'csv_import',
|
||||
'{"max_rows_per_file": 50000, "max_file_bytes": 52428800}'::jsonb,
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
Reference in new issue
Block a user