feat(csv-import): Refactor CSV import logic and enhance data handling
Deploy Development / deploy (push) Successful in 48s
Build Test / pytest-backend-csv (push) Failing after 3s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 15s

- Updated the CSV import architecture to clarify the distinction between import and data layer responsibilities, as outlined in the new section of ARCHITECTURE.md.
- Enhanced the build_row_after_mapping function to include module-specific context for improved data processing.
- Introduced source unit options in the admin CSV template editor to facilitate user-defined conversions, improving flexibility in handling various data formats.
- Added new tests to validate the handling of source units and ensure accurate conversions during CSV imports.
- Updated module definitions to include unit specifications for nutritional and activity data fields, enhancing data integrity.
This commit is contained in:
Lars committed 2026-04-10 09:54:32 +02:00
1 parent 41cc0ed2a8
commit d6d7e738a5
10 files changed
+324 -27

No files matched your search

+31 -3
View File
@@ -12,6 +12,7 @@ from csv_parser.core import (
get_csv_import_limits,
iter_csv_dict_rows,
)
from csv_parser.mapping_suggest import build_type_conversions_for_mapping
from csv_parser.type_converter import convert_value, build_row_after_mapping
@@ -73,6 +74,25 @@ def test_convert_date_and_kcal_factor():
assert abs(k - 8000 * 0.239) < 0.01
def test_convert_kcal_via_source_unit_kj():
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
k = convert_value("4184", "kcal", spec, module="nutrition")
assert abs(k - 1000.0) < 0.05
def test_convert_protein_kg_to_g():
spec = {"type": "float", "source_unit": "kg", "decimal_separator": "."}
g = convert_value("0.1", "protein_g", spec, module="nutrition")
assert abs(g - 100.0) < 0.001
def test_build_row_source_unit_without_module_no_factor():
"""Ohne module bleibt source_unit wirkungslos (Abwärtskompatibilität)."""
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
k = convert_value("4184", "kcal", spec, module=None)
assert abs(k - 4184.0) < 0.01
def test_iter_csv_dict_rows_full_file():
text = "a;b\n1;2\n3;4\n"
rows = list(iter_csv_dict_rows(text, ";", has_header=True))
@@ -86,9 +106,16 @@ def test_build_row_after_mapping():
"date": {"type": "date", "format": "dd.mm.yyyy"},
"kcal": {"type": "float", "conversion_factor": 0.239, "decimal_separator": "."},
}
out = build_row_after_mapping(csv_row, fm, tc)
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
assert out["date"].month == 1
assert out["kcal"] is not None
assert abs(float(out["kcal"]) - 4200 * 0.239) < 0.02
def test_build_type_conversions_kj_header_sets_source_unit():
fm = {"kJ": "kcal", "Datum": "date"}
tc = build_type_conversions_for_mapping("nutrition", fm, None)
assert tc["kcal"].get("source_unit") == "kj"
def test_build_row_fddb_raw_header_keys_match_normalized_template():
@@ -111,15 +138,16 @@ def test_build_row_fddb_raw_header_keys_match_normalized_template():
"date": {"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
"kcal": {
"type": "float",
"conversion_factor": 0.239,
"source_unit": "kj",
"decimal_separator": ",",
},
"fat_g": {"type": "float", "decimal_separator": ","},
"carbs_g": {"type": "float", "decimal_separator": ","},
"protein_g": {"type": "float", "decimal_separator": ","},
}
out = build_row_after_mapping(csv_row, fm, tc)
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
assert out["date"].year == 2024 and out["date"].month == 1 and out["date"].day == 1
assert out["kcal"] is not None and abs(float(out["kcal"]) - (42000 / 4.184)) < 0.1
def test_convert_date_ddmm_with_seconds():