feat(csv-import): Refactor CSV import logic and enhance data handling
- Updated the CSV import architecture to clarify the distinction between import and data layer responsibilities, as outlined in the new section of ARCHITECTURE.md. - Enhanced the build_row_after_mapping function to include module-specific context for improved data processing. - Introduced source unit options in the admin CSV template editor to facilitate user-defined conversions, improving flexibility in handling various data formats. - Added new tests to validate the handling of source units and ensure accurate conversions during CSV imports. - Updated module definitions to include unit specifications for nutritional and activity data fields, enhancing data integrity.
This commit is contained in:
1 parent
41cc0ed2a8
commit
d6d7e738a5
10 files changed
+324
-27
No files matched your search
@@ -12,6 +12,7 @@ from csv_parser.core import (
|
||||
get_csv_import_limits,
|
||||
iter_csv_dict_rows,
|
||||
)
|
||||
from csv_parser.mapping_suggest import build_type_conversions_for_mapping
|
||||
from csv_parser.type_converter import convert_value, build_row_after_mapping
|
||||
|
||||
|
||||
@@ -73,6 +74,25 @@ def test_convert_date_and_kcal_factor():
|
||||
assert abs(k - 8000 * 0.239) < 0.01
|
||||
|
||||
|
||||
def test_convert_kcal_via_source_unit_kj():
|
||||
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
|
||||
k = convert_value("4184", "kcal", spec, module="nutrition")
|
||||
assert abs(k - 1000.0) < 0.05
|
||||
|
||||
|
||||
def test_convert_protein_kg_to_g():
|
||||
spec = {"type": "float", "source_unit": "kg", "decimal_separator": "."}
|
||||
g = convert_value("0.1", "protein_g", spec, module="nutrition")
|
||||
assert abs(g - 100.0) < 0.001
|
||||
|
||||
|
||||
def test_build_row_source_unit_without_module_no_factor():
|
||||
"""Ohne module bleibt source_unit wirkungslos (Abwärtskompatibilität)."""
|
||||
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
|
||||
k = convert_value("4184", "kcal", spec, module=None)
|
||||
assert abs(k - 4184.0) < 0.01
|
||||
|
||||
|
||||
def test_iter_csv_dict_rows_full_file():
|
||||
text = "a;b\n1;2\n3;4\n"
|
||||
rows = list(iter_csv_dict_rows(text, ";", has_header=True))
|
||||
@@ -86,9 +106,16 @@ def test_build_row_after_mapping():
|
||||
"date": {"type": "date", "format": "dd.mm.yyyy"},
|
||||
"kcal": {"type": "float", "conversion_factor": 0.239, "decimal_separator": "."},
|
||||
}
|
||||
out = build_row_after_mapping(csv_row, fm, tc)
|
||||
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
|
||||
assert out["date"].month == 1
|
||||
assert out["kcal"] is not None
|
||||
assert abs(float(out["kcal"]) - 4200 * 0.239) < 0.02
|
||||
|
||||
|
||||
def test_build_type_conversions_kj_header_sets_source_unit():
|
||||
fm = {"kJ": "kcal", "Datum": "date"}
|
||||
tc = build_type_conversions_for_mapping("nutrition", fm, None)
|
||||
assert tc["kcal"].get("source_unit") == "kj"
|
||||
|
||||
|
||||
def test_build_row_fddb_raw_header_keys_match_normalized_template():
|
||||
@@ -111,15 +138,16 @@ def test_build_row_fddb_raw_header_keys_match_normalized_template():
|
||||
"date": {"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
|
||||
"kcal": {
|
||||
"type": "float",
|
||||
"conversion_factor": 0.239,
|
||||
"source_unit": "kj",
|
||||
"decimal_separator": ",",
|
||||
},
|
||||
"fat_g": {"type": "float", "decimal_separator": ","},
|
||||
"carbs_g": {"type": "float", "decimal_separator": ","},
|
||||
"protein_g": {"type": "float", "decimal_separator": ","},
|
||||
}
|
||||
out = build_row_after_mapping(csv_row, fm, tc)
|
||||
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
|
||||
assert out["date"].year == 2024 and out["date"].month == 1 and out["date"].day == 1
|
||||
assert out["kcal"] is not None and abs(float(out["kcal"]) - (42000 / 4.184)) < 0.1
|
||||
|
||||
|
||||
def test_convert_date_ddmm_with_seconds():
|
||||
|
||||
Reference in new issue
Block a user