feat(csv-parser): Enhance CSV parsing with header normalization and flexible date handling
Deploy Development / deploy (push) Successful in 53s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 17s

- Added a new function to strip header keys of unwanted characters, improving CSV import consistency.
- Updated CSV row iteration to utilize the new header normalization function, ensuring cleaner data processing.
- Enhanced date parsing capabilities to support flexible formats, accommodating various date representations in CSV files.
- Introduced additional tests to validate the new header normalization and date parsing functionalities.
This commit is contained in:
Lars committed 2026-04-10 06:23:46 +02:00
1 parent 5e5f3b4e5a
commit 338163ac0b
3 files changed
+403 -42

No files matched your search

+83
View File
@@ -74,3 +74,86 @@ def test_build_row_after_mapping():
out = build_row_after_mapping(csv_row, fm, tc)
assert out["date"].month == 1
assert out["kcal"] is not None
def test_build_row_fddb_raw_header_keys_match_normalized_template():
"""FDDB: DictReader liefert deutsche Überschrift, Seed nutzt normalisierten Key."""
csv_row = {
"Datum Tag Monat Jahr Stunde Minute": "01.01.2024 8:30",
"kJ": "42000",
"Fett (g)": "50",
"KH (g)": "200",
"Protein (g)": "100",
}
fm = {
"datum_tag_monat_jahr_stunde_minute": "date",
"kj": "kcal",
"fett_g": "fat_g",
"kh_g": "carbs_g",
"protein_g": "protein_g",
}
tc = {
"date": {"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
"kcal": {
"type": "float",
"conversion_factor": 0.239,
"decimal_separator": ",",
},
"fat_g": {"type": "float", "decimal_separator": ","},
"carbs_g": {"type": "float", "decimal_separator": ","},
"protein_g": {"type": "float", "decimal_separator": ","},
}
out = build_row_after_mapping(csv_row, fm, tc)
assert out["date"].year == 2024 and out["date"].month == 1 and out["date"].day == 1
def test_convert_date_ddmm_with_seconds():
d = convert_value(
"15.01.2024 14:30:00",
"date",
{"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
)
assert d.month == 1 and d.day == 15
def test_float_decimal_separator_auto_eu_us():
assert abs(convert_value("1.234,56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
assert abs(convert_value("1,234.56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
assert abs(convert_value("1234,5", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.5) < 1e-9
def test_float_flexible_falls_back_to_auto():
spec = {"type": "float", "decimal_separator": ",", "flexible": True}
assert abs(convert_value("1234.56", "x", spec) - 1234.56) < 1e-9
def test_date_flexible_iso_while_primary_ddmm():
spec = {
"type": "date",
"format": "dd.mm.yyyy",
"flexible": True,
"extract": "date_only",
}
d1 = convert_value("2024-03-15", "d", spec)
d2 = convert_value("15.03.2024", "d", spec)
assert d1 == d2
def test_date_extra_formats_without_days_in_name():
spec = {
"type": "date",
"format": "yyyy-mm-dd",
"formats": ["%d.%m.%Y"],
"extract": "date_only",
}
assert convert_value("08.04.2026", "d", spec).day == 8
def test_int_flexible_thousands():
assert convert_value("1.234", "n", {"type": "int", "flexible": True}) == 1234
def test_datetime_flexible():
spec = {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}
dtv = convert_value("15.01.2024 14:30:00", "t", spec)
assert dtv.month == 1 and dtv.day == 15 and dtv.hour == 14