feat(csv-parser): Enhance CSV parsing with header normalization and flexible date handling
- Added a new function to strip header keys of unwanted characters, improving CSV import consistency. - Updated CSV row iteration to utilize the new header normalization function, ensuring cleaner data processing. - Enhanced date parsing capabilities to support flexible formats, accommodating various date representations in CSV files. - Introduced additional tests to validate the new header normalization and date parsing functionalities.
This commit is contained in:
1 parent
5e5f3b4e5a
commit
338163ac0b
3 files changed
+403
-42
No files matched your search
@@ -74,3 +74,86 @@ def test_build_row_after_mapping():
|
||||
out = build_row_after_mapping(csv_row, fm, tc)
|
||||
assert out["date"].month == 1
|
||||
assert out["kcal"] is not None
|
||||
|
||||
|
||||
def test_build_row_fddb_raw_header_keys_match_normalized_template():
|
||||
"""FDDB: DictReader liefert deutsche Überschrift, Seed nutzt normalisierten Key."""
|
||||
csv_row = {
|
||||
"Datum Tag Monat Jahr Stunde Minute": "01.01.2024 8:30",
|
||||
"kJ": "42000",
|
||||
"Fett (g)": "50",
|
||||
"KH (g)": "200",
|
||||
"Protein (g)": "100",
|
||||
}
|
||||
fm = {
|
||||
"datum_tag_monat_jahr_stunde_minute": "date",
|
||||
"kj": "kcal",
|
||||
"fett_g": "fat_g",
|
||||
"kh_g": "carbs_g",
|
||||
"protein_g": "protein_g",
|
||||
}
|
||||
tc = {
|
||||
"date": {"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
|
||||
"kcal": {
|
||||
"type": "float",
|
||||
"conversion_factor": 0.239,
|
||||
"decimal_separator": ",",
|
||||
},
|
||||
"fat_g": {"type": "float", "decimal_separator": ","},
|
||||
"carbs_g": {"type": "float", "decimal_separator": ","},
|
||||
"protein_g": {"type": "float", "decimal_separator": ","},
|
||||
}
|
||||
out = build_row_after_mapping(csv_row, fm, tc)
|
||||
assert out["date"].year == 2024 and out["date"].month == 1 and out["date"].day == 1
|
||||
|
||||
|
||||
def test_convert_date_ddmm_with_seconds():
|
||||
d = convert_value(
|
||||
"15.01.2024 14:30:00",
|
||||
"date",
|
||||
{"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
|
||||
)
|
||||
assert d.month == 1 and d.day == 15
|
||||
|
||||
|
||||
def test_float_decimal_separator_auto_eu_us():
|
||||
assert abs(convert_value("1.234,56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
|
||||
assert abs(convert_value("1,234.56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
|
||||
assert abs(convert_value("1234,5", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.5) < 1e-9
|
||||
|
||||
|
||||
def test_float_flexible_falls_back_to_auto():
|
||||
spec = {"type": "float", "decimal_separator": ",", "flexible": True}
|
||||
assert abs(convert_value("1234.56", "x", spec) - 1234.56) < 1e-9
|
||||
|
||||
|
||||
def test_date_flexible_iso_while_primary_ddmm():
|
||||
spec = {
|
||||
"type": "date",
|
||||
"format": "dd.mm.yyyy",
|
||||
"flexible": True,
|
||||
"extract": "date_only",
|
||||
}
|
||||
d1 = convert_value("2024-03-15", "d", spec)
|
||||
d2 = convert_value("15.03.2024", "d", spec)
|
||||
assert d1 == d2
|
||||
|
||||
|
||||
def test_date_extra_formats_without_days_in_name():
|
||||
spec = {
|
||||
"type": "date",
|
||||
"format": "yyyy-mm-dd",
|
||||
"formats": ["%d.%m.%Y"],
|
||||
"extract": "date_only",
|
||||
}
|
||||
assert convert_value("08.04.2026", "d", spec).day == 8
|
||||
|
||||
|
||||
def test_int_flexible_thousands():
|
||||
assert convert_value("1.234", "n", {"type": "int", "flexible": True}) == 1234
|
||||
|
||||
|
||||
def test_datetime_flexible():
|
||||
spec = {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}
|
||||
dtv = convert_value("15.01.2024 14:30:00", "t", spec)
|
||||
assert dtv.month == 1 and dtv.day == 15 and dtv.hour == 14
|
||||
Reference in new issue
Block a user