Lars
6945b748cb
- Increased precision for `kcal_active`, `kcal_resting`, `hr_avg`, and `hr_max` fields in the activity log schema. - Added a new function `_activity_hr_bpm` to validate heart rate values during CSV import, ensuring they fall within plausible ranges. - Updated the CSV parser to utilize the new heart rate validation function for improved data integrity. - Enhanced the type converter to accommodate additional aliases for energy fields in CSV imports. - Added a test to verify conversion of active energy from kJ to kcal, ensuring accurate data handling.
285 lines
10 KiB
Python
285 lines
10 KiB
Python
"""Tests für CSV-Parser Foundation (Issue #21)."""
|
|
|
|
import pytest
|
|
|
|
from csv_parser.core import (
|
|
decode_raw_bytes,
|
|
sniff_delimiter,
|
|
parse_csv_sample,
|
|
column_signature,
|
|
headers_signature_match_score,
|
|
headers_signature_rank_metrics,
|
|
get_csv_import_limits,
|
|
iter_csv_dict_rows,
|
|
)
|
|
from csv_parser.field_units import source_unit_choices_for_field
|
|
from csv_parser.mapping_suggest import build_type_conversions_for_mapping
|
|
from csv_parser.type_converter import convert_value, build_row_after_mapping
|
|
|
|
|
|
def test_decode_bom_utf8():
|
|
raw = "\ufeffa;b;c\n1;2;3".encode("utf-8-sig")
|
|
t = decode_raw_bytes(raw)
|
|
assert not t.startswith("\ufeff")
|
|
assert "a;b;c" in t
|
|
|
|
|
|
def test_sniff_delimiter():
|
|
assert sniff_delimiter("a;b;c;d") == ";"
|
|
assert sniff_delimiter("a,b,c") == ","
|
|
|
|
|
|
def test_parse_csv_sample_header():
|
|
text = "Date;kcal\n2024-01-01;2000\n"
|
|
headers, rows, delim = parse_csv_sample(text, delimiter=";", max_data_rows=3)
|
|
assert headers == ["Date", "kcal"]
|
|
assert delim == ";"
|
|
assert rows[0]["Date"] == "2024-01-01"
|
|
assert rows[0]["kcal"] == "2000"
|
|
|
|
|
|
def test_column_signature_sorted_unique():
|
|
sig = column_signature(["B", "a", "a"])
|
|
assert sig == ["a", "b"]
|
|
|
|
|
|
def test_jaccard():
|
|
s1 = column_signature(["Date", "Calories"])
|
|
s2 = column_signature(["Date", "Calories", "Fat"])
|
|
assert headers_signature_match_score(s1, s2) == pytest.approx(2 / 3)
|
|
|
|
|
|
def test_template_recall_full_when_csv_has_extra_columns():
|
|
"""Alle Template-Spalten in der CSV → Recall 1.0; Jaccard niedriger bei vielen Zusatzspalten."""
|
|
csv_sig = column_signature(
|
|
["D", "E", "F", "Extra1", "Extra2", "Extra3", "Extra4", "Extra5"]
|
|
)
|
|
tmpl_sig = column_signature(["d", "e", "f"])
|
|
m = headers_signature_rank_metrics(csv_sig, tmpl_sig)
|
|
assert m["confidence"] == 1.0
|
|
assert m["template_recall"] == 1.0
|
|
assert m["columns_matched"] == 3
|
|
assert m["columns_in_template"] == 3
|
|
assert m["jaccard"] == pytest.approx(3 / 8)
|
|
|
|
|
|
def test_get_csv_import_limits_default():
|
|
assert get_csv_import_limits(None)["max_rows_per_file"] == 50_000
|
|
|
|
|
|
def test_convert_date_and_kcal_factor():
|
|
d = convert_value("15.01.2024", "date", {"type": "date", "format": "dd.mm.yyyy"})
|
|
assert d.year == 2024 and d.month == 1 and d.day == 15
|
|
|
|
k = convert_value("8000", "kcal", {"type": "float", "conversion_factor": 0.239, "decimal_separator": "."})
|
|
assert abs(k - 8000 * 0.239) < 0.01
|
|
|
|
|
|
def test_convert_kcal_via_source_unit_kj():
|
|
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
|
|
k = convert_value("4184", "kcal", spec, module="nutrition")
|
|
assert abs(k - 1000.0) < 0.05
|
|
|
|
|
|
def test_convert_protein_kg_to_g():
|
|
spec = {"type": "float", "source_unit": "kg", "decimal_separator": "."}
|
|
g = convert_value("0.1", "protein_g", spec, module="nutrition")
|
|
assert abs(g - 100.0) < 0.001
|
|
|
|
|
|
def test_convert_custom_source_unit_only_conversion_factor():
|
|
"""Nicht vordefinierte Umrechnung: conversion_factor (optional mit source_unit: custom)."""
|
|
spec = {"type": "float", "source_unit": "custom", "conversion_factor": 2.5, "decimal_separator": "."}
|
|
k = convert_value("100", "kcal", spec, module="nutrition")
|
|
assert abs(k - 250.0) < 0.001
|
|
|
|
|
|
def test_convert_unknown_source_unit_uses_conversion_factor_only():
|
|
spec = {"type": "float", "source_unit": "exotic_unit", "conversion_factor": 0.5, "decimal_separator": "."}
|
|
k = convert_value("200", "kcal", spec, module="nutrition")
|
|
assert abs(k - 100.0) < 0.001
|
|
|
|
|
|
def test_build_row_source_unit_without_module_no_factor():
|
|
"""Ohne module bleibt source_unit wirkungslos (Abwärtskompatibilität)."""
|
|
spec = {"type": "float", "source_unit": "kj", "decimal_separator": "."}
|
|
k = convert_value("4184", "kcal", spec, module=None)
|
|
assert abs(k - 4184.0) < 0.01
|
|
|
|
|
|
def test_iter_csv_dict_rows_full_file():
|
|
text = "a;b\n1;2\n3;4\n"
|
|
rows = list(iter_csv_dict_rows(text, ";", has_header=True))
|
|
assert rows == [{"a": "1", "b": "2"}, {"a": "3", "b": "4"}]
|
|
|
|
|
|
def test_build_row_after_mapping():
|
|
csv_row = {"Datum": "01.01.2024", "kj": "4200"}
|
|
fm = {"Datum": "date", "kj": "kcal"}
|
|
tc = {
|
|
"date": {"type": "date", "format": "dd.mm.yyyy"},
|
|
"kcal": {"type": "float", "conversion_factor": 0.239, "decimal_separator": "."},
|
|
}
|
|
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
|
|
assert out["date"].month == 1
|
|
assert out["kcal"] is not None
|
|
assert abs(float(out["kcal"]) - 4200 * 0.239) < 0.02
|
|
|
|
|
|
def test_build_type_conversions_kj_header_sets_source_unit():
|
|
fm = {"kJ": "kcal", "Datum": "date"}
|
|
tc = build_type_conversions_for_mapping("nutrition", fm, None)
|
|
assert tc["kcal"].get("source_unit") == "kj"
|
|
|
|
|
|
def test_build_row_fddb_raw_header_keys_match_normalized_template():
|
|
"""FDDB: DictReader liefert deutsche Überschrift, Seed nutzt normalisierten Key."""
|
|
csv_row = {
|
|
"Datum Tag Monat Jahr Stunde Minute": "01.01.2024 8:30",
|
|
"kJ": "42000",
|
|
"Fett (g)": "50",
|
|
"KH (g)": "200",
|
|
"Protein (g)": "100",
|
|
}
|
|
fm = {
|
|
"datum_tag_monat_jahr_stunde_minute": "date",
|
|
"kj": "kcal",
|
|
"fett_g": "fat_g",
|
|
"kh_g": "carbs_g",
|
|
"protein_g": "protein_g",
|
|
}
|
|
tc = {
|
|
"date": {"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
|
|
"kcal": {
|
|
"type": "float",
|
|
"source_unit": "kj",
|
|
"decimal_separator": ",",
|
|
},
|
|
"fat_g": {"type": "float", "decimal_separator": ","},
|
|
"carbs_g": {"type": "float", "decimal_separator": ","},
|
|
"protein_g": {"type": "float", "decimal_separator": ","},
|
|
}
|
|
out = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
|
|
assert out["date"].year == 2024 and out["date"].month == 1 and out["date"].day == 1
|
|
assert out["kcal"] is not None and abs(float(out["kcal"]) - (42000 / 4.184)) < 0.1
|
|
|
|
|
|
def test_convert_date_ddmm_with_seconds():
|
|
d = convert_value(
|
|
"15.01.2024 14:30:00",
|
|
"date",
|
|
{"type": "date", "format": "dd.mm.yyyy HH:MM", "extract": "date_only"},
|
|
)
|
|
assert d.month == 1 and d.day == 15
|
|
|
|
|
|
def test_float_decimal_separator_auto_eu_us():
|
|
assert abs(convert_value("1.234,56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
|
|
assert abs(convert_value("1,234.56", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.56) < 1e-9
|
|
assert abs(convert_value("1234,5", "x", {"type": "float", "decimal_separator": "auto"}) - 1234.5) < 1e-9
|
|
|
|
|
|
def test_float_flexible_falls_back_to_auto():
|
|
spec = {"type": "float", "decimal_separator": ",", "flexible": True}
|
|
assert abs(convert_value("1234.56", "x", spec) - 1234.56) < 1e-9
|
|
|
|
|
|
def test_date_flexible_iso_while_primary_ddmm():
|
|
spec = {
|
|
"type": "date",
|
|
"format": "dd.mm.yyyy",
|
|
"flexible": True,
|
|
"extract": "date_only",
|
|
}
|
|
d1 = convert_value("2024-03-15", "d", spec)
|
|
d2 = convert_value("15.03.2024", "d", spec)
|
|
assert d1 == d2
|
|
|
|
|
|
def test_date_extra_formats_without_days_in_name():
|
|
spec = {
|
|
"type": "date",
|
|
"format": "yyyy-mm-dd",
|
|
"formats": ["%d.%m.%Y"],
|
|
"extract": "date_only",
|
|
}
|
|
assert convert_value("08.04.2026", "d", spec).day == 8
|
|
|
|
|
|
def test_int_flexible_thousands():
|
|
assert convert_value("1.234", "n", {"type": "int", "flexible": True}) == 1234
|
|
|
|
|
|
def test_build_row_after_mapping_column_order_independent():
|
|
fm = {"Spalte B": "resting_hr", "Spalte A": "date"}
|
|
tc = {
|
|
"date": {"type": "date", "format": "yyyy-mm-dd", "flexible": True},
|
|
"resting_hr": {"type": "int", "flexible": True},
|
|
}
|
|
r1 = build_row_after_mapping(
|
|
{"Spalte A": "2026-01-15", "Spalte B": "58"}, fm, tc, module="vitals_baseline"
|
|
)
|
|
r2 = build_row_after_mapping(
|
|
{"Spalte B": "58", "Spalte A": "2026-01-15"}, fm, tc, module="vitals_baseline"
|
|
)
|
|
assert r1 == r2
|
|
assert r1["resting_hr"] == 58
|
|
|
|
|
|
def test_omron_report_date_formats_without_flexible_flag():
|
|
"""Omron «Bericht»-Export: engl. Month abbrev + deutscher Monatsname; Vorlage oft nur dd.mm.yyyy."""
|
|
spec = {"type": "date", "format": "dd.mm.yyyy"}
|
|
assert convert_value("10 Apr. 2026", "measured_date", spec).isoformat() == "2026-04-10"
|
|
assert convert_value("31 März 2026", "measured_date", spec).isoformat() == "2026-03-31"
|
|
assert convert_value("11 März 2026", "measured_date", spec).isoformat() == "2026-03-11"
|
|
|
|
|
|
def test_int_flexible_german_decimal_rounds():
|
|
"""Apple-DE: HRV/SpO2 als «37,26» / «95,22» — nicht 3726 aus Ziffern konkatenieren."""
|
|
spec = {"type": "int", "flexible": True}
|
|
assert convert_value("37,26", "hrv", spec) == 37
|
|
assert convert_value("95,22", "spo2", spec) == 95
|
|
|
|
|
|
def test_datetime_flexible():
|
|
spec = {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}
|
|
dtv = convert_value("15.01.2024 14:30:00", "t", spec)
|
|
assert dtv.month == 1 and dtv.day == 15 and dtv.hour == 14
|
|
|
|
|
|
def test_apple_workout_datetime_without_seconds_iso_not_swapped():
|
|
"""Apple Export: 2026-04-09 16:48 — ohne :SS; kein dayfirst-Fehlparser (09↔04)."""
|
|
spec = {
|
|
"type": "datetime",
|
|
"format": "yyyy-mm-dd HH:MM:SS",
|
|
"extract": "date_and_time",
|
|
}
|
|
dtv = convert_value("2026-04-09 16:48", "start_time", spec, module="activity")
|
|
assert dtv.year == 2026 and dtv.month == 4 and dtv.day == 9
|
|
assert dtv.hour == 16 and dtv.minute == 48
|
|
|
|
|
|
def test_iso_yyyy_mm_dd_dateutil_fallback_not_dayfirst_swapped():
|
|
"""Nur dateutil: ISO-Datum darf mit Default-dayfirst nicht vertauscht werden."""
|
|
spec = {"type": "date", "format": "dd.mm.yyyy", "flexible": True}
|
|
d = convert_value("2026-04-09", "d", spec)
|
|
assert d.month == 4 and d.day == 9
|
|
|
|
|
|
def test_activity_kcal_active_kj_source_unit_to_kcal():
|
|
"""Apple-DE: Aktive Energie als kJ; DB-Feld ist kcal."""
|
|
spec = {
|
|
"type": "float",
|
|
"decimal_separator": ".",
|
|
"flexible": True,
|
|
"source_unit": "kj",
|
|
}
|
|
v = convert_value("4184", "kcal_active", spec, module="activity")
|
|
assert abs(v - 1000.0) < 0.02
|
|
|
|
|
|
def test_source_unit_choices_include_custom_at_end():
|
|
opts = source_unit_choices_for_field("nutrition", "protein_g")
|
|
assert opts[-1]["id"] == "custom"
|
|
assert any(o["id"] == "mg" for o in opts)
|