From 6945b748cb112f164b456fd9f3bc78014638b523 Mon Sep 17 00:00:00 2001 From: Lars Date: Sat, 11 Apr 2026 06:41:23 +0200 Subject: [PATCH] feat(schema, csv_parser): Update activity log schema and parsing logic - Increased precision for `kcal_active`, `kcal_resting`, `hr_avg`, and `hr_max` fields in the activity log schema. - Added a new function `_activity_hr_bpm` to validate heart rate values during CSV import, ensuring they fall within plausible ranges. - Updated the CSV parser to utilize the new heart rate validation function for improved data integrity. - Enhanced the type converter to accommodate additional aliases for energy fields in CSV imports. - Added a test to verify conversion of active energy from kJ to kcal, ensuring accurate data handling. --- backend/csv_parser/executor.py | 14 ++++++++-- backend/csv_parser/type_converter.py | 2 +- .../043_csv_parser_seed_templates.sql | 11 +++++--- ...tivity_log_widen_energy_and_hr_numeric.sql | 8 ++++++ .../053_csv_activity_apple_de_kj_energy.sql | 26 +++++++++++++++++++ backend/schema.sql | 8 +++--- backend/tests/test_csv_parser_core.py | 12 +++++++++ 7 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 backend/migrations/052_activity_log_widen_energy_and_hr_numeric.sql create mode 100644 backend/migrations/053_csv_activity_apple_de_kj_energy.sql diff --git a/backend/csv_parser/executor.py b/backend/csv_parser/executor.py index 13631af..5a7d630 100644 --- a/backend/csv_parser/executor.py +++ b/backend/csv_parser/executor.py @@ -776,6 +776,16 @@ def _sf_act(val: Any) -> float | None: return None +def _activity_hr_bpm(val: Any) -> float | None: + """Plausible Herzfrequenz (Import); größere Werte oft Fehlzuordnung (z. B. Schrittzahl) → NUMERIC-Overflow.""" + v = _sf_act(val) + if v is None: + return None + if v < 20 or v > 280: + return None + return v + + def _looks_like_time_only(s: str) -> bool: t = s.strip() if not t or " " in t: @@ -867,8 +877,8 @@ def _import_activity( kcal_a = _sf_act(mapped.get("kcal_active")) kcal_r = _sf_act(mapped.get("kcal_resting")) - hr_a = _sf_act(mapped.get("hr_avg")) - hr_m = _sf_act(mapped.get("hr_max")) + hr_a = _activity_hr_bpm(mapped.get("hr_avg")) + hr_m = _activity_hr_bpm(mapped.get("hr_max")) dist = _sf_act(mapped.get("distance_km")) wtype = str(activity_type).strip() diff --git a/backend/csv_parser/type_converter.py b/backend/csv_parser/type_converter.py index 90af16d..a29e5c7 100644 --- a/backend/csv_parser/type_converter.py +++ b/backend/csv_parser/type_converter.py @@ -526,7 +526,7 @@ def _activity_alias_db_field(csv_col: str) -> str | None: return "distance_km" if "aktive_energie" in n or "active_energy" in n: return "kcal_active" - if "ruheenergie" in n or "resting_energy" in n: + if "ruheeintr" in n or "ruheenergie" in n or "resting_energy" in n: return "kcal_resting" if ("herzfrequenz" in n or "heart_rate" in n) and ("max" in low or "max" in n): return "hr_max" diff --git a/backend/migrations/043_csv_parser_seed_templates.sql b/backend/migrations/043_csv_parser_seed_templates.sql index ba843ba..7c4114a 100644 --- a/backend/migrations/043_csv_parser_seed_templates.sql +++ b/backend/migrations/043_csv_parser_seed_templates.sql @@ -145,7 +145,7 @@ INSERT INTO csv_field_mappings ( SELECT NULL, true, 'activity', 'Apple Health Workout Export (Deutsch)', 'Apple Health CSV-Export für Workouts (Deutsch). Automatisches Training-Type-Mapping.', - ARRAY['Aktive Energie (kcal)', 'Dauer', 'Durchschnittliche Herzfrequenz (bpm)', 'Ende', 'Start', 'Strecke (km)', 'Trainingsart']::TEXT[], + ARRAY['Aktive Energie (kJ)', 'Aktive Energie (kcal)', 'Dauer', 'Durchschnittliche Herzfrequenz (bpm)', 'Ende', 'Ruheeinträge (kJ)', 'Start', 'Strecke (km)', 'Trainingsart']::TEXT[], ',', 'utf-8', true, '{ "Trainingsart": "activity_type", @@ -154,15 +154,18 @@ SELECT "Dauer": "duration_min", "Strecke (km)": "distance_km", "Aktive Energie (kcal)": "kcal_active", + "Aktive Energie (kJ)": "kcal_active", + "Ruheeinträge (kJ)": "kcal_resting", "Durchschnittliche Herzfrequenz (bpm)": "hr_avg" }'::JSONB, '{ "start_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "extract": "date_and_time", "flexible": true}, "end_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": true}, "duration_min": {"type": "duration", "format": "HH:MM:SS", "target_unit": "minutes"}, - "distance_km": {"type": "float", "decimal_separator": ","}, - "kcal_active": {"type": "float", "decimal_separator": ","}, - "hr_avg": {"type": "int"} + "distance_km": {"type": "float", "decimal_separator": ",", "flexible": true}, + "kcal_active": {"type": "float", "decimal_separator": ".", "flexible": true, "source_unit": "kj"}, + "kcal_resting": {"type": "float", "decimal_separator": ".", "flexible": true, "source_unit": "kj"}, + "hr_avg": {"type": "int", "flexible": true} }'::JSONB WHERE NOT EXISTS ( SELECT 1 FROM csv_field_mappings f diff --git a/backend/migrations/052_activity_log_widen_energy_and_hr_numeric.sql b/backend/migrations/052_activity_log_widen_energy_and_hr_numeric.sql new file mode 100644 index 0000000..2c5ec8f --- /dev/null +++ b/backend/migrations/052_activity_log_widen_energy_and_hr_numeric.sql @@ -0,0 +1,8 @@ +-- Apple-/Import-Werte: Energie oft >1000 (kJ oder kcal), HR-Felder NUMERIC(5,2) zu eng für Fehlzuordnungen. +-- Idempotent: Typ nur anheben, nie verkleinern. + +ALTER TABLE activity_log + ALTER COLUMN kcal_active TYPE NUMERIC(12, 2), + ALTER COLUMN kcal_resting TYPE NUMERIC(12, 2), + ALTER COLUMN hr_avg TYPE NUMERIC(6, 2), + ALTER COLUMN hr_max TYPE NUMERIC(6, 2); diff --git a/backend/migrations/053_csv_activity_apple_de_kj_energy.sql b/backend/migrations/053_csv_activity_apple_de_kj_energy.sql new file mode 100644 index 0000000..ce0e642 --- /dev/null +++ b/backend/migrations/053_csv_activity_apple_de_kj_energy.sql @@ -0,0 +1,26 @@ +-- Apple Health Workout (Deutsch): „Aktive Energie (kJ)“ / „Ruheeinträge (kJ)“ → DB speichert kcal. +-- type_conversions.source_unit „kj“ nutzt field_units (1/4.184). + +UPDATE csv_field_mappings +SET + field_mappings = COALESCE(field_mappings, '{}'::jsonb) + || '{"Aktive Energie (kJ)": "kcal_active", "Ruheeinträge (kJ)": "kcal_resting"}'::jsonb, + type_conversions = jsonb_set( + jsonb_set( + COALESCE(type_conversions, '{}'::jsonb), + '{kcal_active}', + COALESCE(type_conversions->'kcal_active', '{}'::jsonb) || '{"source_unit": "kj"}'::jsonb, + true + ), + '{kcal_resting}', + COALESCE( + type_conversions->'kcal_resting', + '{"type": "float", "decimal_separator": ",", "flexible": true}'::jsonb + ) || '{"source_unit": "kj"}'::jsonb, + true + ) +WHERE is_system = true + AND profile_id IS NULL + AND module = 'activity' + AND mapping_name = 'Apple Health Workout Export (Deutsch)' + AND type_conversions IS NOT NULL; diff --git a/backend/schema.sql b/backend/schema.sql index a921b78..d0bc600 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -160,10 +160,10 @@ CREATE TABLE IF NOT EXISTS activity_log ( end_time TIME, activity_type VARCHAR(50) NOT NULL, duration_min NUMERIC(6,2), - kcal_active NUMERIC(7,2), - kcal_resting NUMERIC(7,2), - hr_avg NUMERIC(5,2), - hr_max NUMERIC(5,2), + kcal_active NUMERIC(12,2), + kcal_resting NUMERIC(12,2), + hr_avg NUMERIC(6,2), + hr_max NUMERIC(6,2), distance_km NUMERIC(7,2), rpe INTEGER CHECK (rpe >= 1 AND rpe <= 10), source VARCHAR(20) DEFAULT 'manual', diff --git a/backend/tests/test_csv_parser_core.py b/backend/tests/test_csv_parser_core.py index 93eb18a..e9b962d 100644 --- a/backend/tests/test_csv_parser_core.py +++ b/backend/tests/test_csv_parser_core.py @@ -266,6 +266,18 @@ def test_iso_yyyy_mm_dd_dateutil_fallback_not_dayfirst_swapped(): assert d.month == 4 and d.day == 9 +def test_activity_kcal_active_kj_source_unit_to_kcal(): + """Apple-DE: Aktive Energie als kJ; DB-Feld ist kcal.""" + spec = { + "type": "float", + "decimal_separator": ".", + "flexible": True, + "source_unit": "kj", + } + v = convert_value("4184", "kcal_active", spec, module="activity") + assert abs(v - 1000.0) < 0.02 + + def test_source_unit_choices_include_custom_at_end(): opts = source_unit_choices_for_field("nutrition", "protein_g") assert opts[-1]["id"] == "custom"