From 26ab11eb7b17a8a519ca9b8e07d98f430cd647d3 Mon Sep 17 00:00:00 2001 From: Lars Date: Fri, 10 Apr 2026 07:30:48 +0200 Subject: [PATCH] feat(csv-import): Enhance CSV import functionality with new modules and tests - Added support for new CSV import modules: sleep and vitals_baseline, expanding the import capabilities. - Implemented backend logic for handling CSV imports related to sleep and vitals baseline, including error handling and data processing. - Updated frontend components to include new modules in the CSV import interface, improving user experience. - Introduced unit tests for the new import functionalities to ensure reliability and correctness. - Enhanced existing CSV analysis features to accommodate the new modules, ensuring consistent behavior across the application. --- .gitea/workflows/test.yml | 17 + backend/csv_parser/executor.py | 398 +++++++++++++++++- backend/csv_parser/mapping_suggest.py | 33 +- backend/csv_parser/module_registry.py | 34 +- backend/csv_parser/sleep_apple_import.py | 329 +++++++++++++++ .../044_csv_parser_sleep_vitals_templates.sql | 155 +++++++ backend/pytest.ini | 5 + backend/requirements-dev.txt | 6 + backend/routers/csv_import.py | 17 +- backend/routers/sleep.py | 314 +------------- backend/tests/test_csv_import_executor.py | 178 ++++++++ .../src/pages/AdminCsvTemplateEditorPage.jsx | 2 + frontend/src/pages/AdminCsvTemplatesPage.jsx | 2 + frontend/src/pages/UniversalCsvImportPage.jsx | 111 +++-- 14 files changed, 1261 insertions(+), 340 deletions(-) create mode 100644 backend/csv_parser/sleep_apple_import.py create mode 100644 backend/migrations/044_csv_parser_sleep_vitals_templates.sql create mode 100644 backend/pytest.ini create mode 100644 backend/requirements-dev.txt create mode 100644 backend/tests/test_csv_import_executor.py diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 9ff0e60..8d30fab 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -5,6 +5,23 @@ on: branches: [main, develop] jobs: + pytest-backend-csv: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + pip install -r backend/requirements.txt -r backend/requirements-dev.txt + - name: Pytest — CSV-Import & Parser (Smoke) + run: | + cd backend + python -m pytest tests/test_csv_parser_core.py tests/test_csv_import_executor.py tests/test_mapping_suggest.py -q --tb=short + lint-backend: runs-on: ubuntu-latest steps: diff --git a/backend/csv_parser/executor.py b/backend/csv_parser/executor.py index 53ddb0c..a186ec0 100644 --- a/backend/csv_parser/executor.py +++ b/backend/csv_parser/executor.py @@ -9,10 +9,29 @@ import uuid from collections import defaultdict from typing import Any +import logging + from csv_parser.core import iter_csv_dict_rows from csv_parser.module_registry import get_module_definition from csv_parser.type_converter import build_row_after_mapping +logger = logging.getLogger(__name__) + +try: + from evaluation_helper import evaluate_and_save_activity as _evaluate_and_save_activity + + _EVALUATION_AVAILABLE = True +except Exception: # pragma: no cover + _evaluate_and_save_activity = None + _EVALUATION_AVAILABLE = False + + +def _resolve_training_type_for_activity(activity_type: str, profile_id: str): + """Lazy import — ermöglicht Tests ohne Laden von routers.activity (bcrypt).""" + from routers.activity import get_training_type_for_activity + + return get_training_type_for_activity(activity_type, profile_id) + def coerce_date(val: Any) -> dt.date | None: if val is None: @@ -48,6 +67,31 @@ def run_universal_csv_import( if not mod: raise ValueError(f"Unbekanntes Modul: {module}") + rows_total = 0 + error_details: list[dict[str, Any]] = [] + affected_ids: dict[str, list[str]] = defaultdict(list) + + if module == "sleep": + from csv_parser.sleep_apple_import import import_apple_sleep_nights + + try: + r = import_apple_sleep_nights(cur, profile_id, text) + except ValueError as e: + raise ValueError(str(e)) from e + error_details.extend(r.get("error_details") or []) + for sid in r.get("affected_ids") or []: + affected_ids["sleep_log"].append(sid) + return { + "rows_total": r["rows_total"], + "rows_imported": r["inserted"], + "rows_updated": r["updated"], + "rows_skipped": r["skipped"], + "rows_errors": len(error_details), + "error_details": error_details[:50], + "new_entries": r.get("new_entries", r["inserted"]), + "affected_ids": dict(affected_ids), + } + fm = mapping.get("field_mappings") or {} if isinstance(fm, str): raise ValueError("field_mappings muss ein Objekt sein") @@ -58,10 +102,6 @@ def run_universal_csv_import( delim = mapping.get("delimiter") or "," has_header = mapping.get("has_header", True) - rows_total = 0 - error_details: list[dict[str, Any]] = [] - affected_ids: dict[str, list[str]] = defaultdict(list) - if module == "nutrition": stats = _import_nutrition( cur, @@ -101,6 +141,32 @@ def run_universal_csv_import( affected_ids, ) rows_total = stats.pop("rows_total") + elif module == "activity": + stats = _import_activity( + cur, + profile_id, + text, + delim, + bool(has_header), + fm, + tc, + error_details, + affected_ids, + ) + rows_total = stats.pop("rows_total") + elif module == "vitals_baseline": + stats = _import_vitals_baseline( + cur, + profile_id, + text, + delim, + bool(has_header), + fm, + tc, + error_details, + affected_ids, + ) + rows_total = stats.pop("rows_total") else: raise ValueError(f"Modul '{module}' wird für Universal-Import noch nicht unterstützt") @@ -375,3 +441,327 @@ def _import_blood_pressure( "skipped": skipped, "new_entries": inserted, } + + +def _v_safe_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + if isinstance(value, float): + return int(value) + s = str(value).strip() + if "." in s: + return int(float(s)) + return int(s) + except (ValueError, TypeError): + return None + + +def _v_safe_float(value: Any) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + +def _import_vitals_baseline( + cur, + profile_id: str, + text: str, + delim: str, + has_header: bool, + fm: dict, + tc: dict | None, + error_details: list, + affected_ids: dict, +) -> dict[str, int]: + rows_total = 0 + inserted = 0 + updated = 0 + skipped = 0 + for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header): + rows_total += 1 + mapped = build_row_after_mapping(csv_row, fm, tc) + d = coerce_date(mapped.get("date")) + if d is None: + error_details.append({"row": rows_total, "error": "Datum fehlt"}) + continue + rhr = _v_safe_int(mapped.get("resting_hr")) + hrv = _v_safe_int(mapped.get("hrv")) + vo2 = _v_safe_float(mapped.get("vo2_max")) + spo2 = _v_safe_int(mapped.get("spo2")) + resp = _v_safe_float(mapped.get("respiratory_rate")) + if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)): + skipped += 1 + continue + iso = d.isoformat() + try: + cur.execute( + """ + INSERT INTO vitals_baseline ( + profile_id, date, + resting_hr, hrv, vo2_max, spo2, respiratory_rate, + source + ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'csv') + ON CONFLICT (profile_id, date) + DO UPDATE SET + resting_hr = COALESCE(EXCLUDED.resting_hr, vitals_baseline.resting_hr), + hrv = COALESCE(EXCLUDED.hrv, vitals_baseline.hrv), + vo2_max = COALESCE(EXCLUDED.vo2_max, vitals_baseline.vo2_max), + spo2 = COALESCE(EXCLUDED.spo2, vitals_baseline.spo2), + respiratory_rate = COALESCE(EXCLUDED.respiratory_rate, vitals_baseline.respiratory_rate), + updated_at = NOW() + WHERE vitals_baseline.source != 'manual' + RETURNING (xmax = 0) AS inserted, id + """, + (profile_id, iso, rhr, hrv, vo2, spo2, resp), + ) + result = cur.fetchone() + if result is None: + skipped += 1 + elif result.get("inserted"): + inserted += 1 + if result.get("id"): + affected_ids["vitals_baseline"].append(str(result["id"])) + else: + updated += 1 + if result.get("id"): + affected_ids["vitals_baseline"].append(str(result["id"])) + except Exception as e: + error_details.append({"row": rows_total, "error": str(e)}) + + return { + "rows_total": rows_total, + "inserted": inserted, + "updated": updated, + "skipped": skipped, + "new_entries": inserted, + } + + +def _sf_act(val: Any) -> float | None: + try: + return round(float(val), 1) if val is not None else None + except (TypeError, ValueError): + return None + + +def _looks_like_time_only(s: str) -> bool: + t = s.strip() + if not t or " " in t: + return False + parts = t.split(":") + if len(parts) < 2 or len(parts) > 3: + return False + try: + for p in parts: + int(p) + return True + except ValueError: + return False + + +def _import_activity( + cur, + profile_id: str, + text: str, + delim: str, + has_header: bool, + fm: dict, + tc: dict | None, + error_details: list, + affected_ids: dict, +) -> dict[str, int]: + rows_total = 0 + inserted = 0 + updated = 0 + new_entries = 0 + + for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header): + rows_total += 1 + mapped = build_row_after_mapping(csv_row, fm, tc) + activity_type = mapped.get("activity_type") + if not activity_type or not str(activity_type).strip(): + error_details.append({"row": rows_total, "error": "Trainingsart fehlt"}) + continue + + start_raw = mapped.get("start_time") + date_d = coerce_date(mapped.get("date")) + start_key: str | None = None + if isinstance(start_raw, dt.datetime): + start_key = start_raw.strftime("%Y-%m-%d %H:%M:%S") + if date_d is None: + date_d = start_raw.date() + elif isinstance(start_raw, dt.time): + if date_d is None: + error_details.append( + {"row": rows_total, "error": "Startzeit (Uhrzeit) ohne Datumsspalte"} + ) + continue + start_key = f"{date_d.isoformat()} {start_raw.strftime('%H:%M:%S')}" + elif isinstance(start_raw, str) and start_raw.strip(): + s = start_raw.strip() + if date_d is not None and _looks_like_time_only(s): + start_key = f"{date_d.isoformat()} {s}" + else: + start_key = s + if date_d is None and len(start_key) >= 10: + for fmt in ("%Y-%m-%d", "%d.%m.%Y"): + try: + date_d = dt.datetime.strptime(start_key[:10], fmt).date() + break + except ValueError: + continue + + if date_d is None or not start_key: + error_details.append({"row": rows_total, "error": "Datum/Startzeit fehlt oder ungültig"}) + continue + + end_raw = mapped.get("end_time") + if isinstance(end_raw, dt.datetime): + end_str = end_raw.strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(end_raw, str): + end_str = end_raw.strip() + else: + end_str = "" + + duration_min = mapped.get("duration_min") + if duration_min is not None: + try: + duration_min = round(float(duration_min), 1) + except (TypeError, ValueError): + duration_min = None + + kcal_a = _sf_act(mapped.get("kcal_active")) + kcal_r = _sf_act(mapped.get("kcal_resting")) + hr_a = _sf_act(mapped.get("hr_avg")) + hr_m = _sf_act(mapped.get("hr_max")) + dist = _sf_act(mapped.get("distance_km")) + + wtype = str(activity_type).strip() + training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity( + wtype, profile_id + ) + + iso = date_d.isoformat() + + try: + cur.execute( + """ + SELECT id FROM activity_log + WHERE profile_id = %s AND date = %s AND start_time = %s + """, + (profile_id, iso, start_key), + ) + existing = cur.fetchone() + + if existing: + eid = existing["id"] + cur.execute( + """ + UPDATE activity_log + SET end_time = %s, + activity_type = %s, + duration_min = %s, + kcal_active = %s, + kcal_resting = %s, + hr_avg = %s, + hr_max = %s, + distance_km = %s, + training_type_id = %s, + training_category = %s, + training_subcategory = %s, + source = 'csv' + WHERE id = %s + RETURNING id + """, + ( + end_str or None, + wtype, + duration_min, + kcal_a, + kcal_r, + hr_a, + hr_m, + dist, + training_type_id, + training_category, + training_subcategory, + eid, + ), + ) + row = cur.fetchone() + updated += 1 + if row and row.get("id"): + affected_ids["activity_log"].append(str(row["id"])) + aid = eid + else: + eid = str(uuid.uuid4()) + cur.execute( + """ + INSERT INTO activity_log ( + id, profile_id, date, start_time, end_time, activity_type, duration_min, + kcal_active, kcal_resting, hr_avg, hr_max, distance_km, + source, training_type_id, training_category, training_subcategory, created + ) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'csv',%s,%s,%s,CURRENT_TIMESTAMP) + RETURNING id + """, + ( + eid, + profile_id, + iso, + start_key, + end_str or None, + wtype, + duration_min, + kcal_a, + kcal_r, + hr_a, + hr_m, + dist, + training_type_id, + training_category, + training_subcategory, + ), + ) + row = cur.fetchone() + inserted += 1 + new_entries += 1 + if row and row.get("id"): + affected_ids["activity_log"].append(str(row["id"])) + aid = eid + + if _EVALUATION_AVAILABLE and training_type_id and _evaluate_and_save_activity: + try: + activity_dict = { + "id": aid, + "profile_id": profile_id, + "date": iso, + "training_type_id": training_type_id, + "duration_min": duration_min, + "hr_avg": hr_a, + "hr_max": hr_m, + "distance_km": dist, + "kcal_active": kcal_a, + "kcal_resting": kcal_r, + "rpe": None, + "pace_min_per_km": None, + "cadence": None, + "elevation_gain": None, + } + _evaluate_and_save_activity(cur, aid, activity_dict, training_type_id, profile_id) + except Exception as eval_err: + logger.warning("[csv activity] Auto-Eval fehlgeschlagen: %s", eval_err) + except Exception as e: + error_details.append({"row": rows_total, "error": str(e)}) + + return { + "rows_total": rows_total, + "inserted": inserted, + "updated": updated, + "skipped": 0, + "new_entries": new_entries, + } diff --git a/backend/csv_parser/mapping_suggest.py b/backend/csv_parser/mapping_suggest.py index bc13203..b88104b 100644 --- a/backend/csv_parser/mapping_suggest.py +++ b/backend/csv_parser/mapping_suggest.py @@ -41,7 +41,17 @@ _MODULE_HEADER_ALIASES: dict[str, dict[str, frozenset[str]]] = { "duration_min": frozenset({"dauer", "duration", "min"}), "distance_km": frozenset({"strecke", "distance", "km", "distanz"}), "kcal_active": frozenset({"kcal", "kalorie", "energie", "active"}), - "hr_avg": frozenset({"puls", "heart", "hr", "bpm", "herzfrequenz"}), + "kcal_resting": frozenset({"ruhe", "resting"}), + "hr_avg": frozenset({"puls", "heart", "hr", "bpm", "herzfrequenz", "durchschn"}), + "hr_max": frozenset({"max", "peak"}), + }, + "vitals_baseline": { + "date": frozenset({"datum", "date", "tag", "start", "zeit"}), + "resting_hr": frozenset({"ruhepuls", "resting", "rhr"}), + "hrv": frozenset({"hrv", "variabilit", "vfc"}), + "vo2_max": frozenset({"vo2"}), + "spo2": frozenset({"sauerstoff", "spo2", "oxygen"}), + "respiratory_rate": frozenset({"atem", "respiratory"}), }, } @@ -73,7 +83,22 @@ _DEFAULT_TYPE_CONVERSIONS: dict[str, dict[str, dict[str, Any]]] = { "duration_min": {"type": "duration", "format": "HH:MM:SS", "target_unit": "minutes", "flexible": True}, "distance_km": {"type": "float", "decimal_separator": "auto", "flexible": True}, "kcal_active": {"type": "float", "decimal_separator": "auto", "flexible": True}, + "kcal_resting": {"type": "float", "decimal_separator": "auto", "flexible": True}, "hr_avg": {"type": "int", "flexible": True}, + "hr_max": {"type": "int", "flexible": True}, + }, + "vitals_baseline": { + "date": { + "type": "datetime", + "format": "yyyy-mm-dd HH:MM:SS", + "extract": "date_only", + "flexible": True, + }, + "resting_hr": {"type": "int", "flexible": True}, + "hrv": {"type": "int", "flexible": True}, + "vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": True}, + "spo2": {"type": "int", "flexible": True}, + "respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": True}, }, } @@ -131,6 +156,9 @@ def suggest_field_mappings( Mappt jede CSV-Spalte (Roh-Header als Key) auf DB-Feld oder '-'. Nutzt zuerst eine passende Seed-Vorlage, dann Alias-Heuristik. """ + if module == "sleep": + return {h: "-" for h in headers} + mod = get_module_definition(module) if not mod: return {h: "-" for h in headers} @@ -163,6 +191,9 @@ def build_type_conversions_for_mapping( seed_tc: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """type_conversions nur für zugewiesene Zielfelder; Seed überschreibt Defaults.""" + if module == "sleep": + return {} + defaults = _DEFAULT_TYPE_CONVERSIONS.get(module, {}) out: dict[str, Any] = {} targets = {v for v in field_mappings.values() if v and v not in ("-", "_skip")} diff --git a/backend/csv_parser/module_registry.py b/backend/csv_parser/module_registry.py index fb31104..0f2ba0e 100644 --- a/backend/csv_parser/module_registry.py +++ b/backend/csv_parser/module_registry.py @@ -27,19 +27,39 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = { "activity": { "table": "activity_log", "fields": { - "date": {"type": "date", "required": True}, - "start_time": {"type": "time", "required": False}, - "end_time": {"type": "time", "required": False}, + "date": {"type": "date", "required": False}, + "start_time": {"type": "datetime", "required": False}, + "end_time": {"type": "datetime", "required": False}, "activity_type": {"type": "string", "required": True}, "duration_min": {"type": "float", "required": False, "min": 0}, "kcal_active": {"type": "float", "required": False}, + "kcal_resting": {"type": "float", "required": False}, "distance_km": {"type": "float", "required": False}, "hr_avg": {"type": "float", "required": False, "min": 30, "max": 220}, + "hr_max": {"type": "float", "required": False, "min": 30, "max": 220}, }, "derive_date_from_datetime_field": "start_time", "duplicate_key": ["profile_id", "date", "start_time"], "duplicate_strategy": "update", }, + "sleep": { + "table": "sleep_log", + "fields": {}, + "import_mode": "apple_sleep_aggregate", + }, + "vitals_baseline": { + "table": "vitals_baseline", + "fields": { + "date": {"type": "date", "required": True}, + "resting_hr": {"type": "int", "required": False}, + "hrv": {"type": "int", "required": False}, + "vo2_max": {"type": "float", "required": False}, + "spo2": {"type": "int", "required": False}, + "respiratory_rate": {"type": "float", "required": False}, + }, + "duplicate_key": ["profile_id", "date"], + "duplicate_strategy": "update", + }, "blood_pressure": { "table": "blood_pressure_log", "fields": { @@ -81,6 +101,14 @@ def validate_field_mappings(module: str, field_mappings: dict) -> None: raise ValueError(f"Unbekanntes Modul: {module}") fields = cast(dict, mod["fields"]) allowed = set(fields.keys()) + if not allowed: + for _csv_col, db_field in field_mappings.items(): + if db_field not in ("", None, "-", "_skip"): + raise ValueError( + f"Modul '{module}' nutzt einen Aggregat-Import ohne Spalten-Mapping; " + f"alle Spalten müssen „ignorieren“ sein." + ) + return for _csv_col, db_field in field_mappings.items(): if db_field in ("", None, "-", "_skip"): continue diff --git a/backend/csv_parser/sleep_apple_import.py b/backend/csv_parser/sleep_apple_import.py new file mode 100644 index 0000000..ae22e94 --- /dev/null +++ b/backend/csv_parser/sleep_apple_import.py @@ -0,0 +1,329 @@ +""" +Apple-Health-Schlaf-CSV → sleep_log (für Universal-Import und /api/sleep/import). +Nutzt dieselbe Logik wie der Sleep-Router, ohne HTTPException — arbeitet auf einem übergebenen Cursor. +""" + +from __future__ import annotations + +import csv +import io +import json +import logging +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from typing import Any, Literal + +logger = logging.getLogger(__name__) + + +def _strip_row_keys(row: dict) -> dict: + return {(k or "").strip(): (v.strip() if isinstance(v, str) else v) for k, v in row.items()} + + +def _safe_float(value: Any) -> float | None: + if value is None or value == "": + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, Decimal): + return float(value) + try: + s = str(value).strip().replace(",", ".") + return float(s) + except (ValueError, InvalidOperation): + return None + + +def _parse_apple_sleep_datetime(value: str) -> datetime: + raw = (value or "").strip() + if not raw: + raise ValueError("empty datetime") + fmts = ( + "%Y-%m-%d %H:%M:%S %z", + "%d.%m.%y %H:%M:%S", + "%d.%m.%Y %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ) + last_err = None + for fmt in fmts: + try: + return datetime.strptime(raw, fmt) + except ValueError as e: + last_err = e + raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from last_err + + +def _hr_to_minutes(hours: float | None) -> int: + if hours is None: + return 0 + return int(round(float(hours) * 60)) + + +def detect_apple_sleep_csv_format(fieldnames: list[str] | None) -> Literal["segments", "summary"]: + if not fieldnames: + raise ValueError("CSV enthält keine Spaltenüberschriften.") + fn = {(f or "").strip() for f in fieldnames} + if {"Start", "End", "Duration (hr)", "Value"}.issubset(fn): + return "segments" + if "Start" in fn and "End" in fn and "Total Sleep (hr)" in fn: + return "summary" + raise ValueError( + "Unbekanntes Apple-Health-Schlaf-CSV. Erwartet: Segment-Export oder Schlafanalyse-Zusammenfassung." + ) + + +def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]: + nights_dict: dict[date, dict] = {} + for raw in reader: + row = _strip_row_keys(raw) + start_s = row.get("Start") or "" + end_s = row.get("End") or "" + if not start_s or not end_s: + continue + try: + start_dt = _parse_apple_sleep_datetime(start_s) + end_dt = _parse_apple_sleep_datetime(end_s) + except ValueError: + continue + + dt_key = (row.get("Date/Time") or row.get("Datum/Uhrzeit") or "").strip() + if dt_key: + try: + wake_d = datetime.strptime(dt_key[:10], "%Y-%m-%d").date() + except ValueError: + wake_d = end_dt.date() + else: + wake_d = end_dt.date() + + core_hr = _safe_float(row.get("Core (hr)")) + if core_hr is None: + core_hr = _safe_float(row.get("Light (hr)")) or 0.0 + deep_min = _hr_to_minutes(_safe_float(row.get("Deep (hr)"))) + rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)"))) + light_min = _hr_to_minutes(core_hr) + awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)"))) + nights_dict[wake_d] = { + "bedtime": start_dt, + "wake_time": end_dt, + "segments": [], + "deep_minutes": deep_min, + "rem_minutes": rem_min, + "light_minutes": light_min, + "awake_minutes": awake_min, + } + return nights_dict + + +def _build_nights_from_apple_segments(reader: csv.DictReader, phase_map: dict) -> dict[date, dict]: + segments = [] + for raw in reader: + row = _strip_row_keys(raw) + phase_key = (row.get("Value") or "").strip() + phase_en = phase_map.get(phase_key) + if phase_en is None: + continue + try: + start_dt = _parse_apple_sleep_datetime(row.get("Start") or "") + end_dt = _parse_apple_sleep_datetime(row.get("End") or "") + duration_hr = _safe_float(row.get("Duration (hr)")) + if duration_hr is None: + continue + except (ValueError, TypeError): + continue + duration_min = int(duration_hr * 60) + segments.append({ + "start": start_dt, + "end": end_dt, + "duration_min": duration_min, + "phase": phase_en, + }) + + segments.sort(key=lambda s: s["start"]) + nights = [] + current_night = None + + for seg in segments: + if current_night is None or (seg["start"] - current_night["wake_time"]).total_seconds() > 7200: + current_night = { + "bedtime": seg["start"], + "wake_time": seg["end"], + "segments": [], + "deep_minutes": 0, + "rem_minutes": 0, + "light_minutes": 0, + "awake_minutes": 0, + } + nights.append(current_night) + + current_night["segments"].append(seg) + current_night["wake_time"] = max(current_night["wake_time"], seg["end"]) + current_night["bedtime"] = min(current_night["bedtime"], seg["start"]) + + if seg["phase"] == "deep": + current_night["deep_minutes"] += seg["duration_min"] + elif seg["phase"] == "rem": + current_night["rem_minutes"] += seg["duration_min"] + elif seg["phase"] == "light": + current_night["light_minutes"] += seg["duration_min"] + elif seg["phase"] == "awake": + current_night["awake_minutes"] += seg["duration_min"] + + nights_dict: dict[date, dict] = {} + for night in nights: + wake_date = night["wake_time"].date() + nights_dict[wake_date] = night + return nights_dict + + +def import_apple_sleep_nights(cur, profile_id: str, text: str) -> dict[str, Any]: + """ + Schreibt in sleep_log. Kein conn.commit — Aufrufer rollt Transaktion. + Gibt Statistik im Executor-Format zurück. + """ + csv_text = text.replace("\r\n", "\n").replace("\r", "\n") + if csv_text.startswith("\ufeff"): + csv_text = csv_text[1:] + reader = csv.DictReader(io.StringIO(csv_text)) + fmt = detect_apple_sleep_csv_format(reader.fieldnames) + + phase_map = { + "Kern": "light", + "Core": "light", + "Light": "light", + "REM": "rem", + "Tief": "deep", + "Deep": "deep", + "Wach": "awake", + "Awake": "awake", + "Schlafend": None, + "Asleep": None, + "In Bed": None, + } + + if fmt == "summary": + nights_dict = _build_nights_from_apple_summary(reader) + else: + nights_dict = _build_nights_from_apple_segments(reader, phase_map) + + if not nights_dict: + raise ValueError( + "Keine importierbaren Schlafzeilen gefunden (prüfe Start/Ende und Format)." + ) + + inserted = 0 + updated = 0 + skipped = 0 + error_details: list[dict[str, Any]] = [] + affected_ids: list[str] = [] + + row_hint = 0 + for wake_date, night in nights_dict.items(): + row_hint += 1 + duration_minutes = ( + night["deep_minutes"] + night["rem_minutes"] + night["light_minutes"] + ) + wake_count = sum(1 for seg in night["segments"] if seg["phase"] == "awake") + + sleep_segments = [ + { + "phase": seg["phase"], + "start": seg["start"].isoformat(), + "end": seg["end"].isoformat(), + "duration_min": seg["duration_min"], + } + for seg in night["segments"] + ] + + cur.execute( + """ + SELECT id, source FROM sleep_log + WHERE profile_id = %s AND date = %s + """, + (profile_id, wake_date), + ) + existing = cur.fetchone() + + if existing and existing["source"] == "manual": + skipped += 1 + continue + + try: + if existing: + cur.execute( + """ + UPDATE sleep_log SET + bedtime = %s, + wake_time = %s, + duration_minutes = %s, + wake_count = %s, + deep_minutes = %s, + rem_minutes = %s, + light_minutes = %s, + awake_minutes = %s, + sleep_segments = %s, + source = 'apple_health', + updated_at = CURRENT_TIMESTAMP + WHERE id = %s AND profile_id = %s + RETURNING id + """, + ( + night["bedtime"].time(), + night["wake_time"].time(), + duration_minutes, + wake_count, + night["deep_minutes"], + night["rem_minutes"], + night["light_minutes"], + night["awake_minutes"], + json.dumps(sleep_segments), + existing["id"], + profile_id, + ), + ) + row = cur.fetchone() + updated += 1 + if row and row.get("id"): + affected_ids.append(str(row["id"])) + else: + cur.execute( + """ + INSERT INTO sleep_log ( + profile_id, date, bedtime, wake_time, duration_minutes, + wake_count, deep_minutes, rem_minutes, light_minutes, awake_minutes, + sleep_segments, source, created_at, updated_at + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'apple_health', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + RETURNING id + """, + ( + profile_id, + wake_date, + night["bedtime"].time(), + night["wake_time"].time(), + duration_minutes, + wake_count, + night["deep_minutes"], + night["rem_minutes"], + night["light_minutes"], + night["awake_minutes"], + json.dumps(sleep_segments), + ), + ) + row = cur.fetchone() + inserted += 1 + if row and row.get("id"): + affected_ids.append(str(row["id"])) + except Exception as e: + logger.warning("Sleep import row failed: %s", e) + error_details.append({"row": row_hint, "error": str(e)}) + + return { + "rows_total": len(nights_dict), + "inserted": inserted, + "updated": updated, + "skipped": skipped, + "new_entries": inserted, + "error_details": error_details, + "affected_ids": affected_ids, + } diff --git a/backend/migrations/044_csv_parser_sleep_vitals_templates.sql b/backend/migrations/044_csv_parser_sleep_vitals_templates.sql new file mode 100644 index 0000000..5cf141f --- /dev/null +++ b/backend/migrations/044_csv_parser_sleep_vitals_templates.sql @@ -0,0 +1,155 @@ +-- Migration 044: CSV Parser — System-Vorlagen Schlaf (Apple) + Vitalwerte Baseline (Issue #21) +-- Idempotent wie 043. + +INSERT INTO csv_field_mappings ( + profile_id, is_system, module, mapping_name, description, + column_signature, delimiter, encoding, has_header, + field_mappings, type_conversions +) +SELECT + NULL, + true, + 'sleep', + 'Apple Health Schlaf (Segment-Export)', + 'Apple-Health-Schlaf-CSV mit Phasen-Zeilen (Start, End, Duration (hr), Value). Import läuft aggregiert pro Nacht.', + ARRAY['Start', 'End', 'Duration (hr)', 'Value']::TEXT[], + ',', + 'utf-8', + true, + '{ + "Start": "-", + "End": "-", + "Duration (hr)": "-", + "Value": "-" + }'::JSONB, + '{}'::JSONB +WHERE NOT EXISTS ( + SELECT 1 FROM csv_field_mappings f + WHERE f.is_system AND f.profile_id IS NULL + AND f.module = 'sleep' AND f.mapping_name = 'Apple Health Schlaf (Segment-Export)' +); + +INSERT INTO csv_field_mappings ( + profile_id, is_system, module, mapping_name, description, + column_signature, delimiter, encoding, has_header, + field_mappings, type_conversions +) +SELECT + NULL, + true, + 'sleep', + 'Apple Health Schlaf (Schlafanalyse / Nacht)', + 'Apple-Health-Schlafanalyse mit Nacht-Zusammenfassung (u. a. Total Sleep (hr), Start, End).', + ARRAY['Start', 'End', 'Total Sleep (hr)']::TEXT[], + ',', + 'utf-8', + true, + '{ + "Start": "-", + "End": "-", + "Total Sleep (hr)": "-" + }'::JSONB, + '{}'::JSONB +WHERE NOT EXISTS ( + SELECT 1 FROM csv_field_mappings f + WHERE f.is_system AND f.profile_id IS NULL + AND f.module = 'sleep' AND f.mapping_name = 'Apple Health Schlaf (Schlafanalyse / Nacht)' +); + +INSERT INTO csv_field_mappings ( + profile_id, is_system, module, mapping_name, description, + column_signature, delimiter, encoding, has_header, + field_mappings, type_conversions +) +SELECT + NULL, + true, + 'vitals_baseline', + 'Apple Health Baseline Vitals (English)', + 'Tägliche Baseline-Messungen aus Apple Health (Ruhepuls, HRV, VO2, SpO2, Atemfrequenz).', + ARRAY[ + 'Start', + 'Resting Heart Rate', + 'Heart Rate Variability', + 'VO2 Max', + 'Oxygen Saturation', + 'Respiratory Rate' + ]::TEXT[], + ',', + 'utf-8', + true, + '{ + "Start": "date", + "Resting Heart Rate": "resting_hr", + "Heart Rate Variability": "hrv", + "VO2 Max": "vo2_max", + "Oxygen Saturation": "spo2", + "Respiratory Rate": "respiratory_rate" + }'::JSONB, + '{ + "date": { + "type": "datetime", + "format": "yyyy-mm-dd HH:MM:SS", + "extract": "date_only", + "flexible": true + }, + "resting_hr": {"type": "int", "flexible": true}, + "hrv": {"type": "int", "flexible": true}, + "vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": true}, + "spo2": {"type": "int", "flexible": true}, + "respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": true} + }'::JSONB +WHERE NOT EXISTS ( + SELECT 1 FROM csv_field_mappings f + WHERE f.is_system AND f.profile_id IS NULL + AND f.module = 'vitals_baseline' AND f.mapping_name = 'Apple Health Baseline Vitals (English)' +); + +INSERT INTO csv_field_mappings ( + profile_id, is_system, module, mapping_name, description, + column_signature, delimiter, encoding, has_header, + field_mappings, type_conversions +) +SELECT + NULL, + true, + 'vitals_baseline', + 'Apple Health Baseline Vitals (Deutsch)', + 'Tägliche Baseline-Messungen aus Apple Health (deutsche Spaltenbezeichnungen).', + ARRAY[ + 'Datum/Uhrzeit', + 'Ruhepuls (count/min)', + 'Herzfrequenzvariabilität (ms)', + 'VO2 max (ml/(kg·min))', + 'Blutsauerstoffsättigung (%)', + 'Atemfrequenz (count/min)' + ]::TEXT[], + ',', + 'utf-8', + true, + '{ + "Datum/Uhrzeit": "date", + "Ruhepuls (count/min)": "resting_hr", + "Herzfrequenzvariabilität (ms)": "hrv", + "VO2 max (ml/(kg·min))": "vo2_max", + "Blutsauerstoffsättigung (%)": "spo2", + "Atemfrequenz (count/min)": "respiratory_rate" + }'::JSONB, + '{ + "date": { + "type": "datetime", + "format": "yyyy-mm-dd HH:MM:SS", + "extract": "date_only", + "flexible": true + }, + "resting_hr": {"type": "int", "flexible": true}, + "hrv": {"type": "int", "flexible": true}, + "vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": true}, + "spo2": {"type": "int", "flexible": true}, + "respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": true} + }'::JSONB +WHERE NOT EXISTS ( + SELECT 1 FROM csv_field_mappings f + WHERE f.is_system AND f.profile_id IS NULL + AND f.module = 'vitals_baseline' AND f.mapping_name = 'Apple Health Baseline Vitals (Deutsch)' +); diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..cb97653 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +addopts = -q --tb=short diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..6592570 --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,6 @@ +# Entwicklung / CI: nach requirements.txt installieren +# pip install -r requirements.txt -r requirements-dev.txt +# +# Empfohlene Smoke-Suite (CSV-Import + Parser, ohne DB/Workflow-Integration): +# python -m pytest tests/test_csv_parser_core.py tests/test_csv_import_executor.py tests/test_mapping_suggest.py +pytest>=8.0 diff --git a/backend/routers/csv_import.py b/backend/routers/csv_import.py index 20c23fc..5d1e4d7 100644 --- a/backend/routers/csv_import.py +++ b/backend/routers/csv_import.py @@ -328,6 +328,14 @@ def _check_module_feature_access(pid: str, module: str) -> None: 403, f"Limit erreicht (Gewichtseinträge): {access.get('used')}/{access.get('limit')}", ) + elif module == "activity": + access = check_feature_access(pid, "activity_entries") + log_feature_usage(pid, "activity_entries", access, "csv_universal_import") + if not access["allowed"]: + raise HTTPException( + 403, + f"Limit erreicht (Aktivitätseinträge): {access.get('used')}/{access.get('limit')}", + ) @router.post("/import") @@ -383,12 +391,6 @@ async def csv_import_execute( exec_module = m["module"] resolved_module = exec_module - if exec_module == "activity": - raise HTTPException( - 501, - "Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet " - "(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.", - ) if not get_module_definition(exec_module): raise HTTPException( 400, @@ -492,6 +494,9 @@ async def csv_import_execute( elif resolved_module == "weight": for _ in range(ne): increment_feature_usage(pid, "weight_entries") + elif resolved_module == "activity": + for _ in range(ne): + increment_feature_usage(pid, "activity_entries") return { "success": True, diff --git a/backend/routers/sleep.py b/backend/routers/sleep.py index da5b559..7d864e2 100644 --- a/backend/routers/sleep.py +++ b/backend/routers/sleep.py @@ -8,186 +8,16 @@ Endpoints: """ from fastapi import APIRouter, Depends, HTTPException, UploadFile, File from pydantic import BaseModel -from typing import Literal from datetime import datetime, timedelta, date -from decimal import Decimal, InvalidOperation -import csv -import io -import json +from typing import Literal from auth import require_auth +from csv_parser.sleep_apple_import import import_apple_sleep_nights from db import get_db, get_cursor router = APIRouter(prefix="/api/sleep", tags=["sleep"]) -def _strip_row_keys(row: dict) -> dict: - return {(k or "").strip(): (v.strip() if isinstance(v, str) else v) for k, v in row.items()} - - -def _safe_float(value) -> float | None: - if value is None or value == "": - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, Decimal): - return float(value) - try: - s = str(value).strip().replace(",", ".") - return float(s) - except (ValueError, InvalidOperation): - return None - - -def _parse_apple_sleep_datetime(value: str) -> datetime: - raw = (value or "").strip() - if not raw: - raise ValueError("empty datetime") - fmts = ( - "%Y-%m-%d %H:%M:%S %z", - "%d.%m.%y %H:%M:%S", - "%d.%m.%Y %H:%M:%S", - "%Y-%m-%d %H:%M:%S", - ) - last_err = None - for fmt in fmts: - try: - return datetime.strptime(raw, fmt) - except ValueError as e: - last_err = e - raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from last_err - - -def _hr_to_minutes(hours: float | None) -> int: - if hours is None: - return 0 - return int(round(float(hours) * 60)) - - -def _detect_apple_sleep_csv_format(fieldnames: list[str] | None) -> Literal["segments", "summary"]: - if not fieldnames: - raise HTTPException( - status_code=400, - detail="CSV enthält keine Spaltenüberschriften.", - ) - fn = {(f or "").strip() for f in fieldnames} - if {"Start", "End", "Duration (hr)", "Value"}.issubset(fn): - return "segments" - if "Start" in fn and "End" in fn and "Total Sleep (hr)" in fn: - return "summary" - raise HTTPException( - status_code=400, - detail=( - "Unbekanntes Apple-Health-Schlaf-CSV. " - "Erwartet wird entweder der Segment-Export (Spalten Start, End, Duration (hr), Value) " - "oder die Schlafanalyse mit Nacht-Zusammenfassung (u. a. Total Sleep (hr), Core/Light, Deep, REM)." - ), - ) - - -def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]: - nights_dict: dict[date, dict] = {} - for raw in reader: - row = _strip_row_keys(raw) - start_s = row.get("Start") or "" - end_s = row.get("End") or "" - if not start_s or not end_s: - continue - try: - start_dt = _parse_apple_sleep_datetime(start_s) - end_dt = _parse_apple_sleep_datetime(end_s) - except ValueError: - continue - - dt_key = (row.get("Date/Time") or row.get("Datum/Uhrzeit") or "").strip() - if dt_key: - try: - wake_d = datetime.strptime(dt_key[:10], "%Y-%m-%d").date() - except ValueError: - wake_d = end_dt.date() - else: - wake_d = end_dt.date() - - core_hr = _safe_float(row.get("Core (hr)")) - if core_hr is None: - core_hr = _safe_float(row.get("Light (hr)")) or 0.0 - deep_min = _hr_to_minutes(_safe_float(row.get("Deep (hr)"))) - rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)"))) - light_min = _hr_to_minutes(core_hr) - awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)"))) - nights_dict[wake_d] = { - "bedtime": start_dt, - "wake_time": end_dt, - "segments": [], - "deep_minutes": deep_min, - "rem_minutes": rem_min, - "light_minutes": light_min, - "awake_minutes": awake_min, - } - return nights_dict - - -def _build_nights_from_apple_segments(reader: csv.DictReader, phase_map: dict) -> dict[date, dict]: - segments = [] - for raw in reader: - row = _strip_row_keys(raw) - phase_key = (row.get("Value") or "").strip() - phase_en = phase_map.get(phase_key) - if phase_en is None: - continue - try: - start_dt = _parse_apple_sleep_datetime(row.get("Start") or "") - end_dt = _parse_apple_sleep_datetime(row.get("End") or "") - duration_hr = _safe_float(row.get("Duration (hr)")) - if duration_hr is None: - continue - except (ValueError, TypeError): - continue - duration_min = int(duration_hr * 60) - segments.append({ - "start": start_dt, - "end": end_dt, - "duration_min": duration_min, - "phase": phase_en, - }) - - segments.sort(key=lambda s: s["start"]) - nights = [] - current_night = None - - for seg in segments: - if current_night is None or (seg["start"] - current_night["wake_time"]).total_seconds() > 7200: - current_night = { - "bedtime": seg["start"], - "wake_time": seg["end"], - "segments": [], - "deep_minutes": 0, - "rem_minutes": 0, - "light_minutes": 0, - "awake_minutes": 0, - } - nights.append(current_night) - - current_night["segments"].append(seg) - current_night["wake_time"] = max(current_night["wake_time"], seg["end"]) - current_night["bedtime"] = min(current_night["bedtime"], seg["start"]) - - if seg["phase"] == "deep": - current_night["deep_minutes"] += seg["duration_min"] - elif seg["phase"] == "rem": - current_night["rem_minutes"] += seg["duration_min"] - elif seg["phase"] == "light": - current_night["light_minutes"] += seg["duration_min"] - elif seg["phase"] == "awake": - current_night["awake_minutes"] += seg["duration_min"] - - nights_dict: dict[date, dict] = {} - for night in nights: - wake_date = night["wake_time"].date() - nights_dict[wake_date] = night - return nights_dict - - # ── Models ──────────────────────────────────────────────────────────────────── class SleepCreate(BaseModel): @@ -641,139 +471,25 @@ async def import_apple_health_sleep( pid = session['profile_id'] content = await file.read() - csv_text = content.decode('utf-8-sig') - reader = csv.DictReader(io.StringIO(csv_text)) - fmt = _detect_apple_sleep_csv_format(reader.fieldnames) - - phase_map = { - "Kern": "light", - "Core": "light", - "Light": "light", - "REM": "rem", - "Tief": "deep", - "Deep": "deep", - "Wach": "awake", - "Awake": "awake", - "Schlafend": None, - "Asleep": None, - "In Bed": None, - } - - if fmt == "summary": - nights_dict = _build_nights_from_apple_summary(reader) - else: - nights_dict = _build_nights_from_apple_segments(reader, phase_map) - - if not nights_dict: - raise HTTPException( - status_code=400, - detail="Keine importierbaren Schlafzeilen gefunden (prüfe Start/Ende und Format).", - ) - - # Insert nights - imported = 0 - skipped = 0 + try: + csv_text = content.decode("utf-8-sig") + except UnicodeDecodeError: + csv_text = content.decode("latin-1") with get_db() as conn: cur = get_cursor(conn) - - for date, night in nights_dict.items(): - # Calculate sleep duration (deep + rem + light, WITHOUT awake) - # Note: awake_minutes tracked separately, not part of sleep duration - duration_minutes = ( - night['deep_minutes'] + - night['rem_minutes'] + - night['light_minutes'] - ) - - # Calculate wake_count (number of awake segments) - wake_count = sum(1 for seg in night['segments'] if seg['phase'] == 'awake') - - # Prepare JSONB segments with full datetime - sleep_segments = [ - { - 'phase': seg['phase'], - 'start': seg['start'].isoformat(), # Full datetime: 2026-03-21T22:30:00 - 'end': seg['end'].isoformat(), # Full datetime: 2026-03-21T23:15:00 - 'duration_min': seg['duration_min'] - } - for seg in night['segments'] - ] - - # Check if manual entry exists - do NOT overwrite - cur.execute(""" - SELECT id, source FROM sleep_log - WHERE profile_id = %s AND date = %s - """, (pid, date)) - existing = cur.fetchone() - - if existing and existing['source'] == 'manual': - skipped += 1 - continue # Skip - don't overwrite manual entries - - # Upsert (only if not manual) - # If entry exists and is NOT manual → update - # If entry doesn't exist → insert - if existing: - # Update existing non-manual entry - cur.execute(""" - UPDATE sleep_log SET - bedtime = %s, - wake_time = %s, - duration_minutes = %s, - wake_count = %s, - deep_minutes = %s, - rem_minutes = %s, - light_minutes = %s, - awake_minutes = %s, - sleep_segments = %s, - source = 'apple_health', - updated_at = CURRENT_TIMESTAMP - WHERE id = %s AND profile_id = %s - """, ( - night['bedtime'].time(), - night['wake_time'].time(), - duration_minutes, - wake_count, - night['deep_minutes'], - night['rem_minutes'], - night['light_minutes'], - night['awake_minutes'], - json.dumps(sleep_segments), - existing['id'], - pid - )) - else: - # Insert new entry - cur.execute(""" - INSERT INTO sleep_log ( - profile_id, date, bedtime, wake_time, duration_minutes, - wake_count, deep_minutes, rem_minutes, light_minutes, awake_minutes, - sleep_segments, source, created_at, updated_at - ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'apple_health', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - ) - """, ( - pid, - date, - night['bedtime'].time(), - night['wake_time'].time(), - duration_minutes, - wake_count, - night['deep_minutes'], - night['rem_minutes'], - night['light_minutes'], - night['awake_minutes'], - json.dumps(sleep_segments) - )) - - imported += 1 - + try: + r = import_apple_sleep_nights(cur, pid, csv_text) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) conn.commit() + imported = r["inserted"] + r["updated"] + skipped = r["skipped"] + total = r["rows_total"] return { "imported": imported, "skipped": skipped, - "total_nights": len(nights_dict), - "message": f"{imported} Nächte importiert, {skipped} übersprungen (manuelle Einträge)" + "total_nights": total, + "message": f"{imported} Nächte importiert, {skipped} übersprungen (manuelle Einträge)", } diff --git a/backend/tests/test_csv_import_executor.py b/backend/tests/test_csv_import_executor.py new file mode 100644 index 0000000..b70252a --- /dev/null +++ b/backend/tests/test_csv_import_executor.py @@ -0,0 +1,178 @@ +""" +Smoke-Tests für Universal-CSV-Import (Executor + Apple-Schlaf-Parser). + +Nutzt einen minimalen Fake-Cursor (kein PostgreSQL), damit die Pipelines bei jedem +pytest-Lauf mitlaufen. +""" + +from __future__ import annotations + +import uuid + +import pytest + +from csv_parser.executor import run_universal_csv_import +from csv_parser.sleep_apple_import import detect_apple_sleep_csv_format + + +class _SeqCursor: + """Minimaler Cursor: execute protokolliert; fetchone liefert vorgegebene Sequenz.""" + + def __init__(self, fetch_sequence: list) -> None: + self.executes: list[tuple[str, tuple | None]] = [] + self._fetch = list(fetch_sequence) + + def execute(self, sql: str, params=None) -> None: + self.executes.append((sql, params)) + + def fetchone(self): + if self._fetch: + return self._fetch.pop(0) + return None + + +PID = "00000000-0000-0000-0000-000000000001" + + +def test_detect_apple_sleep_summary_vs_segment(): + assert detect_apple_sleep_csv_format(["Start", "End", "Total Sleep (hr)", "Core (hr)"]) == "summary" + assert detect_apple_sleep_csv_format(["Start", "End", "Duration (hr)", "Value"]) == "segments" + + +def test_run_universal_import_sleep_one_night_inserts(monkeypatch): + """Eine Summary-Zeile → INSERT; SELECT vorher ohne Treffer.""" + text = ( + "Start,End,Total Sleep (hr),Core (hr),Deep (hr),REM (hr),Awake (hr)\n" + "2024-01-15 22:00:00,2024-01-16 06:00:00,8.0,5.0,1.0,1.5,0.5\n" + ) + cur = _SeqCursor( + [ + None, + {"id": 101}, + ] + ) + out = run_universal_csv_import(cur, PID, "sleep", text, "sleep.csv", {}) + assert out["rows_total"] >= 1 + assert out["rows_imported"] >= 1 + assert any("INSERT INTO sleep_log" in q[0] for q in cur.executes) + + +def test_run_universal_import_activity_insert(monkeypatch): + monkeypatch.setattr( + "csv_parser.executor._resolve_training_type_for_activity", + lambda *_a, **_k: (None, None, None), + ) + text = ( + "Workout Type,Start,End,Duration,Distance (km),Active Energy (kcal)\n" + "Running,2024-01-15 08:00:00,2024-01-15 09:00:00,1:00:00,10.0,500\n" + ) + mapping = { + "delimiter": ",", + "has_header": True, + "field_mappings": { + "Workout Type": "activity_type", + "Start": "start_time", + "End": "end_time", + "Duration": "duration_min", + "Distance (km)": "distance_km", + "Active Energy (kcal)": "kcal_active", + }, + "type_conversions": { + "start_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}, + "end_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}, + "duration_min": { + "type": "duration", + "format": "HH:MM:SS", + "target_unit": "minutes", + "flexible": True, + }, + "distance_km": {"type": "float", "decimal_separator": ".", "flexible": True}, + "kcal_active": {"type": "float", "decimal_separator": ".", "flexible": True}, + }, + } + new_id = str(uuid.uuid4()) + cur = _SeqCursor([None, {"id": new_id}]) + out = run_universal_csv_import(cur, PID, "activity", text, "act.csv", mapping) + assert out["rows_imported"] == 1 + assert out["new_entries"] == 1 + assert any("INSERT INTO activity_log" in q[0] for q in cur.executes) + + +def test_run_universal_import_vitals_baseline_upsert_insert_path(): + text = ( + "Start,Resting Heart Rate,Heart Rate Variability,VO2 Max\n" + "2024-01-15 07:00:00,55,45,42.5\n" + ) + mapping = { + "delimiter": ",", + "has_header": True, + "field_mappings": { + "Start": "date", + "Resting Heart Rate": "resting_hr", + "Heart Rate Variability": "hrv", + "VO2 Max": "vo2_max", + }, + "type_conversions": { + "date": { + "type": "datetime", + "format": "yyyy-mm-dd HH:MM:SS", + "extract": "date_only", + "flexible": True, + }, + "resting_hr": {"type": "int", "flexible": True}, + "hrv": {"type": "int", "flexible": True}, + "vo2_max": {"type": "float", "decimal_separator": ".", "flexible": True}, + }, + } + cur = _SeqCursor([{"inserted": True, "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"}]) + out = run_universal_csv_import(cur, PID, "vitals_baseline", text, "v.csv", mapping) + assert out["rows_imported"] == 1 + assert any("INSERT INTO vitals_baseline" in q[0] for q in cur.executes) + + +def test_run_universal_import_activity_garmin_time_plus_date_columns(monkeypatch): + """Datum in eigener Spalte, Uhrzeit wie bei Garmin nur als Uhrzeit.""" + monkeypatch.setattr( + "csv_parser.executor._resolve_training_type_for_activity", + lambda *_a, **_k: (None, None, None), + ) + text = ( + "Activity Type,Date,Time,Duration,Distance,Calories,Avg HR\n" + "Run,2024-01-20,08:30:00,0:45:00,8.0,400,140\n" + ) + mapping = { + "delimiter": ",", + "has_header": True, + "field_mappings": { + "Activity Type": "activity_type", + "Date": "date", + "Time": "start_time", + "Duration": "duration_min", + "Distance": "distance_km", + "Calories": "kcal_active", + "Avg HR": "hr_avg", + }, + "type_conversions": { + "date": {"type": "date", "format": "yyyy-mm-dd", "flexible": True}, + "start_time": {"type": "time", "format": "HH:MM:SS", "flexible": True}, + "duration_min": { + "type": "duration", + "format": "HH:MM:SS", + "target_unit": "minutes", + "flexible": True, + }, + "distance_km": {"type": "float", "decimal_separator": ".", "flexible": True}, + "kcal_active": {"type": "float", "decimal_separator": ".", "flexible": True}, + "hr_avg": {"type": "int", "flexible": True}, + }, + } + new_id = str(uuid.uuid4()) + cur = _SeqCursor([None, {"id": new_id}]) + out = run_universal_csv_import(cur, PID, "activity", text, "garmin.csv", mapping) + assert out["rows_imported"] == 1 + # Duplicate-Key muss Datum + kombinierte Startzeit enthalten + assert any( + params and "2024-01-20 08:30:00" in str(params) + for _sql, params in cur.executes + if params + ) diff --git a/frontend/src/pages/AdminCsvTemplateEditorPage.jsx b/frontend/src/pages/AdminCsvTemplateEditorPage.jsx index 5b03a3a..510bd90 100644 --- a/frontend/src/pages/AdminCsvTemplateEditorPage.jsx +++ b/frontend/src/pages/AdminCsvTemplateEditorPage.jsx @@ -9,6 +9,8 @@ const MODULE_LABEL = { weight: 'Gewicht', blood_pressure: 'Blutdruck', activity: 'Aktivität', + sleep: 'Schlaf', + vitals_baseline: 'Vitalwerte (Baseline)', } function SampleTable({ sampleRows, columns }) { diff --git a/frontend/src/pages/AdminCsvTemplatesPage.jsx b/frontend/src/pages/AdminCsvTemplatesPage.jsx index 0bf4a92..ace9e49 100644 --- a/frontend/src/pages/AdminCsvTemplatesPage.jsx +++ b/frontend/src/pages/AdminCsvTemplatesPage.jsx @@ -8,6 +8,8 @@ const MODULE_LABEL = { weight: 'Gewicht', blood_pressure: 'Blutdruck', activity: 'Aktivität', + sleep: 'Schlaf', + vitals_baseline: 'Vitalwerte (Baseline)', } export default function AdminCsvTemplatesPage() { diff --git a/frontend/src/pages/UniversalCsvImportPage.jsx b/frontend/src/pages/UniversalCsvImportPage.jsx index 9525115..38cb26e 100644 --- a/frontend/src/pages/UniversalCsvImportPage.jsx +++ b/frontend/src/pages/UniversalCsvImportPage.jsx @@ -1,17 +1,26 @@ -import { useState, useMemo } from 'react' +import { useState, useMemo, useRef } from 'react' import { useNavigate } from 'react-router-dom' -import { ArrowLeft, FileSpreadsheet, Loader2 } from 'lucide-react' +import { ArrowLeft, FileSpreadsheet, Loader2, Upload } from 'lucide-react' import { api } from '../utils/api' import { csvPreviewTdStyle } from '../utils/csvPreviewCells' /** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */ -const EXECUTOR_READY = new Set(['nutrition', 'weight', 'blood_pressure']) +const EXECUTOR_READY = new Set([ + 'nutrition', + 'weight', + 'blood_pressure', + 'activity', + 'sleep', + 'vitals_baseline', +]) const MODULE_LABEL = { nutrition: 'Ernährung', weight: 'Gewicht', blood_pressure: 'Blutdruck', activity: 'Aktivität', + sleep: 'Schlaf', + vitals_baseline: 'Vitalwerte (Baseline)', } function mergeMappingChoices(detected, mapData) { @@ -98,7 +107,9 @@ function SampleTable({ sampleRows, columns }) { export default function UniversalCsvImportPage() { const navigate = useNavigate() + const fileInputRef = useRef(null) const [file, setFile] = useState(null) + const [dragActive, setDragActive] = useState(false) const [analyzeResult, setAnalyzeResult] = useState(null) const [mappingChoices, setMappingChoices] = useState([]) const [mappingId, setMappingId] = useState('') @@ -113,6 +124,21 @@ export default function UniversalCsvImportPage() { ) const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module) + const assignCsvFile = (f) => { + if (!f) return + const name = (f.name || '').toLowerCase() + if (!name.endsWith('.csv')) { + setError('Bitte eine .csv-Datei wählen.') + return + } + setFile(f) + setAnalyzeResult(null) + setMappingChoices([]) + setMappingId('') + setSuccess(null) + setError(null) + } + const handleAnalyze = async () => { if (!file) { setError('Bitte eine CSV-Datei wählen') @@ -153,7 +179,7 @@ export default function UniversalCsvImportPage() { return } if (!importAllowed) { - setError('Diese Vorlage kann hier noch nicht importiert werden (z. B. Aktivität → Seite „Aktivität“).') + setError('Diese Vorlage wird vom Universal-Importer noch nicht unterstützt.') return } setLoadingImport(true) @@ -197,19 +223,11 @@ export default function UniversalCsvImportPage() { CSV-Import

- Datei hochladen: Die App vergleicht die Spalten-Struktur mit allen gespeicherten Vorlagen und schlägt - passende Ziele vor (Ernährung, Gewicht, Blutdruck, …). Du bestätigst nur noch die Vorlage — ohne - vorher ein „Modul“ raten zu müssen.{' '} - Später: eine Datei, mehrere Zieltabellen. Trainingseinheiten aktuell weiter unter{' '} - - . + CSV hier ablegen oder die Fläche antippen: Die App vergleicht die Spalten mit gespeicherten Vorlagen + (Ernährung, Gewicht, Blutdruck, Aktivität, Schlaf, Vitalwerte) und schlägt passende Ziele vor. Du + bestätigst die Vorlage — ohne zuerst ein Modell raten zu müssen. Schlaf-Import erwartet das + Apple-Health-Schlaf-CSV (Segment- oder Zusammenfassungs-Export).{' '} + Ausblick: Eine Datei → mehrere Zieltabellen.

{error && ( @@ -243,18 +261,58 @@ export default function UniversalCsvImportPage() {
1. CSV-Datei
{ - setFile(e.target.files?.[0] || null) - setAnalyzeResult(null) - setMappingChoices([]) - setMappingId('') - setSuccess(null) - }} + style={{ position: 'absolute', width: 0, height: 0, opacity: 0, pointerEvents: 'none' }} + onChange={(e) => assignCsvFile(e.target.files?.[0] || null)} /> +