feat(csv-import): Enhance CSV import functionality with new modules and tests
Some checks failed
Deploy Development / deploy (push) Successful in 55s
Build Test / pytest-backend-csv (push) Failing after 1m4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Added support for new CSV import modules: sleep and vitals_baseline, expanding the import capabilities.
- Implemented backend logic for handling CSV imports related to sleep and vitals baseline, including error handling and data processing.
- Updated frontend components to include new modules in the CSV import interface, improving user experience.
- Introduced unit tests for the new import functionalities to ensure reliability and correctness.
- Enhanced existing CSV analysis features to accommodate the new modules, ensuring consistent behavior across the application.
This commit is contained in:
Lars 2026-04-10 07:30:48 +02:00
parent b4cc3cb934
commit 26ab11eb7b
14 changed files with 1261 additions and 340 deletions

View File

@ -5,6 +5,23 @@ on:
branches: [main, develop] branches: [main, develop]
jobs: jobs:
pytest-backend-csv:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: |
pip install -r backend/requirements.txt -r backend/requirements-dev.txt
- name: Pytest — CSV-Import & Parser (Smoke)
run: |
cd backend
python -m pytest tests/test_csv_parser_core.py tests/test_csv_import_executor.py tests/test_mapping_suggest.py -q --tb=short
lint-backend: lint-backend:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

View File

@ -9,10 +9,29 @@ import uuid
from collections import defaultdict from collections import defaultdict
from typing import Any from typing import Any
import logging
from csv_parser.core import iter_csv_dict_rows from csv_parser.core import iter_csv_dict_rows
from csv_parser.module_registry import get_module_definition from csv_parser.module_registry import get_module_definition
from csv_parser.type_converter import build_row_after_mapping from csv_parser.type_converter import build_row_after_mapping
logger = logging.getLogger(__name__)
try:
from evaluation_helper import evaluate_and_save_activity as _evaluate_and_save_activity
_EVALUATION_AVAILABLE = True
except Exception: # pragma: no cover
_evaluate_and_save_activity = None
_EVALUATION_AVAILABLE = False
def _resolve_training_type_for_activity(activity_type: str, profile_id: str):
"""Lazy import — ermöglicht Tests ohne Laden von routers.activity (bcrypt)."""
from routers.activity import get_training_type_for_activity
return get_training_type_for_activity(activity_type, profile_id)
def coerce_date(val: Any) -> dt.date | None: def coerce_date(val: Any) -> dt.date | None:
if val is None: if val is None:
@ -48,6 +67,31 @@ def run_universal_csv_import(
if not mod: if not mod:
raise ValueError(f"Unbekanntes Modul: {module}") raise ValueError(f"Unbekanntes Modul: {module}")
rows_total = 0
error_details: list[dict[str, Any]] = []
affected_ids: dict[str, list[str]] = defaultdict(list)
if module == "sleep":
from csv_parser.sleep_apple_import import import_apple_sleep_nights
try:
r = import_apple_sleep_nights(cur, profile_id, text)
except ValueError as e:
raise ValueError(str(e)) from e
error_details.extend(r.get("error_details") or [])
for sid in r.get("affected_ids") or []:
affected_ids["sleep_log"].append(sid)
return {
"rows_total": r["rows_total"],
"rows_imported": r["inserted"],
"rows_updated": r["updated"],
"rows_skipped": r["skipped"],
"rows_errors": len(error_details),
"error_details": error_details[:50],
"new_entries": r.get("new_entries", r["inserted"]),
"affected_ids": dict(affected_ids),
}
fm = mapping.get("field_mappings") or {} fm = mapping.get("field_mappings") or {}
if isinstance(fm, str): if isinstance(fm, str):
raise ValueError("field_mappings muss ein Objekt sein") raise ValueError("field_mappings muss ein Objekt sein")
@ -58,10 +102,6 @@ def run_universal_csv_import(
delim = mapping.get("delimiter") or "," delim = mapping.get("delimiter") or ","
has_header = mapping.get("has_header", True) has_header = mapping.get("has_header", True)
rows_total = 0
error_details: list[dict[str, Any]] = []
affected_ids: dict[str, list[str]] = defaultdict(list)
if module == "nutrition": if module == "nutrition":
stats = _import_nutrition( stats = _import_nutrition(
cur, cur,
@ -101,6 +141,32 @@ def run_universal_csv_import(
affected_ids, affected_ids,
) )
rows_total = stats.pop("rows_total") rows_total = stats.pop("rows_total")
elif module == "activity":
stats = _import_activity(
cur,
profile_id,
text,
delim,
bool(has_header),
fm,
tc,
error_details,
affected_ids,
)
rows_total = stats.pop("rows_total")
elif module == "vitals_baseline":
stats = _import_vitals_baseline(
cur,
profile_id,
text,
delim,
bool(has_header),
fm,
tc,
error_details,
affected_ids,
)
rows_total = stats.pop("rows_total")
else: else:
raise ValueError(f"Modul '{module}' wird für Universal-Import noch nicht unterstützt") raise ValueError(f"Modul '{module}' wird für Universal-Import noch nicht unterstützt")
@ -375,3 +441,327 @@ def _import_blood_pressure(
"skipped": skipped, "skipped": skipped,
"new_entries": inserted, "new_entries": inserted,
} }
def _v_safe_int(value: Any) -> int | None:
if value is None or value == "":
return None
try:
if isinstance(value, float):
return int(value)
s = str(value).strip()
if "." in s:
return int(float(s))
return int(s)
except (ValueError, TypeError):
return None
def _v_safe_float(value: Any) -> float | None:
if value is None or value == "":
return None
try:
return float(value)
except (ValueError, TypeError):
return None
def _import_vitals_baseline(
cur,
profile_id: str,
text: str,
delim: str,
has_header: bool,
fm: dict,
tc: dict | None,
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
rows_total = 0
inserted = 0
updated = 0
skipped = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
d = coerce_date(mapped.get("date"))
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt"})
continue
rhr = _v_safe_int(mapped.get("resting_hr"))
hrv = _v_safe_int(mapped.get("hrv"))
vo2 = _v_safe_float(mapped.get("vo2_max"))
spo2 = _v_safe_int(mapped.get("spo2"))
resp = _v_safe_float(mapped.get("respiratory_rate"))
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
skipped += 1
continue
iso = d.isoformat()
try:
cur.execute(
"""
INSERT INTO vitals_baseline (
profile_id, date,
resting_hr, hrv, vo2_max, spo2, respiratory_rate,
source
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'csv')
ON CONFLICT (profile_id, date)
DO UPDATE SET
resting_hr = COALESCE(EXCLUDED.resting_hr, vitals_baseline.resting_hr),
hrv = COALESCE(EXCLUDED.hrv, vitals_baseline.hrv),
vo2_max = COALESCE(EXCLUDED.vo2_max, vitals_baseline.vo2_max),
spo2 = COALESCE(EXCLUDED.spo2, vitals_baseline.spo2),
respiratory_rate = COALESCE(EXCLUDED.respiratory_rate, vitals_baseline.respiratory_rate),
updated_at = NOW()
WHERE vitals_baseline.source != 'manual'
RETURNING (xmax = 0) AS inserted, id
""",
(profile_id, iso, rhr, hrv, vo2, spo2, resp),
)
result = cur.fetchone()
if result is None:
skipped += 1
elif result.get("inserted"):
inserted += 1
if result.get("id"):
affected_ids["vitals_baseline"].append(str(result["id"]))
else:
updated += 1
if result.get("id"):
affected_ids["vitals_baseline"].append(str(result["id"]))
except Exception as e:
error_details.append({"row": rows_total, "error": str(e)})
return {
"rows_total": rows_total,
"inserted": inserted,
"updated": updated,
"skipped": skipped,
"new_entries": inserted,
}
def _sf_act(val: Any) -> float | None:
try:
return round(float(val), 1) if val is not None else None
except (TypeError, ValueError):
return None
def _looks_like_time_only(s: str) -> bool:
t = s.strip()
if not t or " " in t:
return False
parts = t.split(":")
if len(parts) < 2 or len(parts) > 3:
return False
try:
for p in parts:
int(p)
return True
except ValueError:
return False
def _import_activity(
cur,
profile_id: str,
text: str,
delim: str,
has_header: bool,
fm: dict,
tc: dict | None,
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
rows_total = 0
inserted = 0
updated = 0
new_entries = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
activity_type = mapped.get("activity_type")
if not activity_type or not str(activity_type).strip():
error_details.append({"row": rows_total, "error": "Trainingsart fehlt"})
continue
start_raw = mapped.get("start_time")
date_d = coerce_date(mapped.get("date"))
start_key: str | None = None
if isinstance(start_raw, dt.datetime):
start_key = start_raw.strftime("%Y-%m-%d %H:%M:%S")
if date_d is None:
date_d = start_raw.date()
elif isinstance(start_raw, dt.time):
if date_d is None:
error_details.append(
{"row": rows_total, "error": "Startzeit (Uhrzeit) ohne Datumsspalte"}
)
continue
start_key = f"{date_d.isoformat()} {start_raw.strftime('%H:%M:%S')}"
elif isinstance(start_raw, str) and start_raw.strip():
s = start_raw.strip()
if date_d is not None and _looks_like_time_only(s):
start_key = f"{date_d.isoformat()} {s}"
else:
start_key = s
if date_d is None and len(start_key) >= 10:
for fmt in ("%Y-%m-%d", "%d.%m.%Y"):
try:
date_d = dt.datetime.strptime(start_key[:10], fmt).date()
break
except ValueError:
continue
if date_d is None or not start_key:
error_details.append({"row": rows_total, "error": "Datum/Startzeit fehlt oder ungültig"})
continue
end_raw = mapped.get("end_time")
if isinstance(end_raw, dt.datetime):
end_str = end_raw.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(end_raw, str):
end_str = end_raw.strip()
else:
end_str = ""
duration_min = mapped.get("duration_min")
if duration_min is not None:
try:
duration_min = round(float(duration_min), 1)
except (TypeError, ValueError):
duration_min = None
kcal_a = _sf_act(mapped.get("kcal_active"))
kcal_r = _sf_act(mapped.get("kcal_resting"))
hr_a = _sf_act(mapped.get("hr_avg"))
hr_m = _sf_act(mapped.get("hr_max"))
dist = _sf_act(mapped.get("distance_km"))
wtype = str(activity_type).strip()
training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity(
wtype, profile_id
)
iso = date_d.isoformat()
try:
cur.execute(
"""
SELECT id FROM activity_log
WHERE profile_id = %s AND date = %s AND start_time = %s
""",
(profile_id, iso, start_key),
)
existing = cur.fetchone()
if existing:
eid = existing["id"]
cur.execute(
"""
UPDATE activity_log
SET end_time = %s,
activity_type = %s,
duration_min = %s,
kcal_active = %s,
kcal_resting = %s,
hr_avg = %s,
hr_max = %s,
distance_km = %s,
training_type_id = %s,
training_category = %s,
training_subcategory = %s,
source = 'csv'
WHERE id = %s
RETURNING id
""",
(
end_str or None,
wtype,
duration_min,
kcal_a,
kcal_r,
hr_a,
hr_m,
dist,
training_type_id,
training_category,
training_subcategory,
eid,
),
)
row = cur.fetchone()
updated += 1
if row and row.get("id"):
affected_ids["activity_log"].append(str(row["id"]))
aid = eid
else:
eid = str(uuid.uuid4())
cur.execute(
"""
INSERT INTO activity_log (
id, profile_id, date, start_time, end_time, activity_type, duration_min,
kcal_active, kcal_resting, hr_avg, hr_max, distance_km,
source, training_type_id, training_category, training_subcategory, created
)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'csv',%s,%s,%s,CURRENT_TIMESTAMP)
RETURNING id
""",
(
eid,
profile_id,
iso,
start_key,
end_str or None,
wtype,
duration_min,
kcal_a,
kcal_r,
hr_a,
hr_m,
dist,
training_type_id,
training_category,
training_subcategory,
),
)
row = cur.fetchone()
inserted += 1
new_entries += 1
if row and row.get("id"):
affected_ids["activity_log"].append(str(row["id"]))
aid = eid
if _EVALUATION_AVAILABLE and training_type_id and _evaluate_and_save_activity:
try:
activity_dict = {
"id": aid,
"profile_id": profile_id,
"date": iso,
"training_type_id": training_type_id,
"duration_min": duration_min,
"hr_avg": hr_a,
"hr_max": hr_m,
"distance_km": dist,
"kcal_active": kcal_a,
"kcal_resting": kcal_r,
"rpe": None,
"pace_min_per_km": None,
"cadence": None,
"elevation_gain": None,
}
_evaluate_and_save_activity(cur, aid, activity_dict, training_type_id, profile_id)
except Exception as eval_err:
logger.warning("[csv activity] Auto-Eval fehlgeschlagen: %s", eval_err)
except Exception as e:
error_details.append({"row": rows_total, "error": str(e)})
return {
"rows_total": rows_total,
"inserted": inserted,
"updated": updated,
"skipped": 0,
"new_entries": new_entries,
}

View File

@ -41,7 +41,17 @@ _MODULE_HEADER_ALIASES: dict[str, dict[str, frozenset[str]]] = {
"duration_min": frozenset({"dauer", "duration", "min"}), "duration_min": frozenset({"dauer", "duration", "min"}),
"distance_km": frozenset({"strecke", "distance", "km", "distanz"}), "distance_km": frozenset({"strecke", "distance", "km", "distanz"}),
"kcal_active": frozenset({"kcal", "kalorie", "energie", "active"}), "kcal_active": frozenset({"kcal", "kalorie", "energie", "active"}),
"hr_avg": frozenset({"puls", "heart", "hr", "bpm", "herzfrequenz"}), "kcal_resting": frozenset({"ruhe", "resting"}),
"hr_avg": frozenset({"puls", "heart", "hr", "bpm", "herzfrequenz", "durchschn"}),
"hr_max": frozenset({"max", "peak"}),
},
"vitals_baseline": {
"date": frozenset({"datum", "date", "tag", "start", "zeit"}),
"resting_hr": frozenset({"ruhepuls", "resting", "rhr"}),
"hrv": frozenset({"hrv", "variabilit", "vfc"}),
"vo2_max": frozenset({"vo2"}),
"spo2": frozenset({"sauerstoff", "spo2", "oxygen"}),
"respiratory_rate": frozenset({"atem", "respiratory"}),
}, },
} }
@ -73,7 +83,22 @@ _DEFAULT_TYPE_CONVERSIONS: dict[str, dict[str, dict[str, Any]]] = {
"duration_min": {"type": "duration", "format": "HH:MM:SS", "target_unit": "minutes", "flexible": True}, "duration_min": {"type": "duration", "format": "HH:MM:SS", "target_unit": "minutes", "flexible": True},
"distance_km": {"type": "float", "decimal_separator": "auto", "flexible": True}, "distance_km": {"type": "float", "decimal_separator": "auto", "flexible": True},
"kcal_active": {"type": "float", "decimal_separator": "auto", "flexible": True}, "kcal_active": {"type": "float", "decimal_separator": "auto", "flexible": True},
"kcal_resting": {"type": "float", "decimal_separator": "auto", "flexible": True},
"hr_avg": {"type": "int", "flexible": True}, "hr_avg": {"type": "int", "flexible": True},
"hr_max": {"type": "int", "flexible": True},
},
"vitals_baseline": {
"date": {
"type": "datetime",
"format": "yyyy-mm-dd HH:MM:SS",
"extract": "date_only",
"flexible": True,
},
"resting_hr": {"type": "int", "flexible": True},
"hrv": {"type": "int", "flexible": True},
"vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": True},
"spo2": {"type": "int", "flexible": True},
"respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": True},
}, },
} }
@ -131,6 +156,9 @@ def suggest_field_mappings(
Mappt jede CSV-Spalte (Roh-Header als Key) auf DB-Feld oder '-'. Mappt jede CSV-Spalte (Roh-Header als Key) auf DB-Feld oder '-'.
Nutzt zuerst eine passende Seed-Vorlage, dann Alias-Heuristik. Nutzt zuerst eine passende Seed-Vorlage, dann Alias-Heuristik.
""" """
if module == "sleep":
return {h: "-" for h in headers}
mod = get_module_definition(module) mod = get_module_definition(module)
if not mod: if not mod:
return {h: "-" for h in headers} return {h: "-" for h in headers}
@ -163,6 +191,9 @@ def build_type_conversions_for_mapping(
seed_tc: Mapping[str, Any] | None = None, seed_tc: Mapping[str, Any] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""type_conversions nur für zugewiesene Zielfelder; Seed überschreibt Defaults.""" """type_conversions nur für zugewiesene Zielfelder; Seed überschreibt Defaults."""
if module == "sleep":
return {}
defaults = _DEFAULT_TYPE_CONVERSIONS.get(module, {}) defaults = _DEFAULT_TYPE_CONVERSIONS.get(module, {})
out: dict[str, Any] = {} out: dict[str, Any] = {}
targets = {v for v in field_mappings.values() if v and v not in ("-", "_skip")} targets = {v for v in field_mappings.values() if v and v not in ("-", "_skip")}

View File

@ -27,19 +27,39 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
"activity": { "activity": {
"table": "activity_log", "table": "activity_log",
"fields": { "fields": {
"date": {"type": "date", "required": True}, "date": {"type": "date", "required": False},
"start_time": {"type": "time", "required": False}, "start_time": {"type": "datetime", "required": False},
"end_time": {"type": "time", "required": False}, "end_time": {"type": "datetime", "required": False},
"activity_type": {"type": "string", "required": True}, "activity_type": {"type": "string", "required": True},
"duration_min": {"type": "float", "required": False, "min": 0}, "duration_min": {"type": "float", "required": False, "min": 0},
"kcal_active": {"type": "float", "required": False}, "kcal_active": {"type": "float", "required": False},
"kcal_resting": {"type": "float", "required": False},
"distance_km": {"type": "float", "required": False}, "distance_km": {"type": "float", "required": False},
"hr_avg": {"type": "float", "required": False, "min": 30, "max": 220}, "hr_avg": {"type": "float", "required": False, "min": 30, "max": 220},
"hr_max": {"type": "float", "required": False, "min": 30, "max": 220},
}, },
"derive_date_from_datetime_field": "start_time", "derive_date_from_datetime_field": "start_time",
"duplicate_key": ["profile_id", "date", "start_time"], "duplicate_key": ["profile_id", "date", "start_time"],
"duplicate_strategy": "update", "duplicate_strategy": "update",
}, },
"sleep": {
"table": "sleep_log",
"fields": {},
"import_mode": "apple_sleep_aggregate",
},
"vitals_baseline": {
"table": "vitals_baseline",
"fields": {
"date": {"type": "date", "required": True},
"resting_hr": {"type": "int", "required": False},
"hrv": {"type": "int", "required": False},
"vo2_max": {"type": "float", "required": False},
"spo2": {"type": "int", "required": False},
"respiratory_rate": {"type": "float", "required": False},
},
"duplicate_key": ["profile_id", "date"],
"duplicate_strategy": "update",
},
"blood_pressure": { "blood_pressure": {
"table": "blood_pressure_log", "table": "blood_pressure_log",
"fields": { "fields": {
@ -81,6 +101,14 @@ def validate_field_mappings(module: str, field_mappings: dict) -> None:
raise ValueError(f"Unbekanntes Modul: {module}") raise ValueError(f"Unbekanntes Modul: {module}")
fields = cast(dict, mod["fields"]) fields = cast(dict, mod["fields"])
allowed = set(fields.keys()) allowed = set(fields.keys())
if not allowed:
for _csv_col, db_field in field_mappings.items():
if db_field not in ("", None, "-", "_skip"):
raise ValueError(
f"Modul '{module}' nutzt einen Aggregat-Import ohne Spalten-Mapping; "
f"alle Spalten müssen „ignorieren“ sein."
)
return
for _csv_col, db_field in field_mappings.items(): for _csv_col, db_field in field_mappings.items():
if db_field in ("", None, "-", "_skip"): if db_field in ("", None, "-", "_skip"):
continue continue

View File

@ -0,0 +1,329 @@
"""
Apple-Health-Schlaf-CSV sleep_log (für Universal-Import und /api/sleep/import).
Nutzt dieselbe Logik wie der Sleep-Router, ohne HTTPException arbeitet auf einem übergebenen Cursor.
"""
from __future__ import annotations
import csv
import io
import json
import logging
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Any, Literal
logger = logging.getLogger(__name__)
def _strip_row_keys(row: dict) -> dict:
return {(k or "").strip(): (v.strip() if isinstance(v, str) else v) for k, v in row.items()}
def _safe_float(value: Any) -> float | None:
if value is None or value == "":
return None
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, Decimal):
return float(value)
try:
s = str(value).strip().replace(",", ".")
return float(s)
except (ValueError, InvalidOperation):
return None
def _parse_apple_sleep_datetime(value: str) -> datetime:
raw = (value or "").strip()
if not raw:
raise ValueError("empty datetime")
fmts = (
"%Y-%m-%d %H:%M:%S %z",
"%d.%m.%y %H:%M:%S",
"%d.%m.%Y %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
)
last_err = None
for fmt in fmts:
try:
return datetime.strptime(raw, fmt)
except ValueError as e:
last_err = e
raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from last_err
def _hr_to_minutes(hours: float | None) -> int:
if hours is None:
return 0
return int(round(float(hours) * 60))
def detect_apple_sleep_csv_format(fieldnames: list[str] | None) -> Literal["segments", "summary"]:
if not fieldnames:
raise ValueError("CSV enthält keine Spaltenüberschriften.")
fn = {(f or "").strip() for f in fieldnames}
if {"Start", "End", "Duration (hr)", "Value"}.issubset(fn):
return "segments"
if "Start" in fn and "End" in fn and "Total Sleep (hr)" in fn:
return "summary"
raise ValueError(
"Unbekanntes Apple-Health-Schlaf-CSV. Erwartet: Segment-Export oder Schlafanalyse-Zusammenfassung."
)
def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]:
nights_dict: dict[date, dict] = {}
for raw in reader:
row = _strip_row_keys(raw)
start_s = row.get("Start") or ""
end_s = row.get("End") or ""
if not start_s or not end_s:
continue
try:
start_dt = _parse_apple_sleep_datetime(start_s)
end_dt = _parse_apple_sleep_datetime(end_s)
except ValueError:
continue
dt_key = (row.get("Date/Time") or row.get("Datum/Uhrzeit") or "").strip()
if dt_key:
try:
wake_d = datetime.strptime(dt_key[:10], "%Y-%m-%d").date()
except ValueError:
wake_d = end_dt.date()
else:
wake_d = end_dt.date()
core_hr = _safe_float(row.get("Core (hr)"))
if core_hr is None:
core_hr = _safe_float(row.get("Light (hr)")) or 0.0
deep_min = _hr_to_minutes(_safe_float(row.get("Deep (hr)")))
rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)")))
light_min = _hr_to_minutes(core_hr)
awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)")))
nights_dict[wake_d] = {
"bedtime": start_dt,
"wake_time": end_dt,
"segments": [],
"deep_minutes": deep_min,
"rem_minutes": rem_min,
"light_minutes": light_min,
"awake_minutes": awake_min,
}
return nights_dict
def _build_nights_from_apple_segments(reader: csv.DictReader, phase_map: dict) -> dict[date, dict]:
segments = []
for raw in reader:
row = _strip_row_keys(raw)
phase_key = (row.get("Value") or "").strip()
phase_en = phase_map.get(phase_key)
if phase_en is None:
continue
try:
start_dt = _parse_apple_sleep_datetime(row.get("Start") or "")
end_dt = _parse_apple_sleep_datetime(row.get("End") or "")
duration_hr = _safe_float(row.get("Duration (hr)"))
if duration_hr is None:
continue
except (ValueError, TypeError):
continue
duration_min = int(duration_hr * 60)
segments.append({
"start": start_dt,
"end": end_dt,
"duration_min": duration_min,
"phase": phase_en,
})
segments.sort(key=lambda s: s["start"])
nights = []
current_night = None
for seg in segments:
if current_night is None or (seg["start"] - current_night["wake_time"]).total_seconds() > 7200:
current_night = {
"bedtime": seg["start"],
"wake_time": seg["end"],
"segments": [],
"deep_minutes": 0,
"rem_minutes": 0,
"light_minutes": 0,
"awake_minutes": 0,
}
nights.append(current_night)
current_night["segments"].append(seg)
current_night["wake_time"] = max(current_night["wake_time"], seg["end"])
current_night["bedtime"] = min(current_night["bedtime"], seg["start"])
if seg["phase"] == "deep":
current_night["deep_minutes"] += seg["duration_min"]
elif seg["phase"] == "rem":
current_night["rem_minutes"] += seg["duration_min"]
elif seg["phase"] == "light":
current_night["light_minutes"] += seg["duration_min"]
elif seg["phase"] == "awake":
current_night["awake_minutes"] += seg["duration_min"]
nights_dict: dict[date, dict] = {}
for night in nights:
wake_date = night["wake_time"].date()
nights_dict[wake_date] = night
return nights_dict
def import_apple_sleep_nights(cur, profile_id: str, text: str) -> dict[str, Any]:
"""
Schreibt in sleep_log. Kein conn.commit Aufrufer rollt Transaktion.
Gibt Statistik im Executor-Format zurück.
"""
csv_text = text.replace("\r\n", "\n").replace("\r", "\n")
if csv_text.startswith("\ufeff"):
csv_text = csv_text[1:]
reader = csv.DictReader(io.StringIO(csv_text))
fmt = detect_apple_sleep_csv_format(reader.fieldnames)
phase_map = {
"Kern": "light",
"Core": "light",
"Light": "light",
"REM": "rem",
"Tief": "deep",
"Deep": "deep",
"Wach": "awake",
"Awake": "awake",
"Schlafend": None,
"Asleep": None,
"In Bed": None,
}
if fmt == "summary":
nights_dict = _build_nights_from_apple_summary(reader)
else:
nights_dict = _build_nights_from_apple_segments(reader, phase_map)
if not nights_dict:
raise ValueError(
"Keine importierbaren Schlafzeilen gefunden (prüfe Start/Ende und Format)."
)
inserted = 0
updated = 0
skipped = 0
error_details: list[dict[str, Any]] = []
affected_ids: list[str] = []
row_hint = 0
for wake_date, night in nights_dict.items():
row_hint += 1
duration_minutes = (
night["deep_minutes"] + night["rem_minutes"] + night["light_minutes"]
)
wake_count = sum(1 for seg in night["segments"] if seg["phase"] == "awake")
sleep_segments = [
{
"phase": seg["phase"],
"start": seg["start"].isoformat(),
"end": seg["end"].isoformat(),
"duration_min": seg["duration_min"],
}
for seg in night["segments"]
]
cur.execute(
"""
SELECT id, source FROM sleep_log
WHERE profile_id = %s AND date = %s
""",
(profile_id, wake_date),
)
existing = cur.fetchone()
if existing and existing["source"] == "manual":
skipped += 1
continue
try:
if existing:
cur.execute(
"""
UPDATE sleep_log SET
bedtime = %s,
wake_time = %s,
duration_minutes = %s,
wake_count = %s,
deep_minutes = %s,
rem_minutes = %s,
light_minutes = %s,
awake_minutes = %s,
sleep_segments = %s,
source = 'apple_health',
updated_at = CURRENT_TIMESTAMP
WHERE id = %s AND profile_id = %s
RETURNING id
""",
(
night["bedtime"].time(),
night["wake_time"].time(),
duration_minutes,
wake_count,
night["deep_minutes"],
night["rem_minutes"],
night["light_minutes"],
night["awake_minutes"],
json.dumps(sleep_segments),
existing["id"],
profile_id,
),
)
row = cur.fetchone()
updated += 1
if row and row.get("id"):
affected_ids.append(str(row["id"]))
else:
cur.execute(
"""
INSERT INTO sleep_log (
profile_id, date, bedtime, wake_time, duration_minutes,
wake_count, deep_minutes, rem_minutes, light_minutes, awake_minutes,
sleep_segments, source, created_at, updated_at
) VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'apple_health', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
)
RETURNING id
""",
(
profile_id,
wake_date,
night["bedtime"].time(),
night["wake_time"].time(),
duration_minutes,
wake_count,
night["deep_minutes"],
night["rem_minutes"],
night["light_minutes"],
night["awake_minutes"],
json.dumps(sleep_segments),
),
)
row = cur.fetchone()
inserted += 1
if row and row.get("id"):
affected_ids.append(str(row["id"]))
except Exception as e:
logger.warning("Sleep import row failed: %s", e)
error_details.append({"row": row_hint, "error": str(e)})
return {
"rows_total": len(nights_dict),
"inserted": inserted,
"updated": updated,
"skipped": skipped,
"new_entries": inserted,
"error_details": error_details,
"affected_ids": affected_ids,
}

View File

@ -0,0 +1,155 @@
-- Migration 044: CSV Parser — System-Vorlagen Schlaf (Apple) + Vitalwerte Baseline (Issue #21)
-- Idempotent wie 043.
INSERT INTO csv_field_mappings (
profile_id, is_system, module, mapping_name, description,
column_signature, delimiter, encoding, has_header,
field_mappings, type_conversions
)
SELECT
NULL,
true,
'sleep',
'Apple Health Schlaf (Segment-Export)',
'Apple-Health-Schlaf-CSV mit Phasen-Zeilen (Start, End, Duration (hr), Value). Import läuft aggregiert pro Nacht.',
ARRAY['Start', 'End', 'Duration (hr)', 'Value']::TEXT[],
',',
'utf-8',
true,
'{
"Start": "-",
"End": "-",
"Duration (hr)": "-",
"Value": "-"
}'::JSONB,
'{}'::JSONB
WHERE NOT EXISTS (
SELECT 1 FROM csv_field_mappings f
WHERE f.is_system AND f.profile_id IS NULL
AND f.module = 'sleep' AND f.mapping_name = 'Apple Health Schlaf (Segment-Export)'
);
INSERT INTO csv_field_mappings (
profile_id, is_system, module, mapping_name, description,
column_signature, delimiter, encoding, has_header,
field_mappings, type_conversions
)
SELECT
NULL,
true,
'sleep',
'Apple Health Schlaf (Schlafanalyse / Nacht)',
'Apple-Health-Schlafanalyse mit Nacht-Zusammenfassung (u. a. Total Sleep (hr), Start, End).',
ARRAY['Start', 'End', 'Total Sleep (hr)']::TEXT[],
',',
'utf-8',
true,
'{
"Start": "-",
"End": "-",
"Total Sleep (hr)": "-"
}'::JSONB,
'{}'::JSONB
WHERE NOT EXISTS (
SELECT 1 FROM csv_field_mappings f
WHERE f.is_system AND f.profile_id IS NULL
AND f.module = 'sleep' AND f.mapping_name = 'Apple Health Schlaf (Schlafanalyse / Nacht)'
);
INSERT INTO csv_field_mappings (
profile_id, is_system, module, mapping_name, description,
column_signature, delimiter, encoding, has_header,
field_mappings, type_conversions
)
SELECT
NULL,
true,
'vitals_baseline',
'Apple Health Baseline Vitals (English)',
'Tägliche Baseline-Messungen aus Apple Health (Ruhepuls, HRV, VO2, SpO2, Atemfrequenz).',
ARRAY[
'Start',
'Resting Heart Rate',
'Heart Rate Variability',
'VO2 Max',
'Oxygen Saturation',
'Respiratory Rate'
]::TEXT[],
',',
'utf-8',
true,
'{
"Start": "date",
"Resting Heart Rate": "resting_hr",
"Heart Rate Variability": "hrv",
"VO2 Max": "vo2_max",
"Oxygen Saturation": "spo2",
"Respiratory Rate": "respiratory_rate"
}'::JSONB,
'{
"date": {
"type": "datetime",
"format": "yyyy-mm-dd HH:MM:SS",
"extract": "date_only",
"flexible": true
},
"resting_hr": {"type": "int", "flexible": true},
"hrv": {"type": "int", "flexible": true},
"vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": true},
"spo2": {"type": "int", "flexible": true},
"respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": true}
}'::JSONB
WHERE NOT EXISTS (
SELECT 1 FROM csv_field_mappings f
WHERE f.is_system AND f.profile_id IS NULL
AND f.module = 'vitals_baseline' AND f.mapping_name = 'Apple Health Baseline Vitals (English)'
);
INSERT INTO csv_field_mappings (
profile_id, is_system, module, mapping_name, description,
column_signature, delimiter, encoding, has_header,
field_mappings, type_conversions
)
SELECT
NULL,
true,
'vitals_baseline',
'Apple Health Baseline Vitals (Deutsch)',
'Tägliche Baseline-Messungen aus Apple Health (deutsche Spaltenbezeichnungen).',
ARRAY[
'Datum/Uhrzeit',
'Ruhepuls (count/min)',
'Herzfrequenzvariabilität (ms)',
'VO2 max (ml/(kg·min))',
'Blutsauerstoffsättigung (%)',
'Atemfrequenz (count/min)'
]::TEXT[],
',',
'utf-8',
true,
'{
"Datum/Uhrzeit": "date",
"Ruhepuls (count/min)": "resting_hr",
"Herzfrequenzvariabilität (ms)": "hrv",
"VO2 max (ml/(kg·min))": "vo2_max",
"Blutsauerstoffsättigung (%)": "spo2",
"Atemfrequenz (count/min)": "respiratory_rate"
}'::JSONB,
'{
"date": {
"type": "datetime",
"format": "yyyy-mm-dd HH:MM:SS",
"extract": "date_only",
"flexible": true
},
"resting_hr": {"type": "int", "flexible": true},
"hrv": {"type": "int", "flexible": true},
"vo2_max": {"type": "float", "decimal_separator": "auto", "flexible": true},
"spo2": {"type": "int", "flexible": true},
"respiratory_rate": {"type": "float", "decimal_separator": "auto", "flexible": true}
}'::JSONB
WHERE NOT EXISTS (
SELECT 1 FROM csv_field_mappings f
WHERE f.is_system AND f.profile_id IS NULL
AND f.module = 'vitals_baseline' AND f.mapping_name = 'Apple Health Baseline Vitals (Deutsch)'
);

5
backend/pytest.ini Normal file
View File

@ -0,0 +1,5 @@
[pytest]
testpaths = tests
python_files = test_*.py
python_functions = test_*
addopts = -q --tb=short

View File

@ -0,0 +1,6 @@
# Entwicklung / CI: nach requirements.txt installieren
# pip install -r requirements.txt -r requirements-dev.txt
#
# Empfohlene Smoke-Suite (CSV-Import + Parser, ohne DB/Workflow-Integration):
# python -m pytest tests/test_csv_parser_core.py tests/test_csv_import_executor.py tests/test_mapping_suggest.py
pytest>=8.0

View File

@ -328,6 +328,14 @@ def _check_module_feature_access(pid: str, module: str) -> None:
403, 403,
f"Limit erreicht (Gewichtseinträge): {access.get('used')}/{access.get('limit')}", f"Limit erreicht (Gewichtseinträge): {access.get('used')}/{access.get('limit')}",
) )
elif module == "activity":
access = check_feature_access(pid, "activity_entries")
log_feature_usage(pid, "activity_entries", access, "csv_universal_import")
if not access["allowed"]:
raise HTTPException(
403,
f"Limit erreicht (Aktivitätseinträge): {access.get('used')}/{access.get('limit')}",
)
@router.post("/import") @router.post("/import")
@ -383,12 +391,6 @@ async def csv_import_execute(
exec_module = m["module"] exec_module = m["module"]
resolved_module = exec_module resolved_module = exec_module
if exec_module == "activity":
raise HTTPException(
501,
"Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet "
"(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.",
)
if not get_module_definition(exec_module): if not get_module_definition(exec_module):
raise HTTPException( raise HTTPException(
400, 400,
@ -492,6 +494,9 @@ async def csv_import_execute(
elif resolved_module == "weight": elif resolved_module == "weight":
for _ in range(ne): for _ in range(ne):
increment_feature_usage(pid, "weight_entries") increment_feature_usage(pid, "weight_entries")
elif resolved_module == "activity":
for _ in range(ne):
increment_feature_usage(pid, "activity_entries")
return { return {
"success": True, "success": True,

View File

@ -8,186 +8,16 @@ Endpoints:
""" """
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
from pydantic import BaseModel from pydantic import BaseModel
from typing import Literal
from datetime import datetime, timedelta, date from datetime import datetime, timedelta, date
from decimal import Decimal, InvalidOperation from typing import Literal
import csv
import io
import json
from auth import require_auth from auth import require_auth
from csv_parser.sleep_apple_import import import_apple_sleep_nights
from db import get_db, get_cursor from db import get_db, get_cursor
router = APIRouter(prefix="/api/sleep", tags=["sleep"]) router = APIRouter(prefix="/api/sleep", tags=["sleep"])
def _strip_row_keys(row: dict) -> dict:
return {(k or "").strip(): (v.strip() if isinstance(v, str) else v) for k, v in row.items()}
def _safe_float(value) -> float | None:
if value is None or value == "":
return None
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, Decimal):
return float(value)
try:
s = str(value).strip().replace(",", ".")
return float(s)
except (ValueError, InvalidOperation):
return None
def _parse_apple_sleep_datetime(value: str) -> datetime:
raw = (value or "").strip()
if not raw:
raise ValueError("empty datetime")
fmts = (
"%Y-%m-%d %H:%M:%S %z",
"%d.%m.%y %H:%M:%S",
"%d.%m.%Y %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
)
last_err = None
for fmt in fmts:
try:
return datetime.strptime(raw, fmt)
except ValueError as e:
last_err = e
raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from last_err
def _hr_to_minutes(hours: float | None) -> int:
if hours is None:
return 0
return int(round(float(hours) * 60))
def _detect_apple_sleep_csv_format(fieldnames: list[str] | None) -> Literal["segments", "summary"]:
if not fieldnames:
raise HTTPException(
status_code=400,
detail="CSV enthält keine Spaltenüberschriften.",
)
fn = {(f or "").strip() for f in fieldnames}
if {"Start", "End", "Duration (hr)", "Value"}.issubset(fn):
return "segments"
if "Start" in fn and "End" in fn and "Total Sleep (hr)" in fn:
return "summary"
raise HTTPException(
status_code=400,
detail=(
"Unbekanntes Apple-Health-Schlaf-CSV. "
"Erwartet wird entweder der Segment-Export (Spalten Start, End, Duration (hr), Value) "
"oder die Schlafanalyse mit Nacht-Zusammenfassung (u. a. Total Sleep (hr), Core/Light, Deep, REM)."
),
)
def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]:
nights_dict: dict[date, dict] = {}
for raw in reader:
row = _strip_row_keys(raw)
start_s = row.get("Start") or ""
end_s = row.get("End") or ""
if not start_s or not end_s:
continue
try:
start_dt = _parse_apple_sleep_datetime(start_s)
end_dt = _parse_apple_sleep_datetime(end_s)
except ValueError:
continue
dt_key = (row.get("Date/Time") or row.get("Datum/Uhrzeit") or "").strip()
if dt_key:
try:
wake_d = datetime.strptime(dt_key[:10], "%Y-%m-%d").date()
except ValueError:
wake_d = end_dt.date()
else:
wake_d = end_dt.date()
core_hr = _safe_float(row.get("Core (hr)"))
if core_hr is None:
core_hr = _safe_float(row.get("Light (hr)")) or 0.0
deep_min = _hr_to_minutes(_safe_float(row.get("Deep (hr)")))
rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)")))
light_min = _hr_to_minutes(core_hr)
awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)")))
nights_dict[wake_d] = {
"bedtime": start_dt,
"wake_time": end_dt,
"segments": [],
"deep_minutes": deep_min,
"rem_minutes": rem_min,
"light_minutes": light_min,
"awake_minutes": awake_min,
}
return nights_dict
def _build_nights_from_apple_segments(reader: csv.DictReader, phase_map: dict) -> dict[date, dict]:
segments = []
for raw in reader:
row = _strip_row_keys(raw)
phase_key = (row.get("Value") or "").strip()
phase_en = phase_map.get(phase_key)
if phase_en is None:
continue
try:
start_dt = _parse_apple_sleep_datetime(row.get("Start") or "")
end_dt = _parse_apple_sleep_datetime(row.get("End") or "")
duration_hr = _safe_float(row.get("Duration (hr)"))
if duration_hr is None:
continue
except (ValueError, TypeError):
continue
duration_min = int(duration_hr * 60)
segments.append({
"start": start_dt,
"end": end_dt,
"duration_min": duration_min,
"phase": phase_en,
})
segments.sort(key=lambda s: s["start"])
nights = []
current_night = None
for seg in segments:
if current_night is None or (seg["start"] - current_night["wake_time"]).total_seconds() > 7200:
current_night = {
"bedtime": seg["start"],
"wake_time": seg["end"],
"segments": [],
"deep_minutes": 0,
"rem_minutes": 0,
"light_minutes": 0,
"awake_minutes": 0,
}
nights.append(current_night)
current_night["segments"].append(seg)
current_night["wake_time"] = max(current_night["wake_time"], seg["end"])
current_night["bedtime"] = min(current_night["bedtime"], seg["start"])
if seg["phase"] == "deep":
current_night["deep_minutes"] += seg["duration_min"]
elif seg["phase"] == "rem":
current_night["rem_minutes"] += seg["duration_min"]
elif seg["phase"] == "light":
current_night["light_minutes"] += seg["duration_min"]
elif seg["phase"] == "awake":
current_night["awake_minutes"] += seg["duration_min"]
nights_dict: dict[date, dict] = {}
for night in nights:
wake_date = night["wake_time"].date()
nights_dict[wake_date] = night
return nights_dict
# ── Models ──────────────────────────────────────────────────────────────────── # ── Models ────────────────────────────────────────────────────────────────────
class SleepCreate(BaseModel): class SleepCreate(BaseModel):
@ -641,139 +471,25 @@ async def import_apple_health_sleep(
pid = session['profile_id'] pid = session['profile_id']
content = await file.read() content = await file.read()
csv_text = content.decode('utf-8-sig') try:
reader = csv.DictReader(io.StringIO(csv_text)) csv_text = content.decode("utf-8-sig")
fmt = _detect_apple_sleep_csv_format(reader.fieldnames) except UnicodeDecodeError:
csv_text = content.decode("latin-1")
phase_map = {
"Kern": "light",
"Core": "light",
"Light": "light",
"REM": "rem",
"Tief": "deep",
"Deep": "deep",
"Wach": "awake",
"Awake": "awake",
"Schlafend": None,
"Asleep": None,
"In Bed": None,
}
if fmt == "summary":
nights_dict = _build_nights_from_apple_summary(reader)
else:
nights_dict = _build_nights_from_apple_segments(reader, phase_map)
if not nights_dict:
raise HTTPException(
status_code=400,
detail="Keine importierbaren Schlafzeilen gefunden (prüfe Start/Ende und Format).",
)
# Insert nights
imported = 0
skipped = 0
with get_db() as conn: with get_db() as conn:
cur = get_cursor(conn) cur = get_cursor(conn)
try:
for date, night in nights_dict.items(): r = import_apple_sleep_nights(cur, pid, csv_text)
# Calculate sleep duration (deep + rem + light, WITHOUT awake) except ValueError as e:
# Note: awake_minutes tracked separately, not part of sleep duration raise HTTPException(status_code=400, detail=str(e))
duration_minutes = (
night['deep_minutes'] +
night['rem_minutes'] +
night['light_minutes']
)
# Calculate wake_count (number of awake segments)
wake_count = sum(1 for seg in night['segments'] if seg['phase'] == 'awake')
# Prepare JSONB segments with full datetime
sleep_segments = [
{
'phase': seg['phase'],
'start': seg['start'].isoformat(), # Full datetime: 2026-03-21T22:30:00
'end': seg['end'].isoformat(), # Full datetime: 2026-03-21T23:15:00
'duration_min': seg['duration_min']
}
for seg in night['segments']
]
# Check if manual entry exists - do NOT overwrite
cur.execute("""
SELECT id, source FROM sleep_log
WHERE profile_id = %s AND date = %s
""", (pid, date))
existing = cur.fetchone()
if existing and existing['source'] == 'manual':
skipped += 1
continue # Skip - don't overwrite manual entries
# Upsert (only if not manual)
# If entry exists and is NOT manual → update
# If entry doesn't exist → insert
if existing:
# Update existing non-manual entry
cur.execute("""
UPDATE sleep_log SET
bedtime = %s,
wake_time = %s,
duration_minutes = %s,
wake_count = %s,
deep_minutes = %s,
rem_minutes = %s,
light_minutes = %s,
awake_minutes = %s,
sleep_segments = %s,
source = 'apple_health',
updated_at = CURRENT_TIMESTAMP
WHERE id = %s AND profile_id = %s
""", (
night['bedtime'].time(),
night['wake_time'].time(),
duration_minutes,
wake_count,
night['deep_minutes'],
night['rem_minutes'],
night['light_minutes'],
night['awake_minutes'],
json.dumps(sleep_segments),
existing['id'],
pid
))
else:
# Insert new entry
cur.execute("""
INSERT INTO sleep_log (
profile_id, date, bedtime, wake_time, duration_minutes,
wake_count, deep_minutes, rem_minutes, light_minutes, awake_minutes,
sleep_segments, source, created_at, updated_at
) VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'apple_health', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
)
""", (
pid,
date,
night['bedtime'].time(),
night['wake_time'].time(),
duration_minutes,
wake_count,
night['deep_minutes'],
night['rem_minutes'],
night['light_minutes'],
night['awake_minutes'],
json.dumps(sleep_segments)
))
imported += 1
conn.commit() conn.commit()
imported = r["inserted"] + r["updated"]
skipped = r["skipped"]
total = r["rows_total"]
return { return {
"imported": imported, "imported": imported,
"skipped": skipped, "skipped": skipped,
"total_nights": len(nights_dict), "total_nights": total,
"message": f"{imported} Nächte importiert, {skipped} übersprungen (manuelle Einträge)" "message": f"{imported} Nächte importiert, {skipped} übersprungen (manuelle Einträge)",
} }

View File

@ -0,0 +1,178 @@
"""
Smoke-Tests für Universal-CSV-Import (Executor + Apple-Schlaf-Parser).
Nutzt einen minimalen Fake-Cursor (kein PostgreSQL), damit die Pipelines bei jedem
pytest-Lauf mitlaufen.
"""
from __future__ import annotations
import uuid
import pytest
from csv_parser.executor import run_universal_csv_import
from csv_parser.sleep_apple_import import detect_apple_sleep_csv_format
class _SeqCursor:
"""Minimaler Cursor: execute protokolliert; fetchone liefert vorgegebene Sequenz."""
def __init__(self, fetch_sequence: list) -> None:
self.executes: list[tuple[str, tuple | None]] = []
self._fetch = list(fetch_sequence)
def execute(self, sql: str, params=None) -> None:
self.executes.append((sql, params))
def fetchone(self):
if self._fetch:
return self._fetch.pop(0)
return None
PID = "00000000-0000-0000-0000-000000000001"
def test_detect_apple_sleep_summary_vs_segment():
assert detect_apple_sleep_csv_format(["Start", "End", "Total Sleep (hr)", "Core (hr)"]) == "summary"
assert detect_apple_sleep_csv_format(["Start", "End", "Duration (hr)", "Value"]) == "segments"
def test_run_universal_import_sleep_one_night_inserts(monkeypatch):
"""Eine Summary-Zeile → INSERT; SELECT vorher ohne Treffer."""
text = (
"Start,End,Total Sleep (hr),Core (hr),Deep (hr),REM (hr),Awake (hr)\n"
"2024-01-15 22:00:00,2024-01-16 06:00:00,8.0,5.0,1.0,1.5,0.5\n"
)
cur = _SeqCursor(
[
None,
{"id": 101},
]
)
out = run_universal_csv_import(cur, PID, "sleep", text, "sleep.csv", {})
assert out["rows_total"] >= 1
assert out["rows_imported"] >= 1
assert any("INSERT INTO sleep_log" in q[0] for q in cur.executes)
def test_run_universal_import_activity_insert(monkeypatch):
monkeypatch.setattr(
"csv_parser.executor._resolve_training_type_for_activity",
lambda *_a, **_k: (None, None, None),
)
text = (
"Workout Type,Start,End,Duration,Distance (km),Active Energy (kcal)\n"
"Running,2024-01-15 08:00:00,2024-01-15 09:00:00,1:00:00,10.0,500\n"
)
mapping = {
"delimiter": ",",
"has_header": True,
"field_mappings": {
"Workout Type": "activity_type",
"Start": "start_time",
"End": "end_time",
"Duration": "duration_min",
"Distance (km)": "distance_km",
"Active Energy (kcal)": "kcal_active",
},
"type_conversions": {
"start_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True},
"end_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True},
"duration_min": {
"type": "duration",
"format": "HH:MM:SS",
"target_unit": "minutes",
"flexible": True,
},
"distance_km": {"type": "float", "decimal_separator": ".", "flexible": True},
"kcal_active": {"type": "float", "decimal_separator": ".", "flexible": True},
},
}
new_id = str(uuid.uuid4())
cur = _SeqCursor([None, {"id": new_id}])
out = run_universal_csv_import(cur, PID, "activity", text, "act.csv", mapping)
assert out["rows_imported"] == 1
assert out["new_entries"] == 1
assert any("INSERT INTO activity_log" in q[0] for q in cur.executes)
def test_run_universal_import_vitals_baseline_upsert_insert_path():
text = (
"Start,Resting Heart Rate,Heart Rate Variability,VO2 Max\n"
"2024-01-15 07:00:00,55,45,42.5\n"
)
mapping = {
"delimiter": ",",
"has_header": True,
"field_mappings": {
"Start": "date",
"Resting Heart Rate": "resting_hr",
"Heart Rate Variability": "hrv",
"VO2 Max": "vo2_max",
},
"type_conversions": {
"date": {
"type": "datetime",
"format": "yyyy-mm-dd HH:MM:SS",
"extract": "date_only",
"flexible": True,
},
"resting_hr": {"type": "int", "flexible": True},
"hrv": {"type": "int", "flexible": True},
"vo2_max": {"type": "float", "decimal_separator": ".", "flexible": True},
},
}
cur = _SeqCursor([{"inserted": True, "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"}])
out = run_universal_csv_import(cur, PID, "vitals_baseline", text, "v.csv", mapping)
assert out["rows_imported"] == 1
assert any("INSERT INTO vitals_baseline" in q[0] for q in cur.executes)
def test_run_universal_import_activity_garmin_time_plus_date_columns(monkeypatch):
"""Datum in eigener Spalte, Uhrzeit wie bei Garmin nur als Uhrzeit."""
monkeypatch.setattr(
"csv_parser.executor._resolve_training_type_for_activity",
lambda *_a, **_k: (None, None, None),
)
text = (
"Activity Type,Date,Time,Duration,Distance,Calories,Avg HR\n"
"Run,2024-01-20,08:30:00,0:45:00,8.0,400,140\n"
)
mapping = {
"delimiter": ",",
"has_header": True,
"field_mappings": {
"Activity Type": "activity_type",
"Date": "date",
"Time": "start_time",
"Duration": "duration_min",
"Distance": "distance_km",
"Calories": "kcal_active",
"Avg HR": "hr_avg",
},
"type_conversions": {
"date": {"type": "date", "format": "yyyy-mm-dd", "flexible": True},
"start_time": {"type": "time", "format": "HH:MM:SS", "flexible": True},
"duration_min": {
"type": "duration",
"format": "HH:MM:SS",
"target_unit": "minutes",
"flexible": True,
},
"distance_km": {"type": "float", "decimal_separator": ".", "flexible": True},
"kcal_active": {"type": "float", "decimal_separator": ".", "flexible": True},
"hr_avg": {"type": "int", "flexible": True},
},
}
new_id = str(uuid.uuid4())
cur = _SeqCursor([None, {"id": new_id}])
out = run_universal_csv_import(cur, PID, "activity", text, "garmin.csv", mapping)
assert out["rows_imported"] == 1
# Duplicate-Key muss Datum + kombinierte Startzeit enthalten
assert any(
params and "2024-01-20 08:30:00" in str(params)
for _sql, params in cur.executes
if params
)

View File

@ -9,6 +9,8 @@ const MODULE_LABEL = {
weight: 'Gewicht', weight: 'Gewicht',
blood_pressure: 'Blutdruck', blood_pressure: 'Blutdruck',
activity: 'Aktivität', activity: 'Aktivität',
sleep: 'Schlaf',
vitals_baseline: 'Vitalwerte (Baseline)',
} }
function SampleTable({ sampleRows, columns }) { function SampleTable({ sampleRows, columns }) {

View File

@ -8,6 +8,8 @@ const MODULE_LABEL = {
weight: 'Gewicht', weight: 'Gewicht',
blood_pressure: 'Blutdruck', blood_pressure: 'Blutdruck',
activity: 'Aktivität', activity: 'Aktivität',
sleep: 'Schlaf',
vitals_baseline: 'Vitalwerte (Baseline)',
} }
export default function AdminCsvTemplatesPage() { export default function AdminCsvTemplatesPage() {

View File

@ -1,17 +1,26 @@
import { useState, useMemo } from 'react' import { useState, useMemo, useRef } from 'react'
import { useNavigate } from 'react-router-dom' import { useNavigate } from 'react-router-dom'
import { ArrowLeft, FileSpreadsheet, Loader2 } from 'lucide-react' import { ArrowLeft, FileSpreadsheet, Loader2, Upload } from 'lucide-react'
import { api } from '../utils/api' import { api } from '../utils/api'
import { csvPreviewTdStyle } from '../utils/csvPreviewCells' import { csvPreviewTdStyle } from '../utils/csvPreviewCells'
/** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */ /** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */
const EXECUTOR_READY = new Set(['nutrition', 'weight', 'blood_pressure']) const EXECUTOR_READY = new Set([
'nutrition',
'weight',
'blood_pressure',
'activity',
'sleep',
'vitals_baseline',
])
const MODULE_LABEL = { const MODULE_LABEL = {
nutrition: 'Ernährung', nutrition: 'Ernährung',
weight: 'Gewicht', weight: 'Gewicht',
blood_pressure: 'Blutdruck', blood_pressure: 'Blutdruck',
activity: 'Aktivität', activity: 'Aktivität',
sleep: 'Schlaf',
vitals_baseline: 'Vitalwerte (Baseline)',
} }
function mergeMappingChoices(detected, mapData) { function mergeMappingChoices(detected, mapData) {
@ -98,7 +107,9 @@ function SampleTable({ sampleRows, columns }) {
export default function UniversalCsvImportPage() { export default function UniversalCsvImportPage() {
const navigate = useNavigate() const navigate = useNavigate()
const fileInputRef = useRef(null)
const [file, setFile] = useState(null) const [file, setFile] = useState(null)
const [dragActive, setDragActive] = useState(false)
const [analyzeResult, setAnalyzeResult] = useState(null) const [analyzeResult, setAnalyzeResult] = useState(null)
const [mappingChoices, setMappingChoices] = useState([]) const [mappingChoices, setMappingChoices] = useState([])
const [mappingId, setMappingId] = useState('') const [mappingId, setMappingId] = useState('')
@ -113,6 +124,21 @@ export default function UniversalCsvImportPage() {
) )
const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module) const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module)
const assignCsvFile = (f) => {
if (!f) return
const name = (f.name || '').toLowerCase()
if (!name.endsWith('.csv')) {
setError('Bitte eine .csv-Datei wählen.')
return
}
setFile(f)
setAnalyzeResult(null)
setMappingChoices([])
setMappingId('')
setSuccess(null)
setError(null)
}
const handleAnalyze = async () => { const handleAnalyze = async () => {
if (!file) { if (!file) {
setError('Bitte eine CSV-Datei wählen') setError('Bitte eine CSV-Datei wählen')
@ -153,7 +179,7 @@ export default function UniversalCsvImportPage() {
return return
} }
if (!importAllowed) { if (!importAllowed) {
setError('Diese Vorlage kann hier noch nicht importiert werden (z.B. Aktivität → Seite „Aktivität“).') setError('Diese Vorlage wird vom Universal-Importer noch nicht unterstützt.')
return return
} }
setLoadingImport(true) setLoadingImport(true)
@ -197,19 +223,11 @@ export default function UniversalCsvImportPage() {
CSV-Import CSV-Import
</h1> </h1>
<p style={{ fontSize: 14, color: 'var(--text2)', marginBottom: 20, lineHeight: 1.55 }}> <p style={{ fontSize: 14, color: 'var(--text2)', marginBottom: 20, lineHeight: 1.55 }}>
Datei hochladen: Die App vergleicht die Spalten-Struktur mit allen gespeicherten Vorlagen und schlägt CSV hier ablegen oder die Fläche antippen: Die App vergleicht die Spalten mit gespeicherten Vorlagen
passende Ziele vor (Ernährung, Gewicht, Blutdruck, ). Du bestätigst nur noch die Vorlage ohne (Ernährung, Gewicht, Blutdruck, Aktivität, Schlaf, Vitalwerte) und schlägt passende Ziele vor. Du
vorher ein Modul raten zu müssen.{' '} bestätigst die Vorlage ohne zuerst ein Modell raten zu müssen. Schlaf-Import erwartet das
<strong>Später:</strong> eine Datei, mehrere Zieltabellen. Trainingseinheiten aktuell weiter unter{' '} Apple-Health-Schlaf-CSV (Segment- oder Zusammenfassungs-Export).{' '}
<button <strong>Ausblick:</strong> Eine Datei mehrere Zieltabellen.
type="button"
className="btn btn-secondary"
style={{ padding: '2px 8px', fontSize: 13 }}
onClick={() => navigate('/activity')}
>
Aktivität
</button>
.
</p> </p>
{error && ( {error && (
@ -243,18 +261,58 @@ export default function UniversalCsvImportPage() {
<div className="card" style={{ marginBottom: 16, padding: 16 }}> <div className="card" style={{ marginBottom: 16, padding: 16 }}>
<div className="form-label">1. CSV-Datei</div> <div className="form-label">1. CSV-Datei</div>
<input <input
ref={fileInputRef}
type="file" type="file"
accept=".csv,text/csv" accept=".csv,text/csv"
className="form-input" className="form-input"
style={{ marginTop: 8, width: '100%' }} style={{ position: 'absolute', width: 0, height: 0, opacity: 0, pointerEvents: 'none' }}
onChange={(e) => { onChange={(e) => assignCsvFile(e.target.files?.[0] || null)}
setFile(e.target.files?.[0] || null)
setAnalyzeResult(null)
setMappingChoices([])
setMappingId('')
setSuccess(null)
}}
/> />
<button
type="button"
onClick={() => fileInputRef.current?.click()}
onDragEnter={(e) => {
e.preventDefault()
setDragActive(true)
}}
onDragOver={(e) => {
e.preventDefault()
setDragActive(true)
}}
onDragLeave={(e) => {
e.preventDefault()
if (!e.currentTarget.contains(e.relatedTarget)) setDragActive(false)
}}
onDrop={(e) => {
e.preventDefault()
setDragActive(false)
assignCsvFile(e.dataTransfer?.files?.[0] || null)
}}
className="btn btn-secondary"
style={{
marginTop: 8,
width: '100%',
minHeight: 120,
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
gap: 10,
border: `2px dashed ${dragActive ? 'var(--accent)' : 'var(--border)'}`,
borderRadius: 12,
background: dragActive ? 'var(--surface2)' : 'var(--surface)',
cursor: 'pointer',
padding: 16,
}}
>
<Upload size={28} strokeWidth={1.75} color="var(--accent)" />
<span style={{ fontSize: 15, color: 'var(--text1)', fontWeight: 600 }}>
Datei ablegen oder tippen zum Auswählen
</span>
<span style={{ fontSize: 13, color: 'var(--text3)' }}>
{file ? file.name : 'Noch keine Datei gewählt'}
</span>
</button>
<button <button
type="button" type="button"
className="btn btn-primary" className="btn btn-primary"
@ -367,8 +425,7 @@ export default function UniversalCsvImportPage() {
{selectedChoice && !importAllowed && ( {selectedChoice && !importAllowed && (
<p style={{ fontSize: 13, color: 'var(--text3)', marginTop: 10 }}> <p style={{ fontSize: 13, color: 'var(--text3)', marginTop: 10 }}>
Diese Vorlage passt strukturell, wird aber an der passenden Stelle importiert (z.B. Aktivität Diese Vorlage passt strukturell; dieser Import-Weg unterstützt das Zielmodul noch nicht.
Apple-Health-CSV dort).
</p> </p>
)} )}