feat(csv-import): Enhance CSV import functionality with new modules and tests
Deploy Development / deploy (push) Successful in 55s
Build Test / pytest-backend-csv (push) Failing after 1m4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Added support for new CSV import modules: sleep and vitals_baseline, expanding the import capabilities.
- Implemented backend logic for handling CSV imports related to sleep and vitals baseline, including error handling and data processing.
- Updated frontend components to include new modules in the CSV import interface, improving user experience.
- Introduced unit tests for the new import functionalities to ensure reliability and correctness.
- Enhanced existing CSV analysis features to accommodate the new modules, ensuring consistent behavior across the application.
This commit is contained in:
Lars committed 2026-04-10 07:30:48 +02:00
1 parent b4cc3cb934
commit 26ab11eb7b
14 files changed
+1261 -340

No files matched your search

+394 -4
View File
@@ -9,10 +9,29 @@ import uuid
from collections import defaultdict
from typing import Any
import logging
from csv_parser.core import iter_csv_dict_rows
from csv_parser.module_registry import get_module_definition
from csv_parser.type_converter import build_row_after_mapping
logger = logging.getLogger(__name__)
try:
from evaluation_helper import evaluate_and_save_activity as _evaluate_and_save_activity
_EVALUATION_AVAILABLE = True
except Exception: # pragma: no cover
_evaluate_and_save_activity = None
_EVALUATION_AVAILABLE = False
def _resolve_training_type_for_activity(activity_type: str, profile_id: str):
"""Lazy import — ermöglicht Tests ohne Laden von routers.activity (bcrypt)."""
from routers.activity import get_training_type_for_activity
return get_training_type_for_activity(activity_type, profile_id)
def coerce_date(val: Any) -> dt.date | None:
if val is None:
@@ -48,6 +67,31 @@ def run_universal_csv_import(
if not mod:
raise ValueError(f"Unbekanntes Modul: {module}")
rows_total = 0
error_details: list[dict[str, Any]] = []
affected_ids: dict[str, list[str]] = defaultdict(list)
if module == "sleep":
from csv_parser.sleep_apple_import import import_apple_sleep_nights
try:
r = import_apple_sleep_nights(cur, profile_id, text)
except ValueError as e:
raise ValueError(str(e)) from e
error_details.extend(r.get("error_details") or [])
for sid in r.get("affected_ids") or []:
affected_ids["sleep_log"].append(sid)
return {
"rows_total": r["rows_total"],
"rows_imported": r["inserted"],
"rows_updated": r["updated"],
"rows_skipped": r["skipped"],
"rows_errors": len(error_details),
"error_details": error_details[:50],
"new_entries": r.get("new_entries", r["inserted"]),
"affected_ids": dict(affected_ids),
}
fm = mapping.get("field_mappings") or {}
if isinstance(fm, str):
raise ValueError("field_mappings muss ein Objekt sein")
@@ -58,10 +102,6 @@ def run_universal_csv_import(
delim = mapping.get("delimiter") or ","
has_header = mapping.get("has_header", True)
rows_total = 0
error_details: list[dict[str, Any]] = []
affected_ids: dict[str, list[str]] = defaultdict(list)
if module == "nutrition":
stats = _import_nutrition(
cur,
@@ -101,6 +141,32 @@ def run_universal_csv_import(
affected_ids,
)
rows_total = stats.pop("rows_total")
elif module == "activity":
stats = _import_activity(
cur,
profile_id,
text,
delim,
bool(has_header),
fm,
tc,
error_details,
affected_ids,
)
rows_total = stats.pop("rows_total")
elif module == "vitals_baseline":
stats = _import_vitals_baseline(
cur,
profile_id,
text,
delim,
bool(has_header),
fm,
tc,
error_details,
affected_ids,
)
rows_total = stats.pop("rows_total")
else:
raise ValueError(f"Modul '{module}' wird für Universal-Import noch nicht unterstützt")
@@ -375,3 +441,327 @@ def _import_blood_pressure(
"skipped": skipped,
"new_entries": inserted,
}
def _v_safe_int(value: Any) -> int | None:
if value is None or value == "":
return None
try:
if isinstance(value, float):
return int(value)
s = str(value).strip()
if "." in s:
return int(float(s))
return int(s)
except (ValueError, TypeError):
return None
def _v_safe_float(value: Any) -> float | None:
if value is None or value == "":
return None
try:
return float(value)
except (ValueError, TypeError):
return None
def _import_vitals_baseline(
cur,
profile_id: str,
text: str,
delim: str,
has_header: bool,
fm: dict,
tc: dict | None,
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
rows_total = 0
inserted = 0
updated = 0
skipped = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
d = coerce_date(mapped.get("date"))
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt"})
continue
rhr = _v_safe_int(mapped.get("resting_hr"))
hrv = _v_safe_int(mapped.get("hrv"))
vo2 = _v_safe_float(mapped.get("vo2_max"))
spo2 = _v_safe_int(mapped.get("spo2"))
resp = _v_safe_float(mapped.get("respiratory_rate"))
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
skipped += 1
continue
iso = d.isoformat()
try:
cur.execute(
"""
INSERT INTO vitals_baseline (
profile_id, date,
resting_hr, hrv, vo2_max, spo2, respiratory_rate,
source
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'csv')
ON CONFLICT (profile_id, date)
DO UPDATE SET
resting_hr = COALESCE(EXCLUDED.resting_hr, vitals_baseline.resting_hr),
hrv = COALESCE(EXCLUDED.hrv, vitals_baseline.hrv),
vo2_max = COALESCE(EXCLUDED.vo2_max, vitals_baseline.vo2_max),
spo2 = COALESCE(EXCLUDED.spo2, vitals_baseline.spo2),
respiratory_rate = COALESCE(EXCLUDED.respiratory_rate, vitals_baseline.respiratory_rate),
updated_at = NOW()
WHERE vitals_baseline.source != 'manual'
RETURNING (xmax = 0) AS inserted, id
""",
(profile_id, iso, rhr, hrv, vo2, spo2, resp),
)
result = cur.fetchone()
if result is None:
skipped += 1
elif result.get("inserted"):
inserted += 1
if result.get("id"):
affected_ids["vitals_baseline"].append(str(result["id"]))
else:
updated += 1
if result.get("id"):
affected_ids["vitals_baseline"].append(str(result["id"]))
except Exception as e:
error_details.append({"row": rows_total, "error": str(e)})
return {
"rows_total": rows_total,
"inserted": inserted,
"updated": updated,
"skipped": skipped,
"new_entries": inserted,
}
def _sf_act(val: Any) -> float | None:
try:
return round(float(val), 1) if val is not None else None
except (TypeError, ValueError):
return None
def _looks_like_time_only(s: str) -> bool:
t = s.strip()
if not t or " " in t:
return False
parts = t.split(":")
if len(parts) < 2 or len(parts) > 3:
return False
try:
for p in parts:
int(p)
return True
except ValueError:
return False
def _import_activity(
cur,
profile_id: str,
text: str,
delim: str,
has_header: bool,
fm: dict,
tc: dict | None,
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
rows_total = 0
inserted = 0
updated = 0
new_entries = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
activity_type = mapped.get("activity_type")
if not activity_type or not str(activity_type).strip():
error_details.append({"row": rows_total, "error": "Trainingsart fehlt"})
continue
start_raw = mapped.get("start_time")
date_d = coerce_date(mapped.get("date"))
start_key: str | None = None
if isinstance(start_raw, dt.datetime):
start_key = start_raw.strftime("%Y-%m-%d %H:%M:%S")
if date_d is None:
date_d = start_raw.date()
elif isinstance(start_raw, dt.time):
if date_d is None:
error_details.append(
{"row": rows_total, "error": "Startzeit (Uhrzeit) ohne Datumsspalte"}
)
continue
start_key = f"{date_d.isoformat()} {start_raw.strftime('%H:%M:%S')}"
elif isinstance(start_raw, str) and start_raw.strip():
s = start_raw.strip()
if date_d is not None and _looks_like_time_only(s):
start_key = f"{date_d.isoformat()} {s}"
else:
start_key = s
if date_d is None and len(start_key) >= 10:
for fmt in ("%Y-%m-%d", "%d.%m.%Y"):
try:
date_d = dt.datetime.strptime(start_key[:10], fmt).date()
break
except ValueError:
continue
if date_d is None or not start_key:
error_details.append({"row": rows_total, "error": "Datum/Startzeit fehlt oder ungültig"})
continue
end_raw = mapped.get("end_time")
if isinstance(end_raw, dt.datetime):
end_str = end_raw.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(end_raw, str):
end_str = end_raw.strip()
else:
end_str = ""
duration_min = mapped.get("duration_min")
if duration_min is not None:
try:
duration_min = round(float(duration_min), 1)
except (TypeError, ValueError):
duration_min = None
kcal_a = _sf_act(mapped.get("kcal_active"))
kcal_r = _sf_act(mapped.get("kcal_resting"))
hr_a = _sf_act(mapped.get("hr_avg"))
hr_m = _sf_act(mapped.get("hr_max"))
dist = _sf_act(mapped.get("distance_km"))
wtype = str(activity_type).strip()
training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity(
wtype, profile_id
)
iso = date_d.isoformat()
try:
cur.execute(
"""
SELECT id FROM activity_log
WHERE profile_id = %s AND date = %s AND start_time = %s
""",
(profile_id, iso, start_key),
)
existing = cur.fetchone()
if existing:
eid = existing["id"]
cur.execute(
"""
UPDATE activity_log
SET end_time = %s,
activity_type = %s,
duration_min = %s,
kcal_active = %s,
kcal_resting = %s,
hr_avg = %s,
hr_max = %s,
distance_km = %s,
training_type_id = %s,
training_category = %s,
training_subcategory = %s,
source = 'csv'
WHERE id = %s
RETURNING id
""",
(
end_str or None,
wtype,
duration_min,
kcal_a,
kcal_r,
hr_a,
hr_m,
dist,
training_type_id,
training_category,
training_subcategory,
eid,
),
)
row = cur.fetchone()
updated += 1
if row and row.get("id"):
affected_ids["activity_log"].append(str(row["id"]))
aid = eid
else:
eid = str(uuid.uuid4())
cur.execute(
"""
INSERT INTO activity_log (
id, profile_id, date, start_time, end_time, activity_type, duration_min,
kcal_active, kcal_resting, hr_avg, hr_max, distance_km,
source, training_type_id, training_category, training_subcategory, created
)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'csv',%s,%s,%s,CURRENT_TIMESTAMP)
RETURNING id
""",
(
eid,
profile_id,
iso,
start_key,
end_str or None,
wtype,
duration_min,
kcal_a,
kcal_r,
hr_a,
hr_m,
dist,
training_type_id,
training_category,
training_subcategory,
),
)
row = cur.fetchone()
inserted += 1
new_entries += 1
if row and row.get("id"):
affected_ids["activity_log"].append(str(row["id"]))
aid = eid
if _EVALUATION_AVAILABLE and training_type_id and _evaluate_and_save_activity:
try:
activity_dict = {
"id": aid,
"profile_id": profile_id,
"date": iso,
"training_type_id": training_type_id,
"duration_min": duration_min,
"hr_avg": hr_a,
"hr_max": hr_m,
"distance_km": dist,
"kcal_active": kcal_a,
"kcal_resting": kcal_r,
"rpe": None,
"pace_min_per_km": None,
"cadence": None,
"elevation_gain": None,
}
_evaluate_and_save_activity(cur, aid, activity_dict, training_type_id, profile_id)
except Exception as eval_err:
logger.warning("[csv activity] Auto-Eval fehlgeschlagen: %s", eval_err)
except Exception as e:
error_details.append({"row": rows_total, "error": str(e)})
return {
"rows_total": rows_total,
"inserted": inserted,
"updated": updated,
"skipped": 0,
"new_entries": new_entries,
}