feat(csv-import): Enhance CSV import functionality with new modules and tests
- Added support for new CSV import modules: sleep and vitals_baseline, expanding the import capabilities. - Implemented backend logic for handling CSV imports related to sleep and vitals baseline, including error handling and data processing. - Updated frontend components to include new modules in the CSV import interface, improving user experience. - Introduced unit tests for the new import functionalities to ensure reliability and correctness. - Enhanced existing CSV analysis features to accommodate the new modules, ensuring consistent behavior across the application.
This commit is contained in:
1 parent
b4cc3cb934
commit
26ab11eb7b
14 files changed
+1261
-340
No files matched your search
@@ -9,10 +9,29 @@ import uuid
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
import logging
|
||||
|
||||
from csv_parser.core import iter_csv_dict_rows
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
from csv_parser.type_converter import build_row_after_mapping
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from evaluation_helper import evaluate_and_save_activity as _evaluate_and_save_activity
|
||||
|
||||
_EVALUATION_AVAILABLE = True
|
||||
except Exception: # pragma: no cover
|
||||
_evaluate_and_save_activity = None
|
||||
_EVALUATION_AVAILABLE = False
|
||||
|
||||
|
||||
def _resolve_training_type_for_activity(activity_type: str, profile_id: str):
|
||||
"""Lazy import — ermöglicht Tests ohne Laden von routers.activity (bcrypt)."""
|
||||
from routers.activity import get_training_type_for_activity
|
||||
|
||||
return get_training_type_for_activity(activity_type, profile_id)
|
||||
|
||||
|
||||
def coerce_date(val: Any) -> dt.date | None:
|
||||
if val is None:
|
||||
@@ -48,6 +67,31 @@ def run_universal_csv_import(
|
||||
if not mod:
|
||||
raise ValueError(f"Unbekanntes Modul: {module}")
|
||||
|
||||
rows_total = 0
|
||||
error_details: list[dict[str, Any]] = []
|
||||
affected_ids: dict[str, list[str]] = defaultdict(list)
|
||||
|
||||
if module == "sleep":
|
||||
from csv_parser.sleep_apple_import import import_apple_sleep_nights
|
||||
|
||||
try:
|
||||
r = import_apple_sleep_nights(cur, profile_id, text)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
error_details.extend(r.get("error_details") or [])
|
||||
for sid in r.get("affected_ids") or []:
|
||||
affected_ids["sleep_log"].append(sid)
|
||||
return {
|
||||
"rows_total": r["rows_total"],
|
||||
"rows_imported": r["inserted"],
|
||||
"rows_updated": r["updated"],
|
||||
"rows_skipped": r["skipped"],
|
||||
"rows_errors": len(error_details),
|
||||
"error_details": error_details[:50],
|
||||
"new_entries": r.get("new_entries", r["inserted"]),
|
||||
"affected_ids": dict(affected_ids),
|
||||
}
|
||||
|
||||
fm = mapping.get("field_mappings") or {}
|
||||
if isinstance(fm, str):
|
||||
raise ValueError("field_mappings muss ein Objekt sein")
|
||||
@@ -58,10 +102,6 @@ def run_universal_csv_import(
|
||||
delim = mapping.get("delimiter") or ","
|
||||
has_header = mapping.get("has_header", True)
|
||||
|
||||
rows_total = 0
|
||||
error_details: list[dict[str, Any]] = []
|
||||
affected_ids: dict[str, list[str]] = defaultdict(list)
|
||||
|
||||
if module == "nutrition":
|
||||
stats = _import_nutrition(
|
||||
cur,
|
||||
@@ -101,6 +141,32 @@ def run_universal_csv_import(
|
||||
affected_ids,
|
||||
)
|
||||
rows_total = stats.pop("rows_total")
|
||||
elif module == "activity":
|
||||
stats = _import_activity(
|
||||
cur,
|
||||
profile_id,
|
||||
text,
|
||||
delim,
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
rows_total = stats.pop("rows_total")
|
||||
elif module == "vitals_baseline":
|
||||
stats = _import_vitals_baseline(
|
||||
cur,
|
||||
profile_id,
|
||||
text,
|
||||
delim,
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
rows_total = stats.pop("rows_total")
|
||||
else:
|
||||
raise ValueError(f"Modul '{module}' wird für Universal-Import noch nicht unterstützt")
|
||||
|
||||
@@ -375,3 +441,327 @@ def _import_blood_pressure(
|
||||
"skipped": skipped,
|
||||
"new_entries": inserted,
|
||||
}
|
||||
|
||||
|
||||
def _v_safe_int(value: Any) -> int | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
s = str(value).strip()
|
||||
if "." in s:
|
||||
return int(float(s))
|
||||
return int(s)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _v_safe_float(value: Any) -> float | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _import_vitals_baseline(
|
||||
cur,
|
||||
profile_id: str,
|
||||
text: str,
|
||||
delim: str,
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
rows_total = 0
|
||||
inserted = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
d = coerce_date(mapped.get("date"))
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt"})
|
||||
continue
|
||||
rhr = _v_safe_int(mapped.get("resting_hr"))
|
||||
hrv = _v_safe_int(mapped.get("hrv"))
|
||||
vo2 = _v_safe_float(mapped.get("vo2_max"))
|
||||
spo2 = _v_safe_int(mapped.get("spo2"))
|
||||
resp = _v_safe_float(mapped.get("respiratory_rate"))
|
||||
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
|
||||
skipped += 1
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
try:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO vitals_baseline (
|
||||
profile_id, date,
|
||||
resting_hr, hrv, vo2_max, spo2, respiratory_rate,
|
||||
source
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'csv')
|
||||
ON CONFLICT (profile_id, date)
|
||||
DO UPDATE SET
|
||||
resting_hr = COALESCE(EXCLUDED.resting_hr, vitals_baseline.resting_hr),
|
||||
hrv = COALESCE(EXCLUDED.hrv, vitals_baseline.hrv),
|
||||
vo2_max = COALESCE(EXCLUDED.vo2_max, vitals_baseline.vo2_max),
|
||||
spo2 = COALESCE(EXCLUDED.spo2, vitals_baseline.spo2),
|
||||
respiratory_rate = COALESCE(EXCLUDED.respiratory_rate, vitals_baseline.respiratory_rate),
|
||||
updated_at = NOW()
|
||||
WHERE vitals_baseline.source != 'manual'
|
||||
RETURNING (xmax = 0) AS inserted, id
|
||||
""",
|
||||
(profile_id, iso, rhr, hrv, vo2, spo2, resp),
|
||||
)
|
||||
result = cur.fetchone()
|
||||
if result is None:
|
||||
skipped += 1
|
||||
elif result.get("inserted"):
|
||||
inserted += 1
|
||||
if result.get("id"):
|
||||
affected_ids["vitals_baseline"].append(str(result["id"]))
|
||||
else:
|
||||
updated += 1
|
||||
if result.get("id"):
|
||||
affected_ids["vitals_baseline"].append(str(result["id"]))
|
||||
except Exception as e:
|
||||
error_details.append({"row": rows_total, "error": str(e)})
|
||||
|
||||
return {
|
||||
"rows_total": rows_total,
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": skipped,
|
||||
"new_entries": inserted,
|
||||
}
|
||||
|
||||
|
||||
def _sf_act(val: Any) -> float | None:
|
||||
try:
|
||||
return round(float(val), 1) if val is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_time_only(s: str) -> bool:
|
||||
t = s.strip()
|
||||
if not t or " " in t:
|
||||
return False
|
||||
parts = t.split(":")
|
||||
if len(parts) < 2 or len(parts) > 3:
|
||||
return False
|
||||
try:
|
||||
for p in parts:
|
||||
int(p)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _import_activity(
|
||||
cur,
|
||||
profile_id: str,
|
||||
text: str,
|
||||
delim: str,
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
rows_total = 0
|
||||
inserted = 0
|
||||
updated = 0
|
||||
new_entries = 0
|
||||
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
activity_type = mapped.get("activity_type")
|
||||
if not activity_type or not str(activity_type).strip():
|
||||
error_details.append({"row": rows_total, "error": "Trainingsart fehlt"})
|
||||
continue
|
||||
|
||||
start_raw = mapped.get("start_time")
|
||||
date_d = coerce_date(mapped.get("date"))
|
||||
start_key: str | None = None
|
||||
if isinstance(start_raw, dt.datetime):
|
||||
start_key = start_raw.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if date_d is None:
|
||||
date_d = start_raw.date()
|
||||
elif isinstance(start_raw, dt.time):
|
||||
if date_d is None:
|
||||
error_details.append(
|
||||
{"row": rows_total, "error": "Startzeit (Uhrzeit) ohne Datumsspalte"}
|
||||
)
|
||||
continue
|
||||
start_key = f"{date_d.isoformat()} {start_raw.strftime('%H:%M:%S')}"
|
||||
elif isinstance(start_raw, str) and start_raw.strip():
|
||||
s = start_raw.strip()
|
||||
if date_d is not None and _looks_like_time_only(s):
|
||||
start_key = f"{date_d.isoformat()} {s}"
|
||||
else:
|
||||
start_key = s
|
||||
if date_d is None and len(start_key) >= 10:
|
||||
for fmt in ("%Y-%m-%d", "%d.%m.%Y"):
|
||||
try:
|
||||
date_d = dt.datetime.strptime(start_key[:10], fmt).date()
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if date_d is None or not start_key:
|
||||
error_details.append({"row": rows_total, "error": "Datum/Startzeit fehlt oder ungültig"})
|
||||
continue
|
||||
|
||||
end_raw = mapped.get("end_time")
|
||||
if isinstance(end_raw, dt.datetime):
|
||||
end_str = end_raw.strftime("%Y-%m-%d %H:%M:%S")
|
||||
elif isinstance(end_raw, str):
|
||||
end_str = end_raw.strip()
|
||||
else:
|
||||
end_str = ""
|
||||
|
||||
duration_min = mapped.get("duration_min")
|
||||
if duration_min is not None:
|
||||
try:
|
||||
duration_min = round(float(duration_min), 1)
|
||||
except (TypeError, ValueError):
|
||||
duration_min = None
|
||||
|
||||
kcal_a = _sf_act(mapped.get("kcal_active"))
|
||||
kcal_r = _sf_act(mapped.get("kcal_resting"))
|
||||
hr_a = _sf_act(mapped.get("hr_avg"))
|
||||
hr_m = _sf_act(mapped.get("hr_max"))
|
||||
dist = _sf_act(mapped.get("distance_km"))
|
||||
|
||||
wtype = str(activity_type).strip()
|
||||
training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity(
|
||||
wtype, profile_id
|
||||
)
|
||||
|
||||
iso = date_d.isoformat()
|
||||
|
||||
try:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id FROM activity_log
|
||||
WHERE profile_id = %s AND date = %s AND start_time = %s
|
||||
""",
|
||||
(profile_id, iso, start_key),
|
||||
)
|
||||
existing = cur.fetchone()
|
||||
|
||||
if existing:
|
||||
eid = existing["id"]
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE activity_log
|
||||
SET end_time = %s,
|
||||
activity_type = %s,
|
||||
duration_min = %s,
|
||||
kcal_active = %s,
|
||||
kcal_resting = %s,
|
||||
hr_avg = %s,
|
||||
hr_max = %s,
|
||||
distance_km = %s,
|
||||
training_type_id = %s,
|
||||
training_category = %s,
|
||||
training_subcategory = %s,
|
||||
source = 'csv'
|
||||
WHERE id = %s
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
end_str or None,
|
||||
wtype,
|
||||
duration_min,
|
||||
kcal_a,
|
||||
kcal_r,
|
||||
hr_a,
|
||||
hr_m,
|
||||
dist,
|
||||
training_type_id,
|
||||
training_category,
|
||||
training_subcategory,
|
||||
eid,
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
updated += 1
|
||||
if row and row.get("id"):
|
||||
affected_ids["activity_log"].append(str(row["id"]))
|
||||
aid = eid
|
||||
else:
|
||||
eid = str(uuid.uuid4())
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO activity_log (
|
||||
id, profile_id, date, start_time, end_time, activity_type, duration_min,
|
||||
kcal_active, kcal_resting, hr_avg, hr_max, distance_km,
|
||||
source, training_type_id, training_category, training_subcategory, created
|
||||
)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'csv',%s,%s,%s,CURRENT_TIMESTAMP)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
eid,
|
||||
profile_id,
|
||||
iso,
|
||||
start_key,
|
||||
end_str or None,
|
||||
wtype,
|
||||
duration_min,
|
||||
kcal_a,
|
||||
kcal_r,
|
||||
hr_a,
|
||||
hr_m,
|
||||
dist,
|
||||
training_type_id,
|
||||
training_category,
|
||||
training_subcategory,
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
inserted += 1
|
||||
new_entries += 1
|
||||
if row and row.get("id"):
|
||||
affected_ids["activity_log"].append(str(row["id"]))
|
||||
aid = eid
|
||||
|
||||
if _EVALUATION_AVAILABLE and training_type_id and _evaluate_and_save_activity:
|
||||
try:
|
||||
activity_dict = {
|
||||
"id": aid,
|
||||
"profile_id": profile_id,
|
||||
"date": iso,
|
||||
"training_type_id": training_type_id,
|
||||
"duration_min": duration_min,
|
||||
"hr_avg": hr_a,
|
||||
"hr_max": hr_m,
|
||||
"distance_km": dist,
|
||||
"kcal_active": kcal_a,
|
||||
"kcal_resting": kcal_r,
|
||||
"rpe": None,
|
||||
"pace_min_per_km": None,
|
||||
"cadence": None,
|
||||
"elevation_gain": None,
|
||||
}
|
||||
_evaluate_and_save_activity(cur, aid, activity_dict, training_type_id, profile_id)
|
||||
except Exception as eval_err:
|
||||
logger.warning("[csv activity] Auto-Eval fehlgeschlagen: %s", eval_err)
|
||||
except Exception as e:
|
||||
error_details.append({"row": rows_total, "error": str(e)})
|
||||
|
||||
return {
|
||||
"rows_total": rows_total,
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": 0,
|
||||
"new_entries": new_entries,
|
||||
}
|
||||
Reference in new issue
Block a user