feat(csv-import): Enhance CSV import processing and validation
Deploy Development / deploy (push) Successful in 56s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 1s
Build Test / build-frontend (push) Successful in 16s

- Updated the CSV import logic to support new row processing specifications for weight and vitals baseline, allowing for better data aggregation and validation.
- Implemented handling for multiple rows on the same day, enabling aggregation of values such as averages for vitals and last values for weight.
- Enhanced test coverage for the new import functionalities, ensuring correct behavior during data processing and validation.
- Refactored the module registry to include default import row processing options for better flexibility in handling CSV data.
This commit is contained in:
Lars committed 2026-04-10 15:09:34 +02:00
1 parent c0fcdea1fe
commit e35d167055
5 files changed
+165 -9

No files matched your search

+64 -8
View File
@@ -130,6 +130,7 @@ def run_universal_csv_import(
bool(has_header),
fm,
tc,
mapping,
error_details,
affected_ids,
)
@@ -169,6 +170,7 @@ def run_universal_csv_import(
bool(has_header),
fm,
tc,
mapping,
error_details,
affected_ids,
)
@@ -296,19 +298,18 @@ def _import_weight(
has_header: bool,
fm: dict,
tc: dict | None,
mapping: dict[str, Any],
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
spec = resolve_import_row_processing("weight", mapping)
mapped_rows: list[dict[str, Any]] = []
rows_total = 0
inserted = 0
updated = 0
new_entries = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc, module="weight")
d = coerce_date(mapped.get("date"))
w = mapped.get("weight")
note = mapped.get("note")
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt"})
continue
@@ -316,10 +317,37 @@ def _import_weight(
error_details.append({"row": rows_total, "error": "Gewicht fehlt"})
continue
try:
w = float(w)
float(w)
except (TypeError, ValueError):
error_details.append({"row": rows_total, "error": "Gewicht ungültig"})
continue
mapped["date"] = d
mapped_rows.append(mapped)
if spec:
try:
validate_import_row_processing("weight", spec, fm)
except ValueError as e:
raise ValueError(str(e)) from e
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
else:
merged_rows = list(mapped_rows)
inserted = 0
updated = 0
new_entries = 0
for merged in merged_rows:
d = coerce_date(merged.get("date"))
w = merged.get("weight")
note = merged.get("note")
if d is None:
continue
if w is None:
continue
try:
w = float(w)
except (TypeError, ValueError):
continue
iso = d.isoformat()
cur.execute(
"SELECT id FROM weight_log WHERE profile_id=%s AND date=%s",
@@ -496,13 +524,14 @@ def _import_vitals_baseline(
has_header: bool,
fm: dict,
tc: dict | None,
mapping: dict[str, Any],
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
spec = resolve_import_row_processing("vitals_baseline", mapping)
mapped_rows: list[dict[str, Any]] = []
rows_total = 0
inserted = 0
updated = 0
skipped = 0
skipped_prefilter = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc, module="vitals_baseline")
@@ -515,6 +544,33 @@ def _import_vitals_baseline(
vo2 = _v_safe_float(mapped.get("vo2_max"))
spo2 = _v_safe_int(mapped.get("spo2"))
resp = _v_safe_float(mapped.get("respiratory_rate"))
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
skipped_prefilter += 1
continue
mapped["date"] = d
mapped_rows.append(mapped)
if spec:
try:
validate_import_row_processing("vitals_baseline", spec, fm)
except ValueError as e:
raise ValueError(str(e)) from e
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
else:
merged_rows = list(mapped_rows)
inserted = 0
updated = 0
skipped = skipped_prefilter
for merged in merged_rows:
d = coerce_date(merged.get("date"))
if d is None:
continue
rhr = _v_safe_int(merged.get("resting_hr"))
hrv = _v_safe_int(merged.get("hrv"))
vo2 = _v_safe_float(merged.get("vo2_max"))
spo2 = _v_safe_int(merged.get("spo2"))
resp = _v_safe_float(merged.get("respiratory_rate"))
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
skipped += 1
continue