feat(csv-import): Enhance CSV import processing and validation
- Updated the CSV import logic to support new row processing specifications for weight and vitals baseline, allowing for better data aggregation and validation. - Implemented handling for multiple rows on the same day, enabling aggregation of values such as averages for vitals and last values for weight. - Enhanced test coverage for the new import functionalities, ensuring correct behavior during data processing and validation. - Refactored the module registry to include default import row processing options for better flexibility in handling CSV data.
This commit is contained in:
1 parent
c0fcdea1fe
commit
e35d167055
5 files changed
+165
-9
No files matched your search
@@ -130,6 +130,7 @@ def run_universal_csv_import(
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
mapping,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
@@ -169,6 +170,7 @@ def run_universal_csv_import(
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
mapping,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
@@ -296,19 +298,18 @@ def _import_weight(
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
mapping: dict[str, Any],
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
spec = resolve_import_row_processing("weight", mapping)
|
||||
mapped_rows: list[dict[str, Any]] = []
|
||||
rows_total = 0
|
||||
inserted = 0
|
||||
updated = 0
|
||||
new_entries = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="weight")
|
||||
d = coerce_date(mapped.get("date"))
|
||||
w = mapped.get("weight")
|
||||
note = mapped.get("note")
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt"})
|
||||
continue
|
||||
@@ -316,10 +317,37 @@ def _import_weight(
|
||||
error_details.append({"row": rows_total, "error": "Gewicht fehlt"})
|
||||
continue
|
||||
try:
|
||||
w = float(w)
|
||||
float(w)
|
||||
except (TypeError, ValueError):
|
||||
error_details.append({"row": rows_total, "error": "Gewicht ungültig"})
|
||||
continue
|
||||
mapped["date"] = d
|
||||
mapped_rows.append(mapped)
|
||||
|
||||
if spec:
|
||||
try:
|
||||
validate_import_row_processing("weight", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
new_entries = 0
|
||||
for merged in merged_rows:
|
||||
d = coerce_date(merged.get("date"))
|
||||
w = merged.get("weight")
|
||||
note = merged.get("note")
|
||||
if d is None:
|
||||
continue
|
||||
if w is None:
|
||||
continue
|
||||
try:
|
||||
w = float(w)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
cur.execute(
|
||||
"SELECT id FROM weight_log WHERE profile_id=%s AND date=%s",
|
||||
@@ -496,13 +524,14 @@ def _import_vitals_baseline(
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
mapping: dict[str, Any],
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
spec = resolve_import_row_processing("vitals_baseline", mapping)
|
||||
mapped_rows: list[dict[str, Any]] = []
|
||||
rows_total = 0
|
||||
inserted = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
skipped_prefilter = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="vitals_baseline")
|
||||
@@ -515,6 +544,33 @@ def _import_vitals_baseline(
|
||||
vo2 = _v_safe_float(mapped.get("vo2_max"))
|
||||
spo2 = _v_safe_int(mapped.get("spo2"))
|
||||
resp = _v_safe_float(mapped.get("respiratory_rate"))
|
||||
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
|
||||
skipped_prefilter += 1
|
||||
continue
|
||||
mapped["date"] = d
|
||||
mapped_rows.append(mapped)
|
||||
|
||||
if spec:
|
||||
try:
|
||||
validate_import_row_processing("vitals_baseline", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
skipped = skipped_prefilter
|
||||
for merged in merged_rows:
|
||||
d = coerce_date(merged.get("date"))
|
||||
if d is None:
|
||||
continue
|
||||
rhr = _v_safe_int(merged.get("resting_hr"))
|
||||
hrv = _v_safe_int(merged.get("hrv"))
|
||||
vo2 = _v_safe_float(merged.get("vo2_max"))
|
||||
spo2 = _v_safe_int(merged.get("spo2"))
|
||||
resp = _v_safe_float(merged.get("respiratory_rate"))
|
||||
if not any(x is not None for x in (rhr, hrv, vo2, spo2, resp)):
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user