feat(csv-import): Refactor CSV import logic and enhance data handling
Deploy Development / deploy (push) Successful in 48s
Build Test / pytest-backend-csv (push) Failing after 3s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 15s

- Updated the CSV import architecture to clarify the distinction between import and data layer responsibilities, as outlined in the new section of ARCHITECTURE.md.
- Enhanced the build_row_after_mapping function to include module-specific context for improved data processing.
- Introduced source unit options in the admin CSV template editor to facilitate user-defined conversions, improving flexibility in handling various data formats.
- Added new tests to validate the handling of source units and ensure accurate conversions during CSV imports.
- Updated module definitions to include unit specifications for nutritional and activity data fields, enhancing data integrity.
This commit is contained in:
Lars committed 2026-04-10 09:54:32 +02:00
1 parent 41cc0ed2a8
commit d6d7e738a5
10 files changed
+324 -27

No files matched your search

+5 -5
View File
@@ -200,7 +200,7 @@ def _import_nutrition(
rows_total = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
mapped = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
d = coerce_date(mapped.get("date"))
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt oder ungültig"})
@@ -283,7 +283,7 @@ def _import_weight(
new_entries = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
mapped = build_row_after_mapping(csv_row, fm, tc, module="weight")
d = coerce_date(mapped.get("date"))
w = mapped.get("weight")
note = mapped.get("note")
@@ -356,7 +356,7 @@ def _import_blood_pressure(
skipped = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
mapped = build_row_after_mapping(csv_row, fm, tc, module="blood_pressure")
md = coerce_date(mapped.get("measured_date"))
mt = mapped.get("measured_time")
if md is None:
@@ -483,7 +483,7 @@ def _import_vitals_baseline(
skipped = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
mapped = build_row_after_mapping(csv_row, fm, tc, module="vitals_baseline")
d = coerce_date(mapped.get("date"))
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt"})
@@ -581,7 +581,7 @@ def _import_activity(
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
mapped = build_row_after_mapping(csv_row, fm, tc)
mapped = build_row_after_mapping(csv_row, fm, tc, module="activity")
activity_type = mapped.get("activity_type")
if not activity_type or not str(activity_type).strip():
error_details.append({"row": rows_total, "error": "Trainingsart fehlt"})
+101
View File
@@ -0,0 +1,101 @@
"""
Kanonische Speichereinheiten pro CSV-Zielfeld (module_registry: field.unit) und
abwählbare Quelleinheiten → Faktor für type_conversions.source_unit.
"""
from __future__ import annotations
from typing import Any
from csv_parser.module_registry import get_module_definition
# 1 kcal = 4.184 kJ (IEC/ISO 80000)
_KJ_TO_KCAL = 1.0 / 4.184
# — Energie (Ziel kcal) —
_ENERGY: list[dict[str, Any]] = [
{"id": "kcal", "label": "Kilokalorien (kcal), wie in DB", "factor": 1.0},
{"id": "kj", "label": "Kilojoule (kJ) → kcal", "factor": _KJ_TO_KCAL},
{"id": "j", "label": "Joule (J) → kcal", "factor": _KJ_TO_KCAL / 1000.0},
]
# — Masse klein (Ziel g): Makronährstoffe —
_GRAM: list[dict[str, Any]] = [
{"id": "g", "label": "Gramm (g), wie in DB", "factor": 1.0},
{"id": "kg", "label": "Kilogramm (kg) → g", "factor": 1000.0},
{"id": "mg", "label": "Milligramm (mg) → g", "factor": 0.001},
]
# — Körpergewicht (Ziel kg) —
_KG: list[dict[str, Any]] = [
{"id": "kg", "label": "Kilogramm (kg), wie in DB", "factor": 1.0},
{"id": "g", "label": "Gramm (g) → kg", "factor": 0.001},
{"id": "lb", "label": "Pfund / lb → kg", "factor": 0.45359237},
{"id": "oz", "label": "Unze / oz → kg", "factor": 0.028349523125},
]
# — Strecke (Ziel km) —
_KM: list[dict[str, Any]] = [
{"id": "km", "label": "Kilometer (km), wie in DB", "factor": 1.0},
{"id": "m", "label": "Meter (m) → km", "factor": 0.001},
{"id": "mi", "label": "Meilen (mi) → km", "factor": 1.609344},
]
_UNIT_FAMILY: dict[str, list[dict[str, Any]]] = {
"kcal": _ENERGY,
"g": _GRAM,
"kg": _KG,
"km": _KM,
}
def get_canonical_unit(module: str, db_field: str) -> str | None:
mod = get_module_definition(module)
if not mod:
return None
finfo: dict[str, Any] | None = mod.get("fields", {}).get(db_field)
if not finfo:
return None
u = finfo.get("unit")
return str(u) if u else None
def source_unit_choices_for_field(module: str, db_field: str) -> list[dict[str, Any]]:
"""Optionen für GUI: id, label, canonical_unit, is_canonical (Umrechnung serverseitig)."""
cu = get_canonical_unit(module, db_field)
if not cu:
return []
choices = _UNIT_FAMILY.get(cu)
if not choices:
return []
return [
{
"id": c["id"],
"label": c["label"],
"canonical_unit": cu,
"is_canonical": c["id"] == cu,
}
for c in choices
]
def factor_source_to_canonical(module: str, db_field: str, source_unit: str | None) -> float:
"""
Multiplikator: CSV-Zahl * Faktor → Wert in kanonischer DB-Einheit.
Unbekannte/None/leer/Passthrough → 1.0
"""
if source_unit is None:
return 1.0
su = str(source_unit).strip().lower()
if not su:
return 1.0
cu = get_canonical_unit(module, db_field)
if not cu:
return 1.0
choices = _UNIT_FAMILY.get(cu)
if not choices:
return 1.0
for c in choices:
if str(c["id"]).lower() == su:
return float(c["factor"])
return 1.0
+29
View File
@@ -207,4 +207,33 @@ def build_type_conversions_for_mapping(
if t not in out and t in defaults:
out[t] = deepcopy(defaults[t])
_apply_energy_kj_hint_from_headers(module, field_mappings, out)
return out
_ENERGY_FIELDS = frozenset({"kcal", "kcal_active", "kcal_resting"})
def _apply_energy_kj_hint_from_headers(
module: str,
field_mappings: Mapping[str, str],
out: dict[str, Any],
) -> None:
"""Wenn Überschrift kJ/Kilojoule nahelegt (nicht kcal), source_unit kj setzen (FDDB & Co.)."""
if module not in ("nutrition", "activity"):
return
for csv_col, db_field in field_mappings.items():
if db_field not in _ENERGY_FIELDS:
continue
spec = out.get(db_field)
if not isinstance(spec, dict):
continue
if spec.get("source_unit"):
continue
norm = normalize_header_for_signature(str(csv_col)).lower()
if "kcal" in norm:
continue
if "kj" in norm or "kilojoule" in norm:
spec2 = deepcopy(spec)
spec2["source_unit"] = "kj"
out[db_field] = spec2
+8 -8
View File
@@ -16,10 +16,10 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
"table": "nutrition_log",
"fields": {
"date": {"type": "date", "required": True},
"kcal": {"type": "float", "required": False},
"protein_g": {"type": "float", "required": False, "min": 0},
"fat_g": {"type": "float", "required": False, "min": 0},
"carbs_g": {"type": "float", "required": False, "min": 0},
"kcal": {"type": "float", "required": False, "unit": "kcal"},
"protein_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
"fat_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
"carbs_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
},
"duplicate_key": ["profile_id", "date"],
"duplicate_strategy": "update",
@@ -32,9 +32,9 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
"end_time": {"type": "datetime", "required": False},
"activity_type": {"type": "string", "required": True},
"duration_min": {"type": "float", "required": False, "min": 0},
"kcal_active": {"type": "float", "required": False},
"kcal_resting": {"type": "float", "required": False},
"distance_km": {"type": "float", "required": False},
"kcal_active": {"type": "float", "required": False, "unit": "kcal"},
"kcal_resting": {"type": "float", "required": False, "unit": "kcal"},
"distance_km": {"type": "float", "required": False, "unit": "km"},
"hr_avg": {"type": "float", "required": False, "min": 30, "max": 220},
"hr_max": {"type": "float", "required": False, "min": 30, "max": 220},
},
@@ -77,7 +77,7 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
"table": "weight_log",
"fields": {
"date": {"type": "date", "required": True},
"weight": {"type": "float", "required": True, "min": 20, "max": 400},
"weight": {"type": "float", "required": True, "min": 20, "max": 400, "unit": "kg"},
"note": {"type": "string", "required": False, "max_length": 2000},
},
"duplicate_key": ["profile_id", "date"],
+12 -1
View File
@@ -15,6 +15,7 @@ from typing import Any, Mapping, Sequence
from dateutil import parser as dateutil_parser
from csv_parser.core import normalize_header_for_signature
from csv_parser.field_units import factor_source_to_canonical
# Alias → strptime (JSON in Kleinbuchstaben)
DATE_FORMAT_STRPTIME: dict[str, str] = {
@@ -300,6 +301,7 @@ def convert_value(
raw: str,
db_field: str,
spec: Mapping[str, Any] | None,
module: str | None = None,
) -> Any:
"""
Konvertiert eine Roh-Zelle in einen Python-Wert.
@@ -310,6 +312,8 @@ def convert_value(
- decimal_separator: ".", ",", "auto" — bei auto Heuristik EU/US-Mischformen.
- formats: [ "yyyy-mm-dd", "%d.%m.%y", ... ] — weitere strptime-/Alias-Ketten.
- dayfirst: true|false — nur für dateutil-Fallback; Standard: true dann false.
- source_unit: z. B. "kj", "kg" — Quelleinheit; Ziel aus Modul-Feld (unit in module_registry);
Faktor wird serverseitig ermittelt. Optional zusätzlich conversion_factor (legacy/zusätzlich).
"""
if spec is None:
return raw.strip() if raw else None
@@ -325,6 +329,10 @@ def convert_value(
if t in ("float", "number"):
v = _float_from_spec(raw, spec)
if module:
su = spec.get("source_unit")
if su is not None:
v = float(v) * factor_source_to_canonical(module, db_field, str(su))
factor = spec.get("conversion_factor")
if factor is not None:
v = float(v) * float(factor)
@@ -395,6 +403,7 @@ def build_row_after_mapping(
csv_row: Mapping[str, str],
field_mappings: Mapping[str, str],
type_conversions: Mapping[str, Any] | None,
module: str | None = None,
) -> dict[str, Any]:
"""
Wendet Zuordnung csv_spalte → db_feld und Typkonvertierung an.
@@ -408,7 +417,9 @@ def build_row_after_mapping(
continue
spec = tc.get(db_field)
try:
out[db_field] = convert_value(raw, db_field, spec if isinstance(spec, dict) else None)
out[db_field] = convert_value(
raw, db_field, spec if isinstance(spec, dict) else None, module=module
)
except Exception:
out[db_field] = None
return out