feat(csv-import): Refactor CSV import logic and enhance data handling
- Updated the CSV import architecture to clarify the distinction between import and data layer responsibilities, as outlined in the new section of ARCHITECTURE.md. - Enhanced the build_row_after_mapping function to include module-specific context for improved data processing. - Introduced source unit options in the admin CSV template editor to facilitate user-defined conversions, improving flexibility in handling various data formats. - Added new tests to validate the handling of source units and ensure accurate conversions during CSV imports. - Updated module definitions to include unit specifications for nutritional and activity data fields, enhancing data integrity.
This commit is contained in:
1 parent
41cc0ed2a8
commit
d6d7e738a5
10 files changed
+324
-27
No files matched your search
@@ -200,7 +200,7 @@ def _import_nutrition(
|
||||
rows_total = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="nutrition")
|
||||
d = coerce_date(mapped.get("date"))
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt oder ungültig"})
|
||||
@@ -283,7 +283,7 @@ def _import_weight(
|
||||
new_entries = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="weight")
|
||||
d = coerce_date(mapped.get("date"))
|
||||
w = mapped.get("weight")
|
||||
note = mapped.get("note")
|
||||
@@ -356,7 +356,7 @@ def _import_blood_pressure(
|
||||
skipped = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="blood_pressure")
|
||||
md = coerce_date(mapped.get("measured_date"))
|
||||
mt = mapped.get("measured_time")
|
||||
if md is None:
|
||||
@@ -483,7 +483,7 @@ def _import_vitals_baseline(
|
||||
skipped = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="vitals_baseline")
|
||||
d = coerce_date(mapped.get("date"))
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt"})
|
||||
@@ -581,7 +581,7 @@ def _import_activity(
|
||||
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc)
|
||||
mapped = build_row_after_mapping(csv_row, fm, tc, module="activity")
|
||||
activity_type = mapped.get("activity_type")
|
||||
if not activity_type or not str(activity_type).strip():
|
||||
error_details.append({"row": rows_total, "error": "Trainingsart fehlt"})
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
Kanonische Speichereinheiten pro CSV-Zielfeld (module_registry: field.unit) und
|
||||
abwählbare Quelleinheiten → Faktor für type_conversions.source_unit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
|
||||
# 1 kcal = 4.184 kJ (IEC/ISO 80000)
|
||||
_KJ_TO_KCAL = 1.0 / 4.184
|
||||
|
||||
# — Energie (Ziel kcal) —
|
||||
_ENERGY: list[dict[str, Any]] = [
|
||||
{"id": "kcal", "label": "Kilokalorien (kcal), wie in DB", "factor": 1.0},
|
||||
{"id": "kj", "label": "Kilojoule (kJ) → kcal", "factor": _KJ_TO_KCAL},
|
||||
{"id": "j", "label": "Joule (J) → kcal", "factor": _KJ_TO_KCAL / 1000.0},
|
||||
]
|
||||
|
||||
# — Masse klein (Ziel g): Makronährstoffe —
|
||||
_GRAM: list[dict[str, Any]] = [
|
||||
{"id": "g", "label": "Gramm (g), wie in DB", "factor": 1.0},
|
||||
{"id": "kg", "label": "Kilogramm (kg) → g", "factor": 1000.0},
|
||||
{"id": "mg", "label": "Milligramm (mg) → g", "factor": 0.001},
|
||||
]
|
||||
|
||||
# — Körpergewicht (Ziel kg) —
|
||||
_KG: list[dict[str, Any]] = [
|
||||
{"id": "kg", "label": "Kilogramm (kg), wie in DB", "factor": 1.0},
|
||||
{"id": "g", "label": "Gramm (g) → kg", "factor": 0.001},
|
||||
{"id": "lb", "label": "Pfund / lb → kg", "factor": 0.45359237},
|
||||
{"id": "oz", "label": "Unze / oz → kg", "factor": 0.028349523125},
|
||||
]
|
||||
|
||||
# — Strecke (Ziel km) —
|
||||
_KM: list[dict[str, Any]] = [
|
||||
{"id": "km", "label": "Kilometer (km), wie in DB", "factor": 1.0},
|
||||
{"id": "m", "label": "Meter (m) → km", "factor": 0.001},
|
||||
{"id": "mi", "label": "Meilen (mi) → km", "factor": 1.609344},
|
||||
]
|
||||
|
||||
_UNIT_FAMILY: dict[str, list[dict[str, Any]]] = {
|
||||
"kcal": _ENERGY,
|
||||
"g": _GRAM,
|
||||
"kg": _KG,
|
||||
"km": _KM,
|
||||
}
|
||||
|
||||
|
||||
def get_canonical_unit(module: str, db_field: str) -> str | None:
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
return None
|
||||
finfo: dict[str, Any] | None = mod.get("fields", {}).get(db_field)
|
||||
if not finfo:
|
||||
return None
|
||||
u = finfo.get("unit")
|
||||
return str(u) if u else None
|
||||
|
||||
|
||||
def source_unit_choices_for_field(module: str, db_field: str) -> list[dict[str, Any]]:
|
||||
"""Optionen für GUI: id, label, canonical_unit, is_canonical (Umrechnung serverseitig)."""
|
||||
cu = get_canonical_unit(module, db_field)
|
||||
if not cu:
|
||||
return []
|
||||
choices = _UNIT_FAMILY.get(cu)
|
||||
if not choices:
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"id": c["id"],
|
||||
"label": c["label"],
|
||||
"canonical_unit": cu,
|
||||
"is_canonical": c["id"] == cu,
|
||||
}
|
||||
for c in choices
|
||||
]
|
||||
|
||||
|
||||
def factor_source_to_canonical(module: str, db_field: str, source_unit: str | None) -> float:
|
||||
"""
|
||||
Multiplikator: CSV-Zahl * Faktor → Wert in kanonischer DB-Einheit.
|
||||
Unbekannte/None/leer/Passthrough → 1.0
|
||||
"""
|
||||
if source_unit is None:
|
||||
return 1.0
|
||||
su = str(source_unit).strip().lower()
|
||||
if not su:
|
||||
return 1.0
|
||||
cu = get_canonical_unit(module, db_field)
|
||||
if not cu:
|
||||
return 1.0
|
||||
choices = _UNIT_FAMILY.get(cu)
|
||||
if not choices:
|
||||
return 1.0
|
||||
for c in choices:
|
||||
if str(c["id"]).lower() == su:
|
||||
return float(c["factor"])
|
||||
return 1.0
|
||||
@@ -207,4 +207,33 @@ def build_type_conversions_for_mapping(
|
||||
if t not in out and t in defaults:
|
||||
out[t] = deepcopy(defaults[t])
|
||||
|
||||
_apply_energy_kj_hint_from_headers(module, field_mappings, out)
|
||||
return out
|
||||
|
||||
|
||||
_ENERGY_FIELDS = frozenset({"kcal", "kcal_active", "kcal_resting"})
|
||||
|
||||
|
||||
def _apply_energy_kj_hint_from_headers(
|
||||
module: str,
|
||||
field_mappings: Mapping[str, str],
|
||||
out: dict[str, Any],
|
||||
) -> None:
|
||||
"""Wenn Überschrift kJ/Kilojoule nahelegt (nicht kcal), source_unit kj setzen (FDDB & Co.)."""
|
||||
if module not in ("nutrition", "activity"):
|
||||
return
|
||||
for csv_col, db_field in field_mappings.items():
|
||||
if db_field not in _ENERGY_FIELDS:
|
||||
continue
|
||||
spec = out.get(db_field)
|
||||
if not isinstance(spec, dict):
|
||||
continue
|
||||
if spec.get("source_unit"):
|
||||
continue
|
||||
norm = normalize_header_for_signature(str(csv_col)).lower()
|
||||
if "kcal" in norm:
|
||||
continue
|
||||
if "kj" in norm or "kilojoule" in norm:
|
||||
spec2 = deepcopy(spec)
|
||||
spec2["source_unit"] = "kj"
|
||||
out[db_field] = spec2
|
||||
@@ -16,10 +16,10 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
"table": "nutrition_log",
|
||||
"fields": {
|
||||
"date": {"type": "date", "required": True},
|
||||
"kcal": {"type": "float", "required": False},
|
||||
"protein_g": {"type": "float", "required": False, "min": 0},
|
||||
"fat_g": {"type": "float", "required": False, "min": 0},
|
||||
"carbs_g": {"type": "float", "required": False, "min": 0},
|
||||
"kcal": {"type": "float", "required": False, "unit": "kcal"},
|
||||
"protein_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
|
||||
"fat_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
|
||||
"carbs_g": {"type": "float", "required": False, "min": 0, "unit": "g"},
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
"duplicate_strategy": "update",
|
||||
@@ -32,9 +32,9 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
"end_time": {"type": "datetime", "required": False},
|
||||
"activity_type": {"type": "string", "required": True},
|
||||
"duration_min": {"type": "float", "required": False, "min": 0},
|
||||
"kcal_active": {"type": "float", "required": False},
|
||||
"kcal_resting": {"type": "float", "required": False},
|
||||
"distance_km": {"type": "float", "required": False},
|
||||
"kcal_active": {"type": "float", "required": False, "unit": "kcal"},
|
||||
"kcal_resting": {"type": "float", "required": False, "unit": "kcal"},
|
||||
"distance_km": {"type": "float", "required": False, "unit": "km"},
|
||||
"hr_avg": {"type": "float", "required": False, "min": 30, "max": 220},
|
||||
"hr_max": {"type": "float", "required": False, "min": 30, "max": 220},
|
||||
},
|
||||
@@ -77,7 +77,7 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
"table": "weight_log",
|
||||
"fields": {
|
||||
"date": {"type": "date", "required": True},
|
||||
"weight": {"type": "float", "required": True, "min": 20, "max": 400},
|
||||
"weight": {"type": "float", "required": True, "min": 20, "max": 400, "unit": "kg"},
|
||||
"note": {"type": "string", "required": False, "max_length": 2000},
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import Any, Mapping, Sequence
|
||||
from dateutil import parser as dateutil_parser
|
||||
|
||||
from csv_parser.core import normalize_header_for_signature
|
||||
from csv_parser.field_units import factor_source_to_canonical
|
||||
|
||||
# Alias → strptime (JSON in Kleinbuchstaben)
|
||||
DATE_FORMAT_STRPTIME: dict[str, str] = {
|
||||
@@ -300,6 +301,7 @@ def convert_value(
|
||||
raw: str,
|
||||
db_field: str,
|
||||
spec: Mapping[str, Any] | None,
|
||||
module: str | None = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Konvertiert eine Roh-Zelle in einen Python-Wert.
|
||||
@@ -310,6 +312,8 @@ def convert_value(
|
||||
- decimal_separator: ".", ",", "auto" — bei auto Heuristik EU/US-Mischformen.
|
||||
- formats: [ "yyyy-mm-dd", "%d.%m.%y", ... ] — weitere strptime-/Alias-Ketten.
|
||||
- dayfirst: true|false — nur für dateutil-Fallback; Standard: true dann false.
|
||||
- source_unit: z. B. "kj", "kg" — Quelleinheit; Ziel aus Modul-Feld (unit in module_registry);
|
||||
Faktor wird serverseitig ermittelt. Optional zusätzlich conversion_factor (legacy/zusätzlich).
|
||||
"""
|
||||
if spec is None:
|
||||
return raw.strip() if raw else None
|
||||
@@ -325,6 +329,10 @@ def convert_value(
|
||||
|
||||
if t in ("float", "number"):
|
||||
v = _float_from_spec(raw, spec)
|
||||
if module:
|
||||
su = spec.get("source_unit")
|
||||
if su is not None:
|
||||
v = float(v) * factor_source_to_canonical(module, db_field, str(su))
|
||||
factor = spec.get("conversion_factor")
|
||||
if factor is not None:
|
||||
v = float(v) * float(factor)
|
||||
@@ -395,6 +403,7 @@ def build_row_after_mapping(
|
||||
csv_row: Mapping[str, str],
|
||||
field_mappings: Mapping[str, str],
|
||||
type_conversions: Mapping[str, Any] | None,
|
||||
module: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Wendet Zuordnung csv_spalte → db_feld und Typkonvertierung an.
|
||||
@@ -408,7 +417,9 @@ def build_row_after_mapping(
|
||||
continue
|
||||
spec = tc.get(db_field)
|
||||
try:
|
||||
out[db_field] = convert_value(raw, db_field, spec if isinstance(spec, dict) else None)
|
||||
out[db_field] = convert_value(
|
||||
raw, db_field, spec if isinstance(spec, dict) else None, module=module
|
||||
)
|
||||
except Exception:
|
||||
out[db_field] = None
|
||||
return out
|
||||
Reference in new issue
Block a user