refactor(csv-import): Enhance nutrition data processing and template rendering
- Updated the nutrition import logic to utilize a new row processing specification, improving data aggregation and validation. - Refactored the template rendering process in the workflow executor to use Jinja2's Environment with ChainableUndefined for better handling of missing attributes. - Added backward-compatible shortcuts for accessing decision signals in node contexts, enhancing flexibility in template usage. - Introduced import row processing options in CSV templates, allowing for more customizable data handling during imports.
This commit is contained in:
1 parent
8b67f7ab55
commit
c0fcdea1fe
9 files changed
+357
-22
No files matched your search
@@ -12,6 +12,11 @@ from typing import Any
|
||||
import logging
|
||||
|
||||
from csv_parser.core import iter_csv_dict_rows
|
||||
from csv_parser.import_row_processing import (
|
||||
aggregate_mapped_rows,
|
||||
resolve_import_row_processing,
|
||||
validate_import_row_processing,
|
||||
)
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
from csv_parser.type_converter import build_row_after_mapping
|
||||
|
||||
@@ -111,6 +116,7 @@ def run_universal_csv_import(
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
mapping,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
@@ -191,12 +197,12 @@ def _import_nutrition(
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
mapping: dict[str, Any],
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
agg: dict[str, dict[str, float]] = defaultdict(
|
||||
lambda: {"kcal": 0.0, "protein_g": 0.0, "fat_g": 0.0, "carbs_g": 0.0}
|
||||
)
|
||||
spec = resolve_import_row_processing("nutrition", mapping)
|
||||
mapped_rows: list[dict[str, Any]] = []
|
||||
rows_total = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
@@ -205,23 +211,39 @@ def _import_nutrition(
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt oder ungültig"})
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
for key in ("kcal", "protein_g", "fat_g", "carbs_g"):
|
||||
v = mapped.get(key)
|
||||
if v is not None:
|
||||
try:
|
||||
agg[iso][key] += float(v)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
mapped["date"] = d
|
||||
mapped_rows.append(mapped)
|
||||
|
||||
if spec:
|
||||
try:
|
||||
validate_import_row_processing("nutrition", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
new_entries = 0
|
||||
for iso, vals in agg.items():
|
||||
kcal = round(vals["kcal"], 1)
|
||||
fat = round(vals["fat_g"], 1)
|
||||
carbs = round(vals["carbs_g"], 1)
|
||||
prot = round(vals["protein_g"], 1)
|
||||
for merged in merged_rows:
|
||||
d = coerce_date(merged.get("date"))
|
||||
if d is None:
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
|
||||
def _sf_macro(x: Any) -> float:
|
||||
if x is None or x == "":
|
||||
return 0.0
|
||||
try:
|
||||
return float(x)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
kcal = round(_sf_macro(merged.get("kcal")), 1)
|
||||
fat = round(_sf_macro(merged.get("fat_g")), 1)
|
||||
carbs = round(_sf_macro(merged.get("carbs_g")), 1)
|
||||
prot = round(_sf_macro(merged.get("protein_g")), 1)
|
||||
if kcal == 0 and fat == 0 and carbs == 0 and prot == 0:
|
||||
continue
|
||||
|
||||
|
||||
Reference in new issue
Block a user