refactor(csv-import): Enhance nutrition data processing and template rendering
Deploy Development / deploy (push) Successful in 48s
Build Test / pytest-backend (push) Successful in 3s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Updated the nutrition import logic to utilize a new row processing specification, improving data aggregation and validation.
- Refactored the template rendering process in the workflow executor to use Jinja2's Environment with ChainableUndefined for better handling of missing attributes.
- Added backward-compatible shortcuts for accessing decision signals in node contexts, enhancing flexibility in template usage.
- Introduced import row processing options in CSV templates, allowing for more customizable data handling during imports.
This commit is contained in:
Lars committed 2026-04-10 11:56:43 +02:00
1 parent 8b67f7ab55
commit c0fcdea1fe
9 files changed
+357 -22

No files matched your search

+38 -16
View File
@@ -12,6 +12,11 @@ from typing import Any
import logging
from csv_parser.core import iter_csv_dict_rows
from csv_parser.import_row_processing import (
aggregate_mapped_rows,
resolve_import_row_processing,
validate_import_row_processing,
)
from csv_parser.module_registry import get_module_definition
from csv_parser.type_converter import build_row_after_mapping
@@ -111,6 +116,7 @@ def run_universal_csv_import(
bool(has_header),
fm,
tc,
mapping,
error_details,
affected_ids,
)
@@ -191,12 +197,12 @@ def _import_nutrition(
has_header: bool,
fm: dict,
tc: dict | None,
mapping: dict[str, Any],
error_details: list,
affected_ids: dict,
) -> dict[str, int]:
agg: dict[str, dict[str, float]] = defaultdict(
lambda: {"kcal": 0.0, "protein_g": 0.0, "fat_g": 0.0, "carbs_g": 0.0}
)
spec = resolve_import_row_processing("nutrition", mapping)
mapped_rows: list[dict[str, Any]] = []
rows_total = 0
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
rows_total += 1
@@ -205,23 +211,39 @@ def _import_nutrition(
if d is None:
error_details.append({"row": rows_total, "error": "Datum fehlt oder ungültig"})
continue
iso = d.isoformat()
for key in ("kcal", "protein_g", "fat_g", "carbs_g"):
v = mapped.get(key)
if v is not None:
try:
agg[iso][key] += float(v)
except (TypeError, ValueError):
pass
mapped["date"] = d
mapped_rows.append(mapped)
if spec:
try:
validate_import_row_processing("nutrition", spec, fm)
except ValueError as e:
raise ValueError(str(e)) from e
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
else:
merged_rows = list(mapped_rows)
inserted = 0
updated = 0
new_entries = 0
for iso, vals in agg.items():
kcal = round(vals["kcal"], 1)
fat = round(vals["fat_g"], 1)
carbs = round(vals["carbs_g"], 1)
prot = round(vals["protein_g"], 1)
for merged in merged_rows:
d = coerce_date(merged.get("date"))
if d is None:
continue
iso = d.isoformat()
def _sf_macro(x: Any) -> float:
if x is None or x == "":
return 0.0
try:
return float(x)
except (TypeError, ValueError):
return 0.0
kcal = round(_sf_macro(merged.get("kcal")), 1)
fat = round(_sf_macro(merged.get("fat_g")), 1)
carbs = round(_sf_macro(merged.get("carbs_g")), 1)
prot = round(_sf_macro(merged.get("protein_g")), 1)
if kcal == 0 and fat == 0 and carbs == 0 and prot == 0:
continue
+152
View File
@@ -0,0 +1,152 @@
"""
Zeilenaggregation nach CSV-Mapping (group_by + aggregates), vor dem DB-Upsert.
Spezifikation in der Vorlage (import_row_processing JSONB) oder Modul-Default
(import_row_processing_default in module_registry).
"""
from __future__ import annotations
import datetime as dt
import statistics
from typing import Any, Mapping
from csv_parser.module_registry import get_module_definition
ALLOWED_AGGREGATES = frozenset({"sum", "mean", "min", "max", "median", "first", "last"})
def resolve_import_row_processing(module: str, mapping_row: Mapping[str, Any]) -> dict[str, Any] | None:
"""Explizite Vorlage hat Vorrang; sonst Modul-Default; leeres Dict zählt wie „nicht gesetzt“."""
raw = mapping_row.get("import_row_processing")
if isinstance(raw, dict) and raw:
return dict(raw)
mod = get_module_definition(module)
if not mod:
return None
default = mod.get("import_row_processing_default")
if isinstance(default, dict) and default:
return dict(default)
return None
def validate_import_row_processing(
module: str,
spec: Mapping[str, Any],
field_mappings: Mapping[str, Any],
) -> None:
"""Wirft ValueError bei ungültiger Konfiguration."""
mod = get_module_definition(module)
if not mod:
raise ValueError(f"Unbekanntes Modul: {module}")
allowed = set(mod.get("fields") or [])
fm_targets = {str(v) for v in field_mappings.values() if v and v not in ("-", "_skip")}
group_by = spec.get("group_by") or []
if not isinstance(group_by, list) or not all(isinstance(x, str) for x in group_by):
raise ValueError("import_row_processing.group_by muss eine Liste von Feldnamen sein")
aggregates = spec.get("aggregates") or {}
if not isinstance(aggregates, dict):
raise ValueError("import_row_processing.aggregates muss ein Objekt sein")
for g in group_by:
if g not in allowed:
raise ValueError(f"group_by: unbekanntes Feld '{g}' für Modul '{module}'")
if g not in fm_targets:
raise ValueError(
f"group_by: Zielfeld '{g}' ist keiner CSV-Spalte zugeordnet — Aggregation nicht möglich."
)
for field, op in aggregates.items():
if field not in allowed:
raise ValueError(f"aggregates: unbekanntes Feld '{field}' für Modul '{module}'")
if str(op) not in ALLOWED_AGGREGATES:
raise ValueError(
f"aggregates['{field}']: ungültige Operation '{op}'. "
f"Erlaubt: {', '.join(sorted(ALLOWED_AGGREGATES))}"
)
def _sort_key_for_group(v: Any) -> Any:
if isinstance(v, dt.datetime):
return v.isoformat()
if isinstance(v, dt.date):
return v.isoformat()
if isinstance(v, dt.time):
return v.isoformat()
return v
def _apply_aggregate(op: str, values: list[Any]) -> Any:
nums: list[float] = []
for x in values:
if x is None or x == "":
continue
try:
nums.append(float(x))
except (TypeError, ValueError):
continue
if op == "sum":
return sum(nums) if nums else None
if op == "mean":
return statistics.mean(nums) if nums else None
if op == "median":
return float(statistics.median(nums)) if nums else None
if op == "min":
return min(nums) if nums else None
if op == "max":
return max(nums) if nums else None
if op == "first":
for x in values:
if x is not None and x != "":
return x
return None
if op == "last":
for x in reversed(values):
if x is not None and x != "":
return x
return None
raise ValueError(f"Unbekannte Aggregations-Operation: {op}")
def aggregate_mapped_rows(
rows: list[dict[str, Any]],
spec: Mapping[str, Any],
) -> list[dict[str, Any]]:
"""
Gruppiert gemappte Zeilen-Dicts nach group_by und wendet aggregates an.
Felder, die weder in group_by noch in aggregates vorkommen: Wert aus der ersten Zeile der Gruppe.
"""
group_by = spec.get("group_by") or []
aggregates = spec.get("aggregates") or {}
if not group_by:
return rows
buckets: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
order: list[tuple[Any, ...]] = []
for r in rows:
key = tuple(_sort_key_for_group(r.get(g)) for g in group_by)
if key not in buckets:
buckets[key] = []
order.append(key)
buckets[key].append(r)
out: list[dict[str, Any]] = []
for key in order:
group_rows = buckets[key]
first = group_rows[0]
merged: dict[str, Any] = {}
for g in group_by:
merged[g] = first.get(g)
for field, op in aggregates.items():
merged[field] = _apply_aggregate(str(op), [row.get(field) for row in group_rows])
for row in group_rows:
for k, v in row.items():
if k in merged:
continue
if k in group_by or k in aggregates:
continue
merged[k] = v
out.append(merged)
return out
+10
View File
@@ -23,6 +23,16 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
},
"duplicate_key": ["profile_id", "date"],
"duplicate_strategy": "update",
# Mehrere CSV-Zeilen pro Tag (z. B. pro Lebensmittel) → ein nutrition_log-Eintrag
"import_row_processing_default": {
"group_by": ["date"],
"aggregates": {
"kcal": "sum",
"protein_g": "sum",
"fat_g": "sum",
"carbs_g": "sum",
},
},
},
"activity": {
"table": "activity_log",