refactor(csv-import): Enhance nutrition data processing and template rendering
- Updated the nutrition import logic to utilize a new row processing specification, improving data aggregation and validation. - Refactored the template rendering process in the workflow executor to use Jinja2's Environment with ChainableUndefined for better handling of missing attributes. - Added backward-compatible shortcuts for accessing decision signals in node contexts, enhancing flexibility in template usage. - Introduced import row processing options in CSV templates, allowing for more customizable data handling during imports.
This commit is contained in:
1 parent
8b67f7ab55
commit
c0fcdea1fe
9 files changed
+357
-22
No files matched your search
@@ -12,6 +12,11 @@ from typing import Any
|
||||
import logging
|
||||
|
||||
from csv_parser.core import iter_csv_dict_rows
|
||||
from csv_parser.import_row_processing import (
|
||||
aggregate_mapped_rows,
|
||||
resolve_import_row_processing,
|
||||
validate_import_row_processing,
|
||||
)
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
from csv_parser.type_converter import build_row_after_mapping
|
||||
|
||||
@@ -111,6 +116,7 @@ def run_universal_csv_import(
|
||||
bool(has_header),
|
||||
fm,
|
||||
tc,
|
||||
mapping,
|
||||
error_details,
|
||||
affected_ids,
|
||||
)
|
||||
@@ -191,12 +197,12 @@ def _import_nutrition(
|
||||
has_header: bool,
|
||||
fm: dict,
|
||||
tc: dict | None,
|
||||
mapping: dict[str, Any],
|
||||
error_details: list,
|
||||
affected_ids: dict,
|
||||
) -> dict[str, int]:
|
||||
agg: dict[str, dict[str, float]] = defaultdict(
|
||||
lambda: {"kcal": 0.0, "protein_g": 0.0, "fat_g": 0.0, "carbs_g": 0.0}
|
||||
)
|
||||
spec = resolve_import_row_processing("nutrition", mapping)
|
||||
mapped_rows: list[dict[str, Any]] = []
|
||||
rows_total = 0
|
||||
for csv_row in iter_csv_dict_rows(text, delim, has_header=has_header):
|
||||
rows_total += 1
|
||||
@@ -205,23 +211,39 @@ def _import_nutrition(
|
||||
if d is None:
|
||||
error_details.append({"row": rows_total, "error": "Datum fehlt oder ungültig"})
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
for key in ("kcal", "protein_g", "fat_g", "carbs_g"):
|
||||
v = mapped.get(key)
|
||||
if v is not None:
|
||||
try:
|
||||
agg[iso][key] += float(v)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
mapped["date"] = d
|
||||
mapped_rows.append(mapped)
|
||||
|
||||
if spec:
|
||||
try:
|
||||
validate_import_row_processing("nutrition", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
new_entries = 0
|
||||
for iso, vals in agg.items():
|
||||
kcal = round(vals["kcal"], 1)
|
||||
fat = round(vals["fat_g"], 1)
|
||||
carbs = round(vals["carbs_g"], 1)
|
||||
prot = round(vals["protein_g"], 1)
|
||||
for merged in merged_rows:
|
||||
d = coerce_date(merged.get("date"))
|
||||
if d is None:
|
||||
continue
|
||||
iso = d.isoformat()
|
||||
|
||||
def _sf_macro(x: Any) -> float:
|
||||
if x is None or x == "":
|
||||
return 0.0
|
||||
try:
|
||||
return float(x)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
kcal = round(_sf_macro(merged.get("kcal")), 1)
|
||||
fat = round(_sf_macro(merged.get("fat_g")), 1)
|
||||
carbs = round(_sf_macro(merged.get("carbs_g")), 1)
|
||||
prot = round(_sf_macro(merged.get("protein_g")), 1)
|
||||
if kcal == 0 and fat == 0 and carbs == 0 and prot == 0:
|
||||
continue
|
||||
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Zeilenaggregation nach CSV-Mapping (group_by + aggregates), vor dem DB-Upsert.
|
||||
|
||||
Spezifikation in der Vorlage (import_row_processing JSONB) oder Modul-Default
|
||||
(import_row_processing_default in module_registry).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import statistics
|
||||
from typing import Any, Mapping
|
||||
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
|
||||
ALLOWED_AGGREGATES = frozenset({"sum", "mean", "min", "max", "median", "first", "last"})
|
||||
|
||||
|
||||
def resolve_import_row_processing(module: str, mapping_row: Mapping[str, Any]) -> dict[str, Any] | None:
|
||||
"""Explizite Vorlage hat Vorrang; sonst Modul-Default; leeres Dict zählt wie „nicht gesetzt“."""
|
||||
raw = mapping_row.get("import_row_processing")
|
||||
if isinstance(raw, dict) and raw:
|
||||
return dict(raw)
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
return None
|
||||
default = mod.get("import_row_processing_default")
|
||||
if isinstance(default, dict) and default:
|
||||
return dict(default)
|
||||
return None
|
||||
|
||||
|
||||
def validate_import_row_processing(
|
||||
module: str,
|
||||
spec: Mapping[str, Any],
|
||||
field_mappings: Mapping[str, Any],
|
||||
) -> None:
|
||||
"""Wirft ValueError bei ungültiger Konfiguration."""
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
raise ValueError(f"Unbekanntes Modul: {module}")
|
||||
allowed = set(mod.get("fields") or [])
|
||||
fm_targets = {str(v) for v in field_mappings.values() if v and v not in ("-", "_skip")}
|
||||
|
||||
group_by = spec.get("group_by") or []
|
||||
if not isinstance(group_by, list) or not all(isinstance(x, str) for x in group_by):
|
||||
raise ValueError("import_row_processing.group_by muss eine Liste von Feldnamen sein")
|
||||
aggregates = spec.get("aggregates") or {}
|
||||
if not isinstance(aggregates, dict):
|
||||
raise ValueError("import_row_processing.aggregates muss ein Objekt sein")
|
||||
|
||||
for g in group_by:
|
||||
if g not in allowed:
|
||||
raise ValueError(f"group_by: unbekanntes Feld '{g}' für Modul '{module}'")
|
||||
if g not in fm_targets:
|
||||
raise ValueError(
|
||||
f"group_by: Zielfeld '{g}' ist keiner CSV-Spalte zugeordnet — Aggregation nicht möglich."
|
||||
)
|
||||
|
||||
for field, op in aggregates.items():
|
||||
if field not in allowed:
|
||||
raise ValueError(f"aggregates: unbekanntes Feld '{field}' für Modul '{module}'")
|
||||
if str(op) not in ALLOWED_AGGREGATES:
|
||||
raise ValueError(
|
||||
f"aggregates['{field}']: ungültige Operation '{op}'. "
|
||||
f"Erlaubt: {', '.join(sorted(ALLOWED_AGGREGATES))}"
|
||||
)
|
||||
|
||||
|
||||
def _sort_key_for_group(v: Any) -> Any:
|
||||
if isinstance(v, dt.datetime):
|
||||
return v.isoformat()
|
||||
if isinstance(v, dt.date):
|
||||
return v.isoformat()
|
||||
if isinstance(v, dt.time):
|
||||
return v.isoformat()
|
||||
return v
|
||||
|
||||
|
||||
def _apply_aggregate(op: str, values: list[Any]) -> Any:
|
||||
nums: list[float] = []
|
||||
for x in values:
|
||||
if x is None or x == "":
|
||||
continue
|
||||
try:
|
||||
nums.append(float(x))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
if op == "sum":
|
||||
return sum(nums) if nums else None
|
||||
if op == "mean":
|
||||
return statistics.mean(nums) if nums else None
|
||||
if op == "median":
|
||||
return float(statistics.median(nums)) if nums else None
|
||||
if op == "min":
|
||||
return min(nums) if nums else None
|
||||
if op == "max":
|
||||
return max(nums) if nums else None
|
||||
if op == "first":
|
||||
for x in values:
|
||||
if x is not None and x != "":
|
||||
return x
|
||||
return None
|
||||
if op == "last":
|
||||
for x in reversed(values):
|
||||
if x is not None and x != "":
|
||||
return x
|
||||
return None
|
||||
raise ValueError(f"Unbekannte Aggregations-Operation: {op}")
|
||||
|
||||
|
||||
def aggregate_mapped_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
spec: Mapping[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Gruppiert gemappte Zeilen-Dicts nach group_by und wendet aggregates an.
|
||||
Felder, die weder in group_by noch in aggregates vorkommen: Wert aus der ersten Zeile der Gruppe.
|
||||
"""
|
||||
group_by = spec.get("group_by") or []
|
||||
aggregates = spec.get("aggregates") or {}
|
||||
if not group_by:
|
||||
return rows
|
||||
|
||||
buckets: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
|
||||
order: list[tuple[Any, ...]] = []
|
||||
for r in rows:
|
||||
key = tuple(_sort_key_for_group(r.get(g)) for g in group_by)
|
||||
if key not in buckets:
|
||||
buckets[key] = []
|
||||
order.append(key)
|
||||
buckets[key].append(r)
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for key in order:
|
||||
group_rows = buckets[key]
|
||||
first = group_rows[0]
|
||||
merged: dict[str, Any] = {}
|
||||
for g in group_by:
|
||||
merged[g] = first.get(g)
|
||||
for field, op in aggregates.items():
|
||||
merged[field] = _apply_aggregate(str(op), [row.get(field) for row in group_rows])
|
||||
for row in group_rows:
|
||||
for k, v in row.items():
|
||||
if k in merged:
|
||||
continue
|
||||
if k in group_by or k in aggregates:
|
||||
continue
|
||||
merged[k] = v
|
||||
out.append(merged)
|
||||
return out
|
||||
@@ -23,6 +23,16 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
"duplicate_strategy": "update",
|
||||
# Mehrere CSV-Zeilen pro Tag (z. B. pro Lebensmittel) → ein nutrition_log-Eintrag
|
||||
"import_row_processing_default": {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {
|
||||
"kcal": "sum",
|
||||
"protein_g": "sum",
|
||||
"fat_g": "sum",
|
||||
"carbs_g": "sum",
|
||||
},
|
||||
},
|
||||
},
|
||||
"activity": {
|
||||
"table": "activity_log",
|
||||
|
||||
Reference in new issue
Block a user