feat(csv-import): Enhance row aggregation and validation features
- Updated the aggregate_mapped_rows function to support multiple row policies, allowing for flexible handling of duplicate keys during CSV imports. - Introduced deduplication of identical rows before aggregation, improving data integrity. - Enhanced validation for multi_row_policy and dedupe_identical_rows in import_row_processing specifications. - Updated the AdminCsvTemplateEditorPage to include options for multi-row policies and deduplication settings, improving user experience in template management. - Added comprehensive tests to validate new aggregation behaviors and ensure correct error handling for multiple rows.
This commit is contained in:
1 parent
ad7aa2d255
commit
b7cd710c32
5 files changed
+327
-41
No files matched your search
@@ -221,9 +221,13 @@ def _import_nutrition(
|
||||
validate_import_row_processing("nutrition", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
merged_rows, agg_notes = aggregate_mapped_rows(mapped_rows, spec)
|
||||
error_details.extend(agg_notes)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
agg_notes = []
|
||||
|
||||
skipped_groups = sum(n.get("rows_in_group", 0) for n in (agg_notes or []) if n.get("error") == "mehrere_zeilen_pro_schluessel")
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
@@ -285,7 +289,7 @@ def _import_nutrition(
|
||||
"rows_total": rows_total,
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": 0,
|
||||
"skipped": skipped_groups,
|
||||
"new_entries": new_entries,
|
||||
}
|
||||
|
||||
@@ -329,9 +333,13 @@ def _import_weight(
|
||||
validate_import_row_processing("weight", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
merged_rows, agg_notes = aggregate_mapped_rows(mapped_rows, spec)
|
||||
error_details.extend(agg_notes)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
agg_notes = []
|
||||
|
||||
skipped_groups = sum(n.get("rows_in_group", 0) for n in (agg_notes or []) if n.get("error") == "mehrere_zeilen_pro_schluessel")
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
@@ -384,7 +392,7 @@ def _import_weight(
|
||||
"rows_total": rows_total,
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"skipped": 0,
|
||||
"skipped": skipped_groups,
|
||||
"new_entries": new_entries,
|
||||
}
|
||||
|
||||
@@ -555,13 +563,17 @@ def _import_vitals_baseline(
|
||||
validate_import_row_processing("vitals_baseline", spec, fm)
|
||||
except ValueError as e:
|
||||
raise ValueError(str(e)) from e
|
||||
merged_rows = aggregate_mapped_rows(mapped_rows, spec)
|
||||
merged_rows, agg_notes = aggregate_mapped_rows(mapped_rows, spec)
|
||||
error_details.extend(agg_notes)
|
||||
else:
|
||||
merged_rows = list(mapped_rows)
|
||||
agg_notes = []
|
||||
|
||||
skipped_merge = sum(n.get("rows_in_group", 0) for n in (agg_notes or []) if n.get("error") == "mehrere_zeilen_pro_schluessel")
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
skipped = skipped_prefilter
|
||||
skipped = skipped_prefilter + skipped_merge
|
||||
for merged in merged_rows:
|
||||
d = coerce_date(merged.get("date"))
|
||||
if d is None:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""
|
||||
Zeilenaggregation nach CSV-Mapping (group_by + aggregates), vor dem DB-Upsert.
|
||||
|
||||
Spezifikation in der Vorlage (import_row_processing JSONB) oder Modul-Default
|
||||
(import_row_processing_default in module_registry).
|
||||
Spezifikation in der Vorlage (import_row_processing JSONB). Optional: Modul-Default
|
||||
(import_row_processing_default in module_registry) nur als **Legacy-Fallback**, wenn
|
||||
die Vorlage nichts speichert — mittelfristig sollen Vorlagen explizit sein.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,6 +15,8 @@ from typing import Any, Mapping
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
|
||||
ALLOWED_AGGREGATES = frozenset({"sum", "mean", "min", "max", "median", "first", "last"})
|
||||
# Mehr als eine CSV-Zeile pro group_by-Schlüssel
|
||||
ALLOWED_MULTI_ROW_POLICIES = frozenset({"aggregate", "reject", "first_row", "last_row"})
|
||||
|
||||
|
||||
def resolve_import_row_processing(module: str, mapping_row: Mapping[str, Any]) -> dict[str, Any] | None:
|
||||
@@ -66,6 +69,17 @@ def validate_import_row_processing(
|
||||
f"Erlaubt: {', '.join(sorted(ALLOWED_AGGREGATES))}"
|
||||
)
|
||||
|
||||
mrp = spec.get("multi_row_policy")
|
||||
if mrp is not None and str(mrp) not in ALLOWED_MULTI_ROW_POLICIES:
|
||||
raise ValueError(
|
||||
f"multi_row_policy: ungültiger Wert '{mrp}'. "
|
||||
f"Erlaubt: {', '.join(sorted(ALLOWED_MULTI_ROW_POLICIES))}"
|
||||
)
|
||||
|
||||
dedupe = spec.get("dedupe_identical_rows")
|
||||
if dedupe is not None and not isinstance(dedupe, bool):
|
||||
raise ValueError("dedupe_identical_rows muss ein Boolean sein")
|
||||
|
||||
|
||||
def _sort_key_for_group(v: Any) -> Any:
|
||||
if isinstance(v, dt.datetime):
|
||||
@@ -110,18 +124,46 @@ def _apply_aggregate(op: str, values: list[Any]) -> Any:
|
||||
raise ValueError(f"Unbekannte Aggregations-Operation: {op}")
|
||||
|
||||
|
||||
def _row_identity_signature(r: dict[str, Any]) -> tuple[Any, ...]:
|
||||
return tuple(sorted((k, _sort_key_for_group(r.get(k))) for k in sorted(r.keys())))
|
||||
|
||||
|
||||
def _dedupe_identical_mapped_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Exakt gleiche gemappte Zeilen (alle Keys/Werte) — erste behalten."""
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
out: list[dict[str, Any]] = []
|
||||
for r in rows:
|
||||
sig = _row_identity_signature(r)
|
||||
if sig in seen:
|
||||
continue
|
||||
seen.add(sig)
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def aggregate_mapped_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
spec: Mapping[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""
|
||||
Gruppiert gemappte Zeilen-Dicts nach group_by und wendet aggregates an.
|
||||
Felder, die weder in group_by noch in aggregates vorkommen: Wert aus der ersten Zeile der Gruppe.
|
||||
|
||||
Rückgabe: (merged_rows, strukturelle Fehler / Hinweise, z. B. abgelehnte Schlüsselgruppen).
|
||||
"""
|
||||
errors: list[dict[str, Any]] = []
|
||||
rows = list(rows)
|
||||
if spec.get("dedupe_identical_rows"):
|
||||
rows = _dedupe_identical_mapped_rows(rows)
|
||||
|
||||
group_by = spec.get("group_by") or []
|
||||
aggregates = spec.get("aggregates") or {}
|
||||
policy = str(spec.get("multi_row_policy") or "aggregate")
|
||||
if policy not in ALLOWED_MULTI_ROW_POLICIES:
|
||||
policy = "aggregate"
|
||||
|
||||
if not group_by:
|
||||
return rows
|
||||
return rows, errors
|
||||
|
||||
buckets: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
|
||||
order: list[tuple[Any, ...]] = []
|
||||
@@ -132,9 +174,28 @@ def aggregate_mapped_rows(
|
||||
order.append(key)
|
||||
buckets[key].append(r)
|
||||
|
||||
gb_label = ", ".join(group_by)
|
||||
out: list[dict[str, Any]] = []
|
||||
for key in order:
|
||||
group_rows = buckets[key]
|
||||
if len(group_rows) > 1:
|
||||
if policy == "reject":
|
||||
errors.append(
|
||||
{
|
||||
"error": "mehrere_zeilen_pro_schluessel",
|
||||
"message": (
|
||||
f"{len(group_rows)} CSV-Zeilen mit gleichem Schlüssel ({gb_label}); "
|
||||
"laut Vorlage abgelehnt (multi_row_policy=reject)."
|
||||
),
|
||||
"rows_in_group": len(group_rows),
|
||||
}
|
||||
)
|
||||
continue
|
||||
if policy == "first_row":
|
||||
group_rows = [group_rows[0]]
|
||||
elif policy == "last_row":
|
||||
group_rows = [group_rows[-1]]
|
||||
|
||||
first = group_rows[0]
|
||||
merged: dict[str, Any] = {}
|
||||
for g in group_by:
|
||||
@@ -149,4 +210,4 @@ def aggregate_mapped_rows(
|
||||
continue
|
||||
merged[k] = v
|
||||
out.append(merged)
|
||||
return out
|
||||
return out, errors
|
||||
@@ -23,7 +23,7 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
"duplicate_strategy": "update",
|
||||
# Mehrere CSV-Zeilen pro Tag (z. B. pro Lebensmittel) → ein nutrition_log-Eintrag
|
||||
# Legacy-Fallback wenn die Vorlage kein import_row_processing speichert — Vorlagen mittelfristig explizit.
|
||||
"import_row_processing_default": {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {
|
||||
@@ -69,6 +69,7 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
"duplicate_strategy": "update",
|
||||
# Legacy-Fallback — Vorlagen mittelfristig explizit setzen.
|
||||
"import_row_processing_default": {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {
|
||||
@@ -102,7 +103,7 @@ MODULE_DEFINITIONS: Dict[str, Dict[str, Any]] = {
|
||||
},
|
||||
"duplicate_key": ["profile_id", "date"],
|
||||
"duplicate_strategy": "update",
|
||||
# Mehrere CSV-Zeilen pro Tag → ein Eintrag (letzte Zeile im Export zählt)
|
||||
# Legacy-Fallback — Vorlagen mittelfristig explizit setzen.
|
||||
"import_row_processing_default": {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {
|
||||
|
||||
Reference in new issue
Block a user