feat(csv-import): Enhance row aggregation and validation features
Deploy Development / deploy (push) Successful in 47s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 15s

- Updated the aggregate_mapped_rows function to support multiple row policies, allowing for flexible handling of duplicate keys during CSV imports.
- Introduced deduplication of identical rows before aggregation, improving data integrity.
- Enhanced validation for multi_row_policy and dedupe_identical_rows in import_row_processing specifications.
- Updated the AdminCsvTemplateEditorPage to include options for multi-row policies and deduplication settings, improving user experience in template management.
- Added comprehensive tests to validate new aggregation behaviors and ensure correct error handling for multiple rows.
This commit is contained in:
Lars committed 2026-04-10 15:36:12 +02:00
1 parent ad7aa2d255
commit b7cd710c32
5 files changed
+327 -41

No files matched your search

+66 -5
View File
@@ -1,8 +1,9 @@
"""
Zeilenaggregation nach CSV-Mapping (group_by + aggregates), vor dem DB-Upsert.
Spezifikation in der Vorlage (import_row_processing JSONB) oder Modul-Default
(import_row_processing_default in module_registry).
Spezifikation in der Vorlage (import_row_processing JSONB). Optional: Modul-Default
(import_row_processing_default in module_registry) nur als **Legacy-Fallback**, wenn
die Vorlage nichts speichert — mittelfristig sollen Vorlagen explizit sein.
"""
from __future__ import annotations
@@ -14,6 +15,8 @@ from typing import Any, Mapping
from csv_parser.module_registry import get_module_definition
ALLOWED_AGGREGATES = frozenset({"sum", "mean", "min", "max", "median", "first", "last"})
# Mehr als eine CSV-Zeile pro group_by-Schlüssel
ALLOWED_MULTI_ROW_POLICIES = frozenset({"aggregate", "reject", "first_row", "last_row"})
def resolve_import_row_processing(module: str, mapping_row: Mapping[str, Any]) -> dict[str, Any] | None:
@@ -66,6 +69,17 @@ def validate_import_row_processing(
f"Erlaubt: {', '.join(sorted(ALLOWED_AGGREGATES))}"
)
mrp = spec.get("multi_row_policy")
if mrp is not None and str(mrp) not in ALLOWED_MULTI_ROW_POLICIES:
raise ValueError(
f"multi_row_policy: ungültiger Wert '{mrp}'. "
f"Erlaubt: {', '.join(sorted(ALLOWED_MULTI_ROW_POLICIES))}"
)
dedupe = spec.get("dedupe_identical_rows")
if dedupe is not None and not isinstance(dedupe, bool):
raise ValueError("dedupe_identical_rows muss ein Boolean sein")
def _sort_key_for_group(v: Any) -> Any:
if isinstance(v, dt.datetime):
@@ -110,18 +124,46 @@ def _apply_aggregate(op: str, values: list[Any]) -> Any:
raise ValueError(f"Unbekannte Aggregations-Operation: {op}")
def _row_identity_signature(r: dict[str, Any]) -> tuple[Any, ...]:
return tuple(sorted((k, _sort_key_for_group(r.get(k))) for k in sorted(r.keys())))
def _dedupe_identical_mapped_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Exakt gleiche gemappte Zeilen (alle Keys/Werte) — erste behalten."""
seen: set[tuple[Any, ...]] = set()
out: list[dict[str, Any]] = []
for r in rows:
sig = _row_identity_signature(r)
if sig in seen:
continue
seen.add(sig)
out.append(r)
return out
def aggregate_mapped_rows(
rows: list[dict[str, Any]],
spec: Mapping[str, Any],
) -> list[dict[str, Any]]:
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""
Gruppiert gemappte Zeilen-Dicts nach group_by und wendet aggregates an.
Felder, die weder in group_by noch in aggregates vorkommen: Wert aus der ersten Zeile der Gruppe.
Rückgabe: (merged_rows, strukturelle Fehler / Hinweise, z. B. abgelehnte Schlüsselgruppen).
"""
errors: list[dict[str, Any]] = []
rows = list(rows)
if spec.get("dedupe_identical_rows"):
rows = _dedupe_identical_mapped_rows(rows)
group_by = spec.get("group_by") or []
aggregates = spec.get("aggregates") or {}
policy = str(spec.get("multi_row_policy") or "aggregate")
if policy not in ALLOWED_MULTI_ROW_POLICIES:
policy = "aggregate"
if not group_by:
return rows
return rows, errors
buckets: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
order: list[tuple[Any, ...]] = []
@@ -132,9 +174,28 @@ def aggregate_mapped_rows(
order.append(key)
buckets[key].append(r)
gb_label = ", ".join(group_by)
out: list[dict[str, Any]] = []
for key in order:
group_rows = buckets[key]
if len(group_rows) > 1:
if policy == "reject":
errors.append(
{
"error": "mehrere_zeilen_pro_schluessel",
"message": (
f"{len(group_rows)} CSV-Zeilen mit gleichem Schlüssel ({gb_label}); "
"laut Vorlage abgelehnt (multi_row_policy=reject)."
),
"rows_in_group": len(group_rows),
}
)
continue
if policy == "first_row":
group_rows = [group_rows[0]]
elif policy == "last_row":
group_rows = [group_rows[-1]]
first = group_rows[0]
merged: dict[str, Any] = {}
for g in group_by:
@@ -149,4 +210,4 @@ def aggregate_mapped_rows(
continue
merged[k] = v
out.append(merged)
return out
return out, errors