feat(csv-import): Enhance row aggregation and validation features
- Updated the aggregate_mapped_rows function to support multiple row policies, allowing for flexible handling of duplicate keys during CSV imports. - Introduced deduplication of identical rows before aggregation, improving data integrity. - Enhanced validation for multi_row_policy and dedupe_identical_rows in import_row_processing specifications. - Updated the AdminCsvTemplateEditorPage to include options for multi-row policies and deduplication settings, improving user experience in template management. - Added comprehensive tests to validate new aggregation behaviors and ensure correct error handling for multiple rows.
This commit is contained in:
1 parent
ad7aa2d255
commit
b7cd710c32
5 files changed
+327
-41
No files matched your search
@@ -38,12 +38,79 @@ def test_aggregate_mapped_rows_sums_same_group():
|
||||
{"date": d, "kcal": 300.0, "protein_g": 15},
|
||||
]
|
||||
spec = {"group_by": ["date"], "aggregates": {"kcal": "sum", "protein_g": "sum"}}
|
||||
out = aggregate_mapped_rows(rows, spec)
|
||||
out, err = aggregate_mapped_rows(rows, spec)
|
||||
assert err == []
|
||||
assert len(out) == 1
|
||||
assert out[0]["kcal"] == 800.0
|
||||
assert out[0]["protein_g"] == 35
|
||||
|
||||
|
||||
def test_aggregate_mapped_rows_reject_second_group():
|
||||
d = dt.date(2024, 1, 15)
|
||||
rows = [
|
||||
{"date": d, "kcal": 100.0},
|
||||
{"date": d, "kcal": 200.0},
|
||||
]
|
||||
spec = {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {"kcal": "sum"},
|
||||
"multi_row_policy": "reject",
|
||||
}
|
||||
out, err = aggregate_mapped_rows(rows, spec)
|
||||
assert out == []
|
||||
assert len(err) == 1
|
||||
assert err[0].get("error") == "mehrere_zeilen_pro_schluessel"
|
||||
assert err[0].get("rows_in_group") == 2
|
||||
|
||||
|
||||
def test_aggregate_mapped_rows_first_row_no_merge():
|
||||
d = dt.date(2024, 1, 15)
|
||||
rows = [
|
||||
{"date": d, "kcal": 100.0},
|
||||
{"date": d, "kcal": 999.0},
|
||||
]
|
||||
spec = {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {"kcal": "sum"},
|
||||
"multi_row_policy": "first_row",
|
||||
}
|
||||
out, err = aggregate_mapped_rows(rows, spec)
|
||||
assert err == []
|
||||
assert len(out) == 1
|
||||
assert out[0]["kcal"] == 100.0
|
||||
|
||||
|
||||
def test_dedupe_identical_rows_before_group():
|
||||
d = dt.date(2024, 1, 15)
|
||||
rows = [
|
||||
{"date": d, "kcal": 50.0},
|
||||
{"date": d, "kcal": 50.0},
|
||||
{"date": d, "kcal": 50.0},
|
||||
]
|
||||
spec = {
|
||||
"group_by": ["date"],
|
||||
"aggregates": {"kcal": "sum"},
|
||||
"dedupe_identical_rows": True,
|
||||
}
|
||||
out, err = aggregate_mapped_rows(rows, spec)
|
||||
assert err == []
|
||||
assert len(out) == 1
|
||||
assert out[0]["kcal"] == 50.0
|
||||
|
||||
|
||||
def test_validate_multi_row_policy():
|
||||
with pytest.raises(ValueError, match="multi_row_policy"):
|
||||
validate_import_row_processing(
|
||||
"nutrition",
|
||||
{
|
||||
"group_by": ["date"],
|
||||
"aggregates": {"kcal": "sum"},
|
||||
"multi_row_policy": "nope",
|
||||
},
|
||||
{"D": "date", "K": "kcal"},
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_explicit_overrides_default():
|
||||
m = {
|
||||
"import_row_processing": {"group_by": ["date"], "aggregates": {"kcal": "mean"}},
|
||||
|
||||
Reference in new issue
Block a user