feat: Enhance CSV import and validation for activity module
Deploy Development / deploy (push) Successful in 47s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Updated the CSV import logic to merge active training parameters with static fields for the activity module, improving field mapping accuracy.
- Enhanced validation functions to incorporate dynamic field definitions based on active training parameters, ensuring better data integrity during imports.
- Refactored related functions to streamline the process of handling CSV templates and field mappings, improving maintainability and clarity.
- Added new utility functions for resolving activity log column patches and upserting session metrics from CSV, enhancing the overall import functionality.
This commit is contained in:
Lars committed 2026-04-15 08:12:58 +02:00
1 parent 934b915357
commit 574af61349
11 files changed
+325 -80

No files matched your search

+34 -20
View File
@@ -814,6 +814,10 @@ def _import_activity(
run_activity_post_write_hooks_import,
update_activity_columns,
)
from data_layer.activity_session_metrics import (
resolve_activity_log_column_patch_from_csv,
upsert_session_metrics_from_csv_mapped,
)
rows_total = 0
inserted = 0
@@ -894,29 +898,29 @@ def _import_activity(
training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity(
cur, wtype, profile_id
)
column_patch = resolve_activity_log_column_patch_from_csv(
cur, mapped, training_category, training_type_id
)
existing_id = find_activity_duplicate_id(cur, profile_id, iso, workout_start_t)
if existing_id:
update_activity_columns(
cur,
profile_id,
existing_id,
{
"start_time": workout_start_t,
"end_time": end_str or None,
"activity_type": wtype,
"duration_min": duration_min,
"kcal_active": kcal_a,
"kcal_resting": kcal_r,
"hr_avg": hr_a,
"hr_max": hr_m,
"distance_km": dist,
"training_type_id": training_type_id,
"training_category": training_category,
"training_subcategory": training_subcategory,
"source": "csv",
},
)
upd = {
"start_time": workout_start_t,
"end_time": end_str or None,
"activity_type": wtype,
"duration_min": duration_min,
"kcal_active": kcal_a,
"kcal_resting": kcal_r,
"hr_avg": hr_a,
"hr_max": hr_m,
"distance_km": dist,
"training_type_id": training_type_id,
"training_category": training_category,
"training_subcategory": training_subcategory,
"source": "csv",
}
upd.update(column_patch)
update_activity_columns(cur, profile_id, existing_id, upd)
updated += 1
affected_ids["activity_log"].append(str(existing_id))
aid = existing_id
@@ -945,6 +949,8 @@ def _import_activity(
new_entries += 1
affected_ids["activity_log"].append(str(eid))
aid = eid
if column_patch:
update_activity_columns(cur, profile_id, aid, column_patch)
run_activity_post_write_hooks_import(
cur,
@@ -959,6 +965,14 @@ def _import_activity(
kcal_active=kcal_a,
kcal_resting=kcal_r,
)
upsert_session_metrics_from_csv_mapped(
cur,
profile_id,
str(aid),
mapped,
training_category,
training_type_id,
)
cur.execute("RELEASE SAVEPOINT csv_activity_row")
except Exception as e:
try:
@@ -37,12 +37,16 @@ def validate_import_row_processing(
module: str,
spec: Mapping[str, Any],
field_mappings: Mapping[str, Any],
cur=None,
) -> None:
"""Wirft ValueError bei ungültiger Konfiguration."""
mod = get_module_definition(module)
if not mod:
raise ValueError(f"Unbekanntes Modul: {module}")
allowed = set(mod.get("fields") or [])
if module == "activity" and cur is not None:
cur.execute("SELECT key FROM training_parameters WHERE is_active = true")
allowed.update(str(r["key"]) for r in cur.fetchall())
fm_targets = {str(v) for v in field_mappings.values() if v and v not in ("-", "_skip")}
group_by = spec.get("group_by") or []
+33 -5
View File
@@ -127,13 +127,19 @@ def _match_seed_to_db_field(header: str, seed_fm: Mapping[str, str]) -> str | No
return None
def _alias_suggest(norm: str, module: str, used: set[str]) -> str | None:
def _alias_suggest(
norm: str,
module: str,
used: set[str],
*,
field_order: list[str] | None = None,
) -> str | None:
aliases = _MODULE_HEADER_ALIASES.get(module, {})
mod = get_module_definition(module)
if not mod:
return None
field_order = list(mod["fields"].keys())
for db_field in field_order:
order = field_order if field_order is not None else list(mod["fields"].keys())
for db_field in order:
if db_field in used:
continue
tokens = aliases.get(db_field, frozenset())
@@ -152,6 +158,8 @@ def suggest_field_mappings(
headers: list[str],
module: str,
seed_fm: Mapping[str, str] | None = None,
*,
effective_fields: Mapping[str, Any] | None = None,
) -> dict[str, str]:
"""
Mappt jede CSV-Spalte (Roh-Header als Key) auf DB-Feld oder '-'.
@@ -164,13 +172,16 @@ def suggest_field_mappings(
if not mod:
return {h: "-" for h in headers}
fields_map = dict(effective_fields) if effective_fields is not None else dict(mod["fields"])
field_order = list(fields_map.keys())
fm: dict[str, str] = {h: "-" for h in headers}
used: set[str] = set()
if seed_fm:
for h in headers:
db = _match_seed_to_db_field(h, seed_fm)
if db and db not in used:
if db and db not in used and db in fields_map:
fm[h] = db
used.add(db)
@@ -178,7 +189,7 @@ def suggest_field_mappings(
if fm[h] != "-":
continue
norm = _norm_key(h)
db = _alias_suggest(norm, module, used)
db = _alias_suggest(norm, module, used, field_order=field_order)
if db:
fm[h] = db
used.add(db)
@@ -190,6 +201,8 @@ def build_type_conversions_for_mapping(
module: str,
field_mappings: Mapping[str, str],
seed_tc: Mapping[str, Any] | None = None,
*,
effective_fields: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""type_conversions nur für zugewiesene Zielfelder; Seed überschreibt Defaults."""
if module == "sleep":
@@ -198,6 +211,7 @@ def build_type_conversions_for_mapping(
defaults = _DEFAULT_TYPE_CONVERSIONS.get(module, {})
out: dict[str, Any] = {}
targets = {v for v in field_mappings.values() if v and v not in ("-", "_skip")}
field_meta = dict(effective_fields) if effective_fields is not None else None
if seed_tc:
for k, v in seed_tc.items():
@@ -208,6 +222,20 @@ def build_type_conversions_for_mapping(
if t not in out and t in defaults:
out[t] = deepcopy(defaults[t])
for t in sorted(targets):
if t in out:
continue
finfo = (field_meta or {}).get(t) if field_meta else None
if not finfo:
continue
typ = finfo.get("type")
if typ == "int":
out[t] = {"type": "int", "flexible": True}
elif typ == "float":
out[t] = {"type": "float", "decimal_separator": "auto", "flexible": True}
else:
out[t] = {"type": "string"}
_apply_energy_kj_hint_from_headers(module, field_mappings, out)
return out
+4 -1
View File
@@ -125,13 +125,16 @@ def list_modules() -> list[str]:
return sorted(MODULE_DEFINITIONS.keys())
def validate_field_mappings(module: str, field_mappings: dict) -> None:
def validate_field_mappings(module: str, field_mappings: dict, cur=None) -> None:
"""Wirft ValueError bei unbekanntem Modul oder unbekanntem DB-Feld."""
mod = get_module_definition(module)
if not mod:
raise ValueError(f"Unbekanntes Modul: {module}")
fields = cast(dict, mod["fields"])
allowed = set(fields.keys())
if module == "activity" and cur is not None:
cur.execute("SELECT key FROM training_parameters WHERE is_active = true")
allowed.update(str(r["key"]) for r in cur.fetchall())
if not allowed:
for _csv_col, db_field in field_mappings.items():
if db_field not in ("", None, "-", "_skip"):
+9 -3
View File
@@ -15,6 +15,7 @@ from csv_parser.module_registry import (
validate_field_mappings,
validate_required_field_targets,
)
from data_layer.activity_persistence_orchestrator import merge_activity_csv_module_fields
ALLOWED_SPEC_TYPES = frozenset(
{"string", "float", "number", "int", "date", "time", "datetime", "duration"}
@@ -50,6 +51,8 @@ def validate_csv_template(
type_conversions: Mapping[str, Any] | None = None,
import_row_processing: Mapping[str, Any] | None = None,
column_signature: list[str] | None = None,
*,
cur=None,
) -> dict[str, Any]:
"""
Prüft eine Vorlage ohne Datei-Upload.
@@ -74,8 +77,12 @@ def validate_csv_template(
)
return {"valid": False, "errors": errors, "warnings": warnings}
field_defs = dict(mod.get("fields") or {})
if module == "activity" and cur is not None:
field_defs = merge_activity_csv_module_fields(cur, field_defs)
try:
validate_field_mappings(module, fm)
validate_field_mappings(module, fm, cur=cur)
except ValueError as e:
errors.append(
_issue(
@@ -100,7 +107,7 @@ def validate_csv_template(
if import_row_processing:
try:
validate_import_row_processing_spec(module, import_row_processing, fm)
validate_import_row_processing_spec(module, import_row_processing, fm, cur=cur)
except ValueError as e:
errors.append(
_issue(
@@ -111,7 +118,6 @@ def validate_csv_template(
)
)
field_defs = mod.get("fields") or {}
for db_field, spec in tc.items():
if db_field not in field_defs:
errors.append(