feat: Enhance CSV import and validation for activity module
- Updated the CSV import logic to merge active training parameters with static fields for the activity module, improving field mapping accuracy. - Enhanced validation functions to incorporate dynamic field definitions based on active training parameters, ensuring better data integrity during imports. - Refactored related functions to streamline the process of handling CSV templates and field mappings, improving maintainability and clarity. - Added new utility functions for resolving activity log column patches and upserting session metrics from CSV, enhancing the overall import functionality.
This commit is contained in:
1 parent
934b915357
commit
574af61349
11 files changed
+325
-80
No files matched your search
@@ -814,6 +814,10 @@ def _import_activity(
|
||||
run_activity_post_write_hooks_import,
|
||||
update_activity_columns,
|
||||
)
|
||||
from data_layer.activity_session_metrics import (
|
||||
resolve_activity_log_column_patch_from_csv,
|
||||
upsert_session_metrics_from_csv_mapped,
|
||||
)
|
||||
|
||||
rows_total = 0
|
||||
inserted = 0
|
||||
@@ -894,29 +898,29 @@ def _import_activity(
|
||||
training_type_id, training_category, training_subcategory = _resolve_training_type_for_activity(
|
||||
cur, wtype, profile_id
|
||||
)
|
||||
column_patch = resolve_activity_log_column_patch_from_csv(
|
||||
cur, mapped, training_category, training_type_id
|
||||
)
|
||||
existing_id = find_activity_duplicate_id(cur, profile_id, iso, workout_start_t)
|
||||
|
||||
if existing_id:
|
||||
update_activity_columns(
|
||||
cur,
|
||||
profile_id,
|
||||
existing_id,
|
||||
{
|
||||
"start_time": workout_start_t,
|
||||
"end_time": end_str or None,
|
||||
"activity_type": wtype,
|
||||
"duration_min": duration_min,
|
||||
"kcal_active": kcal_a,
|
||||
"kcal_resting": kcal_r,
|
||||
"hr_avg": hr_a,
|
||||
"hr_max": hr_m,
|
||||
"distance_km": dist,
|
||||
"training_type_id": training_type_id,
|
||||
"training_category": training_category,
|
||||
"training_subcategory": training_subcategory,
|
||||
"source": "csv",
|
||||
},
|
||||
)
|
||||
upd = {
|
||||
"start_time": workout_start_t,
|
||||
"end_time": end_str or None,
|
||||
"activity_type": wtype,
|
||||
"duration_min": duration_min,
|
||||
"kcal_active": kcal_a,
|
||||
"kcal_resting": kcal_r,
|
||||
"hr_avg": hr_a,
|
||||
"hr_max": hr_m,
|
||||
"distance_km": dist,
|
||||
"training_type_id": training_type_id,
|
||||
"training_category": training_category,
|
||||
"training_subcategory": training_subcategory,
|
||||
"source": "csv",
|
||||
}
|
||||
upd.update(column_patch)
|
||||
update_activity_columns(cur, profile_id, existing_id, upd)
|
||||
updated += 1
|
||||
affected_ids["activity_log"].append(str(existing_id))
|
||||
aid = existing_id
|
||||
@@ -945,6 +949,8 @@ def _import_activity(
|
||||
new_entries += 1
|
||||
affected_ids["activity_log"].append(str(eid))
|
||||
aid = eid
|
||||
if column_patch:
|
||||
update_activity_columns(cur, profile_id, aid, column_patch)
|
||||
|
||||
run_activity_post_write_hooks_import(
|
||||
cur,
|
||||
@@ -959,6 +965,14 @@ def _import_activity(
|
||||
kcal_active=kcal_a,
|
||||
kcal_resting=kcal_r,
|
||||
)
|
||||
upsert_session_metrics_from_csv_mapped(
|
||||
cur,
|
||||
profile_id,
|
||||
str(aid),
|
||||
mapped,
|
||||
training_category,
|
||||
training_type_id,
|
||||
)
|
||||
cur.execute("RELEASE SAVEPOINT csv_activity_row")
|
||||
except Exception as e:
|
||||
try:
|
||||
|
||||
@@ -37,12 +37,16 @@ def validate_import_row_processing(
|
||||
module: str,
|
||||
spec: Mapping[str, Any],
|
||||
field_mappings: Mapping[str, Any],
|
||||
cur=None,
|
||||
) -> None:
|
||||
"""Wirft ValueError bei ungültiger Konfiguration."""
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
raise ValueError(f"Unbekanntes Modul: {module}")
|
||||
allowed = set(mod.get("fields") or [])
|
||||
if module == "activity" and cur is not None:
|
||||
cur.execute("SELECT key FROM training_parameters WHERE is_active = true")
|
||||
allowed.update(str(r["key"]) for r in cur.fetchall())
|
||||
fm_targets = {str(v) for v in field_mappings.values() if v and v not in ("-", "_skip")}
|
||||
|
||||
group_by = spec.get("group_by") or []
|
||||
|
||||
@@ -127,13 +127,19 @@ def _match_seed_to_db_field(header: str, seed_fm: Mapping[str, str]) -> str | No
|
||||
return None
|
||||
|
||||
|
||||
def _alias_suggest(norm: str, module: str, used: set[str]) -> str | None:
|
||||
def _alias_suggest(
|
||||
norm: str,
|
||||
module: str,
|
||||
used: set[str],
|
||||
*,
|
||||
field_order: list[str] | None = None,
|
||||
) -> str | None:
|
||||
aliases = _MODULE_HEADER_ALIASES.get(module, {})
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
return None
|
||||
field_order = list(mod["fields"].keys())
|
||||
for db_field in field_order:
|
||||
order = field_order if field_order is not None else list(mod["fields"].keys())
|
||||
for db_field in order:
|
||||
if db_field in used:
|
||||
continue
|
||||
tokens = aliases.get(db_field, frozenset())
|
||||
@@ -152,6 +158,8 @@ def suggest_field_mappings(
|
||||
headers: list[str],
|
||||
module: str,
|
||||
seed_fm: Mapping[str, str] | None = None,
|
||||
*,
|
||||
effective_fields: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Mappt jede CSV-Spalte (Roh-Header als Key) auf DB-Feld oder '-'.
|
||||
@@ -164,13 +172,16 @@ def suggest_field_mappings(
|
||||
if not mod:
|
||||
return {h: "-" for h in headers}
|
||||
|
||||
fields_map = dict(effective_fields) if effective_fields is not None else dict(mod["fields"])
|
||||
field_order = list(fields_map.keys())
|
||||
|
||||
fm: dict[str, str] = {h: "-" for h in headers}
|
||||
used: set[str] = set()
|
||||
|
||||
if seed_fm:
|
||||
for h in headers:
|
||||
db = _match_seed_to_db_field(h, seed_fm)
|
||||
if db and db not in used:
|
||||
if db and db not in used and db in fields_map:
|
||||
fm[h] = db
|
||||
used.add(db)
|
||||
|
||||
@@ -178,7 +189,7 @@ def suggest_field_mappings(
|
||||
if fm[h] != "-":
|
||||
continue
|
||||
norm = _norm_key(h)
|
||||
db = _alias_suggest(norm, module, used)
|
||||
db = _alias_suggest(norm, module, used, field_order=field_order)
|
||||
if db:
|
||||
fm[h] = db
|
||||
used.add(db)
|
||||
@@ -190,6 +201,8 @@ def build_type_conversions_for_mapping(
|
||||
module: str,
|
||||
field_mappings: Mapping[str, str],
|
||||
seed_tc: Mapping[str, Any] | None = None,
|
||||
*,
|
||||
effective_fields: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""type_conversions nur für zugewiesene Zielfelder; Seed überschreibt Defaults."""
|
||||
if module == "sleep":
|
||||
@@ -198,6 +211,7 @@ def build_type_conversions_for_mapping(
|
||||
defaults = _DEFAULT_TYPE_CONVERSIONS.get(module, {})
|
||||
out: dict[str, Any] = {}
|
||||
targets = {v for v in field_mappings.values() if v and v not in ("-", "_skip")}
|
||||
field_meta = dict(effective_fields) if effective_fields is not None else None
|
||||
|
||||
if seed_tc:
|
||||
for k, v in seed_tc.items():
|
||||
@@ -208,6 +222,20 @@ def build_type_conversions_for_mapping(
|
||||
if t not in out and t in defaults:
|
||||
out[t] = deepcopy(defaults[t])
|
||||
|
||||
for t in sorted(targets):
|
||||
if t in out:
|
||||
continue
|
||||
finfo = (field_meta or {}).get(t) if field_meta else None
|
||||
if not finfo:
|
||||
continue
|
||||
typ = finfo.get("type")
|
||||
if typ == "int":
|
||||
out[t] = {"type": "int", "flexible": True}
|
||||
elif typ == "float":
|
||||
out[t] = {"type": "float", "decimal_separator": "auto", "flexible": True}
|
||||
else:
|
||||
out[t] = {"type": "string"}
|
||||
|
||||
_apply_energy_kj_hint_from_headers(module, field_mappings, out)
|
||||
return out
|
||||
|
||||
|
||||
@@ -125,13 +125,16 @@ def list_modules() -> list[str]:
|
||||
return sorted(MODULE_DEFINITIONS.keys())
|
||||
|
||||
|
||||
def validate_field_mappings(module: str, field_mappings: dict) -> None:
|
||||
def validate_field_mappings(module: str, field_mappings: dict, cur=None) -> None:
|
||||
"""Wirft ValueError bei unbekanntem Modul oder unbekanntem DB-Feld."""
|
||||
mod = get_module_definition(module)
|
||||
if not mod:
|
||||
raise ValueError(f"Unbekanntes Modul: {module}")
|
||||
fields = cast(dict, mod["fields"])
|
||||
allowed = set(fields.keys())
|
||||
if module == "activity" and cur is not None:
|
||||
cur.execute("SELECT key FROM training_parameters WHERE is_active = true")
|
||||
allowed.update(str(r["key"]) for r in cur.fetchall())
|
||||
if not allowed:
|
||||
for _csv_col, db_field in field_mappings.items():
|
||||
if db_field not in ("", None, "-", "_skip"):
|
||||
|
||||
@@ -15,6 +15,7 @@ from csv_parser.module_registry import (
|
||||
validate_field_mappings,
|
||||
validate_required_field_targets,
|
||||
)
|
||||
from data_layer.activity_persistence_orchestrator import merge_activity_csv_module_fields
|
||||
|
||||
ALLOWED_SPEC_TYPES = frozenset(
|
||||
{"string", "float", "number", "int", "date", "time", "datetime", "duration"}
|
||||
@@ -50,6 +51,8 @@ def validate_csv_template(
|
||||
type_conversions: Mapping[str, Any] | None = None,
|
||||
import_row_processing: Mapping[str, Any] | None = None,
|
||||
column_signature: list[str] | None = None,
|
||||
*,
|
||||
cur=None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Prüft eine Vorlage ohne Datei-Upload.
|
||||
@@ -74,8 +77,12 @@ def validate_csv_template(
|
||||
)
|
||||
return {"valid": False, "errors": errors, "warnings": warnings}
|
||||
|
||||
field_defs = dict(mod.get("fields") or {})
|
||||
if module == "activity" and cur is not None:
|
||||
field_defs = merge_activity_csv_module_fields(cur, field_defs)
|
||||
|
||||
try:
|
||||
validate_field_mappings(module, fm)
|
||||
validate_field_mappings(module, fm, cur=cur)
|
||||
except ValueError as e:
|
||||
errors.append(
|
||||
_issue(
|
||||
@@ -100,7 +107,7 @@ def validate_csv_template(
|
||||
|
||||
if import_row_processing:
|
||||
try:
|
||||
validate_import_row_processing_spec(module, import_row_processing, fm)
|
||||
validate_import_row_processing_spec(module, import_row_processing, fm, cur=cur)
|
||||
except ValueError as e:
|
||||
errors.append(
|
||||
_issue(
|
||||
@@ -111,7 +118,6 @@ def validate_csv_template(
|
||||
)
|
||||
)
|
||||
|
||||
field_defs = mod.get("fields") or {}
|
||||
for db_field, spec in tc.items():
|
||||
if db_field not in field_defs:
|
||||
errors.append(
|
||||
|
||||
Reference in new issue
Block a user