feat: Refactor activity import logic and enhance CSV handling
- Replaced the deprecated `resolve_activity_log_column_patch_from_csv` function with `activity_csv_registry_updates_from_mapped` to streamline updates from CSV mappings. - Updated the `_import_activity` function to utilize the new registry updates, improving data integrity during activity imports. - Enhanced the activity module registry by adding German labels for various fields, improving localization support. - Refactored the session metrics handling to ensure only relevant fields are processed, enhancing the overall robustness of CSV imports.
This commit is contained in:
1 parent
9d47c4ef84
commit
08eae86ddc
6 files changed
+181
-93
No files matched your search
@@ -7,9 +7,10 @@ Feld-Katalog für CSV-Mappings: get_mappable_activity_field_catalog()
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
from models import ActivityEntry
|
||||
|
||||
@@ -45,6 +46,89 @@ def find_activity_duplicate_id(
|
||||
return str(row["id"]) if row else None
|
||||
|
||||
|
||||
# Datum/Start/Ende/Typ setzt der CSV-Executor explizit (Normalisierung); nicht aus diesem Patch überschreiben.
|
||||
_ACTIVITY_CSV_REGISTRY_EXCLUDE = frozenset({"date", "start_time", "end_time", "activity_type"})
|
||||
|
||||
|
||||
def activity_registry_field_keys() -> frozenset[str]:
|
||||
mod = get_module_definition("activity")
|
||||
if not mod:
|
||||
return frozenset()
|
||||
return frozenset((mod.get("fields") or {}).keys())
|
||||
|
||||
|
||||
def activity_csv_registry_updates_from_mapped(mapped: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
activity_log-Updates nur aus Modul-Registry-Feldern (Kernspalten).
|
||||
Trainingsparameter-Keys (nur in training_parameters) laufen über EAV, nicht hier.
|
||||
"""
|
||||
mod = get_module_definition("activity")
|
||||
if not mod:
|
||||
return {}
|
||||
fields = mod.get("fields") or {}
|
||||
out: Dict[str, Any] = {}
|
||||
|
||||
def _sf(v: Any) -> float | None:
|
||||
try:
|
||||
if v is None or (isinstance(v, str) and not str(v).strip()):
|
||||
return None
|
||||
return round(float(v), 1)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def _si(v: Any) -> int | None:
|
||||
try:
|
||||
if v is None or (isinstance(v, str) and not str(v).strip()):
|
||||
return None
|
||||
return int(round(float(v)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def _hr(v: Any) -> float | None:
|
||||
x = _sf(v)
|
||||
if x is None or x < 20 or x > 280:
|
||||
return None
|
||||
return x
|
||||
|
||||
for key, spec in fields.items():
|
||||
if key in _ACTIVITY_CSV_REGISTRY_EXCLUDE:
|
||||
continue
|
||||
if key not in mapped:
|
||||
continue
|
||||
raw = mapped[key]
|
||||
if raw is None or raw == "":
|
||||
continue
|
||||
if isinstance(raw, str) and not raw.strip():
|
||||
continue
|
||||
typ = spec.get("type", "string")
|
||||
if typ == "float":
|
||||
v = _hr(raw) if key in ("hr_avg", "hr_max") else _sf(raw)
|
||||
if v is not None:
|
||||
out[key] = v
|
||||
elif typ == "int":
|
||||
v = _si(raw)
|
||||
if v is not None:
|
||||
out[key] = v
|
||||
elif typ == "datetime":
|
||||
if isinstance(raw, dt.datetime):
|
||||
out[key] = raw.strftime("%Y-%m-%d %H:%M:%S")
|
||||
elif isinstance(raw, dt.date):
|
||||
out[key] = f"{raw.isoformat()} 00:00:00"
|
||||
elif isinstance(raw, str) and raw.strip():
|
||||
out[key] = raw.strip()
|
||||
elif typ == "date":
|
||||
if isinstance(raw, dt.date):
|
||||
out[key] = raw.isoformat()
|
||||
elif isinstance(raw, dt.datetime):
|
||||
out[key] = raw.date().isoformat()
|
||||
elif isinstance(raw, str) and raw.strip():
|
||||
out[key] = raw.strip()
|
||||
else:
|
||||
out[key] = str(raw).strip()
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def insert_activity_from_entry(cur, profile_id: str, eid: str, e: ActivityEntry) -> None:
|
||||
"""INSERT activity_log aus ActivityEntry (manueller API-Pfad)."""
|
||||
d = e.model_dump()
|
||||
@@ -296,7 +380,7 @@ def get_mappable_activity_field_catalog(cur, profile_id: str) -> Dict[str, Any]:
|
||||
"data_type": s.get("type", "string"),
|
||||
"required": bool(s.get("required")),
|
||||
"unit": s.get("unit"),
|
||||
"label_de": key,
|
||||
"label_de": s.get("label_de") or key,
|
||||
}
|
||||
)
|
||||
core_fields.sort(key=lambda x: x["key"])
|
||||
|
||||
@@ -9,6 +9,8 @@ import logging
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence
|
||||
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# activity_log-Spalten, die per training_parameters.source_field aus CSV (Parameter-Key) befüllt werden dürfen.
|
||||
@@ -265,46 +267,6 @@ def _coerce_raw_value_for_parameter(data_type: str, raw: Any) -> Any:
|
||||
raise ValueError(data_type)
|
||||
|
||||
|
||||
def resolve_activity_log_column_patch_from_csv(
|
||||
cur,
|
||||
mapped: Mapping[str, Any],
|
||||
training_category: Optional[str],
|
||||
training_type_id: Optional[int],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Zusätzliche activity_log-Updates aus CSV: Parameter mit source_field → Spalte.
|
||||
"""
|
||||
schema = resolve_activity_attribute_schema(cur, training_category, training_type_id)
|
||||
patch: Dict[str, Any] = {}
|
||||
for spec in schema:
|
||||
src_col = (spec.get("source_field") or "").strip()
|
||||
if not src_col or src_col in ACTIVITY_LOG_PATCH_FORBIDDEN:
|
||||
continue
|
||||
if src_col not in ACTIVITY_LOG_PATCHABLE_COLUMNS:
|
||||
continue
|
||||
pkey = spec["key"]
|
||||
if pkey not in mapped:
|
||||
continue
|
||||
raw = mapped[pkey]
|
||||
if raw is None or raw == "":
|
||||
continue
|
||||
dt = spec["data_type"]
|
||||
rules = _validation_rules_dict(spec["validation_rules"])
|
||||
try:
|
||||
coerced = _coerce_raw_value_for_parameter(dt, raw)
|
||||
_validate_single_value(dt, coerced, rules)
|
||||
except (ActivitySessionMetricsError, TypeError, ValueError) as ex:
|
||||
logger.warning(
|
||||
"CSV activity_log patch skipped %s → %s: %s",
|
||||
pkey,
|
||||
src_col,
|
||||
ex,
|
||||
)
|
||||
continue
|
||||
patch[src_col] = coerced
|
||||
return patch
|
||||
|
||||
|
||||
def upsert_session_metrics_from_csv_mapped(
|
||||
cur,
|
||||
profile_id: str,
|
||||
@@ -314,15 +276,10 @@ def upsert_session_metrics_from_csv_mapped(
|
||||
training_type_id: Optional[int],
|
||||
) -> None:
|
||||
"""
|
||||
EAV für Schema-Parameter aus CSV-mapped Werten.
|
||||
EAV für Trainingsparameter aus CSV (nur Keys, die nicht im activity-Modul-Registry liegen).
|
||||
|
||||
Spalten-gestützte Parameter (source_field ∈ ACTIVITY_LOG_PATCHABLE_COLUMNS) werden
|
||||
ausschließlich über resolve_activity_log_column_patch_from_csv → activity_log
|
||||
geschrieben — hier kein EAV.
|
||||
|
||||
Wenn source_field gesetzt ist, aber **kein** patchbarer Spaltenname (z. B. eigener
|
||||
Key „stola“ wie der Parametername), wäre früher weder Spalten-Update noch EAV erfolgt;
|
||||
dann EAV wie bei reinen Metriken.
|
||||
Kernfelder (Datum, Start, Distanz, HF, …) schreibt der Executor nach activity_log;
|
||||
hier keine doppelten EAV-Zeilen für dieselben Registry-Keys.
|
||||
"""
|
||||
cur.execute(
|
||||
"SELECT profile_id FROM activity_log WHERE id = %s",
|
||||
@@ -331,6 +288,8 @@ def upsert_session_metrics_from_csv_mapped(
|
||||
row = cur.fetchone()
|
||||
if not row or str(row["profile_id"]) != str(profile_id):
|
||||
return
|
||||
mod = get_module_definition("activity") or {}
|
||||
activity_registry_keys = frozenset((mod.get("fields") or {}).keys())
|
||||
schema = resolve_activity_attribute_schema(cur, training_category, training_type_id)
|
||||
for spec in schema:
|
||||
pkey = spec["key"]
|
||||
@@ -339,8 +298,7 @@ def upsert_session_metrics_from_csv_mapped(
|
||||
raw = mapped[pkey]
|
||||
if raw is None or raw == "":
|
||||
continue
|
||||
src_col = (spec.get("source_field") or "").strip()
|
||||
if src_col and src_col in ACTIVITY_LOG_PATCHABLE_COLUMNS:
|
||||
if pkey in activity_registry_keys:
|
||||
continue
|
||||
tid = spec["training_parameter_id"]
|
||||
dt = spec["data_type"]
|
||||
|
||||
Reference in new issue
Block a user