feat: Enhance activity metrics handling and documentation
- Updated the README to include new activity production architecture and phases, improving clarity on the development roadmap. - Enhanced the `ACTIVITY_SESSION_METRICS_EAV_AGENT_GUIDE` with details on the target architecture and phase plan for production readiness. - Introduced a new function `merge_column_backed_and_eav_metrics` to streamline the merging of metrics from column-backed and EAV sources, ensuring data integrity and reducing duplication. - Refactored session metrics handling to eliminate deprecated synchronization methods, improving the overall efficiency of data processing. - Added unit tests for the new merging logic, ensuring robust validation of metrics handling.
This commit is contained in:
1 parent
58ddde6b1e
commit
ca8cee990b
9 files changed
+619
-82
No files matched your search
@@ -10,36 +10,10 @@ from decimal import Decimal
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence
|
||||
|
||||
from csv_parser.module_registry import get_module_definition
|
||||
from data_layer.activity_data_canon import ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# activity_log-Spalten, die per training_parameters.source_field aus CSV (Parameter-Key) befüllt werden dürfen.
|
||||
# Muss mit sync_column_backed_session_metrics übereinstimmen (inkl. Kernmetriken wie hr_avg).
|
||||
ACTIVITY_LOG_PATCHABLE_COLUMNS = frozenset(
|
||||
{
|
||||
"start_time",
|
||||
"end_time",
|
||||
"activity_type",
|
||||
"duration_min",
|
||||
"kcal_active",
|
||||
"kcal_resting",
|
||||
"hr_avg",
|
||||
"hr_max",
|
||||
"hr_min",
|
||||
"distance_km",
|
||||
"rpe",
|
||||
"pace_min_per_km",
|
||||
"cadence",
|
||||
"avg_power",
|
||||
"elevation_gain",
|
||||
"temperature_celsius",
|
||||
"humidity_percent",
|
||||
"avg_hr_percent",
|
||||
"kcal_per_km",
|
||||
"notes",
|
||||
}
|
||||
)
|
||||
|
||||
# Diese Spalten nicht aus CSV-Parameter-Zuordnung überschreiben (kommen aus Typ-Mapping / System).
|
||||
ACTIVITY_LOG_PATCH_FORBIDDEN = frozenset(
|
||||
{
|
||||
@@ -328,13 +302,87 @@ def upsert_session_metrics_from_csv_mapped(
|
||||
)
|
||||
|
||||
|
||||
def merge_column_backed_and_eav_metrics(
|
||||
header: Mapping[str, Any],
|
||||
schema: Sequence[Dict[str, Any]],
|
||||
eav_metrics: Sequence[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Effektive Metrikliste: Pro Schema-Parameter mit source_field gilt activity_log als kanonisch, wenn
|
||||
die Spalte befüllt und koerzierbar ist; sonst Fallback EAV. Reine EAV-Parameter (ohne Spalte oder
|
||||
leere Spalte) kommen aus EAV. Verhindert doppelte Semantik ohne Schreib-Sync.
|
||||
"""
|
||||
eav_by_key = {m["key"]: m for m in eav_metrics}
|
||||
merged: List[Dict[str, Any]] = []
|
||||
keys_handled: set[str] = set()
|
||||
|
||||
for s in schema:
|
||||
k = s["key"]
|
||||
tid = s["training_parameter_id"]
|
||||
dt = s["data_type"]
|
||||
unit = s.get("unit")
|
||||
sf = s.get("source_field")
|
||||
|
||||
used_column = False
|
||||
if sf and isinstance(sf, str) and str(sf).strip():
|
||||
col = str(sf).strip()
|
||||
if col in header and header[col] is not None:
|
||||
try:
|
||||
val = _coerce_raw_value_for_parameter(dt, header[col])
|
||||
merged.append(
|
||||
{
|
||||
"training_parameter_id": tid,
|
||||
"key": k,
|
||||
"data_type": dt,
|
||||
"unit": unit,
|
||||
"value": val,
|
||||
}
|
||||
)
|
||||
used_column = True
|
||||
keys_handled.add(k)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if used_column:
|
||||
continue
|
||||
if k in eav_by_key:
|
||||
merged.append(dict(eav_by_key[k]))
|
||||
keys_handled.add(k)
|
||||
continue
|
||||
|
||||
legacy_col = ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM.get(k)
|
||||
if legacy_col and legacy_col in header and header[legacy_col] is not None:
|
||||
try:
|
||||
val = _coerce_raw_value_for_parameter(dt, header[legacy_col])
|
||||
merged.append(
|
||||
{
|
||||
"training_parameter_id": tid,
|
||||
"key": k,
|
||||
"data_type": dt,
|
||||
"unit": unit,
|
||||
"value": val,
|
||||
}
|
||||
)
|
||||
keys_handled.add(k)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
for m in eav_metrics:
|
||||
if m["key"] in keys_handled:
|
||||
continue
|
||||
merged.append(dict(m))
|
||||
|
||||
merged.sort(key=lambda x: x["key"])
|
||||
return merged
|
||||
|
||||
|
||||
def sync_column_backed_session_metrics(cur, profile_id: str, activity_log_id: str) -> None:
|
||||
"""
|
||||
EAV-Zeilen für alle Schema-Parameter mit gesetztem source_field aus der activity_log-Zeile
|
||||
schreiben (Upsert) bzw. bei NULL in der Quellspalte löschen. Reine Layer-1-Logik; keine Router-Abhängigkeit.
|
||||
[Veraltet / nicht mehr in Schreibpfaden aufgerufen]
|
||||
|
||||
Synchron mit Übergangsphase: activity_log bleibt kanonisch für klassische Spalten; EAV spiegelt dieselben
|
||||
Werte für Profil/Platzhalter/Detail-API, ohne replace_activity_session_metrics aufzurufen.
|
||||
Früher: EAV spiegelte activity_log-Spalten für Parameter mit source_field.
|
||||
Kanon: Spaltenwerte werden bei merge_column_backed_and_eav_metrics beim Lesen berücksichtigt; keine
|
||||
doppelte Speicherung. Funktion bleibt für optionale Admin-/Reparatur-Skripte.
|
||||
"""
|
||||
cur.execute("SELECT * FROM activity_log WHERE id = %s", (activity_log_id,))
|
||||
row = cur.fetchone()
|
||||
@@ -527,36 +575,7 @@ def get_activity_session_logical_unit(cur, profile_id: str, activity_log_id: str
|
||||
cur, header.get("training_category"), header.get("training_type_id")
|
||||
)
|
||||
metrics = fetch_activity_session_metrics(cur, activity_log_id)
|
||||
by_key = {m["key"]: m for m in metrics}
|
||||
merged_metrics: List[Dict[str, Any]] = list(metrics)
|
||||
for s in schema:
|
||||
k = s["key"]
|
||||
if k in by_key:
|
||||
continue
|
||||
sf = s.get("source_field")
|
||||
if not sf or (isinstance(sf, str) and not str(sf).strip()):
|
||||
continue
|
||||
col = str(sf).strip()
|
||||
if col not in header:
|
||||
continue
|
||||
raw = header.get(col)
|
||||
if raw is None:
|
||||
continue
|
||||
dt = s["data_type"]
|
||||
try:
|
||||
val = _coerce_raw_value_for_parameter(dt, raw)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
merged_metrics.append(
|
||||
{
|
||||
"training_parameter_id": s["training_parameter_id"],
|
||||
"key": k,
|
||||
"data_type": dt,
|
||||
"unit": s.get("unit"),
|
||||
"value": val,
|
||||
}
|
||||
)
|
||||
merged_metrics.sort(key=lambda x: x["key"])
|
||||
merged_metrics = merge_column_backed_and_eav_metrics(header, schema, metrics)
|
||||
return {
|
||||
"header": header,
|
||||
"schema": schema,
|
||||
@@ -565,17 +584,33 @@ def get_activity_session_logical_unit(cur, profile_id: str, activity_log_id: str
|
||||
|
||||
|
||||
def enrich_sessions_with_metrics(cur, sessions: List[Dict[str, Any]]) -> None:
|
||||
"""Mutates each session dict: adds key 'session_metrics' (list) when sessions non-empty."""
|
||||
"""
|
||||
Mutates each session dict: adds key 'session_metrics' (list).
|
||||
|
||||
Kombiniert EAV mit activity_log-Spalten für Parameter mit source_field (kanonisch: Spalte),
|
||||
analog zu get_activity_session_logical_unit – ohne doppelte EAV-Speicherung beim Import.
|
||||
"""
|
||||
if not sessions:
|
||||
return
|
||||
ids = [str(s["id"]) for s in sessions if s.get("id")]
|
||||
if not ids:
|
||||
return
|
||||
ph = ",".join(["%s"] * len(ids))
|
||||
|
||||
cur.execute(
|
||||
f"SELECT * FROM activity_log WHERE id IN ({ph})",
|
||||
ids,
|
||||
)
|
||||
headers_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
for r in cur.fetchall():
|
||||
h = dict(r)
|
||||
headers_by_id[str(h["id"])] = h
|
||||
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT
|
||||
m.activity_log_id,
|
||||
m.training_parameter_id,
|
||||
tp.key,
|
||||
tp.data_type,
|
||||
tp.unit,
|
||||
@@ -603,8 +638,33 @@ def enrich_sessions_with_metrics(cur, sessions: List[Dict[str, Any]]) -> None:
|
||||
else:
|
||||
val = r["value_bool"]
|
||||
by_act.setdefault(aid, []).append(
|
||||
{"key": r["key"], "data_type": dt, "unit": r["unit"], "value": val}
|
||||
{
|
||||
"training_parameter_id": r["training_parameter_id"],
|
||||
"key": r["key"],
|
||||
"data_type": dt,
|
||||
"unit": r["unit"],
|
||||
"value": val,
|
||||
}
|
||||
)
|
||||
|
||||
schema_cache: Dict[tuple[Any, Any], List[Dict[str, Any]]] = {}
|
||||
|
||||
def _schema(cat: Any, tid: Any) -> List[Dict[str, Any]]:
|
||||
cache_key = (cat, tid)
|
||||
if cache_key not in schema_cache:
|
||||
schema_cache[cache_key] = resolve_activity_attribute_schema(cur, cat, tid)
|
||||
return schema_cache[cache_key]
|
||||
|
||||
for s in sessions:
|
||||
aid = str(s.get("id"))
|
||||
s["session_metrics"] = by_act.get(aid, [])
|
||||
header = headers_by_id.get(aid)
|
||||
if not header:
|
||||
s["session_metrics"] = []
|
||||
continue
|
||||
schema = _schema(header.get("training_category"), header.get("training_type_id"))
|
||||
eav_list = by_act.get(aid, [])
|
||||
merged = merge_column_backed_and_eav_metrics(header, schema, eav_list)
|
||||
s["session_metrics"] = [
|
||||
{"key": m["key"], "data_type": m["data_type"], "unit": m["unit"], "value": m["value"]}
|
||||
for m in merged
|
||||
]
|
||||
Reference in new issue
Block a user