feat: Enhance activity metrics handling and documentation
Deploy Development / deploy (push) Successful in 1m1s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Updated the README to include new activity production architecture and phases, improving clarity on the development roadmap.
- Enhanced the `ACTIVITY_SESSION_METRICS_EAV_AGENT_GUIDE` with details on the target architecture and phase plan for production readiness.
- Introduced a new function `merge_column_backed_and_eav_metrics` to streamline the merging of metrics from column-backed and EAV sources, ensuring data integrity and reducing duplication.
- Refactored session metrics handling to eliminate deprecated synchronization methods, improving the overall efficiency of data processing.
- Added unit tests for the new merging logic, ensuring robust validation of metrics handling.
This commit is contained in:
Lars committed 2026-04-15 16:59:11 +02:00
1 parent 58ddde6b1e
commit ca8cee990b
9 files changed
+619 -82

No files matched your search

+78
View File
@@ -0,0 +1,78 @@
"""
Kanonische Aufteilung activity_log vs. EAV für Aktivitätssessions.
Single Source für: welche Felder das CSV-/Registry-Modul „activity“ direkt in activity_log schreibt,
und welche training_parameters primär über EAV laufen (mit optionalem Lesefallback auf Legacy-Spalten).
Normative Doku: .claude/docs/technical/ACTIVITY_PRODUCTION_ARCHITECTURE_AND_PHASES.md
"""
from __future__ import annotations
from typing import Dict, Final
# ── activity_log: Modul „activity“ (Universal-CSV-Kern) ───────────────────────
# Nur diese Keys erscheinen in csv_parser.module_registry MODULE_DEFINITIONS["activity"].fields.
# Alles Weitere: training_parameters + EAV (Import über upsert_session_metrics_from_csv_mapped).
ACTIVITY_MODULE_REGISTRY_FIELD_KEYS: Final[frozenset[str]] = frozenset(
{
"date",
"start_time",
"end_time",
"activity_type",
"duration_min",
"kcal_active",
"kcal_resting",
"distance_km",
"hr_avg",
"hr_max",
"rpe",
"notes",
}
)
# Parameter-Keys (training_parameters.key), die primär in EAV geführt werden; source_field nach Migration 057 NULL.
# Lesefallback: activity_log-Spalte unter ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM, falls EAV leer.
ACTIVITY_EAV_PRIMARY_PARAMETER_KEYS: Final[frozenset[str]] = frozenset(
{
"min_hr",
"pace_min_per_km",
"cadence",
"avg_power",
"elevation_gain",
"temperature_celsius",
"humidity_percent",
"avg_hr_percent",
"kcal_per_km",
}
)
# Spaltenname activity_log für Legacy-Lesefallback (Merge), wenn EAV für den Parameter fehlt.
ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM: Final[Dict[str, str]] = {
"min_hr": "hr_min",
"pace_min_per_km": "pace_min_per_km",
"cadence": "cadence",
"avg_power": "avg_power",
"elevation_gain": "elevation_gain",
"temperature_celsius": "temperature_celsius",
"humidity_percent": "humidity_percent",
"avg_hr_percent": "avg_hr_percent",
"kcal_per_km": "kcal_per_km",
}
# Spalten, die mit training_parameters.source_field (nach Migration 057) noch activity_log abbilden.
# Erweiterte Metriken sind EAV-primär — nicht hier auflisten.
ACTIVITY_LOG_PATCHABLE_COLUMNS: Final[frozenset[str]] = frozenset(
{
"start_time",
"end_time",
"activity_type",
"duration_min",
"kcal_active",
"kcal_resting",
"hr_avg",
"hr_max",
"distance_km",
"rpe",
"notes",
}
)
@@ -15,7 +15,6 @@ from typing import Any, Dict, List, Mapping, Optional
from models import ActivityEntry
from csv_parser.module_registry import get_module_definition
from data_layer.activity_session_metrics import sync_column_backed_session_metrics
logger = logging.getLogger(__name__)
@@ -248,7 +247,7 @@ def insert_activity_csv_minimal(
def run_activity_post_write_hooks(cur, profile_id: str, eid: str) -> None:
"""Auto-Eval (falls aktiv) + EAV-Spiegel aus activity_log-Spalten."""
"""Auto-Eval (falls aktiv). Kein Spalte→EAV-Sync: Lesepfad merge_column_backed_and_eav_metrics."""
if _EVALUATION_AVAILABLE and _evaluate_and_save_activity:
cur.execute(
"""
@@ -269,7 +268,6 @@ def run_activity_post_write_hooks(cur, profile_id: str, eid: str) -> None:
_evaluate_and_save_activity(cur, eid, activity_dict, training_type_id, profile_id)
except Exception as eval_error:
logger.error("[AUTO-EVAL] activity %s: %s", eid, eval_error)
sync_column_backed_session_metrics(cur, str(profile_id), str(eid))
def run_activity_post_write_hooks_import(
@@ -286,7 +284,7 @@ def run_activity_post_write_hooks_import(
kcal_active: Any,
kcal_resting: Any,
) -> None:
"""Eval + EAV nach Legacy-Import mit vorgebautem Kontext-Dict."""
"""Auto-Eval nach Import. Kein Spalte→EAV-Sync (siehe run_activity_post_write_hooks)."""
if _EVALUATION_AVAILABLE and training_type_id and _evaluate_and_save_activity:
try:
activity_dict = {
@@ -308,7 +306,6 @@ def run_activity_post_write_hooks_import(
_evaluate_and_save_activity(cur, eid, activity_dict, training_type_id, profile_id)
except Exception as eval_err:
logger.warning("[activity import] Auto-Eval fehlgeschlagen: %s", eval_err)
sync_column_backed_session_metrics(cur, str(profile_id), str(eid))
def merge_activity_csv_module_fields(
+124 -64
View File
@@ -10,36 +10,10 @@ from decimal import Decimal
from typing import Any, Dict, List, Mapping, Optional, Sequence
from csv_parser.module_registry import get_module_definition
from data_layer.activity_data_canon import ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM
logger = logging.getLogger(__name__)
# activity_log-Spalten, die per training_parameters.source_field aus CSV (Parameter-Key) befüllt werden dürfen.
# Muss mit sync_column_backed_session_metrics übereinstimmen (inkl. Kernmetriken wie hr_avg).
ACTIVITY_LOG_PATCHABLE_COLUMNS = frozenset(
{
"start_time",
"end_time",
"activity_type",
"duration_min",
"kcal_active",
"kcal_resting",
"hr_avg",
"hr_max",
"hr_min",
"distance_km",
"rpe",
"pace_min_per_km",
"cadence",
"avg_power",
"elevation_gain",
"temperature_celsius",
"humidity_percent",
"avg_hr_percent",
"kcal_per_km",
"notes",
}
)
# Diese Spalten nicht aus CSV-Parameter-Zuordnung überschreiben (kommen aus Typ-Mapping / System).
ACTIVITY_LOG_PATCH_FORBIDDEN = frozenset(
{
@@ -328,13 +302,87 @@ def upsert_session_metrics_from_csv_mapped(
)
def merge_column_backed_and_eav_metrics(
header: Mapping[str, Any],
schema: Sequence[Dict[str, Any]],
eav_metrics: Sequence[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""
Effektive Metrikliste: Pro Schema-Parameter mit source_field gilt activity_log als kanonisch, wenn
die Spalte befüllt und koerzierbar ist; sonst Fallback EAV. Reine EAV-Parameter (ohne Spalte oder
leere Spalte) kommen aus EAV. Verhindert doppelte Semantik ohne Schreib-Sync.
"""
eav_by_key = {m["key"]: m for m in eav_metrics}
merged: List[Dict[str, Any]] = []
keys_handled: set[str] = set()
for s in schema:
k = s["key"]
tid = s["training_parameter_id"]
dt = s["data_type"]
unit = s.get("unit")
sf = s.get("source_field")
used_column = False
if sf and isinstance(sf, str) and str(sf).strip():
col = str(sf).strip()
if col in header and header[col] is not None:
try:
val = _coerce_raw_value_for_parameter(dt, header[col])
merged.append(
{
"training_parameter_id": tid,
"key": k,
"data_type": dt,
"unit": unit,
"value": val,
}
)
used_column = True
keys_handled.add(k)
except (TypeError, ValueError):
pass
if used_column:
continue
if k in eav_by_key:
merged.append(dict(eav_by_key[k]))
keys_handled.add(k)
continue
legacy_col = ACTIVITY_LOG_LEGACY_COLUMN_FOR_EAV_PRIMARY_PARAM.get(k)
if legacy_col and legacy_col in header and header[legacy_col] is not None:
try:
val = _coerce_raw_value_for_parameter(dt, header[legacy_col])
merged.append(
{
"training_parameter_id": tid,
"key": k,
"data_type": dt,
"unit": unit,
"value": val,
}
)
keys_handled.add(k)
except (TypeError, ValueError):
pass
for m in eav_metrics:
if m["key"] in keys_handled:
continue
merged.append(dict(m))
merged.sort(key=lambda x: x["key"])
return merged
def sync_column_backed_session_metrics(cur, profile_id: str, activity_log_id: str) -> None:
"""
EAV-Zeilen für alle Schema-Parameter mit gesetztem source_field aus der activity_log-Zeile
schreiben (Upsert) bzw. bei NULL in der Quellspalte löschen. Reine Layer-1-Logik; keine Router-Abhängigkeit.
[Veraltet / nicht mehr in Schreibpfaden aufgerufen]
Synchron mit Übergangsphase: activity_log bleibt kanonisch für klassische Spalten; EAV spiegelt dieselben
Werte für Profil/Platzhalter/Detail-API, ohne replace_activity_session_metrics aufzurufen.
Früher: EAV spiegelte activity_log-Spalten für Parameter mit source_field.
Kanon: Spaltenwerte werden bei merge_column_backed_and_eav_metrics beim Lesen berücksichtigt; keine
doppelte Speicherung. Funktion bleibt für optionale Admin-/Reparatur-Skripte.
"""
cur.execute("SELECT * FROM activity_log WHERE id = %s", (activity_log_id,))
row = cur.fetchone()
@@ -527,36 +575,7 @@ def get_activity_session_logical_unit(cur, profile_id: str, activity_log_id: str
cur, header.get("training_category"), header.get("training_type_id")
)
metrics = fetch_activity_session_metrics(cur, activity_log_id)
by_key = {m["key"]: m for m in metrics}
merged_metrics: List[Dict[str, Any]] = list(metrics)
for s in schema:
k = s["key"]
if k in by_key:
continue
sf = s.get("source_field")
if not sf or (isinstance(sf, str) and not str(sf).strip()):
continue
col = str(sf).strip()
if col not in header:
continue
raw = header.get(col)
if raw is None:
continue
dt = s["data_type"]
try:
val = _coerce_raw_value_for_parameter(dt, raw)
except (TypeError, ValueError):
continue
merged_metrics.append(
{
"training_parameter_id": s["training_parameter_id"],
"key": k,
"data_type": dt,
"unit": s.get("unit"),
"value": val,
}
)
merged_metrics.sort(key=lambda x: x["key"])
merged_metrics = merge_column_backed_and_eav_metrics(header, schema, metrics)
return {
"header": header,
"schema": schema,
@@ -565,17 +584,33 @@ def get_activity_session_logical_unit(cur, profile_id: str, activity_log_id: str
def enrich_sessions_with_metrics(cur, sessions: List[Dict[str, Any]]) -> None:
"""Mutates each session dict: adds key 'session_metrics' (list) when sessions non-empty."""
"""
Mutates each session dict: adds key 'session_metrics' (list).
Kombiniert EAV mit activity_log-Spalten für Parameter mit source_field (kanonisch: Spalte),
analog zu get_activity_session_logical_unit – ohne doppelte EAV-Speicherung beim Import.
"""
if not sessions:
return
ids = [str(s["id"]) for s in sessions if s.get("id")]
if not ids:
return
ph = ",".join(["%s"] * len(ids))
cur.execute(
f"SELECT * FROM activity_log WHERE id IN ({ph})",
ids,
)
headers_by_id: Dict[str, Dict[str, Any]] = {}
for r in cur.fetchall():
h = dict(r)
headers_by_id[str(h["id"])] = h
cur.execute(
f"""
SELECT
m.activity_log_id,
m.training_parameter_id,
tp.key,
tp.data_type,
tp.unit,
@@ -603,8 +638,33 @@ def enrich_sessions_with_metrics(cur, sessions: List[Dict[str, Any]]) -> None:
else:
val = r["value_bool"]
by_act.setdefault(aid, []).append(
{"key": r["key"], "data_type": dt, "unit": r["unit"], "value": val}
{
"training_parameter_id": r["training_parameter_id"],
"key": r["key"],
"data_type": dt,
"unit": r["unit"],
"value": val,
}
)
schema_cache: Dict[tuple[Any, Any], List[Dict[str, Any]]] = {}
def _schema(cat: Any, tid: Any) -> List[Dict[str, Any]]:
cache_key = (cat, tid)
if cache_key not in schema_cache:
schema_cache[cache_key] = resolve_activity_attribute_schema(cur, cat, tid)
return schema_cache[cache_key]
for s in sessions:
aid = str(s.get("id"))
s["session_metrics"] = by_act.get(aid, [])
header = headers_by_id.get(aid)
if not header:
s["session_metrics"] = []
continue
schema = _schema(header.get("training_category"), header.get("training_type_id"))
eav_list = by_act.get(aid, [])
merged = merge_column_backed_and_eav_metrics(header, schema, eav_list)
s["session_metrics"] = [
{"key": m["key"], "data_type": m["data_type"], "unit": m["unit"], "value": m["value"]}
for m in merged
]