feat(csv-templates): Add CSV template validation endpoint and enhance error handling
- Introduced a new endpoint for validating CSV templates without saving, allowing users to check field mappings and type conversions. - Updated the `create_system_template` and `update_system_template` functions to include validation reports in responses. - Enhanced error handling in CSV import processes by integrating `enrich_row_error` for more informative error messages. - Improved the AdminCsvTemplateEditorPage to support format checking and display validation results, enhancing user experience. - Incremented version numbers for `csv_import` and `admin_csv_templates` to reflect these updates.
This commit is contained in:
parent
6945b748cb
commit
0629f88b37
|
|
@ -18,6 +18,7 @@ from csv_parser.import_row_processing import (
|
||||||
validate_import_row_processing,
|
validate_import_row_processing,
|
||||||
)
|
)
|
||||||
from csv_parser.module_registry import get_module_definition
|
from csv_parser.module_registry import get_module_definition
|
||||||
|
from csv_parser.import_errors import enrich_row_error
|
||||||
from csv_parser.type_converter import build_row_after_mapping
|
from csv_parser.type_converter import build_row_after_mapping
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -756,8 +757,9 @@ def _import_vitals_baseline(
|
||||||
cur.execute("ROLLBACK TO SAVEPOINT vitals_csv_row")
|
cur.execute("ROLLBACK TO SAVEPOINT vitals_csv_row")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
err = enrich_row_error(str(e), module="vitals_baseline")
|
||||||
error_details.append(
|
error_details.append(
|
||||||
{"row": rows_total, "error": str(e), "context": "vitals_baseline upsert"},
|
{"row": rows_total, "context": "vitals_baseline upsert", **err},
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
@ -1003,7 +1005,8 @@ def _import_activity(
|
||||||
cur.execute("ROLLBACK TO SAVEPOINT csv_activity_row")
|
cur.execute("ROLLBACK TO SAVEPOINT csv_activity_row")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
error_details.append({"row": rows_total, "error": str(e)})
|
err = enrich_row_error(str(e), module="activity")
|
||||||
|
error_details.append({"row": rows_total, **err})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"rows_total": rows_total,
|
"rows_total": rows_total,
|
||||||
|
|
|
||||||
53
backend/csv_parser/import_errors.py
Normal file
53
backend/csv_parser/import_errors.py
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
"""
|
||||||
|
Menschenlesbare Hinweise zu typischen Import-/DB-Fehlern (Universal-CSV).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_row_error(message: str, module: str | None = None) -> dict[str, str | None]:
|
||||||
|
"""
|
||||||
|
Ergänzt eine Rohexception-Zeichenkette um ``code`` und ``hint`` für die Fehlerliste im Import.
|
||||||
|
"""
|
||||||
|
low = (message or "").lower()
|
||||||
|
out: dict[str, str | None] = {"error": message, "code": None, "hint": None}
|
||||||
|
|
||||||
|
if "numeric field overflow" in low or "numeric value out of range" in low:
|
||||||
|
out["code"] = "db_numeric_overflow"
|
||||||
|
out["hint"] = (
|
||||||
|
"Wert passt nicht in die Datenbank-Spalte (z. B. NUMERIC mit begrenzter Größe). "
|
||||||
|
"Häufig: Kilojoule aus dem Export landen im Kalorien-Feld – in der Vorlage für kcal_active/kcal_resting "
|
||||||
|
'"source_unit": "kj" setzen. Oder eine falsche CSV-Spalte ist einem kleinen Zielfeld zugeordnet '
|
||||||
|
"(z. B. große Zahl in einem HF-Feld)."
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
if "violates check constraint" in low and "source" in low:
|
||||||
|
out["code"] = "db_check_constraint_source"
|
||||||
|
out["hint"] = (
|
||||||
|
"Die Tabelle erlaubt den gesetzten «source»-Wert nicht. "
|
||||||
|
"System-Vorlage / Migration zur erlaubten Quelle prüfen (z. B. csv für Universal-Import)."
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
if "current transaction is aborted" in low:
|
||||||
|
out["code"] = "transaction_aborted"
|
||||||
|
out["hint"] = (
|
||||||
|
"Eine frühere Zeile hat einen Datenbankfehler ausgelöst. "
|
||||||
|
"Zuerst die niedrigste Zeilennummer in error_details beheben (Vorlage/Daten prüfen)."
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
if "invalid input syntax" in low and "time" in low:
|
||||||
|
out["code"] = "db_time_cast"
|
||||||
|
out["hint"] = (
|
||||||
|
"start_time/end_time passen nicht zum erwarteten Zeitformat in der Datenbank. "
|
||||||
|
"Vorlage: Datums- und Zeitanteil konsistent (oft nur Uhrzeit, wenn date separat)."
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
if module == "activity" and "foreign key" in low:
|
||||||
|
out["code"] = "db_foreign_key"
|
||||||
|
out["hint"] = "Verknüpfung zur Datenbank verletzt (z. B. training_type). Support kontaktieren."
|
||||||
|
|
||||||
|
return out
|
||||||
223
backend/csv_parser/template_validator.py
Normal file
223
backend/csv_parser/template_validator.py
Normal file
|
|
@ -0,0 +1,223 @@
|
||||||
|
"""
|
||||||
|
Formatprüfung für CSV-Import-Vorlagen (field_mappings, type_conversions).
|
||||||
|
|
||||||
|
Liefert strukturierte Fehler/Warnungen für Admin-UI und Speicher-Guards.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from csv_parser.import_row_processing import validate_import_row_processing as validate_import_row_processing_spec
|
||||||
|
from csv_parser.module_registry import (
|
||||||
|
get_module_definition,
|
||||||
|
validate_field_mappings,
|
||||||
|
validate_required_field_targets,
|
||||||
|
)
|
||||||
|
|
||||||
|
ALLOWED_SPEC_TYPES = frozenset(
|
||||||
|
{"string", "float", "number", "int", "date", "time", "datetime", "duration"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _issue(
|
||||||
|
severity: str,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
hint: str | None = None,
|
||||||
|
field: str | None = None,
|
||||||
|
csv_columns: list[str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
out: dict[str, Any] = {
|
||||||
|
"severity": severity,
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
}
|
||||||
|
if hint:
|
||||||
|
out["hint"] = hint
|
||||||
|
if field:
|
||||||
|
out["field"] = field
|
||||||
|
if csv_columns:
|
||||||
|
out["csv_columns"] = csv_columns
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def validate_csv_template(
|
||||||
|
module: str,
|
||||||
|
field_mappings: Mapping[str, Any] | None,
|
||||||
|
type_conversions: Mapping[str, Any] | None = None,
|
||||||
|
import_row_processing: Mapping[str, Any] | None = None,
|
||||||
|
column_signature: list[str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Prüft eine Vorlage ohne Datei-Upload.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``{"valid": bool, "errors": [...], "warnings": [...]}``
|
||||||
|
"""
|
||||||
|
errors: list[dict[str, Any]] = []
|
||||||
|
warnings: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
fm = dict(field_mappings or {})
|
||||||
|
tc: dict[str, Any] = dict(type_conversions or {}) if type_conversions else {}
|
||||||
|
mod = get_module_definition(module)
|
||||||
|
if not mod:
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"unknown_module",
|
||||||
|
f"Unbekanntes Modul «{module}».",
|
||||||
|
hint="Nur registrierte Module in module_registry sind erlaubt.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {"valid": False, "errors": errors, "warnings": warnings}
|
||||||
|
|
||||||
|
try:
|
||||||
|
validate_field_mappings(module, fm)
|
||||||
|
except ValueError as e:
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"invalid_field_mapping",
|
||||||
|
str(e),
|
||||||
|
hint="Jede Zuordnung muss auf ein bekanntes Zielfeld des Moduls zeigen (oder „–“ / ignorieren).",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
validate_required_field_targets(module, fm)
|
||||||
|
except ValueError as e:
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"missing_required_target",
|
||||||
|
str(e),
|
||||||
|
hint="Pflichtfelder des Moduls müssen mindestens einer CSV-Spalte zugeordnet sein.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if import_row_processing:
|
||||||
|
try:
|
||||||
|
validate_import_row_processing_spec(module, import_row_processing, fm)
|
||||||
|
except ValueError as e:
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"invalid_import_row_processing",
|
||||||
|
str(e),
|
||||||
|
hint="import_row_processing: group_by und aggregates prüfen (siehe Doku Issue #21).",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
field_defs = mod.get("fields") or {}
|
||||||
|
for db_field, spec in tc.items():
|
||||||
|
if db_field not in field_defs:
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"unknown_type_conversion_field",
|
||||||
|
f"type_conversions enthält unbekanntes Zielfeld «{db_field}».",
|
||||||
|
hint="Nur Felder aus der Moduldefinition sind erlaubt.",
|
||||||
|
field=db_field,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if not isinstance(spec, Mapping):
|
||||||
|
errors.append(
|
||||||
|
_issue(
|
||||||
|
"error",
|
||||||
|
"type_conversion_not_object",
|
||||||
|
f"type_conversions[\"{db_field}\"] muss ein JSON-Objekt sein.",
|
||||||
|
field=db_field,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
stype = spec.get("type", "string")
|
||||||
|
if stype not in ALLOWED_SPEC_TYPES:
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"unusual_conversion_type",
|
||||||
|
f"Ungewöhnlicher Typ «{stype}» für «{db_field}» (erwartet u. a. string, float, date, datetime).",
|
||||||
|
field=db_field,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
finfo = field_defs.get(db_field) or {}
|
||||||
|
expected = finfo.get("type")
|
||||||
|
if expected == "date" and stype not in ("date", "datetime"):
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"date_field_conversion",
|
||||||
|
f"Zielfeld «{db_field}» ist ein Datum; der Konvertierungstyp ist «{stype}».",
|
||||||
|
hint="Meist «date» oder «datetime» mit passendem format.",
|
||||||
|
field=db_field,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if expected == "float" and stype == "int" and db_field in ("hr_avg", "hr_max"):
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"hr_as_int",
|
||||||
|
"Herzfrequenz als «int» konvertiert; Nachkommastellen aus Apple-Export gehen verloren.",
|
||||||
|
hint="Optional «float» mit flexible: true verwenden.",
|
||||||
|
field=db_field,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Mehrere CSV-Spalten → dasselbe Zielfeld
|
||||||
|
by_target: dict[str, list[str]] = {}
|
||||||
|
for csv_col, dbf in fm.items():
|
||||||
|
if dbf in (None, "", "-", "_skip"):
|
||||||
|
continue
|
||||||
|
by_target.setdefault(str(dbf), []).append(str(csv_col))
|
||||||
|
for dbf, cols in by_target.items():
|
||||||
|
if len(cols) > 1:
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"duplicate_target_columns",
|
||||||
|
f"Mehrere Spalten mappen auf «{dbf}»: {', '.join(cols)}.",
|
||||||
|
hint="Beim Import gewinnt die letzte Spalte in der CSV-Kopfzeilen-Reihenfolge.",
|
||||||
|
field=dbf,
|
||||||
|
csv_columns=cols,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Kilojoule in kcal-Feldern (häufiger Apple-DE-Fehler)
|
||||||
|
for csv_col, dbf in fm.items():
|
||||||
|
if dbf not in ("kcal_active", "kcal_resting"):
|
||||||
|
continue
|
||||||
|
col_l = str(csv_col).lower()
|
||||||
|
if "kj" in col_l or "kilojoule" in col_l:
|
||||||
|
sub = tc.get(dbf)
|
||||||
|
su = (sub or {}).get("source_unit") if isinstance(sub, Mapping) else None
|
||||||
|
if str(su or "").strip().lower() != "kj":
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"energy_kj_without_source_unit",
|
||||||
|
f"Spalte «{csv_col}» deutet auf Kilojoule, Zielfeld «{dbf}» speichert kcal.",
|
||||||
|
hint='In type_conversions für dieses Feld "source_unit": "kj" setzen (Faktor 1/4.184).',
|
||||||
|
field=str(dbf),
|
||||||
|
csv_columns=[str(csv_col)],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Signatur vs. gemappte Spalten (nur Hinweis)
|
||||||
|
if column_signature:
|
||||||
|
sig_norm = {str(c).strip() for c in column_signature if str(c).strip()}
|
||||||
|
mapped_cols = {str(k).strip() for k in fm.keys()}
|
||||||
|
if sig_norm and not sig_norm.intersection(mapped_cols):
|
||||||
|
warnings.append(
|
||||||
|
_issue(
|
||||||
|
"warning",
|
||||||
|
"signature_vs_mappings_mismatch",
|
||||||
|
"column_signature und die Schlüssel in field_mappings haben keine gemeinsame Spalte.",
|
||||||
|
hint="Signatur dient dem Ranking; für den Import müssen die Kopfzeilen der Datei zu den Keys in field_mappings passen (oder Aliase greifen).",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"valid": len(errors) == 0, "errors": errors, "warnings": warnings}
|
||||||
|
|
@ -23,11 +23,8 @@ from csv_parser.mapping_suggest import build_type_conversions_for_mapping, sugge
|
||||||
from csv_parser.import_row_processing import (
|
from csv_parser.import_row_processing import (
|
||||||
validate_import_row_processing as validate_import_row_processing_spec,
|
validate_import_row_processing as validate_import_row_processing_spec,
|
||||||
)
|
)
|
||||||
from csv_parser.module_registry import (
|
from csv_parser.module_registry import get_module_definition
|
||||||
get_module_definition,
|
from csv_parser.template_validator import validate_csv_template
|
||||||
validate_field_mappings,
|
|
||||||
validate_required_field_targets,
|
|
||||||
)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/admin/csv-templates", tags=["admin", "csv-import"])
|
router = APIRouter(prefix="/api/admin/csv-templates", tags=["admin", "csv-import"])
|
||||||
|
|
||||||
|
|
@ -62,6 +59,16 @@ class CsvImportLimitsBody(BaseModel):
|
||||||
max_file_bytes: int = Field(..., ge=10_000, le=2_147_483_648)
|
max_file_bytes: int = Field(..., ge=10_000, le=2_147_483_648)
|
||||||
|
|
||||||
|
|
||||||
|
class CsvTemplateValidateBody(BaseModel):
|
||||||
|
"""Formatprüfung ohne Speichern (field_mappings + type_conversions + optional row_processing)."""
|
||||||
|
|
||||||
|
module: str
|
||||||
|
field_mappings: dict = Field(default_factory=dict)
|
||||||
|
type_conversions: Optional[dict] = None
|
||||||
|
import_row_processing: Optional[dict] = None
|
||||||
|
column_signature: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
def _row_full(m: dict) -> dict:
|
def _row_full(m: dict) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": m["id"],
|
"id": m["id"],
|
||||||
|
|
@ -255,6 +262,23 @@ def _admin_csv_limits() -> dict[str, int]:
|
||||||
return get_csv_import_limits(r2d(row) if row else None)
|
return get_csv_import_limits(r2d(row) if row else None)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/validate")
|
||||||
|
def validate_system_template_dry_run(body: CsvTemplateValidateBody, session: dict = Depends(require_admin)):
|
||||||
|
"""
|
||||||
|
Validatorlauf für eine Vorlagen-Konfiguration (ohne DB-Schreiben).
|
||||||
|
Nutzbar aus dem Admin-Editor vor dem Speichern.
|
||||||
|
"""
|
||||||
|
if not get_module_definition(body.module):
|
||||||
|
raise HTTPException(400, f"Unbekanntes Modul: {body.module}")
|
||||||
|
return validate_csv_template(
|
||||||
|
body.module,
|
||||||
|
body.field_mappings,
|
||||||
|
body.type_conversions,
|
||||||
|
body.import_row_processing,
|
||||||
|
body.column_signature,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{template_id}")
|
@router.get("/{template_id}")
|
||||||
def get_system_template(template_id: int, session: dict = Depends(require_admin)):
|
def get_system_template(template_id: int, session: dict = Depends(require_admin)):
|
||||||
with get_db() as conn:
|
with get_db() as conn:
|
||||||
|
|
@ -273,17 +297,15 @@ def get_system_template(template_id: int, session: dict = Depends(require_admin)
|
||||||
def create_system_template(body: CsvSystemTemplateCreate, session: dict = Depends(require_admin)):
|
def create_system_template(body: CsvSystemTemplateCreate, session: dict = Depends(require_admin)):
|
||||||
if not get_module_definition(body.module):
|
if not get_module_definition(body.module):
|
||||||
raise HTTPException(400, f"Unbekanntes Modul: {body.module}")
|
raise HTTPException(400, f"Unbekanntes Modul: {body.module}")
|
||||||
try:
|
report = validate_csv_template(
|
||||||
validate_field_mappings(body.module, body.field_mappings)
|
body.module,
|
||||||
validate_required_field_targets(body.module, body.field_mappings)
|
body.field_mappings,
|
||||||
except ValueError as e:
|
body.type_conversions,
|
||||||
raise HTTPException(400, str(e))
|
body.import_row_processing,
|
||||||
|
body.column_signature,
|
||||||
if body.import_row_processing:
|
)
|
||||||
try:
|
if not report["valid"]:
|
||||||
validate_import_row_processing_spec(body.module, body.import_row_processing, body.field_mappings)
|
raise HTTPException(status_code=422, detail=report)
|
||||||
except ValueError as e:
|
|
||||||
raise HTTPException(400, str(e))
|
|
||||||
|
|
||||||
with get_db() as conn:
|
with get_db() as conn:
|
||||||
cur = get_cursor(conn)
|
cur = get_cursor(conn)
|
||||||
|
|
@ -311,7 +333,7 @@ def create_system_template(body: CsvSystemTemplateCreate, session: dict = Depend
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
new_id = cur.fetchone()["id"]
|
new_id = cur.fetchone()["id"]
|
||||||
return {"id": new_id}
|
return {"id": new_id, "validation": report}
|
||||||
|
|
||||||
|
|
||||||
@router.put("/{template_id}")
|
@router.put("/{template_id}")
|
||||||
|
|
@ -335,12 +357,19 @@ def update_system_template(
|
||||||
return _row_full(existing)
|
return _row_full(existing)
|
||||||
|
|
||||||
fm = patch.get("field_mappings", existing["field_mappings"])
|
fm = patch.get("field_mappings", existing["field_mappings"])
|
||||||
if "field_mappings" in patch:
|
tc_eff = patch.get("type_conversions", existing.get("type_conversions"))
|
||||||
try:
|
irp_eff = patch.get("import_row_processing", existing.get("import_row_processing"))
|
||||||
validate_field_mappings(existing["module"], fm)
|
col_eff = patch.get("column_signature", existing.get("column_signature"))
|
||||||
validate_required_field_targets(existing["module"], fm)
|
|
||||||
except ValueError as e:
|
report = validate_csv_template(
|
||||||
raise HTTPException(400, str(e))
|
existing["module"],
|
||||||
|
fm,
|
||||||
|
tc_eff,
|
||||||
|
irp_eff,
|
||||||
|
col_eff if isinstance(col_eff, list) else None,
|
||||||
|
)
|
||||||
|
if not report["valid"]:
|
||||||
|
raise HTTPException(status_code=422, detail=report)
|
||||||
|
|
||||||
fields_sql = []
|
fields_sql = []
|
||||||
vals: list = []
|
vals: list = []
|
||||||
|
|
@ -371,15 +400,6 @@ def update_system_template(
|
||||||
vals.append(Json(tc) if tc is not None else None)
|
vals.append(Json(tc) if tc is not None else None)
|
||||||
if "import_row_processing" in patch:
|
if "import_row_processing" in patch:
|
||||||
irp = patch["import_row_processing"]
|
irp = patch["import_row_processing"]
|
||||||
if irp:
|
|
||||||
try:
|
|
||||||
validate_import_row_processing_spec(
|
|
||||||
existing["module"],
|
|
||||||
irp,
|
|
||||||
patch.get("field_mappings", existing["field_mappings"]),
|
|
||||||
)
|
|
||||||
except ValueError as e:
|
|
||||||
raise HTTPException(400, str(e))
|
|
||||||
fields_sql.append("import_row_processing = %s")
|
fields_sql.append("import_row_processing = %s")
|
||||||
vals.append(Json(irp) if irp is not None else None)
|
vals.append(Json(irp) if irp is not None else None)
|
||||||
|
|
||||||
|
|
@ -393,7 +413,7 @@ def update_system_template(
|
||||||
|
|
||||||
cur.execute("SELECT * FROM csv_field_mappings WHERE id = %s", (template_id,))
|
cur.execute("SELECT * FROM csv_field_mappings WHERE id = %s", (template_id,))
|
||||||
m = r2d(cur.fetchone())
|
m = r2d(cur.fetchone())
|
||||||
return _row_full(m)
|
return {**_row_full(m), "validation": report}
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{template_id}")
|
@router.delete("/{template_id}")
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ from csv_parser.core import (
|
||||||
)
|
)
|
||||||
from csv_parser.type_converter import build_row_after_mapping, diagnose_row_mapping
|
from csv_parser.type_converter import build_row_after_mapping, diagnose_row_mapping
|
||||||
from csv_parser.field_units import source_unit_choices_for_field
|
from csv_parser.field_units import source_unit_choices_for_field
|
||||||
|
from csv_parser.import_errors import enrich_row_error
|
||||||
from csv_parser.module_registry import get_module_definition, list_modules, validate_field_mappings
|
from csv_parser.module_registry import get_module_definition, list_modules, validate_field_mappings
|
||||||
from csv_parser.sleep_apple_import import detect_apple_sleep_csv_format
|
from csv_parser.sleep_apple_import import detect_apple_sleep_csv_format
|
||||||
|
|
||||||
|
|
@ -554,6 +555,7 @@ async def csv_import_execute(
|
||||||
except Exception as exec_err:
|
except Exception as exec_err:
|
||||||
logger.exception("Universal-CSV-Import fehlgeschlagen: %s", exec_err)
|
logger.exception("Universal-CSV-Import fehlgeschlagen: %s", exec_err)
|
||||||
cur.execute("ROLLBACK TO SAVEPOINT csv_import_exec")
|
cur.execute("ROLLBACK TO SAVEPOINT csv_import_exec")
|
||||||
|
err_payload = {"error": str(exec_err), **enrich_row_error(str(exec_err), exec_module)}
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
UPDATE csv_import_log SET
|
UPDATE csv_import_log SET
|
||||||
|
|
@ -562,9 +564,13 @@ async def csv_import_execute(
|
||||||
error_details = %s
|
error_details = %s
|
||||||
WHERE id = %s
|
WHERE id = %s
|
||||||
""",
|
""",
|
||||||
(Json([{"error": str(exec_err)}]), log_id),
|
(Json([err_payload]), log_id),
|
||||||
)
|
)
|
||||||
err_response = HTTPException(500, f"Import fehlgeschlagen: {exec_err}")
|
hint = err_payload.get("hint")
|
||||||
|
msg = f"Import fehlgeschlagen: {exec_err}"
|
||||||
|
if hint:
|
||||||
|
msg = f"{msg} ({hint})"
|
||||||
|
err_response = HTTPException(500, msg)
|
||||||
else:
|
else:
|
||||||
cur.execute("RELEASE SAVEPOINT csv_import_exec")
|
cur.execute("RELEASE SAVEPOINT csv_import_exec")
|
||||||
cur.execute(
|
cur.execute(
|
||||||
|
|
|
||||||
10
backend/tests/test_import_errors.py
Normal file
10
backend/tests/test_import_errors.py
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
from csv_parser.import_errors import enrich_row_error
|
||||||
|
|
||||||
|
|
||||||
|
def test_enrich_numeric_overflow():
|
||||||
|
d = enrich_row_error(
|
||||||
|
"numeric field overflow\nDETAIL: A field with precision 5, scale 2\n",
|
||||||
|
module="activity",
|
||||||
|
)
|
||||||
|
assert d["code"] == "db_numeric_overflow"
|
||||||
|
assert d["hint"] and "kj" in d["hint"].lower()
|
||||||
45
backend/tests/test_template_validator.py
Normal file
45
backend/tests/test_template_validator.py
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
"""Formatprüfung CSV-Vorlagen (template_validator)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from csv_parser.template_validator import validate_csv_template
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_kj_column_warns_without_source_unit():
|
||||||
|
r = validate_csv_template(
|
||||||
|
"activity",
|
||||||
|
{"Aktive Energie (kJ)": "kcal_active", "Start": "start_time", "Trainingsart": "activity_type"},
|
||||||
|
{"start_time": {"type": "datetime", "format": "yyyy-mm-dd HH:MM:SS", "flexible": True}},
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
assert r["valid"] is True
|
||||||
|
codes = {w["code"] for w in r["warnings"]}
|
||||||
|
assert "energy_kj_without_source_unit" in codes
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_invalid_target_error():
|
||||||
|
r = validate_csv_template(
|
||||||
|
"activity",
|
||||||
|
{"X": "not_a_field"},
|
||||||
|
{},
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
assert r["valid"] is False
|
||||||
|
assert any(e["code"] == "invalid_field_mapping" for e in r["errors"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_duplicate_target_warning():
|
||||||
|
r = validate_csv_template(
|
||||||
|
"weight",
|
||||||
|
{"A": "weight", "B": "weight", "Tag": "date"},
|
||||||
|
{
|
||||||
|
"weight": {"type": "float", "decimal_separator": "."},
|
||||||
|
"date": {"type": "date", "format": "yyyy-mm-dd", "flexible": True},
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
assert r["valid"] is True
|
||||||
|
assert any(w["code"] == "duplicate_target_columns" for w in r["warnings"])
|
||||||
|
|
@ -31,8 +31,8 @@ MODULE_VERSIONS = {
|
||||||
"membership": "2.1.0",
|
"membership": "2.1.0",
|
||||||
"workflow": "0.6.0", # Phase 4: End Node Template Engine
|
"workflow": "0.6.0", # Phase 4: End Node Template Engine
|
||||||
"app_dashboard": "1.11.0", # Entitlements: DB-Override widget→features (AND), sonst Katalog
|
"app_dashboard": "1.11.0", # Entitlements: DB-Override widget→features (AND), sonst Katalog
|
||||||
"csv_import": "0.3.1", # GET /csv/modules: import_row_processing_default pro Modul
|
"csv_import": "0.3.2", # Import-Fehler: enrich_row_error / freundlichere 500-Hinweise
|
||||||
"admin_csv_templates": "0.2.0", # Admin-Editor: Zeilenaggregation (Schlüssel + gemeinsame Funktion)
|
"admin_csv_templates": "0.3.0", # POST /validate + Speichern nur bei valid (422 + warnings in Response)
|
||||||
}
|
}
|
||||||
|
|
||||||
CHANGELOG = [
|
CHANGELOG = [
|
||||||
|
|
|
||||||
|
|
@ -278,6 +278,8 @@ export default function AdminCsvTemplateEditorPage() {
|
||||||
const [loading, setLoading] = useState(!isNew)
|
const [loading, setLoading] = useState(!isNew)
|
||||||
const [analyzing, setAnalyzing] = useState(false)
|
const [analyzing, setAnalyzing] = useState(false)
|
||||||
const [saving, setSaving] = useState(false)
|
const [saving, setSaving] = useState(false)
|
||||||
|
const [validating, setValidating] = useState(false)
|
||||||
|
const [validationReport, setValidationReport] = useState(null)
|
||||||
const [error, setError] = useState(null)
|
const [error, setError] = useState(null)
|
||||||
/** Entwurf für „Quelle entspricht Ziel“ (nur source_unit custom); Commit bei Blur/Speichern. */
|
/** Entwurf für „Quelle entspricht Ziel“ (nur source_unit custom); Commit bei Blur/Speichern. */
|
||||||
const [customEquivalenceDraftByField, setCustomEquivalenceDraftByField] = useState({})
|
const [customEquivalenceDraftByField, setCustomEquivalenceDraftByField] = useState({})
|
||||||
|
|
@ -593,6 +595,38 @@ export default function AdminCsvTemplateEditorPage() {
|
||||||
setFieldMappings((prev) => ({ ...prev, [col]: dbField || '-' }))
|
setFieldMappings((prev) => ({ ...prev, [col]: dbField || '-' }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handleFormatCheck = async () => {
|
||||||
|
setError(null)
|
||||||
|
setValidationReport(null)
|
||||||
|
let tc
|
||||||
|
try {
|
||||||
|
tc = JSON.parse(typeConversionsText || '{}')
|
||||||
|
if (tc !== null && typeof tc !== 'object') throw new Error()
|
||||||
|
} catch {
|
||||||
|
setError('type_conversions: ungültiges JSON.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!module) {
|
||||||
|
setError('Modul wählen.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setValidating(true)
|
||||||
|
try {
|
||||||
|
const r = await api.adminValidateCsvTemplate({
|
||||||
|
module,
|
||||||
|
field_mappings: fieldMappings,
|
||||||
|
type_conversions: tc,
|
||||||
|
import_row_processing: null,
|
||||||
|
column_signature: columnSignature.length ? columnSignature : null,
|
||||||
|
})
|
||||||
|
setValidationReport(r)
|
||||||
|
} catch (e) {
|
||||||
|
setError(e.message || 'Formatprüfung fehlgeschlagen')
|
||||||
|
} finally {
|
||||||
|
setValidating(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const handleSave = async () => {
|
const handleSave = async () => {
|
||||||
setError(null)
|
setError(null)
|
||||||
let textForTc = typeConversionsText
|
let textForTc = typeConversionsText
|
||||||
|
|
@ -1432,7 +1466,49 @@ export default function AdminCsvTemplateEditorPage() {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{validationReport ? (
|
||||||
|
<div className="card" style={{ padding: 16, marginBottom: 16, borderColor: validationReport.valid ? 'var(--accent)' : 'var(--danger)' }}>
|
||||||
|
<div className="form-label" style={{ marginBottom: 8 }}>
|
||||||
|
Formatprüfung (Vorlage){' '}
|
||||||
|
{validationReport.valid ? <span style={{ color: 'var(--accent)' }}>— speicherfähig</span> : <span style={{ color: 'var(--danger)' }}>— Fehler beheben</span>}
|
||||||
|
</div>
|
||||||
|
<p style={{ fontSize: 12, color: 'var(--text3)', marginBottom: 10 }}>
|
||||||
|
Ohne Zeilenaggregations-JSON; vollständige Prüfung inkl. Aggregation beim Speichern. Warnungen blockieren nicht.
|
||||||
|
</p>
|
||||||
|
{validationReport.errors?.length ? (
|
||||||
|
<ul style={{ margin: '0 0 12px 1rem', color: 'var(--danger)', fontSize: 14 }}>
|
||||||
|
{validationReport.errors.map((e, i) => (
|
||||||
|
<li key={`e-${i}`}>
|
||||||
|
{e.message}
|
||||||
|
{e.hint ? <span style={{ display: 'block', fontSize: 12, color: 'var(--text2)', marginTop: 4 }}>{e.hint}</span> : null}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
) : null}
|
||||||
|
{validationReport.warnings?.length ? (
|
||||||
|
<ul style={{ margin: '0 0 0 1rem', color: 'var(--text2)', fontSize: 13 }}>
|
||||||
|
{validationReport.warnings.map((w, i) => (
|
||||||
|
<li key={`w-${i}`}>
|
||||||
|
{w.message}
|
||||||
|
{w.hint ? <span style={{ display: 'block', fontSize: 12, color: 'var(--text3)', marginTop: 4 }}>{w.hint}</span> : null}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 12 }}>
|
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 12 }}>
|
||||||
|
<button type="button" className="btn btn-secondary" disabled={validating || saving} onClick={handleFormatCheck} style={{ minWidth: 160 }}>
|
||||||
|
{validating ? (
|
||||||
|
<>
|
||||||
|
<Loader2 size={18} style={{ marginRight: 8, animation: 'spin 0.7s linear infinite' }} />
|
||||||
|
Prüfen …
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
'Format prüfen'
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
<button type="button" className="btn btn-primary" disabled={saving} onClick={handleSave} style={{ flex: 1, minWidth: 160 }}>
|
<button type="button" className="btn btn-primary" disabled={saving} onClick={handleSave} style={{ flex: 1, minWidth: 160 }}>
|
||||||
{saving ? (
|
{saving ? (
|
||||||
<>
|
<>
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,15 @@ export function formatFastApiDetail(detail, fallback = '') {
|
||||||
return parts.length ? parts.join(' · ') : fallback || 'Validierungsfehler'
|
return parts.length ? parts.join(' · ') : fallback || 'Validierungsfehler'
|
||||||
}
|
}
|
||||||
if (typeof detail === 'object') {
|
if (typeof detail === 'object') {
|
||||||
|
if (Array.isArray(detail.errors) && detail.errors.length > 0) {
|
||||||
|
const parts = detail.errors
|
||||||
|
.map((e) => {
|
||||||
|
if (e && typeof e === 'object') return e.message || e.msg || ''
|
||||||
|
return String(e)
|
||||||
|
})
|
||||||
|
.filter(Boolean)
|
||||||
|
if (parts.length) return parts.join(' · ')
|
||||||
|
}
|
||||||
if (typeof detail.msg === 'string') return detail.msg
|
if (typeof detail.msg === 'string') return detail.msg
|
||||||
if (typeof detail.message === 'string') return detail.message
|
if (typeof detail.message === 'string') return detail.message
|
||||||
}
|
}
|
||||||
|
|
@ -554,6 +563,8 @@ export const api = {
|
||||||
return j
|
return j
|
||||||
},
|
},
|
||||||
adminGetCsvTemplate: (id) => req(`/admin/csv-templates/${id}`),
|
adminGetCsvTemplate: (id) => req(`/admin/csv-templates/${id}`),
|
||||||
|
/** Formatprüfung (ohne Speichern); body wie Create, import_row_processing optional */
|
||||||
|
adminValidateCsvTemplate: (d) => req('/admin/csv-templates/validate', json(d)),
|
||||||
adminCreateCsvTemplate: (d) => req('/admin/csv-templates', json(d)),
|
adminCreateCsvTemplate: (d) => req('/admin/csv-templates', json(d)),
|
||||||
adminUpdateCsvTemplate: (id, d) => req(`/admin/csv-templates/${id}`, jput(d)),
|
adminUpdateCsvTemplate: (id, d) => req(`/admin/csv-templates/${id}`, jput(d)),
|
||||||
adminDeleteCsvTemplate: (id) => req(`/admin/csv-templates/${id}`, { method: 'DELETE' }),
|
adminDeleteCsvTemplate: (id) => req(`/admin/csv-templates/${id}`, { method: 'DELETE' }),
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user