feat(csv-import): Update CSV import functionality and enhance analysis features
Deploy Development / deploy (push) Successful in 50s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 17s

- Bumped version of csv_import to 0.3.0, reflecting new analysis capabilities.
- Modified analyze_csv endpoint to allow optional module filtering, improving flexibility in template selection.
- Enhanced the import process to support both system and user-defined templates, ensuring backward compatibility.
- Updated frontend to streamline mapping choices and improve user experience during CSV analysis and import.
- Added detailed error handling and user feedback for import operations.
This commit is contained in:
Lars committed 2026-04-10 06:15:21 +02:00
1 parent 7e9da46fe5
commit 5e5f3b4e5a
4 files changed
+216 -178

No files matched your search

+69 -36
View File
@@ -183,16 +183,19 @@ def copy_csv_mapping(
@router.post("/analyze")
async def analyze_csv(
file: UploadFile = File(...),
module: str = Form(...),
module: Optional[str] = Form(default=None),
delimiter: Optional[str] = Form(default=None),
session: dict = Depends(require_auth),
):
"""
Erste Zeilen parsen, Signatur bilden, System-Templates nach Ähnlichkeit ranken.
Erste Zeilen parsen, Signatur bilden, Vorlagen nach Ähnlichkeit ranken.
Ohne `module`: alle System- + eigene User-Vorlagen (echte Format-Erkennung über Spalten-Signatur).
Mit `module`: nur Vorlagen dieses Moduls (Abwärtskompatibilität).
"""
if not get_module_definition(module):
if module and not get_module_definition(module):
raise HTTPException(400, f"Unbekanntes Modul: {module}")
pid = str(session["profile_id"])
raw = await file.read()
limits = _load_import_limits()
max_bytes = limits.get("max_file_bytes", 52_428_800)
@@ -213,20 +216,32 @@ async def analyze_csv(
headers, sample_rows, used_delim = parse_csv_sample(text, delimiter=delim, max_data_rows=5)
sig = column_signature(headers)
mod_def = get_module_definition(module)
available_fields = mod_def["fields"] if mod_def else {}
mod_def = get_module_definition(module) if module else None
available_fields = mod_def["fields"] if mod_def else None
with get_db() as conn:
cur = get_cursor(conn)
cur.execute(
"""
SELECT id, module, mapping_name, description, column_signature,
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
FROM csv_field_mappings
WHERE is_system = true AND module = %s
""",
(module,),
)
if module:
cur.execute(
"""
SELECT id, module, mapping_name, description, column_signature,
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
FROM csv_field_mappings
WHERE module = %s
AND (is_system = true OR (is_system = false AND profile_id = %s::uuid))
""",
(module, pid),
)
else:
cur.execute(
"""
SELECT id, module, mapping_name, description, column_signature,
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
FROM csv_field_mappings
WHERE is_system = true OR (is_system = false AND profile_id = %s::uuid)
""",
(pid,),
)
templates = [r2d(r) for r in cur.fetchall()]
ranked = []
@@ -237,27 +252,38 @@ async def analyze_csv(
ranked.append(
{
"mapping_id": t["id"],
"module": t["module"],
"mapping_name": t["mapping_name"],
"is_system": bool(t.get("is_system")),
"confidence": round(score, 4),
"match_type": "signature_jaccard",
}
)
ranked.sort(key=lambda x: -x["confidence"])
top = ranked[:25]
recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None
return {
"module": module,
"module_filter": module,
"filename": file.filename,
"encoding_note": "utf-8/latin-1 mit BOM-Strip",
"delimiter": used_delim,
"columns": headers,
"column_signature_normalized": sig,
"sample_rows": sample_rows,
"detected_mappings": ranked[:5],
"detected_mappings": top,
"recommended": recommended,
"available_fields": available_fields,
}
def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> dict:
def _fetch_mapping_row(
cur,
mapping_id: int,
profile_id: str,
module: Optional[str] = None,
) -> dict:
cur.execute(
"""
SELECT * FROM csv_field_mappings WHERE id = %s
@@ -267,7 +293,7 @@ def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> di
m = r2d(cur.fetchone())
if not m:
raise HTTPException(404, "Mapping nicht gefunden")
if m.get("module") != module:
if module is not None and m.get("module") != module:
raise HTTPException(400, "Mapping gehört zu einem anderen Modul")
if not m.get("is_system"):
if str(m.get("profile_id") or "") != profile_id:
@@ -297,24 +323,15 @@ def _check_module_feature_access(pid: str, module: str) -> None:
@router.post("/import")
async def csv_import_execute(
file: UploadFile = File(...),
module: str = Form(...),
mapping_id: int = Form(...),
module: Optional[str] = Form(default=None),
x_profile_id: Optional[str] = Header(default=None),
session: dict = Depends(require_auth),
):
"""
Universal-CSV-Import mit gespeichertem Mapping (Issue #21).
Unterstützt: nutrition, weight, blood_pressure. activity: noch nicht.
Universal-CSV-Import: Zielmodul kommt aus der gewählten Vorlage (`mapping_id`).
Optional `module` dient nur der Absicherung (muss mit Vorlage übereinstimmen).
"""
if module == "activity":
raise HTTPException(
501,
"Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet "
"(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.",
)
if not get_module_definition(module):
raise HTTPException(400, f"Unbekanntes oder nicht unterstütztes Modul: {module}")
pid = get_pid(x_profile_id)
access_di = check_feature_access(pid, "data_import")
@@ -326,8 +343,6 @@ async def csv_import_execute(
f"{access_di.get('used')}/{access_di.get('limit')}",
)
_check_module_feature_access(pid, module)
raw = await file.read()
limits = _load_import_limits()
max_bytes = limits.get("max_file_bytes", 52_428_800)
@@ -349,11 +364,28 @@ async def csv_import_execute(
log_id: int | None = None
err_response: HTTPException | None = None
result: dict | None = None
resolved_module: str | None = None
try:
with get_db() as conn:
cur = get_cursor(conn)
m = _fetch_mapping_row(cur, mapping_id, pid, module)
exec_module = m["module"]
resolved_module = exec_module
if exec_module == "activity":
raise HTTPException(
501,
"Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet "
"(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.",
)
if not get_module_definition(exec_module):
raise HTTPException(
400,
f"Modul der Vorlage wird vom Importer noch nicht unterstützt: {exec_module}",
)
_check_module_feature_access(pid, exec_module)
cur.execute(
"""
@@ -367,7 +399,7 @@ async def csv_import_execute(
'running', NULL, NULL
) RETURNING id
""",
(pid, mapping_id, module, file.filename or "upload.csv"),
(pid, mapping_id, exec_module, file.filename or "upload.csv"),
)
log_id = cur.fetchone()["id"]
@@ -376,7 +408,7 @@ async def csv_import_execute(
result = run_universal_csv_import(
cur,
pid,
module,
exec_module,
text,
file.filename or "upload.csv",
m,
@@ -444,16 +476,17 @@ async def csv_import_execute(
increment_feature_usage(pid, "data_import")
ne = result.get("new_entries", result["rows_imported"])
if module == "nutrition":
if resolved_module == "nutrition":
for _ in range(ne):
increment_feature_usage(pid, "nutrition_entries")
elif module == "weight":
elif resolved_module == "weight":
for _ in range(ne):
increment_feature_usage(pid, "weight_entries")
return {
"success": True,
"import_log_id": log_id,
"module": resolved_module,
"stats": {
"total_rows": result["rows_total"],
"imported": result["rows_imported"],