feat(csv-import): Update CSV import functionality and enhance analysis features
- Bumped version of csv_import to 0.3.0, reflecting new analysis capabilities. - Modified analyze_csv endpoint to allow optional module filtering, improving flexibility in template selection. - Enhanced the import process to support both system and user-defined templates, ensuring backward compatibility. - Updated frontend to streamline mapping choices and improve user experience during CSV analysis and import. - Added detailed error handling and user feedback for import operations.
This commit is contained in:
1 parent
7e9da46fe5
commit
5e5f3b4e5a
4 files changed
+216
-178
No files matched your search
@@ -183,16 +183,19 @@ def copy_csv_mapping(
|
||||
@router.post("/analyze")
|
||||
async def analyze_csv(
|
||||
file: UploadFile = File(...),
|
||||
module: str = Form(...),
|
||||
module: Optional[str] = Form(default=None),
|
||||
delimiter: Optional[str] = Form(default=None),
|
||||
session: dict = Depends(require_auth),
|
||||
):
|
||||
"""
|
||||
Erste Zeilen parsen, Signatur bilden, System-Templates nach Ähnlichkeit ranken.
|
||||
Erste Zeilen parsen, Signatur bilden, Vorlagen nach Ähnlichkeit ranken.
|
||||
Ohne `module`: alle System- + eigene User-Vorlagen (echte Format-Erkennung über Spalten-Signatur).
|
||||
Mit `module`: nur Vorlagen dieses Moduls (Abwärtskompatibilität).
|
||||
"""
|
||||
if not get_module_definition(module):
|
||||
if module and not get_module_definition(module):
|
||||
raise HTTPException(400, f"Unbekanntes Modul: {module}")
|
||||
|
||||
pid = str(session["profile_id"])
|
||||
raw = await file.read()
|
||||
limits = _load_import_limits()
|
||||
max_bytes = limits.get("max_file_bytes", 52_428_800)
|
||||
@@ -213,20 +216,32 @@ async def analyze_csv(
|
||||
headers, sample_rows, used_delim = parse_csv_sample(text, delimiter=delim, max_data_rows=5)
|
||||
sig = column_signature(headers)
|
||||
|
||||
mod_def = get_module_definition(module)
|
||||
available_fields = mod_def["fields"] if mod_def else {}
|
||||
mod_def = get_module_definition(module) if module else None
|
||||
available_fields = mod_def["fields"] if mod_def else None
|
||||
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, module, mapping_name, description, column_signature,
|
||||
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
|
||||
FROM csv_field_mappings
|
||||
WHERE is_system = true AND module = %s
|
||||
""",
|
||||
(module,),
|
||||
)
|
||||
if module:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, module, mapping_name, description, column_signature,
|
||||
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
|
||||
FROM csv_field_mappings
|
||||
WHERE module = %s
|
||||
AND (is_system = true OR (is_system = false AND profile_id = %s::uuid))
|
||||
""",
|
||||
(module, pid),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, module, mapping_name, description, column_signature,
|
||||
delimiter, encoding, has_header, field_mappings, type_conversions, is_system
|
||||
FROM csv_field_mappings
|
||||
WHERE is_system = true OR (is_system = false AND profile_id = %s::uuid)
|
||||
""",
|
||||
(pid,),
|
||||
)
|
||||
templates = [r2d(r) for r in cur.fetchall()]
|
||||
|
||||
ranked = []
|
||||
@@ -237,27 +252,38 @@ async def analyze_csv(
|
||||
ranked.append(
|
||||
{
|
||||
"mapping_id": t["id"],
|
||||
"module": t["module"],
|
||||
"mapping_name": t["mapping_name"],
|
||||
"is_system": bool(t.get("is_system")),
|
||||
"confidence": round(score, 4),
|
||||
"match_type": "signature_jaccard",
|
||||
}
|
||||
)
|
||||
ranked.sort(key=lambda x: -x["confidence"])
|
||||
|
||||
top = ranked[:25]
|
||||
recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None
|
||||
|
||||
return {
|
||||
"module": module,
|
||||
"module_filter": module,
|
||||
"filename": file.filename,
|
||||
"encoding_note": "utf-8/latin-1 mit BOM-Strip",
|
||||
"delimiter": used_delim,
|
||||
"columns": headers,
|
||||
"column_signature_normalized": sig,
|
||||
"sample_rows": sample_rows,
|
||||
"detected_mappings": ranked[:5],
|
||||
"detected_mappings": top,
|
||||
"recommended": recommended,
|
||||
"available_fields": available_fields,
|
||||
}
|
||||
|
||||
|
||||
def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> dict:
|
||||
def _fetch_mapping_row(
|
||||
cur,
|
||||
mapping_id: int,
|
||||
profile_id: str,
|
||||
module: Optional[str] = None,
|
||||
) -> dict:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT * FROM csv_field_mappings WHERE id = %s
|
||||
@@ -267,7 +293,7 @@ def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> di
|
||||
m = r2d(cur.fetchone())
|
||||
if not m:
|
||||
raise HTTPException(404, "Mapping nicht gefunden")
|
||||
if m.get("module") != module:
|
||||
if module is not None and m.get("module") != module:
|
||||
raise HTTPException(400, "Mapping gehört zu einem anderen Modul")
|
||||
if not m.get("is_system"):
|
||||
if str(m.get("profile_id") or "") != profile_id:
|
||||
@@ -297,24 +323,15 @@ def _check_module_feature_access(pid: str, module: str) -> None:
|
||||
@router.post("/import")
|
||||
async def csv_import_execute(
|
||||
file: UploadFile = File(...),
|
||||
module: str = Form(...),
|
||||
mapping_id: int = Form(...),
|
||||
module: Optional[str] = Form(default=None),
|
||||
x_profile_id: Optional[str] = Header(default=None),
|
||||
session: dict = Depends(require_auth),
|
||||
):
|
||||
"""
|
||||
Universal-CSV-Import mit gespeichertem Mapping (Issue #21).
|
||||
Unterstützt: nutrition, weight, blood_pressure. activity: noch nicht.
|
||||
Universal-CSV-Import: Zielmodul kommt aus der gewählten Vorlage (`mapping_id`).
|
||||
Optional `module` dient nur der Absicherung (muss mit Vorlage übereinstimmen).
|
||||
"""
|
||||
if module == "activity":
|
||||
raise HTTPException(
|
||||
501,
|
||||
"Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet "
|
||||
"(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.",
|
||||
)
|
||||
if not get_module_definition(module):
|
||||
raise HTTPException(400, f"Unbekanntes oder nicht unterstütztes Modul: {module}")
|
||||
|
||||
pid = get_pid(x_profile_id)
|
||||
|
||||
access_di = check_feature_access(pid, "data_import")
|
||||
@@ -326,8 +343,6 @@ async def csv_import_execute(
|
||||
f"{access_di.get('used')}/{access_di.get('limit')}",
|
||||
)
|
||||
|
||||
_check_module_feature_access(pid, module)
|
||||
|
||||
raw = await file.read()
|
||||
limits = _load_import_limits()
|
||||
max_bytes = limits.get("max_file_bytes", 52_428_800)
|
||||
@@ -349,11 +364,28 @@ async def csv_import_execute(
|
||||
log_id: int | None = None
|
||||
err_response: HTTPException | None = None
|
||||
result: dict | None = None
|
||||
resolved_module: str | None = None
|
||||
|
||||
try:
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
m = _fetch_mapping_row(cur, mapping_id, pid, module)
|
||||
exec_module = m["module"]
|
||||
resolved_module = exec_module
|
||||
|
||||
if exec_module == "activity":
|
||||
raise HTTPException(
|
||||
501,
|
||||
"Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet "
|
||||
"(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.",
|
||||
)
|
||||
if not get_module_definition(exec_module):
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"Modul der Vorlage wird vom Importer noch nicht unterstützt: {exec_module}",
|
||||
)
|
||||
|
||||
_check_module_feature_access(pid, exec_module)
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
@@ -367,7 +399,7 @@ async def csv_import_execute(
|
||||
'running', NULL, NULL
|
||||
) RETURNING id
|
||||
""",
|
||||
(pid, mapping_id, module, file.filename or "upload.csv"),
|
||||
(pid, mapping_id, exec_module, file.filename or "upload.csv"),
|
||||
)
|
||||
log_id = cur.fetchone()["id"]
|
||||
|
||||
@@ -376,7 +408,7 @@ async def csv_import_execute(
|
||||
result = run_universal_csv_import(
|
||||
cur,
|
||||
pid,
|
||||
module,
|
||||
exec_module,
|
||||
text,
|
||||
file.filename or "upload.csv",
|
||||
m,
|
||||
@@ -444,16 +476,17 @@ async def csv_import_execute(
|
||||
increment_feature_usage(pid, "data_import")
|
||||
|
||||
ne = result.get("new_entries", result["rows_imported"])
|
||||
if module == "nutrition":
|
||||
if resolved_module == "nutrition":
|
||||
for _ in range(ne):
|
||||
increment_feature_usage(pid, "nutrition_entries")
|
||||
elif module == "weight":
|
||||
elif resolved_module == "weight":
|
||||
for _ in range(ne):
|
||||
increment_feature_usage(pid, "weight_entries")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"import_log_id": log_id,
|
||||
"module": resolved_module,
|
||||
"stats": {
|
||||
"total_rows": result["rows_total"],
|
||||
"imported": result["rows_imported"],
|
||||
|
||||
Reference in new issue
Block a user