From 5e5f3b4e5a742dbed69a2732cb8b2ba19a16523f Mon Sep 17 00:00:00 2001 From: Lars Date: Fri, 10 Apr 2026 06:15:21 +0200 Subject: [PATCH] feat(csv-import): Update CSV import functionality and enhance analysis features - Bumped version of csv_import to 0.3.0, reflecting new analysis capabilities. - Modified analyze_csv endpoint to allow optional module filtering, improving flexibility in template selection. - Enhanced the import process to support both system and user-defined templates, ensuring backward compatibility. - Updated frontend to streamline mapping choices and improve user experience during CSV analysis and import. - Added detailed error handling and user feedback for import operations. --- backend/routers/csv_import.py | 105 ++++--- backend/version.py | 3 +- frontend/src/pages/UniversalCsvImportPage.jsx | 276 +++++++++--------- frontend/src/utils/api.js | 10 +- 4 files changed, 216 insertions(+), 178 deletions(-) diff --git a/backend/routers/csv_import.py b/backend/routers/csv_import.py index 916236f..3777884 100644 --- a/backend/routers/csv_import.py +++ b/backend/routers/csv_import.py @@ -183,16 +183,19 @@ def copy_csv_mapping( @router.post("/analyze") async def analyze_csv( file: UploadFile = File(...), - module: str = Form(...), + module: Optional[str] = Form(default=None), delimiter: Optional[str] = Form(default=None), session: dict = Depends(require_auth), ): """ - Erste Zeilen parsen, Signatur bilden, System-Templates nach Ähnlichkeit ranken. + Erste Zeilen parsen, Signatur bilden, Vorlagen nach Ähnlichkeit ranken. + Ohne `module`: alle System- + eigene User-Vorlagen (echte Format-Erkennung über Spalten-Signatur). + Mit `module`: nur Vorlagen dieses Moduls (Abwärtskompatibilität). """ - if not get_module_definition(module): + if module and not get_module_definition(module): raise HTTPException(400, f"Unbekanntes Modul: {module}") + pid = str(session["profile_id"]) raw = await file.read() limits = _load_import_limits() max_bytes = limits.get("max_file_bytes", 52_428_800) @@ -213,20 +216,32 @@ async def analyze_csv( headers, sample_rows, used_delim = parse_csv_sample(text, delimiter=delim, max_data_rows=5) sig = column_signature(headers) - mod_def = get_module_definition(module) - available_fields = mod_def["fields"] if mod_def else {} + mod_def = get_module_definition(module) if module else None + available_fields = mod_def["fields"] if mod_def else None with get_db() as conn: cur = get_cursor(conn) - cur.execute( - """ - SELECT id, module, mapping_name, description, column_signature, - delimiter, encoding, has_header, field_mappings, type_conversions, is_system - FROM csv_field_mappings - WHERE is_system = true AND module = %s - """, - (module,), - ) + if module: + cur.execute( + """ + SELECT id, module, mapping_name, description, column_signature, + delimiter, encoding, has_header, field_mappings, type_conversions, is_system + FROM csv_field_mappings + WHERE module = %s + AND (is_system = true OR (is_system = false AND profile_id = %s::uuid)) + """, + (module, pid), + ) + else: + cur.execute( + """ + SELECT id, module, mapping_name, description, column_signature, + delimiter, encoding, has_header, field_mappings, type_conversions, is_system + FROM csv_field_mappings + WHERE is_system = true OR (is_system = false AND profile_id = %s::uuid) + """, + (pid,), + ) templates = [r2d(r) for r in cur.fetchall()] ranked = [] @@ -237,27 +252,38 @@ async def analyze_csv( ranked.append( { "mapping_id": t["id"], + "module": t["module"], "mapping_name": t["mapping_name"], + "is_system": bool(t.get("is_system")), "confidence": round(score, 4), "match_type": "signature_jaccard", } ) ranked.sort(key=lambda x: -x["confidence"]) + top = ranked[:25] + recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None + return { - "module": module, + "module_filter": module, "filename": file.filename, "encoding_note": "utf-8/latin-1 mit BOM-Strip", "delimiter": used_delim, "columns": headers, "column_signature_normalized": sig, "sample_rows": sample_rows, - "detected_mappings": ranked[:5], + "detected_mappings": top, + "recommended": recommended, "available_fields": available_fields, } -def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> dict: +def _fetch_mapping_row( + cur, + mapping_id: int, + profile_id: str, + module: Optional[str] = None, +) -> dict: cur.execute( """ SELECT * FROM csv_field_mappings WHERE id = %s @@ -267,7 +293,7 @@ def _fetch_mapping_row(cur, mapping_id: int, profile_id: str, module: str) -> di m = r2d(cur.fetchone()) if not m: raise HTTPException(404, "Mapping nicht gefunden") - if m.get("module") != module: + if module is not None and m.get("module") != module: raise HTTPException(400, "Mapping gehört zu einem anderen Modul") if not m.get("is_system"): if str(m.get("profile_id") or "") != profile_id: @@ -297,24 +323,15 @@ def _check_module_feature_access(pid: str, module: str) -> None: @router.post("/import") async def csv_import_execute( file: UploadFile = File(...), - module: str = Form(...), mapping_id: int = Form(...), + module: Optional[str] = Form(default=None), x_profile_id: Optional[str] = Header(default=None), session: dict = Depends(require_auth), ): """ - Universal-CSV-Import mit gespeichertem Mapping (Issue #21). - Unterstützt: nutrition, weight, blood_pressure. activity: noch nicht. + Universal-CSV-Import: Zielmodul kommt aus der gewählten Vorlage (`mapping_id`). + Optional `module` dient nur der Absicherung (muss mit Vorlage übereinstimmen). """ - if module == "activity": - raise HTTPException( - 501, - "Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet " - "(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.", - ) - if not get_module_definition(module): - raise HTTPException(400, f"Unbekanntes oder nicht unterstütztes Modul: {module}") - pid = get_pid(x_profile_id) access_di = check_feature_access(pid, "data_import") @@ -326,8 +343,6 @@ async def csv_import_execute( f"{access_di.get('used')}/{access_di.get('limit')}", ) - _check_module_feature_access(pid, module) - raw = await file.read() limits = _load_import_limits() max_bytes = limits.get("max_file_bytes", 52_428_800) @@ -349,11 +364,28 @@ async def csv_import_execute( log_id: int | None = None err_response: HTTPException | None = None result: dict | None = None + resolved_module: str | None = None try: with get_db() as conn: cur = get_cursor(conn) m = _fetch_mapping_row(cur, mapping_id, pid, module) + exec_module = m["module"] + resolved_module = exec_module + + if exec_module == "activity": + raise HTTPException( + 501, + "Aktivitäts-CSV über den Universal-Importer ist noch nicht freigeschaltet " + "(Training-Type-Mapping). Bitte weiterhin /api/activity/import nutzen.", + ) + if not get_module_definition(exec_module): + raise HTTPException( + 400, + f"Modul der Vorlage wird vom Importer noch nicht unterstützt: {exec_module}", + ) + + _check_module_feature_access(pid, exec_module) cur.execute( """ @@ -367,7 +399,7 @@ async def csv_import_execute( 'running', NULL, NULL ) RETURNING id """, - (pid, mapping_id, module, file.filename or "upload.csv"), + (pid, mapping_id, exec_module, file.filename or "upload.csv"), ) log_id = cur.fetchone()["id"] @@ -376,7 +408,7 @@ async def csv_import_execute( result = run_universal_csv_import( cur, pid, - module, + exec_module, text, file.filename or "upload.csv", m, @@ -444,16 +476,17 @@ async def csv_import_execute( increment_feature_usage(pid, "data_import") ne = result.get("new_entries", result["rows_imported"]) - if module == "nutrition": + if resolved_module == "nutrition": for _ in range(ne): increment_feature_usage(pid, "nutrition_entries") - elif module == "weight": + elif resolved_module == "weight": for _ in range(ne): increment_feature_usage(pid, "weight_entries") return { "success": True, "import_log_id": log_id, + "module": resolved_module, "stats": { "total_rows": result["rows_total"], "imported": result["rows_imported"], diff --git a/backend/version.py b/backend/version.py index 526d353..987a1a5 100644 --- a/backend/version.py +++ b/backend/version.py @@ -31,7 +31,7 @@ MODULE_VERSIONS = { "membership": "2.1.0", "workflow": "0.6.0", # Phase 4: End Node Template Engine "app_dashboard": "1.11.0", # Entitlements: DB-Override widget→features (AND), sonst Katalog - "csv_import": "0.2.0", # Issue #21: + POST /csv/import (nutrition, weight, blood_pressure) + "csv_import": "0.3.0", # Analyse ohne Modul-Filter; Import nur mapping_id (Modul aus Vorlage) "admin_csv_templates": "0.1.0", # Issue #21: System-Templates + Import-Limits (Admin) } @@ -45,6 +45,7 @@ CHANGELOG = [ "API /api/csv: modules, limits, mappings, analyze, copy", "API /api/admin/csv-templates: CRUD System-Templates, import-limits (system_config)", "Issue #21: POST /api/csv/import + executor (nutrition Aggregat/Tag, weight, Blutdruck); activity 501", + "Issue #21: GET-Analyse über alle Vorlagen; Import nur mapping_id; GUI ohne Modul-Dropdown", "v9c_cleanup_features.sql: FK-sichere csv_import→data_import Reihenfolge", ], }, diff --git a/frontend/src/pages/UniversalCsvImportPage.jsx b/frontend/src/pages/UniversalCsvImportPage.jsx index daf4b24..2eb1b38 100644 --- a/frontend/src/pages/UniversalCsvImportPage.jsx +++ b/frontend/src/pages/UniversalCsvImportPage.jsx @@ -1,10 +1,54 @@ -import { useState, useEffect, useCallback } from 'react' +import { useState, useMemo } from 'react' import { useNavigate } from 'react-router-dom' import { ArrowLeft, FileSpreadsheet, Loader2 } from 'lucide-react' import { api } from '../utils/api' -/** Module mit funktionierendem Universal-Executor; Aktivität: Legacy-Import auf der Aktivitätsseite. */ -const UNIVERSAL_IMPORT_IDS = new Set(['nutrition', 'weight', 'blood_pressure']) +/** Ziele, die der Universal-Executor bereits schreiben kann (ohne manuelle Modul-Wahl). */ +const EXECUTOR_READY = new Set(['nutrition', 'weight', 'blood_pressure']) + +const MODULE_LABEL = { + nutrition: 'Ernährung', + weight: 'Gewicht', + blood_pressure: 'Blutdruck', + activity: 'Aktivität', +} + +function mergeMappingChoices(detected, mapData) { + const sys = mapData.system_templates || [] + const usr = mapData.user_mappings || [] + const flat = [ + ...sys.map((m) => ({ ...m, _isSys: true })), + ...usr.map((m) => ({ ...m, _isSys: false })), + ] + const byId = new Map(flat.map((m) => [m.id, m])) + const out = [] + const seen = new Set() + for (const d of detected || []) { + const row = byId.get(d.mapping_id) + if (row) { + out.push({ + id: row.id, + module: row.module, + name: row.name, + is_system: row.is_system, + confidence: d.confidence ?? 0, + }) + seen.add(row.id) + } + } + for (const row of flat) { + if (!seen.has(row.id)) { + out.push({ + id: row.id, + module: row.module, + name: row.name, + is_system: row.is_system, + confidence: 0, + }) + } + } + return out +} function SampleTable({ sampleRows, columns }) { if (!sampleRows?.length || !columns?.length) return null @@ -57,124 +101,75 @@ function SampleTable({ sampleRows, columns }) { export default function UniversalCsvImportPage() { const navigate = useNavigate() - const [modules, setModules] = useState([]) - const [moduleId, setModuleId] = useState('') const [file, setFile] = useState(null) const [analyzeResult, setAnalyzeResult] = useState(null) - const [mappingsList, setMappingsList] = useState({ system_templates: [], user_mappings: [] }) + const [mappingChoices, setMappingChoices] = useState([]) const [mappingId, setMappingId] = useState('') - const [loadingModules, setLoadingModules] = useState(true) const [loadingAnalyze, setLoadingAnalyze] = useState(false) const [loadingImport, setLoadingImport] = useState(false) const [error, setError] = useState(null) const [success, setSuccess] = useState(null) - const loadMappings = useCallback(async (mid) => { - if (!mid) { - setMappingsList({ system_templates: [], user_mappings: [] }) - return - } - const data = await api.getCsvMappings(mid) - setMappingsList({ - system_templates: data.system_templates || [], - user_mappings: data.user_mappings || [], - }) - }, []) - - useEffect(() => { - let cancel = false - ;(async () => { - setLoadingModules(true) - setError(null) - try { - const data = await api.getCsvModules() - const list = (data.modules || []).filter((m) => UNIVERSAL_IMPORT_IDS.has(m.id)) - if (!cancel) { - setModules(list) - if (list.length && !moduleId) setModuleId(list[0].id) - } - } catch (e) { - if (!cancel) setError(e.message || 'Module konnten nicht geladen werden') - } finally { - if (!cancel) setLoadingModules(false) - } - })() - return () => { - cancel = true - } - }, []) - - useEffect(() => { - if (moduleId) { - loadMappings(moduleId) - setAnalyzeResult(null) - setMappingId('') - setSuccess(null) - } - }, [moduleId, loadMappings]) - - const allMappingOptions = [ - ...(mappingsList.system_templates || []).map((m) => ({ - id: m.id, - label: `${m.name} (System)`, - isSystem: true, - })), - ...(mappingsList.user_mappings || []).map((m) => ({ - id: m.id, - label: m.name, - isSystem: false, - })), - ] + const selectedChoice = useMemo( + () => mappingChoices.find((c) => String(c.id) === String(mappingId)), + [mappingChoices, mappingId], + ) + const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module) const handleAnalyze = async () => { - if (!file || !moduleId) { - setError('Bitte Modul und Datei wählen') + if (!file) { + setError('Bitte eine CSV-Datei wählen') return } setLoadingAnalyze(true) setError(null) setSuccess(null) + setMappingId('') try { - const res = await api.analyzeCsv(file, moduleId) + const res = await api.analyzeCsv(file) setAnalyzeResult(res) - const mapData = await api.getCsvMappings(moduleId) - const sys = mapData.system_templates || [] - const usr = mapData.user_mappings || [] - setMappingsList({ system_templates: sys, user_mappings: usr }) - const options = [ - ...sys.map((m) => ({ id: m.id, name: m.name, isSystem: true })), - ...usr.map((m) => ({ id: m.id, name: m.name, isSystem: false })), - ] - const detected = res.detected_mappings || [] - const best = detected[0] - let pick = options[0]?.id - if (best && (best.confidence || 0) >= 0.5 && options.some((o) => o.id === best.mapping_id)) { - pick = best.mapping_id + const mapData = await api.getCsvMappings() + const merged = mergeMappingChoices(res.detected_mappings || [], mapData) + setMappingChoices(merged) + + const best = res.recommended || (res.detected_mappings || [])[0] + let pick = '' + if (best && merged.some((m) => m.id === best.mapping_id)) { + pick = String(best.mapping_id) + } else if (merged.length) { + const firstReady = merged.find((m) => EXECUTOR_READY.has(m.module)) + pick = String((firstReady || merged[0]).id) } - if (pick != null) setMappingId(String(pick)) - else setMappingId('') + setMappingId(pick) } catch (e) { setError(e.message || 'Analyse fehlgeschlagen') setAnalyzeResult(null) + setMappingChoices([]) } finally { setLoadingAnalyze(false) } } const handleImport = async () => { - if (!file || !moduleId || !mappingId) { - setError('Bitte Datei, Modul und Vorlage wählen') + if (!file || !mappingId) { + setError('Bitte Datei und Vorlage wählen') + return + } + if (!importAllowed) { + setError('Diese Vorlage kann hier noch nicht importiert werden (z. B. Aktivität → Seite „Aktivität“).') return } setLoadingImport(true) setError(null) setSuccess(null) try { - const res = await api.importUniversalCsv(file, moduleId, Number(mappingId)) + const res = await api.importUniversalCsv(file, Number(mappingId)) const st = res.stats || {} + const modLabel = MODULE_LABEL[res.module] || res.module || '' setSuccess( - `Import abgeschlossen: ${st.imported ?? 0} neu, ${st.updated ?? 0} aktualisiert, ` + - `${st.skipped ?? 0} übersprungen, ${st.errors ?? 0} Zeilenfehler.` + (modLabel ? `${modLabel}: ` : '') + + `Import fertig — ${st.imported ?? 0} neu, ${st.updated ?? 0} aktualisiert, ` + + `${st.skipped ?? 0} übersprungen, ${st.errors ?? 0} Zeilenfehler.`, ) } catch (e) { setError(e.message || 'Import fehlgeschlagen') @@ -204,9 +199,11 @@ export default function UniversalCsvImportPage() { CSV-Import -

- Bekannte Formate (FDDB, Apple Health, Omron, …) per Vorlage zuordnen und importieren. - Trainingseinheiten weiterhin unter{' '} +

+ Datei hochladen: Die App vergleicht die Spalten-Struktur mit allen gespeicherten Vorlagen und schlägt + passende Ziele vor (Ernährung, Gewicht, Blutdruck, …). Du bestätigst nur noch die Vorlage — ohne + vorher ein „Modul“ raten zu müssen.{' '} + Später: eine Datei, mehrere Zieltabellen. Trainingseinheiten aktuell weiter unter{' '} {analyzeResult && (

-
3. Vorschau
+
2. Erkennung & Vorschau
- Trennzeichen: {analyzeResult.delimiter} · Spalten: {analyzeResult.columns?.length ?? 0} + Trennzeichen: {analyzeResult.delimiter} · Spalten:{' '} + {analyzeResult.columns?.length ?? 0}
- {(analyzeResult.detected_mappings || []).length > 0 && ( -
- Erkannte Vorlagen: -
    - {analyzeResult.detected_mappings.map((d) => ( + + {analyzeResult.recommended && ( +
    + Vorschlag:{' '} + {MODULE_LABEL[analyzeResult.recommended.module] || analyzeResult.recommended.module} —{' '} + {analyzeResult.recommended.mapping_name} ( + {Math.round((analyzeResult.recommended.confidence || 0) * 100)} % Übereinstimmung der Spalten) +
    + )} + + {(analyzeResult.detected_mappings || []).length > 1 && ( +
    + Weitere Treffer +
      + {analyzeResult.detected_mappings.slice(1, 8).map((d) => (
    • - {d.mapping_name} · Übereinstimmung ca. {Math.round((d.confidence || 0) * 100)} % + {MODULE_LABEL[d.module] || d.module}: {d.mapping_name} ·{' '} + {Math.round((d.confidence || 0) * 100)} %
      • ))}
    -
+ )} +
@@ -333,27 +326,38 @@ export default function UniversalCsvImportPage() { onChange={(e) => setMappingId(e.target.value)} style={{ width: '100%', marginTop: 8 }} > - {allMappingOptions.length === 0 ? ( - + {mappingChoices.length === 0 ? ( + ) : ( - allMappingOptions.map((o) => ( + mappingChoices.map((o) => ( )) )} + {selectedChoice && !importAllowed && ( +

+ Diese Vorlage passt strukturell, wird aber an der passenden Stelle importiert (z. B. Aktivität → + Apple-Health-CSV dort). +

+ )} +