feat(csv-import): Enhance Apple sleep CSV import functionality
Some checks failed
Deploy Development / deploy (push) Successful in 54s
Build Test / pytest-backend-csv (push) Failing after 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Integrated date parsing improvements using dateutil for better handling of various date formats in sleep data.
- Added total sleep hours to the nights dictionary for comprehensive sleep analysis.
- Updated the import logic to handle cases where sleep duration is zero, providing appropriate warnings.
- Enhanced the CSV import interface to detect Apple sleep CSV format and provide user feedback on template selection.
- Improved the admin CSV template editor to accommodate new sleep import requirements and clarify usage instructions.
This commit is contained in:
Lars 2026-04-10 07:52:04 +02:00
parent 26ab11eb7b
commit 41cc0ed2a8
6 changed files with 223 additions and 54 deletions

View File

@ -13,6 +13,8 @@ from datetime import date, datetime
from decimal import Decimal, InvalidOperation from decimal import Decimal, InvalidOperation
from typing import Any, Literal from typing import Any, Literal
from dateutil import parser as dateutil_parser
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -44,13 +46,15 @@ def _parse_apple_sleep_datetime(value: str) -> datetime:
"%d.%m.%Y %H:%M:%S", "%d.%m.%Y %H:%M:%S",
"%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S",
) )
last_err = None
for fmt in fmts: for fmt in fmts:
try: try:
return datetime.strptime(raw, fmt) return datetime.strptime(raw, fmt)
except ValueError as e: except ValueError:
last_err = e continue
raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from last_err try:
return dateutil_parser.parse(raw, dayfirst=False)
except (ValueError, TypeError, OverflowError) as e:
raise ValueError(f"Unbekanntes Datumsformat: {raw!r}") from e
def _hr_to_minutes(hours: float | None) -> int: def _hr_to_minutes(hours: float | None) -> int:
@ -102,6 +106,7 @@ def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]
rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)"))) rem_min = _hr_to_minutes(_safe_float(row.get("REM (hr)")))
light_min = _hr_to_minutes(core_hr) light_min = _hr_to_minutes(core_hr)
awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)"))) awake_min = _hr_to_minutes(_safe_float(row.get("Awake (hr)")))
total_sleep_hr = _safe_float(row.get("Total Sleep (hr)"))
nights_dict[wake_d] = { nights_dict[wake_d] = {
"bedtime": start_dt, "bedtime": start_dt,
"wake_time": end_dt, "wake_time": end_dt,
@ -110,6 +115,7 @@ def _build_nights_from_apple_summary(reader: csv.DictReader) -> dict[date, dict]
"rem_minutes": rem_min, "rem_minutes": rem_min,
"light_minutes": light_min, "light_minutes": light_min,
"awake_minutes": awake_min, "awake_minutes": awake_min,
"total_sleep_hr": total_sleep_hr,
} }
return nights_dict return nights_dict
@ -219,9 +225,24 @@ def import_apple_sleep_nights(cur, profile_id: str, text: str) -> dict[str, Any]
row_hint = 0 row_hint = 0
for wake_date, night in nights_dict.items(): for wake_date, night in nights_dict.items():
row_hint += 1 row_hint += 1
duration_minutes = ( phase_sum = (
night["deep_minutes"] + night["rem_minutes"] + night["light_minutes"] night["deep_minutes"] + night["rem_minutes"] + night["light_minutes"]
) )
total_hr = night.get("total_sleep_hr")
fallback_min = int(round(float(total_hr) * 60)) if total_hr is not None else 0
duration_minutes = phase_sum if phase_sum > 0 else fallback_min
if duration_minutes <= 0:
logger.warning(
"Sleep import: überspringe %s — Dauer 0 (Phasen-Summe und Total Sleep (hr) leer/0).",
wake_date,
)
error_details.append(
{
"row": row_hint,
"error": f"Schlafdauer für {wake_date} ist 0 — Phasen oder Total Sleep (hr) fehlen.",
}
)
continue
wake_count = sum(1 for seg in night["segments"] if seg["phase"] == "awake") wake_count = sum(1 for seg in night["segments"] if seg["phase"] == "awake")
sleep_segments = [ sleep_segments = [

View File

@ -0,0 +1,27 @@
-- Migration 045: Schlaf-Systemvorlage — Signatur wie „Schlafanalyse“-Export (Date/Time, Sources, …)
-- Ergänzt die Vorlage aus 044 für besseres Matching; Import-Logik unverändert (Apple-Aggregat-Parser).
UPDATE csv_field_mappings
SET
column_signature = ARRAY[
'Date/Time',
'Start',
'End',
'Total Sleep (hr)',
'Asleep (Unspecified) (hr)',
'In Bed (hr)',
'Core (hr)',
'Deep (hr)',
'REM (hr)',
'Awake (hr)',
'Sources'
]::TEXT[],
description = COALESCE(
description,
'Apple-Health-Schlafanalyse (Nacht-Zusammenfassung): Spalten wie Date/Time, Start/End mit Zeitzone, Kern/Tief/REM etc.'
),
updated_at = CURRENT_TIMESTAMP
WHERE is_system = true
AND profile_id IS NULL
AND module = 'sleep'
AND mapping_name = 'Apple Health Schlaf (Schlafanalyse / Nacht)';

View File

@ -25,6 +25,7 @@ from csv_parser.core import (
parse_csv_sample, parse_csv_sample,
) )
from csv_parser.module_registry import get_module_definition, list_modules, validate_field_mappings from csv_parser.module_registry import get_module_definition, list_modules, validate_field_mappings
from csv_parser.sleep_apple_import import detect_apple_sleep_csv_format
router = APIRouter(prefix="/api/csv", tags=["csv-import"]) router = APIRouter(prefix="/api/csv", tags=["csv-import"])
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -59,7 +60,14 @@ def csv_modules(session: dict = Depends(require_auth)):
for mid in list_modules(): for mid in list_modules():
d = get_module_definition(mid) d = get_module_definition(mid)
if d: if d:
out.append({"id": mid, "table": d["table"], "fields": d["fields"]}) out.append(
{
"id": mid,
"table": d["table"],
"fields": d["fields"],
"import_mode": d.get("import_mode"),
}
)
return {"modules": out} return {"modules": out}
@ -216,6 +224,13 @@ async def analyze_csv(
headers, sample_rows, used_delim = parse_csv_sample(text, delimiter=delim, max_data_rows=5) headers, sample_rows, used_delim = parse_csv_sample(text, delimiter=delim, max_data_rows=5)
sig = column_signature(headers) sig = column_signature(headers)
apple_sleep_csv = False
try:
detect_apple_sleep_csv_format(headers)
apple_sleep_csv = True
except ValueError:
pass
mod_def = get_module_definition(module) if module else None mod_def = get_module_definition(module) if module else None
available_fields = mod_def["fields"] if mod_def else None available_fields = mod_def["fields"] if mod_def else None
@ -248,13 +263,16 @@ async def analyze_csv(
for t in templates: for t in templates:
t_sig = list(t["column_signature"]) if t["column_signature"] else [] t_sig = list(t["column_signature"]) if t["column_signature"] else []
metrics = headers_signature_rank_metrics(sig, t_sig) metrics = headers_signature_rank_metrics(sig, t_sig)
conf = float(metrics["confidence"] or 0)
if apple_sleep_csv and t.get("module") == "sleep":
conf = max(conf, 1.0)
ranked.append( ranked.append(
{ {
"mapping_id": t["id"], "mapping_id": t["id"],
"module": t["module"], "module": t["module"],
"mapping_name": t["mapping_name"], "mapping_name": t["mapping_name"],
"is_system": bool(t.get("is_system")), "is_system": bool(t.get("is_system")),
"confidence": metrics["confidence"], "confidence": round(conf, 4),
"template_recall": metrics["template_recall"], "template_recall": metrics["template_recall"],
"jaccard": metrics["jaccard"], "jaccard": metrics["jaccard"],
"columns_matched": metrics["columns_matched"], "columns_matched": metrics["columns_matched"],
@ -274,6 +292,24 @@ async def analyze_csv(
top = ranked[:25] top = ranked[:25]
recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None
warnings: list[str] = []
if apple_sleep_csv and not any(t.get("module") == "sleep" for t in templates):
warnings.append(
"Diese Datei ist ein Apple-Schlafexport, aber es fehlt eine Vorlage für das Modul «Schlaf» "
"(z. B. Migration 044 / Admin → CSV-Vorlagen)."
)
if (
apple_sleep_csv
and recommended
and recommended.get("module") != "sleep"
and any(t.get("module") == "sleep" for t in templates)
):
warnings.append(
"Apple-Schlaf-CSV erkannt — bitte eine Vorlage unter Modul «Schlaf» wählen, nicht «"
+ str(recommended.get("module") or "")
+ "».",
)
return { return {
"module_filter": module, "module_filter": module,
"filename": file.filename, "filename": file.filename,
@ -285,6 +321,8 @@ async def analyze_csv(
"detected_mappings": top, "detected_mappings": top,
"recommended": recommended, "recommended": recommended,
"available_fields": available_fields, "available_fields": available_fields,
"format_detection": {"apple_sleep": apple_sleep_csv},
"warnings": warnings,
} }

View File

@ -84,13 +84,14 @@ export default function AdminCsvTemplateEditorPage() {
const [error, setError] = useState(null) const [error, setError] = useState(null)
const modMeta = useMemo(() => modules.find((m) => m.id === module), [modules, module]) const modMeta = useMemo(() => modules.find((m) => m.id === module), [modules, module])
const aggregateSleepImport = modMeta?.import_mode === 'apple_sleep_aggregate'
const targetOptions = useMemo(() => { const targetOptions = useMemo(() => {
if (!modMeta?.fields) return [] if (!modMeta?.fields || aggregateSleepImport) return []
return Object.entries(modMeta.fields).map(([key, meta]) => ({ return Object.entries(modMeta.fields).map(([key, meta]) => ({
value: key, value: key,
label: `${key}${meta.required ? ' *' : ''}`, label: `${key}${meta.required ? ' *' : ''}`,
})) }))
}, [modMeta]) }, [modMeta, aggregateSleepImport])
const requiredTargets = useMemo(() => { const requiredTargets = useMemo(() => {
if (!modMeta?.fields) return [] if (!modMeta?.fields) return []
@ -302,7 +303,27 @@ export default function AdminCsvTemplateEditorPage() {
</select> </select>
{!isNew && ( {!isNew && (
<p style={{ fontSize: 13, color: 'var(--text3)', marginTop: 8 }}> <p style={{ fontSize: 13, color: 'var(--text3)', marginTop: 8 }}>
Modul bestehender Vorlagen kann nicht geändert werden. Modul bestehender Vorlagen kann nicht geändert werden. System-Vorlagen können hier bearbeitet und
gespeichert werden (Signatur, Trennzeichen, Zuordnungen).
</p>
)}
{aggregateSleepImport && (
<p
style={{
fontSize: 13,
color: 'var(--text2)',
marginTop: 12,
lineHeight: 1.55,
padding: 12,
background: 'var(--surface2)',
borderRadius: 12,
}}
>
<strong>Schlaf (Apple-Aggregat):</strong> Die Zeilen werden nicht über Spalten-Ziele importiert,
sondern vom Apple-Schlaf-Parser ausgewertet (Schlafanalyse oder Segment-Export). Alle CSV-Spalten
bleiben auf ignorieren. Wichtig sind die{' '}
<strong>gespeicherte Spalten-Signatur</strong> und das passende Datei-Format damit die Datei
in der Nutzer-Auswahl erkannt wird.
</p> </p>
)} )}
</div> </div>
@ -430,12 +451,39 @@ export default function AdminCsvTemplateEditorPage() {
</select> </select>
</label> </label>
</div> </div>
<label className="form-label" style={{ marginTop: 16, display: 'block' }}>
Spalten-Signatur (Vorlagen-Matching eine Original-Überschrift pro Zeile, manuell anpassbar)
</label>
<textarea
className="form-input"
style={{
width: '100%',
minHeight: 100,
marginTop: 8,
fontFamily: 'monospace',
fontSize: 12,
textAlign: 'left',
}}
value={columnSignature.join('\n')}
onChange={(e) =>
setColumnSignature(
e.target.value
.split('\n')
.map((s) => s.trim())
.filter(Boolean),
)
}
placeholder={'Date/Time\nStart\nEnd\nTotal Sleep (hr)\n…'}
/>
</div> </div>
<div className="card" style={{ padding: 16, marginBottom: 16 }}> <div className="card" style={{ padding: 16, marginBottom: 16 }}>
<div className="form-label">3. Spalten Zielfelder (* = Pflicht)</div> <div className="form-label">3. Spalten Zielfelder (* = Pflicht)</div>
{!columns.length ? ( {!columns.length ? (
<p style={{ fontSize: 14, color: 'var(--text2)', marginTop: 8 }}>Nach CSV-Analyse erscheinen die Zeilen hier.</p> <p style={{ fontSize: 14, color: 'var(--text2)', marginTop: 8 }}>
Nach CSV-Analyse erscheinen die Zeilen hier. Bei Schlaf-Vorlagen ohne Analyse: Signatur oben pflegen,
speichern, oder Beispiel-CSV analysieren.
</p>
) : ( ) : (
<div style={{ marginTop: 12, display: 'flex', flexDirection: 'column', gap: 10 }}> <div style={{ marginTop: 12, display: 'flex', flexDirection: 'column', gap: 10 }}>
{columns.map((col) => ( {columns.map((col) => (
@ -455,6 +503,7 @@ export default function AdminCsvTemplateEditorPage() {
className="form-input" className="form-input"
value={fieldMappings[col] || '-'} value={fieldMappings[col] || '-'}
onChange={(e) => updateMapping(col, e.target.value)} onChange={(e) => updateMapping(col, e.target.value)}
disabled={aggregateSleepImport}
style={{ style={{
width: '100%', width: '100%',
minHeight: 46, minHeight: 46,

View File

@ -74,6 +74,8 @@ export default function AdminCsvTemplatesPage() {
<option value="weight">Gewicht</option> <option value="weight">Gewicht</option>
<option value="blood_pressure">Blutdruck</option> <option value="blood_pressure">Blutdruck</option>
<option value="activity">Aktivität</option> <option value="activity">Aktivität</option>
<option value="sleep">Schlaf</option>
<option value="vitals_baseline">Vitalwerte (Baseline)</option>
</select> </select>
</label> </label>
</div> </div>

View File

@ -108,6 +108,7 @@ function SampleTable({ sampleRows, columns }) {
export default function UniversalCsvImportPage() { export default function UniversalCsvImportPage() {
const navigate = useNavigate() const navigate = useNavigate()
const fileInputRef = useRef(null) const fileInputRef = useRef(null)
const analyzeGenRef = useRef(0)
const [file, setFile] = useState(null) const [file, setFile] = useState(null)
const [dragActive, setDragActive] = useState(false) const [dragActive, setDragActive] = useState(false)
const [analyzeResult, setAnalyzeResult] = useState(null) const [analyzeResult, setAnalyzeResult] = useState(null)
@ -124,34 +125,19 @@ export default function UniversalCsvImportPage() {
) )
const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module) const importAllowed = selectedChoice && EXECUTOR_READY.has(selectedChoice.module)
const assignCsvFile = (f) => { const runAnalyze = async (fileToAnalyze) => {
if (!f) return if (!fileToAnalyze) return
const name = (f.name || '').toLowerCase() const gen = ++analyzeGenRef.current
if (!name.endsWith('.csv')) {
setError('Bitte eine .csv-Datei wählen.')
return
}
setFile(f)
setAnalyzeResult(null)
setMappingChoices([])
setMappingId('')
setSuccess(null)
setError(null)
}
const handleAnalyze = async () => {
if (!file) {
setError('Bitte eine CSV-Datei wählen')
return
}
setLoadingAnalyze(true) setLoadingAnalyze(true)
setError(null) setError(null)
setSuccess(null) setSuccess(null)
setMappingId('') setMappingId('')
try { try {
const res = await api.analyzeCsv(file) const res = await api.analyzeCsv(fileToAnalyze)
if (gen !== analyzeGenRef.current) return
setAnalyzeResult(res) setAnalyzeResult(res)
const mapData = await api.getCsvMappings() const mapData = await api.getCsvMappings()
if (gen !== analyzeGenRef.current) return
const merged = mergeMappingChoices(res.detected_mappings || [], mapData) const merged = mergeMappingChoices(res.detected_mappings || [], mapData)
setMappingChoices(merged) setMappingChoices(merged)
@ -165,14 +151,33 @@ export default function UniversalCsvImportPage() {
} }
setMappingId(pick) setMappingId(pick)
} catch (e) { } catch (e) {
if (gen !== analyzeGenRef.current) return
setError(e.message || 'Analyse fehlgeschlagen') setError(e.message || 'Analyse fehlgeschlagen')
setAnalyzeResult(null) setAnalyzeResult(null)
setMappingChoices([]) setMappingChoices([])
} finally { } finally {
setLoadingAnalyze(false) if (gen === analyzeGenRef.current) {
setLoadingAnalyze(false)
}
} }
} }
const assignCsvFile = (f) => {
if (!f) return
const name = (f.name || '').toLowerCase()
if (!name.endsWith('.csv')) {
setError('Bitte eine .csv-Datei wählen.')
return
}
setFile(f)
setAnalyzeResult(null)
setMappingChoices([])
setMappingId('')
setSuccess(null)
setError(null)
void runAnalyze(f)
}
const handleImport = async () => { const handleImport = async () => {
if (!file || !mappingId) { if (!file || !mappingId) {
setError('Bitte Datei und Vorlage wählen') setError('Bitte Datei und Vorlage wählen')
@ -223,10 +228,10 @@ export default function UniversalCsvImportPage() {
CSV-Import CSV-Import
</h1> </h1>
<p style={{ fontSize: 14, color: 'var(--text2)', marginBottom: 20, lineHeight: 1.55 }}> <p style={{ fontSize: 14, color: 'var(--text2)', marginBottom: 20, lineHeight: 1.55 }}>
CSV hier ablegen oder die Fläche antippen: Die App vergleicht die Spalten mit gespeicherten Vorlagen CSV hier ablegen oder die Fläche antippen die Analyse startet sofort. Die App vergleicht die Spalten
(Ernährung, Gewicht, Blutdruck, Aktivität, Schlaf, Vitalwerte) und schlägt passende Ziele vor. Du mit gespeicherten Vorlagen (Ernährung, Gewicht, Blutdruck, Aktivität, Schlaf, Vitalwerte) und schlägt
bestätigst die Vorlage ohne zuerst ein Modell raten zu müssen. Schlaf-Import erwartet das passende Ziele vor. Du bestätigst die Vorlage ohne zuerst ein Modell raten zu müssen. Schlaf-Import
Apple-Health-Schlaf-CSV (Segment- oder Zusammenfassungs-Export).{' '} erwartet das Apple-Health-Schlaf-CSV (Segment- oder Zusammenfassungs-Export).{' '}
<strong>Ausblick:</strong> Eine Datei mehrere Zieltabellen. <strong>Ausblick:</strong> Eine Datei mehrere Zieltabellen.
</p> </p>
@ -310,33 +315,60 @@ export default function UniversalCsvImportPage() {
Datei ablegen oder tippen zum Auswählen Datei ablegen oder tippen zum Auswählen
</span> </span>
<span style={{ fontSize: 13, color: 'var(--text3)' }}> <span style={{ fontSize: 13, color: 'var(--text3)' }}>
{file ? file.name : 'Noch keine Datei gewählt'} {loadingAnalyze ? (
<span style={{ display: 'inline-flex', alignItems: 'center', gap: 8 }}>
<Loader2 size={16} style={{ animation: 'spin 0.7s linear infinite' }} />
Datei wird analysiert
</span>
) : file ? (
file.name
) : (
'Noch keine Datei gewählt'
)}
</span> </span>
</button> </button>
<button {file && !loadingAnalyze && (
type="button" <button
className="btn btn-primary" type="button"
style={{ marginTop: 12, width: '100%' }} className="btn btn-secondary"
disabled={!file || loadingAnalyze} style={{ marginTop: 12, width: '100%' }}
onClick={handleAnalyze} onClick={() => void runAnalyze(file)}
> >
{loadingAnalyze ? ( Dieselbe Datei erneut analysieren
<> </button>
<Loader2 size={18} style={{ marginRight: 8, animation: 'spin 0.7s linear infinite' }} /> Datei )}
wird erkannt
</>
) : (
'Datei analysieren'
)}
</button>
</div> </div>
{analyzeResult && ( {analyzeResult && (
<div className="card" style={{ marginBottom: 16, padding: 16 }}> <div className="card" style={{ marginBottom: 16, padding: 16 }}>
<div className="form-label">2. Erkennung &amp; Vorschau</div> <div className="form-label">2. Erkennung &amp; Vorschau</div>
{(analyzeResult.warnings || []).length > 0 && (
<div
style={{
marginTop: 12,
padding: 12,
borderRadius: 12,
background: 'rgba(216, 90, 48, 0.12)',
border: '1px solid var(--danger)',
color: 'var(--danger-dark, var(--danger))',
fontSize: 14,
lineHeight: 1.5,
}}
>
{(analyzeResult.warnings || []).map((w, i) => (
<div key={i}>{w}</div>
))}
</div>
)}
<div style={{ fontSize: 13, color: 'var(--text2)', marginTop: 8 }}> <div style={{ fontSize: 13, color: 'var(--text2)', marginTop: 8 }}>
Trennzeichen: <strong>{analyzeResult.delimiter}</strong> · Spalten:{' '} Trennzeichen: <strong>{analyzeResult.delimiter}</strong> · Spalten:{' '}
{analyzeResult.columns?.length ?? 0} {analyzeResult.columns?.length ?? 0}
{analyzeResult.format_detection?.apple_sleep ? (
<>
{' '}
· <strong>Format:</strong> Apple-Schlaf (Schlafanalyse oder Segment-Export)
</>
) : null}
</div> </div>
{analyzeResult.recommended && ( {analyzeResult.recommended && (