fix: BLS-Import-UI und Timeout beim Daten-Apply

Datei-Button volle Breite, Import als Aktionsbutton, Batch-Upsert statt Einzel-INSERTs. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-12 14:49:47 +02:00 · 2026-09-12 14:49:47 +02:00 · 8e964509d7
commit 8e964509d7
parent 7ccae33844
7 changed files with 170 additions and 145 deletions
--- a/backend/bls/import_service.py
+++ b/backend/bls/import_service.py
@ -3,6 +3,17 @@ from __future__ import annotations

 from typing import Any

+from psycopg2.extras import execute_values
+
+VALUE_PAGE = 2000
+FOOD_PAGE = 500
+
+
+def should_persist_value(val: dict[str, Any]) -> bool:
+    if val.get("is_trace"):
+        return True
+    return val.get("value_num") is not None
+

 def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
    inserted = updated = 0
@ -39,88 +50,91 @@ def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
 def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]:
    cur.execute("SELECT attr_key, id FROM food_attributes")
    attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()}
-    inserted = updated = values_written = 0
+    codes = [f["bls_code"] for f in foods if f.get("bls_code")]
+    existing: set[str] = set()
+    if codes:
+        cur.execute("SELECT bls_code FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
+        existing = {r["bls_code"] for r in cur.fetchall()}
+    inserted = sum(1 for c in codes if c not in existing)
+    updated = len(codes) - inserted
+    food_rows = [
+        (
+            f["bls_code"], f["name_de"], f.get("name_en"), f.get("food_group"),
+            "official_bls", bls_version, "bls_4.0",
+        )
+        for f in foods if f.get("bls_code")
+    ]
+    if food_rows:
+        execute_values(
+            cur,
+            """
+            INSERT INTO food_catalog
+                (bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
+            VALUES %s
+            ON CONFLICT (bls_code) DO UPDATE SET
+                name_de = EXCLUDED.name_de,
+                name_en = EXCLUDED.name_en,
+                food_group = EXCLUDED.food_group,
+                bls_version = EXCLUDED.bls_version,
+                catalog_kind = 'official_bls',
+                source = 'bls_4.0',
+                is_active = true,
+                updated_at = NOW()
+            WHERE food_catalog.catalog_kind = 'official_bls'
+            """,
+            food_rows,
+            page_size=FOOD_PAGE,
+        )
+
+    food_ids: dict[str, Any] = {}
+    if codes:
+        cur.execute("SELECT bls_code, id FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
+        food_ids = {r["bls_code"]: r["id"] for r in cur.fetchall()}
+
+    value_rows = []
    for f in foods:
-        code = f["bls_code"]
-        cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,))
-        existing = cur.fetchone()
-        if existing:
-            food_id = existing["id"]
-            cur.execute(
-                """
-                UPDATE food_catalog
-                SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s,
-                    catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW()
-                WHERE id=%s AND catalog_kind='official_bls'
-                """,
-                (f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id),
-            )
-            updated += 1
-        else:
-            cur.execute(
-                """
-                INSERT INTO food_catalog
-                    (bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
-                VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0')
-                RETURNING id
-                """,
-                (code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version),
-            )
-            food_id = cur.fetchone()["id"]
-            inserted += 1
+        food_id = food_ids.get(f.get("bls_code"))
+        if not food_id:
+            continue
        for val in f.get("values") or []:
+            if not should_persist_value(val):
+                continue
            aid = attr_ids.get(val["attr_key"])
            if not aid:
                continue
-            if val.get("is_trace"):
-                cur.execute(
-                    """
-                    INSERT INTO food_attribute_values
-                        (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
-                    VALUES (%s, %s, NULL, true, %s, %s, NOW())
-                    ON CONFLICT (food_id, attribute_id) DO UPDATE SET
-                        value_num = NULL, is_trace = true,
-                        origin_code = EXCLUDED.origin_code,
-                        reference_text = EXCLUDED.reference_text,
-                        updated_at = NOW()
-                    """,
-                    (food_id, aid, val.get("origin_code"), val.get("reference_text")),
-                )
-            elif val.get("value_num") is None:
-                cur.execute(
-                    """
-                    INSERT INTO food_attribute_values
-                        (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
-                    VALUES (%s, %s, NULL, false, %s, %s, NOW())
-                    ON CONFLICT (food_id, attribute_id) DO UPDATE SET
-                        value_num = NULL, is_trace = false,
-                        origin_code = EXCLUDED.origin_code,
-                        reference_text = EXCLUDED.reference_text,
-                        updated_at = NOW()
-                    """,
-                    (food_id, aid, val.get("origin_code"), val.get("reference_text")),
-                )
-            else:
-                cur.execute(
-                    """
-                    INSERT INTO food_attribute_values
-                        (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
-                    VALUES (%s, %s, %s, false, %s, %s, NOW())
-                    ON CONFLICT (food_id, attribute_id) DO UPDATE SET
-                        value_num = EXCLUDED.value_num, is_trace = false,
-                        origin_code = EXCLUDED.origin_code,
-                        reference_text = EXCLUDED.reference_text,
-                        updated_at = NOW()
-                    """,
-                    (
-                        food_id, aid, val["value_num"],
-                        val.get("origin_code"), val.get("reference_text"),
-                    ),
-                )
-            values_written += 1
+            value_rows.append((
+                food_id,
+                aid,
+                None if val.get("is_trace") else val.get("value_num"),
+                bool(val.get("is_trace")),
+                val.get("origin_code"),
+                val.get("reference_text"),
+            ))
+
+    if value_rows:
+        execute_values(
+            cur,
+            """
+            INSERT INTO food_attribute_values
+                (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
+            VALUES %s
+            ON CONFLICT (food_id, attribute_id) DO UPDATE SET
+                value_num = EXCLUDED.value_num,
+                is_trace = EXCLUDED.is_trace,
+                origin_code = EXCLUDED.origin_code,
+                reference_text = EXCLUDED.reference_text,
+                updated_at = NOW()
+            """,
+            value_rows,
+            template="(%s, %s, %s, %s, %s, %s, NOW())",
+            page_size=VALUE_PAGE,
+        )
+
    return {
+        "inserted": inserted,
+        "updated": updated,
        "foods_inserted": inserted,
        "foods_updated": updated,
-        "values_written": values_written,
+        "values_written": len(value_rows),
        "foods_total": len(foods),
    }
--- a/backend/routers/admin_bls.py
+++ b/backend/routers/admin_bls.py
@ -4,6 +4,7 @@ from __future__ import annotations
 from typing import Optional

 from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
+from fastapi.concurrency import run_in_threadpool
 from pydantic import BaseModel

 from auth import require_admin
@ -61,14 +62,18 @@ async def import_components(
    if not raw:
        raise HTTPException(400, "Leere Datei")
    try:
-        attrs = parse_components_xlsx(raw)
+        attrs = await run_in_threadpool(parse_components_xlsx, raw)
    except Exception as e:
        raise HTTPException(400, f"Components-Datei unlesbar: {e}") from e
    if dry_run:
        return {"dry_run": True, "attributes": len(attrs), "sample": attrs[:8]}
-    with get_db() as conn:
-        cur = get_cursor(conn)
-        stats = upsert_attributes(cur, attrs)
+
+    def apply():
+        with get_db() as conn:
+            cur = get_cursor(conn)
+            return upsert_attributes(cur, attrs)
+
+    stats = await run_in_threadpool(apply)
    return {"dry_run": False, **stats}


@ -82,7 +87,7 @@ async def import_foods(
    if not raw:
        raise HTTPException(400, "Leere Datei")
    try:
-        parsed = parse_foods_xlsx(raw)
+        parsed = await run_in_threadpool(parse_foods_xlsx, raw)
    except Exception as e:
        raise HTTPException(400, f"Datendatei unlesbar: {e}") from e
    foods = parsed["foods"]
@ -96,9 +101,13 @@ async def import_foods(
                for f in foods[:8]
            ],
        }
-    with get_db() as conn:
-        cur = get_cursor(conn)
-        stats = upsert_foods(cur, foods)
+
+    def apply():
+        with get_db() as conn:
+            cur = get_cursor(conn)
+            return upsert_foods(cur, foods)
+
+    stats = await run_in_threadpool(apply)
    return {"dry_run": False, **stats}


--- a/backend/tests/test_bls_parser.py
+++ b/backend/tests/test_bls_parser.py
@ -2,6 +2,7 @@ from io import BytesIO

 from openpyxl import Workbook

+from bls.import_service import should_persist_value
 from bls.parser import parse_components_xlsx, parse_foods_xlsx


@ -37,3 +38,9 @@ def test_parse_foods_keeps_bls_code():
    assert parsed["foods"][0]["name_de"] == "Hafer roh"
    vals = {v["attr_key"]: v["value_num"] for v in parsed["foods"][0]["values"]}
    assert vals.get("ENERCC") == 350
+
+
+def test_persist_only_numeric_or_trace():
+    assert should_persist_value({"value_num": 1.2, "is_trace": False})
+    assert should_persist_value({"value_num": None, "is_trace": True})
+    assert not should_persist_value({"value_num": None, "is_trace": False})
--- a/frontend/nginx.conf
+++ b/frontend/nginx.conf
@ -7,10 +7,10 @@ server {
        proxy_pass http://backend:8000/api/;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
-        client_max_body_size 20M;
-        proxy_read_timeout 300s;
+        client_max_body_size 50M;
+        proxy_read_timeout 600s;
        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
+        proxy_send_timeout 600s;
    }

    location / {
--- a/frontend/src/pages/AdminBlsImportPage.jsx
+++ b/frontend/src/pages/AdminBlsImportPage.jsx
@ -1,4 +1,4 @@
-import { useEffect, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import { api } from '../utils/api'

 function summarizeCheck(kind, res) {
@ -7,48 +7,8 @@ function summarizeCheck(kind, res) {
  return `${res.foods ?? 0} Lebensmittel, ${res.attribute_columns ?? 0} Stoffspalten`
 }

-function ImportSwitch({ on, disabled, busy, onEnable }) {
-  return (
-    <label style={{ display: 'flex', alignItems: 'center', gap: 10, opacity: disabled ? 0.5 : 1 }}>
-      <button
-        type="button"
-        role="switch"
-        aria-checked={on}
-        aria-label="Import"
-        disabled={disabled || busy || on}
-        onClick={() => { if (!on && !disabled && !busy) onEnable() }}
-        style={{
-          width: 44,
-          height: 24,
-          borderRadius: 12,
-          border: 'none',
-          padding: 0,
-          background: on ? 'var(--accent)' : 'var(--border)',
-          position: 'relative',
-          cursor: disabled || busy || on ? 'not-allowed' : 'pointer',
-        }}
-      >
-        <span
-          style={{
-            position: 'absolute',
-            top: 2,
-            left: on ? 22 : 2,
-            width: 20,
-            height: 20,
-            borderRadius: '50%',
-            background: '#fff',
-            display: 'block',
-          }}
-        />
-      </button>
-      <span style={{ fontSize: 14, color: 'var(--text1)' }}>
-        {on ? 'Importiert' : 'Import'}
-      </span>
-    </label>
-  )
-}
-
 function FileImportBlock({ label, kind, onImported }) {
+  const inputRef = useRef(null)
  const [file, setFile] = useState(null)
  const [check, setCheck] = useState(null)
  const [error, setError] = useState(null)
@ -92,20 +52,29 @@ function FileImportBlock({ label, kind, onImported }) {
  }

  return (
-    <div className="form-row" style={{ flexDirection: 'column', alignItems: 'stretch', gap: 8, marginTop: 16 }}>
-      <label className="form-label">{label}</label>
+    <div className="settings-page__field">
+      <label className="settings-page__field-label">{label}</label>
      <input
+        ref={inputRef}
        type="file"
        accept=".xlsx"
-        className="form-input"
-        disabled={busy}
+        disabled={!!busy}
+        style={{ display: 'none' }}
        onChange={(e) => {
          const next = e.target.files?.[0] || null
          setFile(next)
          checkFile(next)
        }}
      />
-      {busy && <p style={{ fontSize: 13, color: 'var(--text2)', margin: 0 }}>{busy === 'apply' ? 'Importiere…' : 'Prüfe…'}</p>}
+      <button
+        type="button"
+        className="btn btn-secondary btn-full"
+        disabled={!!busy}
+        onClick={() => inputRef.current?.click()}
+      >
+        {file ? file.name : 'Datei auswählen'}
+      </button>
+      {busy && <p style={{ fontSize: 13, color: 'var(--text2)', margin: 0 }}>{busy === 'apply' ? 'Importiere… das kann ein paar Minuten dauern.' : 'Prüfe…'}</p>}
      {error && <p style={{ color: 'var(--danger)', margin: 0 }}>{error}</p>}
      {check && !error && (
        <p style={{ fontSize: 13, color: 'var(--text2)', margin: 0 }}>
@ -114,15 +83,18 @@ function FileImportBlock({ label, kind, onImported }) {
      )}
      {applyResult && (
        <p style={{ fontSize: 13, color: 'var(--text2)', margin: 0 }}>
-          {applyResult.inserted ?? 0} neu · {applyResult.updated ?? 0} aktualisiert
+          {(applyResult.inserted ?? applyResult.foods_inserted) ?? 0} neu · {(applyResult.updated ?? applyResult.foods_updated) ?? 0} aktualisiert
+          {applyResult.values_written != null ? ` · ${applyResult.values_written} Werte` : ''}
        </p>
      )}
-      <ImportSwitch
-        on={imported}
-        disabled={!check || !!error}
-        busy={busy}
-        onEnable={applyImport}
-      />
+      <button
+        type="button"
+        className="btn btn-primary btn-full"
+        disabled={!check || !!error || !!busy || imported}
+        onClick={applyImport}
+      >
+        {imported ? 'Importiert' : 'Importieren'}
+      </button>
    </div>
  )
 }
@ -141,7 +113,7 @@ export default function AdminBlsImportPage() {
      <h1 className="page-title">BLS 4.0 importieren</h1>
      <p style={{ fontSize: 13, color: 'var(--text2)', lineHeight: 1.6 }}>
        Offizielle Dateien von blsdb.de (frei verfügbar, MRI). Codes bleiben erhalten.
-        Datei wählen prüft automatisch. Der Schalter startet den Import. Zuerst Components, dann Daten.
+        Datei wählen prüft automatisch. Danach Importieren. Zuerst Components, dann Daten.
      </p>
      {status && (
        <p style={{ fontSize: 13 }}>
--- a/frontend/src/utils/api.js
+++ b/frontend/src/utils/api.js
@ -73,6 +73,29 @@ async function req(path, opts={}) {
  }
  return res.json()
 }
+async function readJsonResponse(res) {
+  const text = await res.text()
+  const trimmed = (text || '').trim()
+  if (!trimmed) {
+    throw new Error(res.ok ? 'Leere Antwort' : `HTTP ${res.status}`)
+  }
+  if (trimmed.startsWith('<')) {
+    throw new Error(
+      `Server lieferte HTML statt JSON (HTTP ${res.status}). Meist Timeout oder Datei zu groß — erneut versuchen.`
+    )
+  }
+  let parsed
+  try {
+    parsed = JSON.parse(trimmed)
+  } catch {
+    throw new Error(trimmed.slice(0, 180) || `HTTP ${res.status}`)
+  }
+  if (!res.ok) {
+    throw new Error(formatFastApiDetail(parsed.detail, JSON.stringify(parsed)))
+  }
+  return parsed
+}
+
 const json=(d)=>({method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(d)})
 const jput=(d)=>({method:'PUT', headers:{'Content-Type':'application/json'},body:JSON.stringify(d)})

@ -231,12 +254,12 @@ export const api = {
  adminBlsImportComponents: async (file, dryRun=true) => {
    const fd=new FormData();fd.append('file',file)
    const r=await fetch(`${BASE}/admin/bls/import/components?dry_run=${dryRun}`,{method:'POST',body:fd,headers:hdrs()})
-    const d=await r.json();if(!r.ok)throw new Error(formatFastApiDetail(d.detail, JSON.stringify(d)));return d
+    return readJsonResponse(r)
  },
  adminBlsImportFoods: async (file, dryRun=true) => {
    const fd=new FormData();fd.append('file',file)
    const r=await fetch(`${BASE}/admin/bls/import/foods?dry_run=${dryRun}`,{method:'POST',body:fd,headers:hdrs()})
-    const d=await r.json();if(!r.ok)throw new Error(formatFastApiDetail(d.detail, JSON.stringify(d)));return d
+    return readJsonResponse(r)
  },
  adminBlsFoods: (q, kind) => req(`/admin/bls/foods?${q?('q='+encodeURIComponent(q)+'&'):''}${kind?('kind='+kind):''}`),
  adminBlsFoodDetail: (id) => req(`/admin/bls/foods/${id}`),
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@ -51,8 +51,8 @@ server {
        proxy_set_header   X-Real-IP $remote_addr;
        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto $scheme;
-        proxy_read_timeout 120s;  # KI-Calls können länger dauern
-        client_max_body_size 20M; # CSV + Foto Uploads
+        proxy_read_timeout 600s;  # KI-Calls und BLS-Import
+        client_max_body_size 50M; # CSV, Foto, BLS-XLSX
    }

    # Frontend - React PWA