diff --git a/backend/bls/import_service.py b/backend/bls/import_service.py index 8fa872c..5ccf0b0 100644 --- a/backend/bls/import_service.py +++ b/backend/bls/import_service.py @@ -3,6 +3,17 @@ from __future__ import annotations from typing import Any +from psycopg2.extras import execute_values + +VALUE_PAGE = 2000 +FOOD_PAGE = 500 + + +def should_persist_value(val: dict[str, Any]) -> bool: + if val.get("is_trace"): + return True + return val.get("value_num") is not None + def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]: inserted = updated = 0 @@ -39,88 +50,91 @@ def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]: def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]: cur.execute("SELECT attr_key, id FROM food_attributes") attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()} - inserted = updated = values_written = 0 + codes = [f["bls_code"] for f in foods if f.get("bls_code")] + existing: set[str] = set() + if codes: + cur.execute("SELECT bls_code FROM food_catalog WHERE bls_code = ANY(%s)", (codes,)) + existing = {r["bls_code"] for r in cur.fetchall()} + inserted = sum(1 for c in codes if c not in existing) + updated = len(codes) - inserted + food_rows = [ + ( + f["bls_code"], f["name_de"], f.get("name_en"), f.get("food_group"), + "official_bls", bls_version, "bls_4.0", + ) + for f in foods if f.get("bls_code") + ] + if food_rows: + execute_values( + cur, + """ + INSERT INTO food_catalog + (bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source) + VALUES %s + ON CONFLICT (bls_code) DO UPDATE SET + name_de = EXCLUDED.name_de, + name_en = EXCLUDED.name_en, + food_group = EXCLUDED.food_group, + bls_version = EXCLUDED.bls_version, + catalog_kind = 'official_bls', + source = 'bls_4.0', + is_active = true, + updated_at = NOW() + WHERE food_catalog.catalog_kind = 'official_bls' + """, + food_rows, + page_size=FOOD_PAGE, + ) + + food_ids: dict[str, Any] = {} + if codes: + cur.execute("SELECT bls_code, id FROM food_catalog WHERE bls_code = ANY(%s)", (codes,)) + food_ids = {r["bls_code"]: r["id"] for r in cur.fetchall()} + + value_rows = [] for f in foods: - code = f["bls_code"] - cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,)) - existing = cur.fetchone() - if existing: - food_id = existing["id"] - cur.execute( - """ - UPDATE food_catalog - SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s, - catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW() - WHERE id=%s AND catalog_kind='official_bls' - """, - (f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id), - ) - updated += 1 - else: - cur.execute( - """ - INSERT INTO food_catalog - (bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source) - VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0') - RETURNING id - """, - (code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version), - ) - food_id = cur.fetchone()["id"] - inserted += 1 + food_id = food_ids.get(f.get("bls_code")) + if not food_id: + continue for val in f.get("values") or []: + if not should_persist_value(val): + continue aid = attr_ids.get(val["attr_key"]) if not aid: continue - if val.get("is_trace"): - cur.execute( - """ - INSERT INTO food_attribute_values - (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at) - VALUES (%s, %s, NULL, true, %s, %s, NOW()) - ON CONFLICT (food_id, attribute_id) DO UPDATE SET - value_num = NULL, is_trace = true, - origin_code = EXCLUDED.origin_code, - reference_text = EXCLUDED.reference_text, - updated_at = NOW() - """, - (food_id, aid, val.get("origin_code"), val.get("reference_text")), - ) - elif val.get("value_num") is None: - cur.execute( - """ - INSERT INTO food_attribute_values - (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at) - VALUES (%s, %s, NULL, false, %s, %s, NOW()) - ON CONFLICT (food_id, attribute_id) DO UPDATE SET - value_num = NULL, is_trace = false, - origin_code = EXCLUDED.origin_code, - reference_text = EXCLUDED.reference_text, - updated_at = NOW() - """, - (food_id, aid, val.get("origin_code"), val.get("reference_text")), - ) - else: - cur.execute( - """ - INSERT INTO food_attribute_values - (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at) - VALUES (%s, %s, %s, false, %s, %s, NOW()) - ON CONFLICT (food_id, attribute_id) DO UPDATE SET - value_num = EXCLUDED.value_num, is_trace = false, - origin_code = EXCLUDED.origin_code, - reference_text = EXCLUDED.reference_text, - updated_at = NOW() - """, - ( - food_id, aid, val["value_num"], - val.get("origin_code"), val.get("reference_text"), - ), - ) - values_written += 1 + value_rows.append(( + food_id, + aid, + None if val.get("is_trace") else val.get("value_num"), + bool(val.get("is_trace")), + val.get("origin_code"), + val.get("reference_text"), + )) + + if value_rows: + execute_values( + cur, + """ + INSERT INTO food_attribute_values + (food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at) + VALUES %s + ON CONFLICT (food_id, attribute_id) DO UPDATE SET + value_num = EXCLUDED.value_num, + is_trace = EXCLUDED.is_trace, + origin_code = EXCLUDED.origin_code, + reference_text = EXCLUDED.reference_text, + updated_at = NOW() + """, + value_rows, + template="(%s, %s, %s, %s, %s, %s, NOW())", + page_size=VALUE_PAGE, + ) + return { + "inserted": inserted, + "updated": updated, "foods_inserted": inserted, "foods_updated": updated, - "values_written": values_written, + "values_written": len(value_rows), "foods_total": len(foods), } diff --git a/backend/routers/admin_bls.py b/backend/routers/admin_bls.py index b378418..d7479bf 100644 --- a/backend/routers/admin_bls.py +++ b/backend/routers/admin_bls.py @@ -4,6 +4,7 @@ from __future__ import annotations from typing import Optional from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel from auth import require_admin @@ -61,14 +62,18 @@ async def import_components( if not raw: raise HTTPException(400, "Leere Datei") try: - attrs = parse_components_xlsx(raw) + attrs = await run_in_threadpool(parse_components_xlsx, raw) except Exception as e: raise HTTPException(400, f"Components-Datei unlesbar: {e}") from e if dry_run: return {"dry_run": True, "attributes": len(attrs), "sample": attrs[:8]} - with get_db() as conn: - cur = get_cursor(conn) - stats = upsert_attributes(cur, attrs) + + def apply(): + with get_db() as conn: + cur = get_cursor(conn) + return upsert_attributes(cur, attrs) + + stats = await run_in_threadpool(apply) return {"dry_run": False, **stats} @@ -82,7 +87,7 @@ async def import_foods( if not raw: raise HTTPException(400, "Leere Datei") try: - parsed = parse_foods_xlsx(raw) + parsed = await run_in_threadpool(parse_foods_xlsx, raw) except Exception as e: raise HTTPException(400, f"Datendatei unlesbar: {e}") from e foods = parsed["foods"] @@ -96,9 +101,13 @@ async def import_foods( for f in foods[:8] ], } - with get_db() as conn: - cur = get_cursor(conn) - stats = upsert_foods(cur, foods) + + def apply(): + with get_db() as conn: + cur = get_cursor(conn) + return upsert_foods(cur, foods) + + stats = await run_in_threadpool(apply) return {"dry_run": False, **stats} diff --git a/backend/tests/test_bls_parser.py b/backend/tests/test_bls_parser.py index 74a0f2a..c7a2e27 100644 --- a/backend/tests/test_bls_parser.py +++ b/backend/tests/test_bls_parser.py @@ -2,6 +2,7 @@ from io import BytesIO from openpyxl import Workbook +from bls.import_service import should_persist_value from bls.parser import parse_components_xlsx, parse_foods_xlsx @@ -37,3 +38,9 @@ def test_parse_foods_keeps_bls_code(): assert parsed["foods"][0]["name_de"] == "Hafer roh" vals = {v["attr_key"]: v["value_num"] for v in parsed["foods"][0]["values"]} assert vals.get("ENERCC") == 350 + + +def test_persist_only_numeric_or_trace(): + assert should_persist_value({"value_num": 1.2, "is_trace": False}) + assert should_persist_value({"value_num": None, "is_trace": True}) + assert not should_persist_value({"value_num": None, "is_trace": False}) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index d3eedef..d8e49da 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -7,10 +7,10 @@ server { proxy_pass http://backend:8000/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - client_max_body_size 20M; - proxy_read_timeout 300s; + client_max_body_size 50M; + proxy_read_timeout 600s; proxy_connect_timeout 60s; - proxy_send_timeout 60s; + proxy_send_timeout 600s; } location / { diff --git a/frontend/src/pages/AdminBlsImportPage.jsx b/frontend/src/pages/AdminBlsImportPage.jsx index d7addee..0a2e06e 100644 --- a/frontend/src/pages/AdminBlsImportPage.jsx +++ b/frontend/src/pages/AdminBlsImportPage.jsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { api } from '../utils/api' function summarizeCheck(kind, res) { @@ -7,48 +7,8 @@ function summarizeCheck(kind, res) { return `${res.foods ?? 0} Lebensmittel, ${res.attribute_columns ?? 0} Stoffspalten` } -function ImportSwitch({ on, disabled, busy, onEnable }) { - return ( - - ) -} - function FileImportBlock({ label, kind, onImported }) { + const inputRef = useRef(null) const [file, setFile] = useState(null) const [check, setCheck] = useState(null) const [error, setError] = useState(null) @@ -92,20 +52,29 @@ function FileImportBlock({ label, kind, onImported }) { } return ( -
- +
+ { const next = e.target.files?.[0] || null setFile(next) checkFile(next) }} /> - {busy &&

{busy === 'apply' ? 'Importiere…' : 'Prüfe…'}

} + + {busy &&

{busy === 'apply' ? 'Importiere… das kann ein paar Minuten dauern.' : 'Prüfe…'}

} {error &&

{error}

} {check && !error && (

@@ -114,15 +83,18 @@ function FileImportBlock({ label, kind, onImported }) { )} {applyResult && (

- {applyResult.inserted ?? 0} neu · {applyResult.updated ?? 0} aktualisiert + {(applyResult.inserted ?? applyResult.foods_inserted) ?? 0} neu · {(applyResult.updated ?? applyResult.foods_updated) ?? 0} aktualisiert + {applyResult.values_written != null ? ` · ${applyResult.values_written} Werte` : ''}

)} - +
) } @@ -141,7 +113,7 @@ export default function AdminBlsImportPage() {

BLS 4.0 importieren

Offizielle Dateien von blsdb.de (frei verfügbar, MRI). Codes bleiben erhalten. - Datei wählen prüft automatisch. Der Schalter startet den Import. Zuerst Components, dann Daten. + Datei wählen prüft automatisch. Danach Importieren. Zuerst Components, dann Daten.

{status && (

diff --git a/frontend/src/utils/api.js b/frontend/src/utils/api.js index 791ff93..9a3276f 100644 --- a/frontend/src/utils/api.js +++ b/frontend/src/utils/api.js @@ -73,6 +73,29 @@ async function req(path, opts={}) { } return res.json() } +async function readJsonResponse(res) { + const text = await res.text() + const trimmed = (text || '').trim() + if (!trimmed) { + throw new Error(res.ok ? 'Leere Antwort' : `HTTP ${res.status}`) + } + if (trimmed.startsWith('<')) { + throw new Error( + `Server lieferte HTML statt JSON (HTTP ${res.status}). Meist Timeout oder Datei zu groß — erneut versuchen.` + ) + } + let parsed + try { + parsed = JSON.parse(trimmed) + } catch { + throw new Error(trimmed.slice(0, 180) || `HTTP ${res.status}`) + } + if (!res.ok) { + throw new Error(formatFastApiDetail(parsed.detail, JSON.stringify(parsed))) + } + return parsed +} + const json=(d)=>({method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(d)}) const jput=(d)=>({method:'PUT', headers:{'Content-Type':'application/json'},body:JSON.stringify(d)}) @@ -231,12 +254,12 @@ export const api = { adminBlsImportComponents: async (file, dryRun=true) => { const fd=new FormData();fd.append('file',file) const r=await fetch(`${BASE}/admin/bls/import/components?dry_run=${dryRun}`,{method:'POST',body:fd,headers:hdrs()}) - const d=await r.json();if(!r.ok)throw new Error(formatFastApiDetail(d.detail, JSON.stringify(d)));return d + return readJsonResponse(r) }, adminBlsImportFoods: async (file, dryRun=true) => { const fd=new FormData();fd.append('file',file) const r=await fetch(`${BASE}/admin/bls/import/foods?dry_run=${dryRun}`,{method:'POST',body:fd,headers:hdrs()}) - const d=await r.json();if(!r.ok)throw new Error(formatFastApiDetail(d.detail, JSON.stringify(d)));return d + return readJsonResponse(r) }, adminBlsFoods: (q, kind) => req(`/admin/bls/foods?${q?('q='+encodeURIComponent(q)+'&'):''}${kind?('kind='+kind):''}`), adminBlsFoodDetail: (id) => req(`/admin/bls/foods/${id}`), diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e9da7cd..62ec1e5 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -51,8 +51,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 120s; # KI-Calls können länger dauern - client_max_body_size 20M; # CSV + Foto Uploads + proxy_read_timeout 600s; # KI-Calls und BLS-Import + client_max_body_size 50M; # CSV, Foto, BLS-XLSX } # Frontend - React PWA