fix: BLS-Import-UI und Timeout beim Daten-Apply
Deploy Development / deploy (push) Successful in 1m30s
Build Test / pytest-backend (push) Successful in 9s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 21s

Datei-Button volle Breite, Import als Aktionsbutton, Batch-Upsert statt Einzel-INSERTs.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-09-12 14:49:47 +02:00
1 parent 7ccae33844
commit 8e964509d7
7 files changed
+170 -145

No files matched your search

+89 -75
View File
@@ -3,6 +3,17 @@ from __future__ import annotations
from typing import Any
from psycopg2.extras import execute_values
VALUE_PAGE = 2000
FOOD_PAGE = 500
def should_persist_value(val: dict[str, Any]) -> bool:
if val.get("is_trace"):
return True
return val.get("value_num") is not None
def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
inserted = updated = 0
@@ -39,88 +50,91 @@ def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]:
cur.execute("SELECT attr_key, id FROM food_attributes")
attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()}
inserted = updated = values_written = 0
codes = [f["bls_code"] for f in foods if f.get("bls_code")]
existing: set[str] = set()
if codes:
cur.execute("SELECT bls_code FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
existing = {r["bls_code"] for r in cur.fetchall()}
inserted = sum(1 for c in codes if c not in existing)
updated = len(codes) - inserted
food_rows = [
(
f["bls_code"], f["name_de"], f.get("name_en"), f.get("food_group"),
"official_bls", bls_version, "bls_4.0",
)
for f in foods if f.get("bls_code")
]
if food_rows:
execute_values(
cur,
"""
INSERT INTO food_catalog
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
VALUES %s
ON CONFLICT (bls_code) DO UPDATE SET
name_de = EXCLUDED.name_de,
name_en = EXCLUDED.name_en,
food_group = EXCLUDED.food_group,
bls_version = EXCLUDED.bls_version,
catalog_kind = 'official_bls',
source = 'bls_4.0',
is_active = true,
updated_at = NOW()
WHERE food_catalog.catalog_kind = 'official_bls'
""",
food_rows,
page_size=FOOD_PAGE,
)
food_ids: dict[str, Any] = {}
if codes:
cur.execute("SELECT bls_code, id FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
food_ids = {r["bls_code"]: r["id"] for r in cur.fetchall()}
value_rows = []
for f in foods:
code = f["bls_code"]
cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,))
existing = cur.fetchone()
if existing:
food_id = existing["id"]
cur.execute(
"""
UPDATE food_catalog
SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s,
catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW()
WHERE id=%s AND catalog_kind='official_bls'
""",
(f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id),
)
updated += 1
else:
cur.execute(
"""
INSERT INTO food_catalog
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0')
RETURNING id
""",
(code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version),
)
food_id = cur.fetchone()["id"]
inserted += 1
food_id = food_ids.get(f.get("bls_code"))
if not food_id:
continue
for val in f.get("values") or []:
if not should_persist_value(val):
continue
aid = attr_ids.get(val["attr_key"])
if not aid:
continue
if val.get("is_trace"):
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, NULL, true, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = NULL, is_trace = true,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
)
elif val.get("value_num") is None:
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, NULL, false, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = NULL, is_trace = false,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
)
else:
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, %s, false, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = EXCLUDED.value_num, is_trace = false,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(
food_id, aid, val["value_num"],
val.get("origin_code"), val.get("reference_text"),
),
)
values_written += 1
value_rows.append((
food_id,
aid,
None if val.get("is_trace") else val.get("value_num"),
bool(val.get("is_trace")),
val.get("origin_code"),
val.get("reference_text"),
))
if value_rows:
execute_values(
cur,
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES %s
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = EXCLUDED.value_num,
is_trace = EXCLUDED.is_trace,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
value_rows,
template="(%s, %s, %s, %s, %s, %s, NOW())",
page_size=VALUE_PAGE,
)
return {
"inserted": inserted,
"updated": updated,
"foods_inserted": inserted,
"foods_updated": updated,
"values_written": values_written,
"values_written": len(value_rows),
"foods_total": len(foods),
}
+17 -8
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
from typing import Optional
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from auth import require_admin
@@ -61,14 +62,18 @@ async def import_components(
if not raw:
raise HTTPException(400, "Leere Datei")
try:
attrs = parse_components_xlsx(raw)
attrs = await run_in_threadpool(parse_components_xlsx, raw)
except Exception as e:
raise HTTPException(400, f"Components-Datei unlesbar: {e}") from e
if dry_run:
return {"dry_run": True, "attributes": len(attrs), "sample": attrs[:8]}
with get_db() as conn:
cur = get_cursor(conn)
stats = upsert_attributes(cur, attrs)
def apply():
with get_db() as conn:
cur = get_cursor(conn)
return upsert_attributes(cur, attrs)
stats = await run_in_threadpool(apply)
return {"dry_run": False, **stats}
@@ -82,7 +87,7 @@ async def import_foods(
if not raw:
raise HTTPException(400, "Leere Datei")
try:
parsed = parse_foods_xlsx(raw)
parsed = await run_in_threadpool(parse_foods_xlsx, raw)
except Exception as e:
raise HTTPException(400, f"Datendatei unlesbar: {e}") from e
foods = parsed["foods"]
@@ -96,9 +101,13 @@ async def import_foods(
for f in foods[:8]
],
}
with get_db() as conn:
cur = get_cursor(conn)
stats = upsert_foods(cur, foods)
def apply():
with get_db() as conn:
cur = get_cursor(conn)
return upsert_foods(cur, foods)
stats = await run_in_threadpool(apply)
return {"dry_run": False, **stats}
+7
View File
@@ -2,6 +2,7 @@ from io import BytesIO
from openpyxl import Workbook
from bls.import_service import should_persist_value
from bls.parser import parse_components_xlsx, parse_foods_xlsx
@@ -37,3 +38,9 @@ def test_parse_foods_keeps_bls_code():
assert parsed["foods"][0]["name_de"] == "Hafer roh"
vals = {v["attr_key"]: v["value_num"] for v in parsed["foods"][0]["values"]}
assert vals.get("ENERCC") == 350
def test_persist_only_numeric_or_trace():
assert should_persist_value({"value_num": 1.2, "is_trace": False})
assert should_persist_value({"value_num": None, "is_trace": True})
assert not should_persist_value({"value_num": None, "is_trace": False})