fix: BLS-Import-UI und Timeout beim Daten-Apply
Datei-Button volle Breite, Import als Aktionsbutton, Batch-Upsert statt Einzel-INSERTs. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
7ccae33844
commit
8e964509d7
7 files changed
+170
-145
No files matched your search
@@ -3,6 +3,17 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
VALUE_PAGE = 2000
|
||||
FOOD_PAGE = 500
|
||||
|
||||
|
||||
def should_persist_value(val: dict[str, Any]) -> bool:
|
||||
if val.get("is_trace"):
|
||||
return True
|
||||
return val.get("value_num") is not None
|
||||
|
||||
|
||||
def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
|
||||
inserted = updated = 0
|
||||
@@ -39,88 +50,91 @@ def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
|
||||
def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]:
|
||||
cur.execute("SELECT attr_key, id FROM food_attributes")
|
||||
attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()}
|
||||
inserted = updated = values_written = 0
|
||||
codes = [f["bls_code"] for f in foods if f.get("bls_code")]
|
||||
existing: set[str] = set()
|
||||
if codes:
|
||||
cur.execute("SELECT bls_code FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
|
||||
existing = {r["bls_code"] for r in cur.fetchall()}
|
||||
inserted = sum(1 for c in codes if c not in existing)
|
||||
updated = len(codes) - inserted
|
||||
food_rows = [
|
||||
(
|
||||
f["bls_code"], f["name_de"], f.get("name_en"), f.get("food_group"),
|
||||
"official_bls", bls_version, "bls_4.0",
|
||||
)
|
||||
for f in foods if f.get("bls_code")
|
||||
]
|
||||
if food_rows:
|
||||
execute_values(
|
||||
cur,
|
||||
"""
|
||||
INSERT INTO food_catalog
|
||||
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
|
||||
VALUES %s
|
||||
ON CONFLICT (bls_code) DO UPDATE SET
|
||||
name_de = EXCLUDED.name_de,
|
||||
name_en = EXCLUDED.name_en,
|
||||
food_group = EXCLUDED.food_group,
|
||||
bls_version = EXCLUDED.bls_version,
|
||||
catalog_kind = 'official_bls',
|
||||
source = 'bls_4.0',
|
||||
is_active = true,
|
||||
updated_at = NOW()
|
||||
WHERE food_catalog.catalog_kind = 'official_bls'
|
||||
""",
|
||||
food_rows,
|
||||
page_size=FOOD_PAGE,
|
||||
)
|
||||
|
||||
food_ids: dict[str, Any] = {}
|
||||
if codes:
|
||||
cur.execute("SELECT bls_code, id FROM food_catalog WHERE bls_code = ANY(%s)", (codes,))
|
||||
food_ids = {r["bls_code"]: r["id"] for r in cur.fetchall()}
|
||||
|
||||
value_rows = []
|
||||
for f in foods:
|
||||
code = f["bls_code"]
|
||||
cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,))
|
||||
existing = cur.fetchone()
|
||||
if existing:
|
||||
food_id = existing["id"]
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE food_catalog
|
||||
SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s,
|
||||
catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW()
|
||||
WHERE id=%s AND catalog_kind='official_bls'
|
||||
""",
|
||||
(f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id),
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_catalog
|
||||
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
|
||||
VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0')
|
||||
RETURNING id
|
||||
""",
|
||||
(code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version),
|
||||
)
|
||||
food_id = cur.fetchone()["id"]
|
||||
inserted += 1
|
||||
food_id = food_ids.get(f.get("bls_code"))
|
||||
if not food_id:
|
||||
continue
|
||||
for val in f.get("values") or []:
|
||||
if not should_persist_value(val):
|
||||
continue
|
||||
aid = attr_ids.get(val["attr_key"])
|
||||
if not aid:
|
||||
continue
|
||||
if val.get("is_trace"):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, NULL, true, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = NULL, is_trace = true,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
|
||||
)
|
||||
elif val.get("value_num") is None:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, NULL, false, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = NULL, is_trace = false,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, %s, false, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = EXCLUDED.value_num, is_trace = false,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(
|
||||
food_id, aid, val["value_num"],
|
||||
val.get("origin_code"), val.get("reference_text"),
|
||||
),
|
||||
)
|
||||
values_written += 1
|
||||
value_rows.append((
|
||||
food_id,
|
||||
aid,
|
||||
None if val.get("is_trace") else val.get("value_num"),
|
||||
bool(val.get("is_trace")),
|
||||
val.get("origin_code"),
|
||||
val.get("reference_text"),
|
||||
))
|
||||
|
||||
if value_rows:
|
||||
execute_values(
|
||||
cur,
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES %s
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = EXCLUDED.value_num,
|
||||
is_trace = EXCLUDED.is_trace,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
value_rows,
|
||||
template="(%s, %s, %s, %s, %s, %s, NOW())",
|
||||
page_size=VALUE_PAGE,
|
||||
)
|
||||
|
||||
return {
|
||||
"inserted": inserted,
|
||||
"updated": updated,
|
||||
"foods_inserted": inserted,
|
||||
"foods_updated": updated,
|
||||
"values_written": values_written,
|
||||
"values_written": len(value_rows),
|
||||
"foods_total": len(foods),
|
||||
}
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel
|
||||
|
||||
from auth import require_admin
|
||||
@@ -61,14 +62,18 @@ async def import_components(
|
||||
if not raw:
|
||||
raise HTTPException(400, "Leere Datei")
|
||||
try:
|
||||
attrs = parse_components_xlsx(raw)
|
||||
attrs = await run_in_threadpool(parse_components_xlsx, raw)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Components-Datei unlesbar: {e}") from e
|
||||
if dry_run:
|
||||
return {"dry_run": True, "attributes": len(attrs), "sample": attrs[:8]}
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
stats = upsert_attributes(cur, attrs)
|
||||
|
||||
def apply():
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
return upsert_attributes(cur, attrs)
|
||||
|
||||
stats = await run_in_threadpool(apply)
|
||||
return {"dry_run": False, **stats}
|
||||
|
||||
|
||||
@@ -82,7 +87,7 @@ async def import_foods(
|
||||
if not raw:
|
||||
raise HTTPException(400, "Leere Datei")
|
||||
try:
|
||||
parsed = parse_foods_xlsx(raw)
|
||||
parsed = await run_in_threadpool(parse_foods_xlsx, raw)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Datendatei unlesbar: {e}") from e
|
||||
foods = parsed["foods"]
|
||||
@@ -96,9 +101,13 @@ async def import_foods(
|
||||
for f in foods[:8]
|
||||
],
|
||||
}
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
stats = upsert_foods(cur, foods)
|
||||
|
||||
def apply():
|
||||
with get_db() as conn:
|
||||
cur = get_cursor(conn)
|
||||
return upsert_foods(cur, foods)
|
||||
|
||||
stats = await run_in_threadpool(apply)
|
||||
return {"dry_run": False, **stats}
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from io import BytesIO
|
||||
|
||||
from openpyxl import Workbook
|
||||
|
||||
from bls.import_service import should_persist_value
|
||||
from bls.parser import parse_components_xlsx, parse_foods_xlsx
|
||||
|
||||
|
||||
@@ -37,3 +38,9 @@ def test_parse_foods_keeps_bls_code():
|
||||
assert parsed["foods"][0]["name_de"] == "Hafer roh"
|
||||
vals = {v["attr_key"]: v["value_num"] for v in parsed["foods"][0]["values"]}
|
||||
assert vals.get("ENERCC") == 350
|
||||
|
||||
|
||||
def test_persist_only_numeric_or_trace():
|
||||
assert should_persist_value({"value_num": 1.2, "is_trace": False})
|
||||
assert should_persist_value({"value_num": None, "is_trace": True})
|
||||
assert not should_persist_value({"value_num": None, "is_trace": False})
|
||||
Reference in new issue
Block a user