feat: BLS-Stammdaten, FDDB-Mapping und Item-Tagebuch (#106)
Deploy Development / deploy (push) Failing after 52s
Build Test / pytest-backend (push) Successful in 3s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Failing after 9s

Katalog, lernendes Mapping ohne KI, optionale Items und Import-Policy.
Playwright-Smoke und Issue-Audit um Ernährung/Zuordnen/API ergänzt.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-09-12 14:35:57 +02:00
1 parent 72d101415f
commit 919c77fcf8
39 files changed
+2648 -47

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""BLS 4.0 ingest (official MRI XLSX)."""
+126
View File
@@ -0,0 +1,126 @@
"""Apply parsed BLS 4.0 data: upsert attributes and foods, never delete official rows."""
from __future__ import annotations
from typing import Any
def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
inserted = updated = 0
for a in attributes:
cur.execute(
"""
INSERT INTO food_attributes
(attr_key, name_de, name_en, unit, category, data_type, origin, sort_order, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, 'official_bls', %s, NOW())
ON CONFLICT (attr_key) DO UPDATE SET
name_de = EXCLUDED.name_de,
name_en = COALESCE(EXCLUDED.name_en, food_attributes.name_en),
unit = COALESCE(EXCLUDED.unit, food_attributes.unit),
category = COALESCE(EXCLUDED.category, food_attributes.category),
sort_order = EXCLUDED.sort_order,
updated_at = NOW()
WHERE food_attributes.origin = 'official_bls'
RETURNING (xmax = 0) AS inserted
""",
(
a["attr_key"], a["name_de"], a.get("name_en"), a.get("unit"),
a.get("category"), a.get("data_type") or "num_per_100g",
a.get("sort_order") or 0,
),
)
row = cur.fetchone()
if row and row.get("inserted"):
inserted += 1
else:
updated += 1
return {"inserted": inserted, "updated": updated, "total": len(attributes)}
def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]:
cur.execute("SELECT attr_key, id FROM food_attributes")
attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()}
inserted = updated = values_written = 0
for f in foods:
code = f["bls_code"]
cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,))
existing = cur.fetchone()
if existing:
food_id = existing["id"]
cur.execute(
"""
UPDATE food_catalog
SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s,
catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW()
WHERE id=%s AND catalog_kind='official_bls'
""",
(f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id),
)
updated += 1
else:
cur.execute(
"""
INSERT INTO food_catalog
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0')
RETURNING id
""",
(code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version),
)
food_id = cur.fetchone()["id"]
inserted += 1
for val in f.get("values") or []:
aid = attr_ids.get(val["attr_key"])
if not aid:
continue
if val.get("is_trace"):
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, NULL, true, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = NULL, is_trace = true,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
)
elif val.get("value_num") is None:
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, NULL, false, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = NULL, is_trace = false,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
)
else:
cur.execute(
"""
INSERT INTO food_attribute_values
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
VALUES (%s, %s, %s, false, %s, %s, NOW())
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
value_num = EXCLUDED.value_num, is_trace = false,
origin_code = EXCLUDED.origin_code,
reference_text = EXCLUDED.reference_text,
updated_at = NOW()
""",
(
food_id, aid, val["value_num"],
val.get("origin_code"), val.get("reference_text"),
),
)
values_written += 1
return {
"foods_inserted": inserted,
"foods_updated": updated,
"values_written": values_written,
"foods_total": len(foods),
}
+158
View File
@@ -0,0 +1,158 @@
"""Parse official BLS 4.0 XLSX files without a hardcoded nutrient code list."""
from __future__ import annotations
import re
from io import BytesIO
from typing import Any
from openpyxl import load_workbook
# Header like: "ENERCJ Energie (Kilojoule) [kJ/100g]"
ATTR_HEADER_RE = re.compile(
r"^([A-Z][A-Z0-9:]{1,20})\s+(.+?)(?:\s*\[([^\]]+)\])?\s*$"
)
def _cell(v: Any) -> str:
if v is None:
return ""
return str(v).strip()
def parse_components_xlsx(data: bytes) -> list[dict[str, Any]]:
wb = load_workbook(filename=BytesIO(data), read_only=True, data_only=True)
ws = wb.active
rows = ws.iter_rows(values_only=True)
header = [_cell(c) for c in next(rows)]
idx = {h.lower(): i for i, h in enumerate(header) if h}
def col(*names: str) -> int | None:
for n in names:
if n.lower() in idx:
return idx[n.lower()]
for key, i in idx.items():
for n in names:
if n.lower() in key:
return i
return None
i_code = col("code", "schlüssel", "schluessel", "attr_key", "komponente")
i_de = col("name_de", "deutsch", "bezeichnung_de", "name de")
i_en = col("name_en", "english", "bezeichnung_en", "name en")
i_unit = col("unit", "einheit")
i_cat = col("category", "kategorie", "gruppe")
if i_code is None:
i_code = 0
if i_de is None:
i_de = 1 if len(header) > 1 else 0
out = []
sort_order = 0
for raw in rows:
if not raw:
continue
code = _cell(raw[i_code] if i_code < len(raw) else "")
if not code or code.lower() in ("code", "schlüssel", "schluessel"):
continue
name_de = _cell(raw[i_de] if i_de is not None and i_de < len(raw) else "") or code
name_en = _cell(raw[i_en] if i_en is not None and i_en < len(raw) else "") or None
unit = _cell(raw[i_unit] if i_unit is not None and i_unit < len(raw) else "") or None
category = _cell(raw[i_cat] if i_cat is not None and i_cat < len(raw) else "") or None
sort_order += 1
out.append({
"attr_key": code,
"name_de": name_de,
"name_en": name_en,
"unit": unit,
"category": category,
"data_type": "num_per_100g",
"origin": "official_bls",
"sort_order": sort_order,
})
wb.close()
return out
def _parse_value_header(title: str) -> tuple[str | None, str, str | None]:
t = title.strip()
m = ATTR_HEADER_RE.match(t)
if m:
return m.group(1), m.group(2).strip(), m.group(3)
# Fallback: first token
parts = t.split()
if parts and re.match(r"^[A-Z][A-Z0-9:]{1,20}$", parts[0]):
return parts[0], " ".join(parts[1:]) or parts[0], None
return None, t, None
def parse_foods_xlsx(data: bytes) -> dict[str, Any]:
wb = load_workbook(filename=BytesIO(data), read_only=True, data_only=True)
ws = wb.active
rows = ws.iter_rows(values_only=True)
header = [_cell(c) for c in next(rows)]
if len(header) < 3:
wb.close()
raise ValueError("BLS-Datendatei: erwartet mindestens BLS-Code, Name DE, Name EN")
triples: list[dict[str, Any]] = []
i = 3
while i < len(header):
code, name_de, unit = _parse_value_header(header[i])
origin_i = i + 1 if i + 1 < len(header) else None
ref_i = i + 2 if i + 2 < len(header) else None
triples.append({
"attr_key": code or f"COL{i}",
"name_de": name_de,
"unit": unit,
"value_col": i,
"origin_col": origin_i,
"ref_col": ref_i,
})
i += 3 if (origin_i is not None and ref_i is not None) else 1
foods = []
for raw in rows:
if not raw:
continue
code = _cell(raw[0] if len(raw) else "")
if not code:
continue
name_de = _cell(raw[1] if len(raw) > 1 else "") or code
name_en = _cell(raw[2] if len(raw) > 2 else "") or None
values = []
for t in triples:
vc = t["value_col"]
raw_v = raw[vc] if vc < len(raw) else None
is_trace = False
num = None
if raw_v is None or raw_v == "" or raw_v == "-":
num = None
elif str(raw_v).strip().upper() in ("TR", "TRACE", "SPUREN"):
is_trace = True
else:
try:
num = float(str(raw_v).replace(",", "."))
except (TypeError, ValueError):
num = None
origin = ""
if t["origin_col"] is not None and t["origin_col"] < len(raw):
origin = _cell(raw[t["origin_col"]])
ref = ""
if t["ref_col"] is not None and t["ref_col"] < len(raw):
ref = _cell(raw[t["ref_col"]])
values.append({
"attr_key": t["attr_key"],
"value_num": num,
"is_trace": is_trace,
"origin_code": origin or None,
"reference_text": ref or None,
})
foods.append({
"bls_code": code,
"name_de": name_de,
"name_en": name_en,
"food_group": code[0] if code else None,
"values": values,
})
wb.close()
return {"attribute_headers": triples, "foods": foods}