feat: BLS-Stammdaten, FDDB-Mapping und Item-Tagebuch (#106)
Katalog, lernendes Mapping ohne KI, optionale Items und Import-Policy. Playwright-Smoke und Issue-Audit um Ernährung/Zuordnen/API ergänzt. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
72d101415f
commit
919c77fcf8
39 files changed
+2648
-47
No files matched your search
@@ -0,0 +1 @@
|
||||
"""BLS 4.0 ingest (official MRI XLSX)."""
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Apply parsed BLS 4.0 data: upsert attributes and foods, never delete official rows."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def upsert_attributes(cur, attributes: list[dict[str, Any]]) -> dict[str, int]:
|
||||
inserted = updated = 0
|
||||
for a in attributes:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attributes
|
||||
(attr_key, name_de, name_en, unit, category, data_type, origin, sort_order, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'official_bls', %s, NOW())
|
||||
ON CONFLICT (attr_key) DO UPDATE SET
|
||||
name_de = EXCLUDED.name_de,
|
||||
name_en = COALESCE(EXCLUDED.name_en, food_attributes.name_en),
|
||||
unit = COALESCE(EXCLUDED.unit, food_attributes.unit),
|
||||
category = COALESCE(EXCLUDED.category, food_attributes.category),
|
||||
sort_order = EXCLUDED.sort_order,
|
||||
updated_at = NOW()
|
||||
WHERE food_attributes.origin = 'official_bls'
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
""",
|
||||
(
|
||||
a["attr_key"], a["name_de"], a.get("name_en"), a.get("unit"),
|
||||
a.get("category"), a.get("data_type") or "num_per_100g",
|
||||
a.get("sort_order") or 0,
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row and row.get("inserted"):
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
return {"inserted": inserted, "updated": updated, "total": len(attributes)}
|
||||
|
||||
|
||||
def upsert_foods(cur, foods: list[dict[str, Any]], bls_version: str = "4.0") -> dict[str, int]:
|
||||
cur.execute("SELECT attr_key, id FROM food_attributes")
|
||||
attr_ids = {r["attr_key"]: r["id"] for r in cur.fetchall()}
|
||||
inserted = updated = values_written = 0
|
||||
for f in foods:
|
||||
code = f["bls_code"]
|
||||
cur.execute("SELECT id FROM food_catalog WHERE bls_code = %s", (code,))
|
||||
existing = cur.fetchone()
|
||||
if existing:
|
||||
food_id = existing["id"]
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE food_catalog
|
||||
SET name_de=%s, name_en=%s, food_group=%s, bls_version=%s,
|
||||
catalog_kind='official_bls', source='bls_4.0', is_active=true, updated_at=NOW()
|
||||
WHERE id=%s AND catalog_kind='official_bls'
|
||||
""",
|
||||
(f["name_de"], f.get("name_en"), f.get("food_group"), bls_version, food_id),
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_catalog
|
||||
(bls_code, name_de, name_en, food_group, catalog_kind, bls_version, source)
|
||||
VALUES (%s, %s, %s, %s, 'official_bls', %s, 'bls_4.0')
|
||||
RETURNING id
|
||||
""",
|
||||
(code, f["name_de"], f.get("name_en"), f.get("food_group"), bls_version),
|
||||
)
|
||||
food_id = cur.fetchone()["id"]
|
||||
inserted += 1
|
||||
for val in f.get("values") or []:
|
||||
aid = attr_ids.get(val["attr_key"])
|
||||
if not aid:
|
||||
continue
|
||||
if val.get("is_trace"):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, NULL, true, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = NULL, is_trace = true,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
|
||||
)
|
||||
elif val.get("value_num") is None:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, NULL, false, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = NULL, is_trace = false,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(food_id, aid, val.get("origin_code"), val.get("reference_text")),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO food_attribute_values
|
||||
(food_id, attribute_id, value_num, is_trace, origin_code, reference_text, updated_at)
|
||||
VALUES (%s, %s, %s, false, %s, %s, NOW())
|
||||
ON CONFLICT (food_id, attribute_id) DO UPDATE SET
|
||||
value_num = EXCLUDED.value_num, is_trace = false,
|
||||
origin_code = EXCLUDED.origin_code,
|
||||
reference_text = EXCLUDED.reference_text,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
(
|
||||
food_id, aid, val["value_num"],
|
||||
val.get("origin_code"), val.get("reference_text"),
|
||||
),
|
||||
)
|
||||
values_written += 1
|
||||
return {
|
||||
"foods_inserted": inserted,
|
||||
"foods_updated": updated,
|
||||
"values_written": values_written,
|
||||
"foods_total": len(foods),
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Parse official BLS 4.0 XLSX files without a hardcoded nutrient code list."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
# Header like: "ENERCJ Energie (Kilojoule) [kJ/100g]"
|
||||
ATTR_HEADER_RE = re.compile(
|
||||
r"^([A-Z][A-Z0-9:]{1,20})\s+(.+?)(?:\s*\[([^\]]+)\])?\s*$"
|
||||
)
|
||||
|
||||
|
||||
def _cell(v: Any) -> str:
|
||||
if v is None:
|
||||
return ""
|
||||
return str(v).strip()
|
||||
|
||||
|
||||
def parse_components_xlsx(data: bytes) -> list[dict[str, Any]]:
|
||||
wb = load_workbook(filename=BytesIO(data), read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = [_cell(c) for c in next(rows)]
|
||||
idx = {h.lower(): i for i, h in enumerate(header) if h}
|
||||
|
||||
def col(*names: str) -> int | None:
|
||||
for n in names:
|
||||
if n.lower() in idx:
|
||||
return idx[n.lower()]
|
||||
for key, i in idx.items():
|
||||
for n in names:
|
||||
if n.lower() in key:
|
||||
return i
|
||||
return None
|
||||
|
||||
i_code = col("code", "schlüssel", "schluessel", "attr_key", "komponente")
|
||||
i_de = col("name_de", "deutsch", "bezeichnung_de", "name de")
|
||||
i_en = col("name_en", "english", "bezeichnung_en", "name en")
|
||||
i_unit = col("unit", "einheit")
|
||||
i_cat = col("category", "kategorie", "gruppe")
|
||||
if i_code is None:
|
||||
i_code = 0
|
||||
if i_de is None:
|
||||
i_de = 1 if len(header) > 1 else 0
|
||||
|
||||
out = []
|
||||
sort_order = 0
|
||||
for raw in rows:
|
||||
if not raw:
|
||||
continue
|
||||
code = _cell(raw[i_code] if i_code < len(raw) else "")
|
||||
if not code or code.lower() in ("code", "schlüssel", "schluessel"):
|
||||
continue
|
||||
name_de = _cell(raw[i_de] if i_de is not None and i_de < len(raw) else "") or code
|
||||
name_en = _cell(raw[i_en] if i_en is not None and i_en < len(raw) else "") or None
|
||||
unit = _cell(raw[i_unit] if i_unit is not None and i_unit < len(raw) else "") or None
|
||||
category = _cell(raw[i_cat] if i_cat is not None and i_cat < len(raw) else "") or None
|
||||
sort_order += 1
|
||||
out.append({
|
||||
"attr_key": code,
|
||||
"name_de": name_de,
|
||||
"name_en": name_en,
|
||||
"unit": unit,
|
||||
"category": category,
|
||||
"data_type": "num_per_100g",
|
||||
"origin": "official_bls",
|
||||
"sort_order": sort_order,
|
||||
})
|
||||
wb.close()
|
||||
return out
|
||||
|
||||
|
||||
def _parse_value_header(title: str) -> tuple[str | None, str, str | None]:
|
||||
t = title.strip()
|
||||
m = ATTR_HEADER_RE.match(t)
|
||||
if m:
|
||||
return m.group(1), m.group(2).strip(), m.group(3)
|
||||
# Fallback: first token
|
||||
parts = t.split()
|
||||
if parts and re.match(r"^[A-Z][A-Z0-9:]{1,20}$", parts[0]):
|
||||
return parts[0], " ".join(parts[1:]) or parts[0], None
|
||||
return None, t, None
|
||||
|
||||
|
||||
def parse_foods_xlsx(data: bytes) -> dict[str, Any]:
|
||||
wb = load_workbook(filename=BytesIO(data), read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = [_cell(c) for c in next(rows)]
|
||||
if len(header) < 3:
|
||||
wb.close()
|
||||
raise ValueError("BLS-Datendatei: erwartet mindestens BLS-Code, Name DE, Name EN")
|
||||
|
||||
triples: list[dict[str, Any]] = []
|
||||
i = 3
|
||||
while i < len(header):
|
||||
code, name_de, unit = _parse_value_header(header[i])
|
||||
origin_i = i + 1 if i + 1 < len(header) else None
|
||||
ref_i = i + 2 if i + 2 < len(header) else None
|
||||
triples.append({
|
||||
"attr_key": code or f"COL{i}",
|
||||
"name_de": name_de,
|
||||
"unit": unit,
|
||||
"value_col": i,
|
||||
"origin_col": origin_i,
|
||||
"ref_col": ref_i,
|
||||
})
|
||||
i += 3 if (origin_i is not None and ref_i is not None) else 1
|
||||
|
||||
foods = []
|
||||
for raw in rows:
|
||||
if not raw:
|
||||
continue
|
||||
code = _cell(raw[0] if len(raw) else "")
|
||||
if not code:
|
||||
continue
|
||||
name_de = _cell(raw[1] if len(raw) > 1 else "") or code
|
||||
name_en = _cell(raw[2] if len(raw) > 2 else "") or None
|
||||
values = []
|
||||
for t in triples:
|
||||
vc = t["value_col"]
|
||||
raw_v = raw[vc] if vc < len(raw) else None
|
||||
is_trace = False
|
||||
num = None
|
||||
if raw_v is None or raw_v == "" or raw_v == "-":
|
||||
num = None
|
||||
elif str(raw_v).strip().upper() in ("TR", "TRACE", "SPUREN"):
|
||||
is_trace = True
|
||||
else:
|
||||
try:
|
||||
num = float(str(raw_v).replace(",", "."))
|
||||
except (TypeError, ValueError):
|
||||
num = None
|
||||
origin = ""
|
||||
if t["origin_col"] is not None and t["origin_col"] < len(raw):
|
||||
origin = _cell(raw[t["origin_col"]])
|
||||
ref = ""
|
||||
if t["ref_col"] is not None and t["ref_col"] < len(raw):
|
||||
ref = _cell(raw[t["ref_col"]])
|
||||
values.append({
|
||||
"attr_key": t["attr_key"],
|
||||
"value_num": num,
|
||||
"is_trace": is_trace,
|
||||
"origin_code": origin or None,
|
||||
"reference_text": ref or None,
|
||||
})
|
||||
foods.append({
|
||||
"bls_code": code,
|
||||
"name_de": name_de,
|
||||
"name_en": name_en,
|
||||
"food_group": code[0] if code else None,
|
||||
"values": values,
|
||||
})
|
||||
wb.close()
|
||||
return {"attribute_headers": triples, "foods": foods}
|
||||
Reference in new issue
Block a user