feat: implement effective CSV delimiter resolution for imports
Deploy Development / deploy (push) Successful in 1m0s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 17s

- Added `resolve_effective_csv_delimiter` function to determine the correct delimiter based on the uploaded file and template.
- Updated CSV import logic to utilize the new delimiter resolution method, ensuring accurate parsing of CSV files with varying delimiters.
- Enhanced documentation to reflect changes in delimiter handling.
- Added unit tests for the new delimiter resolution functionality.
This commit is contained in:
Lars committed 2026-04-18 10:12:33 +02:00
1 parent 0ad3ddd627
commit 7226e04e9c
5 files changed
+63 -3

No files matched your search

+40
View File
@@ -47,6 +47,46 @@ def sniff_delimiter(sample_line: str) -> str:
return best
def _csv_field_count(line: str, delimiter: str) -> int:
"""Anzahl Felder in einer Zeile (csv.reader, berücksichtigt Anführungszeichen)."""
if not line or not line.strip():
return 0
try:
row = next(csv.reader(io.StringIO(line), delimiter=delimiter))
except StopIteration:
return 0
return len(row)
def resolve_effective_csv_delimiter(text: str, template_delimiter: str | None = None) -> str:
"""
Trennzeichen für die hochgeladene Datei wählen. Gespeicherte Vorlagen haben oft «,»
(Apple EN), tatsächliche Exporte je nach Region «;» (Apple DE / Excel) — mit falschem
Zeichen wird die Kopfzeile zu **einer** Spalte und das Mapping bricht vollständig.
"""
tpl = (template_delimiter or "").strip()
if tpl not in _DEFAULT_DELIMS:
tpl = None
lines = _split_first_lines(text, max_lines=5)
if not lines:
return tpl or ","
header = lines[0]
scores: list[tuple[int, str]] = []
for d in _DEFAULT_DELIMS:
scores.append((_csv_field_count(header, d), d))
max_n = max(n for n, _ in scores)
if max_n <= 1:
return tpl or sniff_delimiter(header)
at_max = [d for n, d in scores if n == max_n]
if tpl and tpl in at_max:
return tpl
return at_max[0]
def _split_first_lines(text: str, max_lines: int = 5) -> List[str]:
lines: List[str] = []
for line in text.splitlines():
+3 -2
View File
@@ -11,7 +11,7 @@ from typing import Any
import logging
from csv_parser.core import iter_csv_dict_rows
from csv_parser.core import iter_csv_dict_rows, resolve_effective_csv_delimiter
from csv_parser.import_row_processing import (
aggregate_mapped_rows,
resolve_import_row_processing,
@@ -97,7 +97,8 @@ def run_universal_csv_import(
if tc is not None and not isinstance(tc, dict):
tc = None
delim = mapping.get("delimiter") or ","
tpl_delim = str(mapping.get("delimiter") or ",").strip() or ","
delim = resolve_effective_csv_delimiter(text, tpl_delim)
has_header = mapping.get("has_header", True)
if module == "nutrition":