feat: implement effective CSV delimiter resolution for imports
All checks were successful
Deploy Development / deploy (push) Successful in 1m0s
Build Test / pytest-backend (push) Successful in 4s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 17s

- Added `resolve_effective_csv_delimiter` function to determine the correct delimiter based on the uploaded file and template.
- Updated CSV import logic to utilize the new delimiter resolution method, ensuring accurate parsing of CSV files with varying delimiters.
- Enhanced documentation to reflect changes in delimiter handling.
- Added unit tests for the new delimiter resolution functionality.
This commit is contained in:
Lars 2026-04-18 10:12:33 +02:00
parent 0ad3ddd627
commit 7226e04e9c
5 changed files with 63 additions and 3 deletions

View File

@ -18,6 +18,7 @@ Dieses Dokument ist **normativ für Agenten**, die ein neues Import-Zielmodul an
| Admin-Systemvorlagen | `backend/routers/admin_csv_templates.py` |
| Nutzer-Import (Profil-Mappings) | `backend/routers/csv_import.py` |
| Vorlagen-Validierung (strukturell + Sample) | `backend/csv_parser/template_validator.py` (`validate_csv_template`) |
| Effektives Listentrennzeichen | `backend/csv_parser/core.py` (`resolve_effective_csv_delimiter`) — Datei kann `;` (z.B. Apple DE) haben, Vorlage `,` (EN); Import/Diagnose **nicht** nur das gespeicherte Trennzeichen blind nutzen. |
**Single Source of Truth** für erlaubte Zielfelder, Typen und Duplikat-Keys ist **`module_registry.py`**. Keine parallele Feldliste in Routern duplizieren.

View File

@ -47,6 +47,46 @@ def sniff_delimiter(sample_line: str) -> str:
return best
def _csv_field_count(line: str, delimiter: str) -> int:
"""Anzahl Felder in einer Zeile (csv.reader, berücksichtigt Anführungszeichen)."""
if not line or not line.strip():
return 0
try:
row = next(csv.reader(io.StringIO(line), delimiter=delimiter))
except StopIteration:
return 0
return len(row)
def resolve_effective_csv_delimiter(text: str, template_delimiter: str | None = None) -> str:
"""
Trennzeichen für die hochgeladene Datei wählen. Gespeicherte Vorlagen haben oft «,»
(Apple EN), tatsächliche Exporte je nach Region «;» (Apple DE / Excel) mit falschem
Zeichen wird die Kopfzeile zu **einer** Spalte und das Mapping bricht vollständig.
"""
tpl = (template_delimiter or "").strip()
if tpl not in _DEFAULT_DELIMS:
tpl = None
lines = _split_first_lines(text, max_lines=5)
if not lines:
return tpl or ","
header = lines[0]
scores: list[tuple[int, str]] = []
for d in _DEFAULT_DELIMS:
scores.append((_csv_field_count(header, d), d))
max_n = max(n for n, _ in scores)
if max_n <= 1:
return tpl or sniff_delimiter(header)
at_max = [d for n, d in scores if n == max_n]
if tpl and tpl in at_max:
return tpl
return at_max[0]
def _split_first_lines(text: str, max_lines: int = 5) -> List[str]:
lines: List[str] = []
for line in text.splitlines():

View File

@ -11,7 +11,7 @@ from typing import Any
import logging
from csv_parser.core import iter_csv_dict_rows
from csv_parser.core import iter_csv_dict_rows, resolve_effective_csv_delimiter
from csv_parser.import_row_processing import (
aggregate_mapped_rows,
resolve_import_row_processing,
@ -97,7 +97,8 @@ def run_universal_csv_import(
if tc is not None and not isinstance(tc, dict):
tc = None
delim = mapping.get("delimiter") or ","
tpl_delim = str(mapping.get("delimiter") or ",").strip() or ","
delim = resolve_effective_csv_delimiter(text, tpl_delim)
has_header = mapping.get("has_header", True)
if module == "nutrition":

View File

@ -29,6 +29,7 @@ from csv_parser.core import (
iter_csv_dict_rows,
normalize_header_for_signature,
parse_csv_sample,
resolve_effective_csv_delimiter,
)
from csv_parser.type_converter import build_row_after_mapping, diagnose_row_mapping
from csv_parser.field_units import source_unit_choices_for_field
@ -393,7 +394,8 @@ async def csv_import_diagnose(
tc = m.get("type_conversions")
if not isinstance(tc, dict):
tc = {}
delim = str(m.get("delimiter") or ",")
tpl_delim = str(m.get("delimiter") or ",").strip() or ","
delim = resolve_effective_csv_delimiter(text, tpl_delim)
exec_module = str(m["module"])
rows_out: list[dict[str, Any]] = []
@ -418,6 +420,7 @@ async def csv_import_diagnose(
"mapping_id": mapping_id,
"mapping_name": m.get("mapping_name"),
"module": exec_module,
"delimiter_template": tpl_delim,
"delimiter_used": delim,
"has_header": bool(m.get("has_header", True)),
"rows_diagnosed": len(rows_out),

View File

@ -11,6 +11,7 @@ from csv_parser.core import (
headers_signature_rank_metrics,
get_csv_import_limits,
iter_csv_dict_rows,
resolve_effective_csv_delimiter,
)
from csv_parser.field_units import source_unit_choices_for_field
from csv_parser.mapping_suggest import build_type_conversions_for_mapping
@ -29,6 +30,20 @@ def test_sniff_delimiter():
assert sniff_delimiter("a,b,c") == ","
def test_resolve_effective_csv_delimiter_semicolon_file_comma_template():
"""DE-Apple: «;» in der Datei, englische Vorlage speichert «,»."""
header = "Workout Type;Start;End;Duration;Aktive Energie (kJ)"
row = "Laufen;2026-04-17 16:25;2026-04-17 17:00;00:30:00;500"
text = header + "\n" + row + "\n"
assert resolve_effective_csv_delimiter(text, ",") == ";"
assert resolve_effective_csv_delimiter(text, None) == ";"
def test_resolve_effective_csv_delimiter_comma_file_keeps_template():
text = "Workout Type,Start,End\nWalk,2026-04-17 16:25,2026-04-17 17:00\n"
assert resolve_effective_csv_delimiter(text, ",") == ","
def test_parse_csv_sample_header():
text = "Date;kcal\n2024-01-01;2000\n"
headers, rows, delim = parse_csv_sample(text, delimiter=";", max_data_rows=3)