feat: Enhance CSV header normalization and mapping for activity data
Deploy Development / deploy (push) Successful in 49s
Build Test / pytest-backend (push) Successful in 5s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Introduced a new utility function `canonical_csv_header_label` to standardize CSV header labels, improving consistency in field mapping.
- Updated the `_lookup_db_field` function to support prefix matching for longer manual keys, enhancing the accuracy of field resolution.
- Added tests to validate handling of non-breaking space characters in CSV headers and ensure correct mapping to normalized keys, improving robustness of CSV parsing.
This commit is contained in:
Lars committed 2026-04-15 10:04:32 +02:00
1 parent c570e67a09
commit e4e8c70cd2
3 files changed
+82 -11

No files matched your search

+22 -10
View File
@@ -57,6 +57,18 @@ def _split_first_lines(text: str, max_lines: int = 5) -> List[str]:
return lines
def canonical_csv_header_label(name: str | None) -> str:
"""
Einheitlicher Spalten-Key für Analyse (Vorlage/Dialog), Import und Signatur.
BOM und NBSP (häufig in Excel/Apple-Exporten) werden vereinheitlicht, damit
field_mappings exakt zu DictReader-Zeilen passt.
"""
if name is None:
return ""
s = str(name).replace("\ufeff", "").replace("\u00a0", " ").strip()
return s
def parse_csv_sample(
text: str,
delimiter: str | None = None,
@@ -85,7 +97,7 @@ def parse_csv_sample(
return [], [], delim
if has_header:
headers = [h.strip() for h in rows_raw[0]]
headers = [canonical_csv_header_label(h) for h in rows_raw[0]]
data = rows_raw[1 : 1 + max_data_rows]
else:
n = len(rows_raw[0])
@@ -103,7 +115,7 @@ def parse_csv_sample(
def normalize_header_for_signature(name: str) -> str:
s = name.strip().lower()
s = canonical_csv_header_label(name).lower()
s = re.sub(r"\s+", "_", s)
s = re.sub(r"[^a-z0-9_äöüß().%-]+", "_", s)
return s.strip("_")
@@ -111,7 +123,9 @@ def normalize_header_for_signature(name: str) -> str:
def column_signature(headers: List[str]) -> List[str]:
"""Sortierte normalisierte Spaltennamen für Signatur-Vergleich."""
return sorted({normalize_header_for_signature(h) for h in headers if h is not None and str(h).strip()})
return sorted(
{normalize_header_for_signature(h) for h in headers if h is not None and canonical_csv_header_label(str(h))}
)
def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -> float:
@@ -178,12 +192,6 @@ def get_csv_import_limits(conn_row: dict | None) -> dict[str, int]:
return defaults
def _strip_header_key(k: str | None) -> str:
if k is None:
return ""
return str(k).strip().removeprefix("\ufeff")
def iter_csv_dict_rows(
text: str,
delimiter: str,
@@ -205,4 +213,8 @@ def iter_csv_dict_rows(
continue
if not any(v and str(v).strip() for v in row.values()):
continue
yield {_strip_header_key(k): (v or "").strip() for k, v in row.items() if _strip_header_key(k)}
yield {
canonical_csv_header_label(k): (v or "").strip()
for k, v in row.items()
if canonical_csv_header_label(k)
}