feat: Enhance CSV header normalization and mapping for activity data
Deploy Development / deploy (push) Successful in 49s
Build Test / pytest-backend (push) Successful in 5s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Introduced a new utility function `canonical_csv_header_label` to standardize CSV header labels, improving consistency in field mapping.
- Updated the `_lookup_db_field` function to support prefix matching for longer manual keys, enhancing the accuracy of field resolution.
- Added tests to validate handling of non-breaking space characters in CSV headers and ensure correct mapping to normalized keys, improving robustness of CSV parsing.
This commit is contained in:
Lars committed 2026-04-15 10:04:32 +02:00
1 parent c570e67a09
commit e4e8c70cd2
3 files changed
+82 -11

No files matched your search

+22 -10
View File
@@ -57,6 +57,18 @@ def _split_first_lines(text: str, max_lines: int = 5) -> List[str]:
return lines
def canonical_csv_header_label(name: str | None) -> str:
"""
Einheitlicher Spalten-Key für Analyse (Vorlage/Dialog), Import und Signatur.
BOM und NBSP (häufig in Excel/Apple-Exporten) werden vereinheitlicht, damit
field_mappings exakt zu DictReader-Zeilen passt.
"""
if name is None:
return ""
s = str(name).replace("\ufeff", "").replace("\u00a0", " ").strip()
return s
def parse_csv_sample(
text: str,
delimiter: str | None = None,
@@ -85,7 +97,7 @@ def parse_csv_sample(
return [], [], delim
if has_header:
headers = [h.strip() for h in rows_raw[0]]
headers = [canonical_csv_header_label(h) for h in rows_raw[0]]
data = rows_raw[1 : 1 + max_data_rows]
else:
n = len(rows_raw[0])
@@ -103,7 +115,7 @@ def parse_csv_sample(
def normalize_header_for_signature(name: str) -> str:
s = name.strip().lower()
s = canonical_csv_header_label(name).lower()
s = re.sub(r"\s+", "_", s)
s = re.sub(r"[^a-z0-9_äöüß().%-]+", "_", s)
return s.strip("_")
@@ -111,7 +123,9 @@ def normalize_header_for_signature(name: str) -> str:
def column_signature(headers: List[str]) -> List[str]:
"""Sortierte normalisierte Spaltennamen für Signatur-Vergleich."""
return sorted({normalize_header_for_signature(h) for h in headers if h is not None and str(h).strip()})
return sorted(
{normalize_header_for_signature(h) for h in headers if h is not None and canonical_csv_header_label(str(h))}
)
def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -> float:
@@ -178,12 +192,6 @@ def get_csv_import_limits(conn_row: dict | None) -> dict[str, int]:
return defaults
def _strip_header_key(k: str | None) -> str:
if k is None:
return ""
return str(k).strip().removeprefix("\ufeff")
def iter_csv_dict_rows(
text: str,
delimiter: str,
@@ -205,4 +213,8 @@ def iter_csv_dict_rows(
continue
if not any(v and str(v).strip() for v in row.values()):
continue
yield {_strip_header_key(k): (v or "").strip() for k, v in row.items() if _strip_header_key(k)}
yield {
canonical_csv_header_label(k): (v or "").strip()
for k, v in row.items()
if canonical_csv_header_label(k)
}
+27 -1
View File
@@ -14,7 +14,7 @@ from typing import Any, Mapping, Sequence
from dateutil import parser as dateutil_parser
from csv_parser.core import normalize_header_for_signature
from csv_parser.core import canonical_csv_header_label, normalize_header_for_signature
from csv_parser.field_units import factor_source_to_canonical
# Alias → strptime (JSON in Kleinbuchstaben)
@@ -477,7 +477,12 @@ def _lookup_db_field(csv_col: str, field_mappings: Mapping[str, str]) -> str | N
CSV-Spaltennamen können Roh-Header sein; Vorlagen-Schlüssel oft normalisiert
(wie column_signature). Exakter Treffer, dann Schlüssel nach Normalisierung,
dann Abgleich aller Vorlagen-Keys über deren Normalform.
Zusätzlich: Präfix-Treffer für lange manuelle Keys (z. B. Apple
„Aufgestiegene Höhe (m)“ → ``aufgestiegene_höhe_(m)`` vs. Mapping
„aufgestiegene Höhe“ → ``aufgestiegene_höhe``) — gewinnt der längste passende Key.
"""
csv_col = canonical_csv_header_label(csv_col)
v = field_mappings.get(csv_col)
if v:
return v if v not in ("-", "_skip") else None
@@ -488,6 +493,27 @@ def _lookup_db_field(csv_col: str, field_mappings: Mapping[str, str]) -> str | N
for k, fv in field_mappings.items():
if normalize_header_for_signature(str(k)) == norm:
return fv if fv not in ("-", "_skip") else None
# Präfix-Match (min. Länge gegen false positives wie „datum“ → „datum_xyz“)
best_fv: str | None = None
best_nk_len = 0
min_prefix = 10
for k, fv in field_mappings.items():
if not fv or fv in ("-", "_skip"):
continue
nk = normalize_header_for_signature(str(k))
if len(nk) < min_prefix or len(nk) >= len(norm):
continue
if not norm.startswith(nk):
continue
boundary = norm[len(nk) : len(nk) + 1]
if boundary not in ("", "_", "("):
continue
if len(nk) > best_nk_len:
best_nk_len = len(nk)
best_fv = fv
if best_fv:
return best_fv
return None