feat: Enhance CSV header normalization and mapping for activity data
- Introduced a new utility function `canonical_csv_header_label` to standardize CSV header labels, improving consistency in field mapping. - Updated the `_lookup_db_field` function to support prefix matching for longer manual keys, enhancing the accuracy of field resolution. - Added tests to validate handling of non-breaking space characters in CSV headers and ensure correct mapping to normalized keys, improving robustness of CSV parsing.
This commit is contained in:
1 parent
c570e67a09
commit
e4e8c70cd2
3 files changed
+82
-11
No files matched your search
+22
-10
@@ -57,6 +57,18 @@ def _split_first_lines(text: str, max_lines: int = 5) -> List[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def canonical_csv_header_label(name: str | None) -> str:
|
||||
"""
|
||||
Einheitlicher Spalten-Key für Analyse (Vorlage/Dialog), Import und Signatur.
|
||||
BOM und NBSP (häufig in Excel/Apple-Exporten) werden vereinheitlicht, damit
|
||||
field_mappings exakt zu DictReader-Zeilen passt.
|
||||
"""
|
||||
if name is None:
|
||||
return ""
|
||||
s = str(name).replace("\ufeff", "").replace("\u00a0", " ").strip()
|
||||
return s
|
||||
|
||||
|
||||
def parse_csv_sample(
|
||||
text: str,
|
||||
delimiter: str | None = None,
|
||||
@@ -85,7 +97,7 @@ def parse_csv_sample(
|
||||
return [], [], delim
|
||||
|
||||
if has_header:
|
||||
headers = [h.strip() for h in rows_raw[0]]
|
||||
headers = [canonical_csv_header_label(h) for h in rows_raw[0]]
|
||||
data = rows_raw[1 : 1 + max_data_rows]
|
||||
else:
|
||||
n = len(rows_raw[0])
|
||||
@@ -103,7 +115,7 @@ def parse_csv_sample(
|
||||
|
||||
|
||||
def normalize_header_for_signature(name: str) -> str:
|
||||
s = name.strip().lower()
|
||||
s = canonical_csv_header_label(name).lower()
|
||||
s = re.sub(r"\s+", "_", s)
|
||||
s = re.sub(r"[^a-z0-9_äöüß().%-]+", "_", s)
|
||||
return s.strip("_")
|
||||
@@ -111,7 +123,9 @@ def normalize_header_for_signature(name: str) -> str:
|
||||
|
||||
def column_signature(headers: List[str]) -> List[str]:
|
||||
"""Sortierte normalisierte Spaltennamen für Signatur-Vergleich."""
|
||||
return sorted({normalize_header_for_signature(h) for h in headers if h is not None and str(h).strip()})
|
||||
return sorted(
|
||||
{normalize_header_for_signature(h) for h in headers if h is not None and canonical_csv_header_label(str(h))}
|
||||
)
|
||||
|
||||
|
||||
def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -> float:
|
||||
@@ -178,12 +192,6 @@ def get_csv_import_limits(conn_row: dict | None) -> dict[str, int]:
|
||||
return defaults
|
||||
|
||||
|
||||
def _strip_header_key(k: str | None) -> str:
|
||||
if k is None:
|
||||
return ""
|
||||
return str(k).strip().removeprefix("\ufeff")
|
||||
|
||||
|
||||
def iter_csv_dict_rows(
|
||||
text: str,
|
||||
delimiter: str,
|
||||
@@ -205,4 +213,8 @@ def iter_csv_dict_rows(
|
||||
continue
|
||||
if not any(v and str(v).strip() for v in row.values()):
|
||||
continue
|
||||
yield {_strip_header_key(k): (v or "").strip() for k, v in row.items() if _strip_header_key(k)}
|
||||
yield {
|
||||
canonical_csv_header_label(k): (v or "").strip()
|
||||
for k, v in row.items()
|
||||
if canonical_csv_header_label(k)
|
||||
}
|
||||
Reference in new issue
Block a user