feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
Deploy Development / deploy (push) Successful in 52s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates.
- Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching.
- Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure.
- Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
Lars committed 2026-04-10 07:08:21 +02:00
1 parent c10da55ec6
commit b4cc3cb934
7 files changed
+223 -60

No files matched your search

+42 -2
View File
@@ -7,7 +7,7 @@ from __future__ import annotations
import csv
import io
import re
from typing import Any, Dict, Iterator, List, Tuple
from typing import Any, Dict, Iterator, List, Sequence, Tuple
_DEFAULT_DELIMS = [",", ";", "\t"]
@@ -115,7 +115,7 @@ def column_signature(headers: List[str]) -> List[str]:
def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -> float:
"""Jaccard-Überlappung 0..1."""
"""Jaccard-Überlappung 0..1 (|A∩B|/|A∪B|). Fällt stark, wenn die CSV viele Zusatzspalten hat."""
a, b = set(sig_csv), set(sig_template)
if not a and not b:
return 1.0
@@ -126,6 +126,46 @@ def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -
return inter / union if union else 0.0
def headers_signature_template_recall(sig_csv: Sequence[str], sig_template: Sequence[str]) -> float:
"""
Anteil der Template-Spalten (Signatur), die in der CSV vorkommen: |A∩B|/|B|.
100 %, sobald alle für die Vorlage relevanten Spalten in der Datei sind — unabhängig von
Zusatzspalten (Gewicht + Ernährung in einer Datei erzeugt keinen „Abzug“ für die jeweilige Vorlage).
"""
a = set(sig_csv)
b = {normalize_header_for_signature(str(x)) for x in sig_template}
b.discard("")
if not b:
return 1.0 if not a else 0.0
inter = len(a & b)
return inter / len(b)
def headers_signature_rank_metrics(sig_csv: List[str], sig_template: List[str]) -> dict[str, Any]:
"""
Einheitliche Kennzahlen für Vorlagen-Ranking und UI.
confidence = template_recall (empfohlen für Anzeige / Sortierung primär).
"""
a = set(sig_csv)
b = {normalize_header_for_signature(str(x)) for x in sig_template}
b.discard("")
inter = a & b
n_inter = len(inter)
n_b = len(b)
n_a = len(a)
union = len(a | b)
template_recall = n_inter / n_b if n_b else (1.0 if not n_a else 0.0)
jaccard = n_inter / union if union else 0.0
return {
"confidence": round(template_recall, 4),
"template_recall": round(template_recall, 4),
"jaccard": round(jaccard, 4),
"columns_matched": n_inter,
"columns_in_template": n_b,
"columns_in_csv": n_a,
}
def get_csv_import_limits(conn_row: dict | None) -> dict[str, int]:
"""Liest Limits aus system_config.csv_import; Fallback bei fehlendem Key."""
defaults = {"max_rows_per_file": 50_000, "max_file_bytes": 52_428_800}