feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates. - Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching. - Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure. - Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
1 parent
c10da55ec6
commit
b4cc3cb934
7 files changed
+223
-60
No files matched your search
@@ -7,7 +7,7 @@ from __future__ import annotations
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from typing import Any, Dict, Iterator, List, Tuple
|
||||
from typing import Any, Dict, Iterator, List, Sequence, Tuple
|
||||
|
||||
_DEFAULT_DELIMS = [",", ";", "\t"]
|
||||
|
||||
@@ -115,7 +115,7 @@ def column_signature(headers: List[str]) -> List[str]:
|
||||
|
||||
|
||||
def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -> float:
|
||||
"""Jaccard-Überlappung 0..1."""
|
||||
"""Jaccard-Überlappung 0..1 (|A∩B|/|A∪B|). Fällt stark, wenn die CSV viele Zusatzspalten hat."""
|
||||
a, b = set(sig_csv), set(sig_template)
|
||||
if not a and not b:
|
||||
return 1.0
|
||||
@@ -126,6 +126,46 @@ def headers_signature_match_score(sig_csv: List[str], sig_template: List[str]) -
|
||||
return inter / union if union else 0.0
|
||||
|
||||
|
||||
def headers_signature_template_recall(sig_csv: Sequence[str], sig_template: Sequence[str]) -> float:
|
||||
"""
|
||||
Anteil der Template-Spalten (Signatur), die in der CSV vorkommen: |A∩B|/|B|.
|
||||
100 %, sobald alle für die Vorlage relevanten Spalten in der Datei sind — unabhängig von
|
||||
Zusatzspalten (Gewicht + Ernährung in einer Datei erzeugt keinen „Abzug“ für die jeweilige Vorlage).
|
||||
"""
|
||||
a = set(sig_csv)
|
||||
b = {normalize_header_for_signature(str(x)) for x in sig_template}
|
||||
b.discard("")
|
||||
if not b:
|
||||
return 1.0 if not a else 0.0
|
||||
inter = len(a & b)
|
||||
return inter / len(b)
|
||||
|
||||
|
||||
def headers_signature_rank_metrics(sig_csv: List[str], sig_template: List[str]) -> dict[str, Any]:
|
||||
"""
|
||||
Einheitliche Kennzahlen für Vorlagen-Ranking und UI.
|
||||
confidence = template_recall (empfohlen für Anzeige / Sortierung primär).
|
||||
"""
|
||||
a = set(sig_csv)
|
||||
b = {normalize_header_for_signature(str(x)) for x in sig_template}
|
||||
b.discard("")
|
||||
inter = a & b
|
||||
n_inter = len(inter)
|
||||
n_b = len(b)
|
||||
n_a = len(a)
|
||||
union = len(a | b)
|
||||
template_recall = n_inter / n_b if n_b else (1.0 if not n_a else 0.0)
|
||||
jaccard = n_inter / union if union else 0.0
|
||||
return {
|
||||
"confidence": round(template_recall, 4),
|
||||
"template_recall": round(template_recall, 4),
|
||||
"jaccard": round(jaccard, 4),
|
||||
"columns_matched": n_inter,
|
||||
"columns_in_template": n_b,
|
||||
"columns_in_csv": n_a,
|
||||
}
|
||||
|
||||
|
||||
def get_csv_import_limits(conn_row: dict | None) -> dict[str, int]:
|
||||
"""Liest Limits aus system_config.csv_import; Fallback bei fehlendem Key."""
|
||||
defaults = {"max_rows_per_file": 50_000, "max_file_bytes": 52_428_800}
|
||||
|
||||
Reference in new issue
Block a user