feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates. - Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching. - Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure. - Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
1 parent
c10da55ec6
commit
b4cc3cb934
7 files changed
+223
-60
No files matched your search
@@ -8,6 +8,7 @@ from csv_parser.core import (
|
||||
parse_csv_sample,
|
||||
column_signature,
|
||||
headers_signature_match_score,
|
||||
headers_signature_rank_metrics,
|
||||
get_csv_import_limits,
|
||||
iter_csv_dict_rows,
|
||||
)
|
||||
@@ -46,6 +47,20 @@ def test_jaccard():
|
||||
assert headers_signature_match_score(s1, s2) == pytest.approx(2 / 3)
|
||||
|
||||
|
||||
def test_template_recall_full_when_csv_has_extra_columns():
|
||||
"""Alle Template-Spalten in der CSV → Recall 1.0; Jaccard niedriger bei vielen Zusatzspalten."""
|
||||
csv_sig = column_signature(
|
||||
["D", "E", "F", "Extra1", "Extra2", "Extra3", "Extra4", "Extra5"]
|
||||
)
|
||||
tmpl_sig = column_signature(["d", "e", "f"])
|
||||
m = headers_signature_rank_metrics(csv_sig, tmpl_sig)
|
||||
assert m["confidence"] == 1.0
|
||||
assert m["template_recall"] == 1.0
|
||||
assert m["columns_matched"] == 3
|
||||
assert m["columns_in_template"] == 3
|
||||
assert m["jaccard"] == pytest.approx(3 / 8)
|
||||
|
||||
|
||||
def test_get_csv_import_limits_default():
|
||||
assert get_csv_import_limits(None)["max_rows_per_file"] == 50_000
|
||||
|
||||
|
||||
Reference in new issue
Block a user