feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates. - Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching. - Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure. - Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
1 parent
c10da55ec6
commit
b4cc3cb934
7 files changed
+223
-60
No files matched your search
@@ -20,7 +20,7 @@ from csv_parser.core import (
|
||||
decode_raw_bytes,
|
||||
column_signature,
|
||||
get_csv_import_limits,
|
||||
headers_signature_match_score,
|
||||
headers_signature_rank_metrics,
|
||||
normalize_header_for_signature,
|
||||
parse_csv_sample,
|
||||
)
|
||||
@@ -247,19 +247,29 @@ async def analyze_csv(
|
||||
ranked = []
|
||||
for t in templates:
|
||||
t_sig = list(t["column_signature"]) if t["column_signature"] else []
|
||||
t_norm = sorted({normalize_header_for_signature(str(s)) for s in t_sig})
|
||||
score = headers_signature_match_score(sig, t_norm)
|
||||
metrics = headers_signature_rank_metrics(sig, t_sig)
|
||||
ranked.append(
|
||||
{
|
||||
"mapping_id": t["id"],
|
||||
"module": t["module"],
|
||||
"mapping_name": t["mapping_name"],
|
||||
"is_system": bool(t.get("is_system")),
|
||||
"confidence": round(score, 4),
|
||||
"match_type": "signature_jaccard",
|
||||
"confidence": metrics["confidence"],
|
||||
"template_recall": metrics["template_recall"],
|
||||
"jaccard": metrics["jaccard"],
|
||||
"columns_matched": metrics["columns_matched"],
|
||||
"columns_in_template": metrics["columns_in_template"],
|
||||
"columns_in_csv": metrics["columns_in_csv"],
|
||||
"match_type": "template_recall",
|
||||
}
|
||||
)
|
||||
ranked.sort(key=lambda x: -x["confidence"])
|
||||
ranked.sort(
|
||||
key=lambda x: (
|
||||
-(x.get("confidence") or 0),
|
||||
-(x.get("columns_matched") or 0),
|
||||
-(x.get("jaccard") or 0),
|
||||
),
|
||||
)
|
||||
|
||||
top = ranked[:25]
|
||||
recommended = top[0] if top and (top[0]["confidence"] or 0) > 0 else None
|
||||
|
||||
Reference in new issue
Block a user