feat(csv-parser): Introduce header signature ranking metrics for enhanced CSV analysis
Deploy Development / deploy (push) Successful in 52s
Build Test / lint-backend (push) Successful in 0s
Build Test / build-frontend (push) Successful in 16s

- Added new functions for calculating header signature recall and ranking metrics, improving the analysis of CSV templates.
- Updated existing CSV analysis endpoints to utilize the new ranking metrics, enhancing the accuracy of template matching.
- Refactored related code to replace Jaccard score calculations with the new metrics, providing a more comprehensive evaluation of CSV structure.
- Improved documentation for new functions to clarify their purpose and usage in the context of CSV template analysis.
This commit is contained in:
Lars committed 2026-04-10 07:08:21 +02:00
1 parent c10da55ec6
commit b4cc3cb934
7 files changed
+223 -60

No files matched your search

+16 -12
View File
@@ -15,7 +15,7 @@ from csv_parser.core import (
column_signature,
decode_raw_bytes,
get_csv_import_limits,
headers_signature_match_score,
headers_signature_rank_metrics,
normalize_header_for_signature,
parse_csv_sample,
)
@@ -134,7 +134,7 @@ async def admin_analyze_csv_for_template(
):
"""
CSV hochladen wie im Nutzer-Import: Spalten + Vorschau + Vorschläge für field_mappings
und type_conversions. Optional Seed-Vorlage (ID) oder beste Jaccard-Systemvorlage für das Modul.
und type_conversions. Optional Seed-Vorlage (ID) oder beste Systemvorlage (Abdeckung, dann Jaccard).
"""
_ = session
if not get_module_definition(module):
@@ -191,15 +191,15 @@ async def admin_analyze_csv_for_template(
)
rows = [r2d(r) for r in cur.fetchall()]
best: dict | None = None
best_score = -1.0
best_key: tuple[float, int, float] = (-1.0, -1, -1.0)
for t in rows:
t_sig = list(t.get("column_signature") or [])
t_norm = sorted({normalize_header_for_signature(str(s)) for s in t_sig})
score = headers_signature_match_score(sig, t_norm)
if score > best_score:
best_score = score
m = headers_signature_rank_metrics(sig, t_sig)
key = (m["confidence"], m["columns_matched"], m["jaccard"])
if key > best_key:
best_key = key
best = t
if best and best_score > 0:
if best and best_key[0] > 0:
seed_row = best
seed_fm = (seed_row or {}).get("field_mappings") or {}
@@ -214,13 +214,17 @@ async def admin_analyze_csv_for_template(
seed_meta = None
if seed_row:
t_sig = [normalize_header_for_signature(str(s)) for s in (seed_row.get("column_signature") or [])]
t_sig = list(seed_row.get("column_signature") or [])
sm = headers_signature_rank_metrics(sig, t_sig)
seed_meta = {
"id": seed_row["id"],
"mapping_name": seed_row.get("mapping_name"),
"confidence": round(headers_signature_match_score(sig, sorted(set(t_sig))), 4)
if t_sig
else 0.0,
"confidence": sm["confidence"],
"template_recall": sm["template_recall"],
"jaccard": sm["jaccard"],
"columns_matched": sm["columns_matched"],
"columns_in_template": sm["columns_in_template"],
"columns_in_csv": sm["columns_in_csv"],
}
return {