Lerne Taxonomie und Trefferliste live von Geizhals, damit Wizard und Kandidaten echte Quelldaten nutzen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
d784248e28
commit
46857715df
25 files changed
+1614
-27
No files matched your search
@@ -100,7 +100,7 @@ class CatalogService:
|
||||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
if source_category is not None and self._should_learn(source_category, source_id):
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
|
||||
@@ -172,7 +172,33 @@ class CatalogService:
|
||||
if definition.id in seen:
|
||||
continue
|
||||
criteria.append(self._criterion_out(definition, None, None, [], None))
|
||||
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
|
||||
return jsonable(
|
||||
{
|
||||
"product_category_id": product_category_id,
|
||||
"source": source_id,
|
||||
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
|
||||
"criteria": criteria,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
|
||||
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
return True
|
||||
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
|
||||
return False
|
||||
from app.settings import get_settings
|
||||
|
||||
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
|
||||
return False
|
||||
keys = {
|
||||
row.external_filter_key
|
||||
for row in self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord).where(
|
||||
SourceFilterDefinitionRecord.source_category_id == source_category.id
|
||||
)
|
||||
)
|
||||
}
|
||||
return keys <= {"17023", "price"}
|
||||
|
||||
def list_templates(self) -> list[dict[str, Any]]:
|
||||
rows, _total = self.catalog.list_profiles(200, 0)
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.discovery.diff import diff_run, next_lifecycle
|
||||
from app.discovery.enums import (
|
||||
CandidateStatus,
|
||||
DiffStatus,
|
||||
FilterType,
|
||||
LifecycleStatus,
|
||||
MappingStatus,
|
||||
MatchStatus,
|
||||
@@ -99,7 +100,20 @@ class TaxonomyService:
|
||||
if catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
adapter = self.adapter or discovery_adapter(source_id)
|
||||
payload = adapter.discover_taxonomy(category_key)
|
||||
try:
|
||||
payload = adapter.discover_taxonomy(category_key)
|
||||
except AppError as exc:
|
||||
category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.external_category_key == category_key,
|
||||
)
|
||||
).first()
|
||||
if category is not None and exc.code == "SOURCE_BLOCKED":
|
||||
category.taxonomy_status = TaxonomyStatus.BLOCKED.value
|
||||
category.last_taxonomy_scan_at = now_utc()
|
||||
self.session.flush()
|
||||
raise
|
||||
now = now_utc()
|
||||
category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
@@ -148,7 +162,7 @@ class TaxonomyService:
|
||||
label=str(raw.get("label") or raw["external_filter_key"]),
|
||||
filter_type=filter_type,
|
||||
canonical_unit=raw.get("canonical_unit"),
|
||||
status=MappingStatus.VERIFIED.value,
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
)
|
||||
self.session.add(filt)
|
||||
self.session.flush()
|
||||
@@ -161,9 +175,14 @@ class TaxonomyService:
|
||||
self._sync_options(filt, raw.get("options") or [])
|
||||
if raw.get("range"):
|
||||
self._sync_range(filt, raw["range"])
|
||||
category.taxonomy_status = (
|
||||
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
|
||||
)
|
||||
self._sync_learned_mapping(category, filt, raw)
|
||||
requested = str(payload.get("taxonomy_status") or "")
|
||||
if requested in {item.value for item in TaxonomyStatus}:
|
||||
category.taxonomy_status = requested
|
||||
else:
|
||||
category.taxonomy_status = (
|
||||
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
|
||||
)
|
||||
self.session.flush()
|
||||
return self.filters(source_id, category.external_category_key)
|
||||
|
||||
@@ -275,6 +294,89 @@ class TaxonomyService:
|
||||
if removed:
|
||||
self._break_mappings(filt.source_category_id, filt.external_filter_key)
|
||||
|
||||
def _sync_learned_mapping(
|
||||
self,
|
||||
category: SourceCategoryRecord,
|
||||
filt: SourceFilterDefinitionRecord,
|
||||
raw: dict[str, Any],
|
||||
) -> None:
|
||||
encoding = raw.get("encoding") or {"url_param": "xf"}
|
||||
definition = upsert_definition(
|
||||
self.session,
|
||||
{
|
||||
"key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}",
|
||||
"origin": "SOURCE",
|
||||
"source": category.source_id,
|
||||
"product_category_id": category.product_category_id,
|
||||
"label": filt.label,
|
||||
"value_type": "number"
|
||||
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
||||
else "string",
|
||||
"unit": filt.canonical_unit,
|
||||
"category": category.product_category_id,
|
||||
},
|
||||
)
|
||||
mapping = self.session.scalars(
|
||||
select(CriterionMappingRecord).where(
|
||||
CriterionMappingRecord.source_category_id == category.id,
|
||||
CriterionMappingRecord.external_filter_key == filt.external_filter_key,
|
||||
)
|
||||
).first()
|
||||
options = raw.get("options") or []
|
||||
source_value = None
|
||||
if options:
|
||||
source_value = str(options[0].get("query_value") or options[0].get("source_value"))
|
||||
now = now_utc()
|
||||
if mapping is None:
|
||||
mapping = CriterionMappingRecord(
|
||||
id=uuid4(),
|
||||
criterion_definition_id=definition.id,
|
||||
criterion_key=definition.key,
|
||||
source_id=category.source_id,
|
||||
source_category_id=category.id,
|
||||
external_filter_key=filt.external_filter_key,
|
||||
source_value=source_value,
|
||||
encoding=encoding,
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
self.session.add(mapping)
|
||||
self.session.flush()
|
||||
else:
|
||||
mapping.criterion_definition_id = definition.id
|
||||
mapping.criterion_key = definition.key
|
||||
mapping.source_value = source_value
|
||||
mapping.encoding = encoding
|
||||
mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value}))
|
||||
mapping.updated_at = now
|
||||
probe_value = source_value or 0
|
||||
if options and options[0].get("normalized_value") is not None:
|
||||
probe_value = options[0]["normalized_value"]
|
||||
operator = (options[0].get("operator") if options else None) or "<="
|
||||
if filt.filter_type == FilterType.NUMERIC_RANGE.value:
|
||||
operator = ">="
|
||||
probe_value = 1000
|
||||
verified = verify_roundtrip(
|
||||
category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}",
|
||||
criterion={
|
||||
"criterion": definition.key,
|
||||
"operator": operator,
|
||||
"value": probe_value,
|
||||
},
|
||||
mapping={
|
||||
"criterion_key": definition.key,
|
||||
"external_filter_key": filt.external_filter_key,
|
||||
"source_value": source_value,
|
||||
"encoding": encoding,
|
||||
},
|
||||
)
|
||||
if mapping.status != MappingStatus.DISABLED.value and verified:
|
||||
mapping.status = MappingStatus.VERIFIED.value
|
||||
elif mapping.status == MappingStatus.BROKEN.value:
|
||||
mapping.status = MappingStatus.LEARNED_UNVERIFIED.value
|
||||
|
||||
def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None:
|
||||
row = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
if row is None:
|
||||
|
||||
Reference in new issue
Block a user