Uebernimm Enum-Listen und Filtergruppen in die Taxonomie, speichere Suchtreffer als Kandidaten und lasse CI das vorhandene API-Image nutzen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
2117ca6b54
commit
68a8e3d945
23 files changed
+481
-118
No files matched your search
@@ -171,6 +171,7 @@ class TaxonomyService:
|
||||
source_category_id=category.id,
|
||||
external_filter_key=str(raw["external_filter_key"]),
|
||||
label=str(raw.get("label") or raw["external_filter_key"]),
|
||||
group_label=raw.get("group_label") or "Allgemein",
|
||||
filter_type=filter_type,
|
||||
canonical_unit=raw.get("canonical_unit"),
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
@@ -181,6 +182,7 @@ class TaxonomyService:
|
||||
if filt.filter_type != filter_type:
|
||||
self._break_mappings(category.id, filt.external_filter_key)
|
||||
filt.label = str(raw.get("label") or filt.label)
|
||||
filt.group_label = raw.get("group_label") or filt.group_label or "Allgemein"
|
||||
filt.filter_type = filter_type
|
||||
filt.canonical_unit = raw.get("canonical_unit")
|
||||
self._sync_options(filt, raw.get("options") or [])
|
||||
@@ -223,12 +225,14 @@ class TaxonomyService:
|
||||
rows = self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord)
|
||||
.where(SourceFilterDefinitionRecord.source_category_id == category.id)
|
||||
.order_by(SourceFilterDefinitionRecord.external_filter_key)
|
||||
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
||||
).all()
|
||||
filters = []
|
||||
for filt in rows:
|
||||
options = self.session.scalars(
|
||||
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
select(SourceFilterOptionRecord)
|
||||
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
.order_by(SourceFilterOptionRecord.source_value)
|
||||
).all()
|
||||
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
filters.append(
|
||||
@@ -236,6 +240,7 @@ class TaxonomyService:
|
||||
"id": filt.id,
|
||||
"external_filter_key": filt.external_filter_key,
|
||||
"label": filt.label,
|
||||
"group_label": filt.group_label or "Allgemein",
|
||||
"filter_type": filt.filter_type,
|
||||
"canonical_unit": filt.canonical_unit,
|
||||
"status": filt.status,
|
||||
@@ -912,7 +917,7 @@ class SearchExecutionService:
|
||||
pages: list[dict[str, Any]] = []
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
pages, items, error_code = self._paginate(adapter, search)
|
||||
pages, items, error_code, advertised = self._paginate(adapter, search)
|
||||
except AppError as exc:
|
||||
run.status = RunStatus.FAILED.value
|
||||
run.error_code = exc.code
|
||||
@@ -921,7 +926,7 @@ class SearchExecutionService:
|
||||
self.session.flush()
|
||||
return self.searches.get_run(run.id)
|
||||
run.page_count = len(pages)
|
||||
run.result_count = len(items)
|
||||
run.result_count = max(len(items), advertised)
|
||||
run.http_status_summary = {"pages": pages}
|
||||
run.error_code = error_code
|
||||
run.finished_at = now_utc()
|
||||
@@ -957,33 +962,38 @@ class SearchExecutionService:
|
||||
self,
|
||||
adapter: DiscoverySourceAdapter,
|
||||
search: DiscoverySearchRecord,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None]:
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None, int]:
|
||||
query = {"url": search.generated_url}
|
||||
collected: list[dict[str, Any]] = []
|
||||
pages: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
expected_pages = 1
|
||||
advertised = 0
|
||||
while page <= expected_pages and page <= MAX_PAGES:
|
||||
try:
|
||||
response = adapter.fetch_results(query, {"page": page})
|
||||
except AppError as exc:
|
||||
if exc.code == "FIXTURE_NOT_FOUND" and page > 1:
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
raise
|
||||
status = int(response.get("http_status") or 200)
|
||||
pages.append({"page": page, "status": status})
|
||||
pages.append({"page": page, "status": status, "result_count": response.get("result_count")})
|
||||
if status == 429:
|
||||
return pages, collected, "BACKOFF"
|
||||
return pages, collected, "BACKOFF", advertised
|
||||
if status == 403 or response.get("challenge") == "captcha" or response.get("blocked"):
|
||||
return pages, collected, "BLOCKED"
|
||||
return pages, collected, "BLOCKED", advertised
|
||||
if status >= 400:
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
collected.extend(adapter.parse_results(response))
|
||||
if response.get("result_count") is not None:
|
||||
advertised = max(advertised, int(response["result_count"]))
|
||||
if advertised > search.broad_result_threshold:
|
||||
return pages, collected, None, advertised
|
||||
expected_pages = int(response.get("page_count") or page)
|
||||
page += 1
|
||||
if page > MAX_PAGES + 1 or (expected_pages > MAX_PAGES and len(pages) >= MAX_PAGES):
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, None
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
return pages, collected, None, advertised
|
||||
|
||||
def _backoff(self, search_id: UUID) -> timedelta:
|
||||
rows = self.session.scalars(
|
||||
@@ -1407,7 +1417,7 @@ class CandidateRankingService:
|
||||
"source_url": None if ref is None else ref.source_url,
|
||||
"current_best_price": None if obs is None else obs.best_price,
|
||||
"currency": None if obs is None else obs.currency,
|
||||
"current_dynamic_attributes": {} if obs is None else _dynamic_attributes(self.session, obs.id),
|
||||
"current_dynamic_attributes": [] if obs is None else _dynamic_attributes(self.session, obs.id),
|
||||
"product_score": row.product_score,
|
||||
"product_eligibility": row.product_eligibility,
|
||||
"ranking_status": ranking_status,
|
||||
|
||||
Reference in new issue
Block a user