Uebernimm Enum-Listen und Filtergruppen in die Taxonomie, speichere Suchtreffer als Kandidaten und lasse CI das vorhandene API-Image nutzen.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
2117ca6b54
commit
68a8e3d945
23 files changed
+481
-118
No files matched your search
+1
-1
@@ -19,7 +19,7 @@ FROM base AS runtime
|
||||
|
||||
COPY . .
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
||||
RUN uv sync --frozen --extra live --no-dev \
|
||||
RUN uv sync --frozen --extra live --extra test --no-dev \
|
||||
&& playwright install --with-deps chromium \
|
||||
&& sed -i 's/\r$//' /app/scripts/startup.sh \
|
||||
&& chmod +x /app/scripts/startup.sh \
|
||||
|
||||
@@ -51,9 +51,7 @@ def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[
|
||||
if encoding.get("encoding_type") == "RANGE_TOKEN":
|
||||
return _encode_range_token(mapping, criterion)
|
||||
if param == "xf":
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
source_value = _xf_value(mapping, criterion)
|
||||
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
||||
return _encode_xf(mapping, criterion)
|
||||
if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}:
|
||||
return [(param, str(criterion.get("value")))]
|
||||
template = encoding.get("value_template")
|
||||
@@ -112,10 +110,30 @@ def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _encode_xf(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
raw = criterion.get("value")
|
||||
values = raw if isinstance(raw, list) else [raw]
|
||||
fragments: list[tuple[str, str]] = []
|
||||
for value in values:
|
||||
if value is None or value == "":
|
||||
continue
|
||||
source_value = _xf_value(mapping, {**criterion, "value": value})
|
||||
fragments.append(("xf", f"{filter_key}_{_plus(source_value)}"))
|
||||
if fragments:
|
||||
return fragments
|
||||
source_value = _xf_value(mapping, criterion)
|
||||
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
||||
|
||||
|
||||
def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
||||
if looked_up is not None:
|
||||
return looked_up
|
||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
||||
value = criterion.get("value")
|
||||
if options and value is not None and value != "":
|
||||
return str(value)
|
||||
return str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
||||
|
||||
|
||||
|
||||
@@ -118,6 +118,7 @@ class SourceFilterDefinitionRecord(Base):
|
||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
group_label: Mapped[str | None] = mapped_column(String(128))
|
||||
filter_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
|
||||
@@ -142,7 +142,7 @@ class CatalogService:
|
||||
filters = self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord)
|
||||
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
|
||||
.order_by(SourceFilterDefinitionRecord.label)
|
||||
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
||||
).all()
|
||||
mappings = {
|
||||
row.external_filter_key: row
|
||||
@@ -173,7 +173,9 @@ class CatalogService:
|
||||
)
|
||||
self.session.flush()
|
||||
options = self.session.scalars(
|
||||
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
select(SourceFilterOptionRecord)
|
||||
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
.order_by(SourceFilterOptionRecord.source_value)
|
||||
).all()
|
||||
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
criteria.append(
|
||||
@@ -254,6 +256,7 @@ class CatalogService:
|
||||
return {
|
||||
"criterion_definition_id": definition.id,
|
||||
"label": definition.label,
|
||||
"group_label": None if filt is None else (filt.group_label or "Allgemein"),
|
||||
"description": definition.description,
|
||||
"origin": definition.origin,
|
||||
"source": definition.source,
|
||||
|
||||
@@ -171,6 +171,7 @@ class TaxonomyService:
|
||||
source_category_id=category.id,
|
||||
external_filter_key=str(raw["external_filter_key"]),
|
||||
label=str(raw.get("label") or raw["external_filter_key"]),
|
||||
group_label=raw.get("group_label") or "Allgemein",
|
||||
filter_type=filter_type,
|
||||
canonical_unit=raw.get("canonical_unit"),
|
||||
status=MappingStatus.LEARNED_UNVERIFIED.value,
|
||||
@@ -181,6 +182,7 @@ class TaxonomyService:
|
||||
if filt.filter_type != filter_type:
|
||||
self._break_mappings(category.id, filt.external_filter_key)
|
||||
filt.label = str(raw.get("label") or filt.label)
|
||||
filt.group_label = raw.get("group_label") or filt.group_label or "Allgemein"
|
||||
filt.filter_type = filter_type
|
||||
filt.canonical_unit = raw.get("canonical_unit")
|
||||
self._sync_options(filt, raw.get("options") or [])
|
||||
@@ -223,12 +225,14 @@ class TaxonomyService:
|
||||
rows = self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord)
|
||||
.where(SourceFilterDefinitionRecord.source_category_id == category.id)
|
||||
.order_by(SourceFilterDefinitionRecord.external_filter_key)
|
||||
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
||||
).all()
|
||||
filters = []
|
||||
for filt in rows:
|
||||
options = self.session.scalars(
|
||||
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
select(SourceFilterOptionRecord)
|
||||
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
.order_by(SourceFilterOptionRecord.source_value)
|
||||
).all()
|
||||
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
filters.append(
|
||||
@@ -236,6 +240,7 @@ class TaxonomyService:
|
||||
"id": filt.id,
|
||||
"external_filter_key": filt.external_filter_key,
|
||||
"label": filt.label,
|
||||
"group_label": filt.group_label or "Allgemein",
|
||||
"filter_type": filt.filter_type,
|
||||
"canonical_unit": filt.canonical_unit,
|
||||
"status": filt.status,
|
||||
@@ -912,7 +917,7 @@ class SearchExecutionService:
|
||||
pages: list[dict[str, Any]] = []
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
pages, items, error_code = self._paginate(adapter, search)
|
||||
pages, items, error_code, advertised = self._paginate(adapter, search)
|
||||
except AppError as exc:
|
||||
run.status = RunStatus.FAILED.value
|
||||
run.error_code = exc.code
|
||||
@@ -921,7 +926,7 @@ class SearchExecutionService:
|
||||
self.session.flush()
|
||||
return self.searches.get_run(run.id)
|
||||
run.page_count = len(pages)
|
||||
run.result_count = len(items)
|
||||
run.result_count = max(len(items), advertised)
|
||||
run.http_status_summary = {"pages": pages}
|
||||
run.error_code = error_code
|
||||
run.finished_at = now_utc()
|
||||
@@ -957,33 +962,38 @@ class SearchExecutionService:
|
||||
self,
|
||||
adapter: DiscoverySourceAdapter,
|
||||
search: DiscoverySearchRecord,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None]:
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str | None, int]:
|
||||
query = {"url": search.generated_url}
|
||||
collected: list[dict[str, Any]] = []
|
||||
pages: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
expected_pages = 1
|
||||
advertised = 0
|
||||
while page <= expected_pages and page <= MAX_PAGES:
|
||||
try:
|
||||
response = adapter.fetch_results(query, {"page": page})
|
||||
except AppError as exc:
|
||||
if exc.code == "FIXTURE_NOT_FOUND" and page > 1:
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
raise
|
||||
status = int(response.get("http_status") or 200)
|
||||
pages.append({"page": page, "status": status})
|
||||
pages.append({"page": page, "status": status, "result_count": response.get("result_count")})
|
||||
if status == 429:
|
||||
return pages, collected, "BACKOFF"
|
||||
return pages, collected, "BACKOFF", advertised
|
||||
if status == 403 or response.get("challenge") == "captcha" or response.get("blocked"):
|
||||
return pages, collected, "BLOCKED"
|
||||
return pages, collected, "BLOCKED", advertised
|
||||
if status >= 400:
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
collected.extend(adapter.parse_results(response))
|
||||
if response.get("result_count") is not None:
|
||||
advertised = max(advertised, int(response["result_count"]))
|
||||
if advertised > search.broad_result_threshold:
|
||||
return pages, collected, None, advertised
|
||||
expected_pages = int(response.get("page_count") or page)
|
||||
page += 1
|
||||
if page > MAX_PAGES + 1 or (expected_pages > MAX_PAGES and len(pages) >= MAX_PAGES):
|
||||
return pages, collected, "PARTIAL"
|
||||
return pages, collected, None
|
||||
return pages, collected, "PARTIAL", advertised
|
||||
return pages, collected, None, advertised
|
||||
|
||||
def _backoff(self, search_id: UUID) -> timedelta:
|
||||
rows = self.session.scalars(
|
||||
@@ -1407,7 +1417,7 @@ class CandidateRankingService:
|
||||
"source_url": None if ref is None else ref.source_url,
|
||||
"current_best_price": None if obs is None else obs.best_price,
|
||||
"currency": None if obs is None else obs.currency,
|
||||
"current_dynamic_attributes": {} if obs is None else _dynamic_attributes(self.session, obs.id),
|
||||
"current_dynamic_attributes": [] if obs is None else _dynamic_attributes(self.session, obs.id),
|
||||
"product_score": row.product_score,
|
||||
"product_eligibility": row.product_eligibility,
|
||||
"ranking_status": ranking_status,
|
||||
|
||||
@@ -20,6 +20,7 @@ EXPAND_LABELS = (
|
||||
"Immer alle Filter anzeigen",
|
||||
"weitere Filter anzeigen",
|
||||
)
|
||||
MORE_OPTION_SELECTOR = 'button:has-text("weitere"), a:has-text("weitere")'
|
||||
|
||||
|
||||
def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]:
|
||||
@@ -101,6 +102,7 @@ def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, s
|
||||
continue
|
||||
if expand_filters:
|
||||
_expand_filter_sections(page)
|
||||
_expand_more_options(page)
|
||||
html = page.content()
|
||||
status = response.status if response is not None else 200
|
||||
browser.close()
|
||||
@@ -119,6 +121,18 @@ def _expand_filter_sections(page: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def _expand_more_options(page: object) -> None:
|
||||
locator = page.locator(MORE_OPTION_SELECTOR)
|
||||
for _ in range(80):
|
||||
if locator.count() == 0:
|
||||
return
|
||||
try:
|
||||
locator.first.click(timeout=2000)
|
||||
page.wait_for_timeout(350)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _blocked(status: int, html: str) -> bool:
|
||||
if status in {401, 403, 429}:
|
||||
return True
|
||||
|
||||
@@ -19,7 +19,26 @@ SKIP_OPTION_LABELS = {
|
||||
"filter überspringen",
|
||||
"einfache filter",
|
||||
"erweiterte filter",
|
||||
"weitere",
|
||||
"weniger",
|
||||
"alle anzeigen",
|
||||
"weniger anzeigen",
|
||||
}
|
||||
PRODUCT_COUNT_RE = re.compile(r"\(\d[\d.]*\s*(produkte)?\)\s*$", re.IGNORECASE)
|
||||
OPTION_TAGS = {"a", "button", "label", "input", "span"}
|
||||
GROUP_ORDER = (
|
||||
"Allgemein",
|
||||
"Display",
|
||||
"CPU",
|
||||
"RAM",
|
||||
"RAM erweitert",
|
||||
"Grafik",
|
||||
"Anschlüsse",
|
||||
"Wireless",
|
||||
"Akku",
|
||||
"Abmessungen",
|
||||
"Sonstiges",
|
||||
)
|
||||
RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"}
|
||||
RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"}
|
||||
TEXT_NAMES = {"asuch", "fs"}
|
||||
@@ -117,7 +136,11 @@ def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dic
|
||||
for group in groups.values()
|
||||
if _usable_group(group)
|
||||
]
|
||||
filters.sort(key=lambda item: (item["label"].lower(), item["external_filter_key"]))
|
||||
for group in filters:
|
||||
section = group.get("group_label") or "Allgemein"
|
||||
group["group_label"] = section
|
||||
group["group_sort"] = _group_sort(section)
|
||||
filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower(), item["external_filter_key"]))
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
@@ -148,6 +171,9 @@ def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) ->
|
||||
source_type = node.attrs.get("data-filter-type") or ""
|
||||
label = _declared_label(node) or f"Filter {filter_id}"
|
||||
group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type))
|
||||
section = _section_label(node)
|
||||
if section:
|
||||
group["group_label"] = section
|
||||
if label and group["label"].startswith("Filter "):
|
||||
group["label"] = label
|
||||
if source_type:
|
||||
@@ -169,19 +195,21 @@ def _collect_xf_links(
|
||||
required_key: str | None = None,
|
||||
allow_histogram: bool = False,
|
||||
) -> None:
|
||||
if node.tag == "a":
|
||||
href = node.attrs.get("href") or ""
|
||||
label = node.attrs.get("aria-label") or _node_text(node)
|
||||
if node.tag in OPTION_TAGS:
|
||||
label = node.attrs.get("aria-label") or node.attrs.get("title") or _node_text(node)
|
||||
if _usable_option(node, label, allow_histogram=allow_histogram):
|
||||
for key, query_value in _xf_tokens(href):
|
||||
for key, query_value in _node_xf_tokens(node):
|
||||
if required_key is not None and key != required_key:
|
||||
continue
|
||||
if required_key is None and not _has_structural_label(node):
|
||||
continue
|
||||
group_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
|
||||
group = groups.setdefault(key, _empty_group(key, group_label, None))
|
||||
if group_label and (group["label"].startswith("Filter ") or group["label"] == key):
|
||||
group["label"] = group_label
|
||||
filter_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
|
||||
group = groups.setdefault(key, _empty_group(key, filter_label, None))
|
||||
section = _section_label(node)
|
||||
if section:
|
||||
group["group_label"] = section
|
||||
if filter_label and (group["label"].startswith("Filter ") or group["label"] == key):
|
||||
group["label"] = filter_label
|
||||
option = _option_from_link(label, query_value)
|
||||
existing = {item["source_value"] for item in group["options"]}
|
||||
if option["source_value"] not in existing:
|
||||
@@ -207,11 +235,14 @@ def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> Non
|
||||
min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin")
|
||||
max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax")
|
||||
label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis"
|
||||
section = _section_label(names.get(min_name) or names.get(max_name) or node) or "Allgemein"
|
||||
groups["price"] = {
|
||||
"external_filter_key": "price",
|
||||
"label": "Preis" if "preis" not in label.lower() else label,
|
||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
||||
"canonical_unit": "EUR",
|
||||
"group_label": section,
|
||||
"group_sort": _group_sort(section),
|
||||
"options": [],
|
||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
||||
"encoding": {
|
||||
@@ -225,11 +256,14 @@ def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> Non
|
||||
if name not in names:
|
||||
continue
|
||||
label = _nearest_group_label(names[name]) or "Suchbegriffe"
|
||||
section = _section_label(names[name]) or "Allgemein"
|
||||
groups[name] = {
|
||||
"external_filter_key": name,
|
||||
"label": label,
|
||||
"filter_type": FilterType.TEXT.value,
|
||||
"canonical_unit": None,
|
||||
"group_label": section,
|
||||
"group_sort": _group_sort(section),
|
||||
"options": [],
|
||||
"encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"},
|
||||
}
|
||||
@@ -243,6 +277,30 @@ def _find_inputs(node: _Node, found: list[_Node]) -> None:
|
||||
_find_inputs(child, found)
|
||||
|
||||
|
||||
def _node_xf_tokens(node: _Node) -> list[tuple[str, str]]:
|
||||
tokens = _xf_tokens(node.attrs.get("href") or "")
|
||||
if tokens:
|
||||
return tokens
|
||||
for attr in ("data-xf", "data-filter-value", "data-value", "value"):
|
||||
raw = node.attrs.get(attr) or ""
|
||||
if attr == "value" and node.attrs.get("name") not in {"xf", ""} and node.tag != "input":
|
||||
continue
|
||||
parsed = _xf_raw_token(raw)
|
||||
if parsed:
|
||||
return [parsed]
|
||||
return []
|
||||
|
||||
|
||||
def _xf_raw_token(raw: str) -> tuple[str, str] | None:
|
||||
value = unquote(raw).replace("|", "~").split("~")[0]
|
||||
if "_" not in value:
|
||||
return None
|
||||
key, token = value.split("_", 1)
|
||||
if key.isdigit() and token:
|
||||
return key, token.replace("+", " ")
|
||||
return None
|
||||
|
||||
|
||||
def _xf_tokens(href: str) -> list[tuple[str, str]]:
|
||||
if not href or href.startswith("#") or href.startswith("javascript:"):
|
||||
return []
|
||||
@@ -261,8 +319,7 @@ def _xf_tokens(href: str) -> list[tuple[str, str]]:
|
||||
|
||||
|
||||
def _option_from_link(label: str, query_value: str) -> dict[str, Any]:
|
||||
cleaned = re.sub(r"\(\d[\d.]*\s*(produkte)?\)\s*$", "", label, flags=re.IGNORECASE).strip()
|
||||
cleaned = re.sub(r"\s+", " ", cleaned)
|
||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
||||
operator = None
|
||||
if cleaned.lower().startswith("ab "):
|
||||
operator = ">="
|
||||
@@ -299,6 +356,8 @@ def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[s
|
||||
"label": label or f"Filter {key}",
|
||||
"filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value),
|
||||
"canonical_unit": None,
|
||||
"group_label": None,
|
||||
"group_sort": _group_sort("Allgemein"),
|
||||
"options": [],
|
||||
"encoding": {
|
||||
"url_param": "xf",
|
||||
@@ -375,18 +434,62 @@ def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool:
|
||||
return False
|
||||
if "bucket" in classes and not allow_histogram:
|
||||
return False
|
||||
cleaned = re.sub(r"\s+", " ", label).strip()
|
||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
||||
if not cleaned or _skip_option(cleaned):
|
||||
return False
|
||||
if "überspringen" in cleaned.lower():
|
||||
if "überspringen" in cleaned.lower() or "aufklappen" in cleaned.lower() or "zuklappen" in cleaned.lower():
|
||||
return False
|
||||
if not allow_histogram and "produkte" in cleaned.lower():
|
||||
if cleaned.lower().startswith("weitere"):
|
||||
return False
|
||||
if len(cleaned.split()) > 8:
|
||||
if not allow_histogram and re.fullmatch(r"\d[\d.]*\s*produkte", cleaned, re.IGNORECASE):
|
||||
return False
|
||||
if len(cleaned.split()) > 12:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _section_label(node: _Node | None) -> str | None:
|
||||
current = node
|
||||
while current is not None:
|
||||
classes = current.attrs.get("class", "").lower()
|
||||
if "filter-row" in classes or current.attrs.get("data-filter-row-id"):
|
||||
text = _strip_section_suffix(
|
||||
current.attrs.get("data-filter-row-name")
|
||||
or current.attrs.get("aria-label")
|
||||
or _row_heading(current)
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
labeled = current.attrs.get("aria-label") or ""
|
||||
if re.search(r"(auf|zu)klappen", labeled, re.IGNORECASE):
|
||||
text = _strip_section_suffix(labeled)
|
||||
if text:
|
||||
return text
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _row_heading(node: _Node) -> str:
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node) and child.tag in {"button", "h2", "h3", "h4", "legend", "strong"}:
|
||||
text = _clean_label(_node_text(child))
|
||||
if text:
|
||||
return text
|
||||
return _clean_label(node.attrs.get("aria-label") or "")
|
||||
|
||||
|
||||
def _strip_section_suffix(text: str | None) -> str:
|
||||
cleaned = _clean_label(text or "")
|
||||
return re.sub(r"\s*(aufklappen|zuklappen|einklappen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
||||
|
||||
|
||||
def _group_sort(label: str) -> int:
|
||||
try:
|
||||
return GROUP_ORDER.index(label)
|
||||
except ValueError:
|
||||
return len(GROUP_ORDER)
|
||||
|
||||
|
||||
def _usable_group(group: dict[str, Any]) -> bool:
|
||||
if group["filter_type"] == FilterType.TEXT.value:
|
||||
return True
|
||||
|
||||
@@ -5,11 +5,20 @@
|
||||
</head>
|
||||
<body>
|
||||
<h1>Notebooks</h1>
|
||||
<div class="filter-row" data-filter-row-id="allgemein">
|
||||
<button aria-label="Allgemein aufklappen">Allgemein</button>
|
||||
<div data-filter-type="text_range" data-filter-id="17023">
|
||||
<span id="filter-element__title-label--17023">RAM</span>
|
||||
<a href="?cat=nb&xf=17023_ab+16GB&promode=true">ab 16GB</a>
|
||||
<a href="?cat=nb&xf=17023_ab+32GB&promode=true">ab 32GB</a>
|
||||
</div>
|
||||
<div data-filter-type="text" data-filter-id="1">
|
||||
<span id="filter-element__title-label--1">Hersteller</span>
|
||||
<a href="?cat=nb&xf=1_Apple" aria-label="Apple (234 Produkte)">Apple</a>
|
||||
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
|
||||
<button type="button" data-xf="1_Lenovo">Lenovo</button>
|
||||
<button type="button">weitere 12</button>
|
||||
</div>
|
||||
<form>
|
||||
<label>Suchbegriffe</label>
|
||||
<input name="asuch" type="text">
|
||||
@@ -17,15 +26,10 @@
|
||||
<input name="bpmin" type="number" placeholder="Preis ab">
|
||||
<input name="bpmax" type="number" placeholder="Preis bis">
|
||||
</form>
|
||||
</div>
|
||||
<div class="filter-row" data-filter-row-id="display">
|
||||
<button aria-label="Display aufklappen">Display</button>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Hersteller:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=1_Apple">Apple</a>
|
||||
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
|
||||
<a href="?cat=nb&xf=1_Lenovo">Lenovo</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Display-Größe:</th>
|
||||
<td>
|
||||
@@ -128,5 +132,6 @@
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Persist source filter groups from the learned taxonomy.
|
||||
|
||||
Revision ID: 0007_filter_groups
|
||||
Revises: 0006_source_category_readiness
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007_filter_groups"
|
||||
down_revision: str | None = "0006_source_category_readiness"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("source_filter_definitions", sa.Column("group_label", sa.String(128)))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source_filter_definitions", "group_label")
|
||||
@@ -95,6 +95,24 @@ def test_multiple_xf_filters_join_with_tilde():
|
||||
assert "xf=12_16384~7436_8192" in url or "xf=7436_8192~12_16384" in url
|
||||
|
||||
|
||||
def test_enum_uses_selected_option_not_first_mapping_value():
|
||||
url, _ = generate_search_url(
|
||||
category_url="https://geizhals.de/?cat=nb",
|
||||
criteria_items=[{"criterion": "hersteller", "operator": "=", "value": "Lenovo"}],
|
||||
mappings=[
|
||||
{
|
||||
"criterion_key": "hersteller",
|
||||
"external_filter_key": "1",
|
||||
"source_value": "Apple",
|
||||
"encoding": {"url_param": "xf", "options": {"Apple": "Apple", "Lenovo": "Lenovo"}},
|
||||
"status": "VERIFIED",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "xf=1_Lenovo" in url
|
||||
assert "Apple" not in url
|
||||
|
||||
|
||||
def test_roundtrip_verifies_xf_and_maxp():
|
||||
assert verify_roundtrip(
|
||||
category_url="https://geizhals.de/?cat=nb",
|
||||
|
||||
@@ -28,6 +28,13 @@ def test_parser_reads_href_filters_and_price_inputs() -> None:
|
||||
assert price["filter_type"] == "NUMERIC_RANGE"
|
||||
assert price["encoding"]["min_parameter"] == "bpmin"
|
||||
assert price["encoding"]["max_parameter"] == "bpmax"
|
||||
hersteller = next(item for item in payload["filters"] if item["external_filter_key"] == "1")
|
||||
assert hersteller["label"] == "Hersteller"
|
||||
assert hersteller["filter_type"] == "ENUM"
|
||||
assert hersteller["group_label"] == "Allgemein"
|
||||
assert {option["source_value"] for option in hersteller["options"]} >= {"Apple", "ASUS", "Lenovo"}
|
||||
display = next(item for item in payload["filters"] if item["external_filter_key"] == "12128")
|
||||
assert display["group_label"] == "Display"
|
||||
|
||||
|
||||
def test_learned_ram_and_price_encode_observed_geizhals_query() -> None:
|
||||
|
||||
Reference in new issue
Block a user