Klappe Geizhals-Filtergruppen beim Lernen auf und nimm neue Kategorien ueber den Quellpfad auf.
Deploy Development / deploy (push) Successful in 1m34s
Test Suite / pytest-backend (push) Failing after 16s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 33s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-04 15:04:19 +02:00
1 parent 46857715df
commit 2117ca6b54
10 files changed
+203 -25

No files matched your search

+7 -4
View File
@@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlparse
from app.discovery.enums import FilterType
from app.errors import AppError
from app.settings import get_settings
from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
from app.sources.geizhals.live_page import load_category_page, load_result_page
from app.sources.geizhals.result_parser import parse_result_html, with_page
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
@@ -23,16 +24,17 @@ class GeizhalsAdapter:
settings = get_settings()
self.live = (not settings.geizhals_use_fixtures) if live is None else live
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
if not self.live:
return self._load(f"{category_ref}-taxonomy.json")
url = f"https://geizhals.de/?cat={category_ref}&promode=true"
url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
category_ref = parse_category_key(url)
html, mode, status = load_category_page(
url,
category_ref=category_ref,
playwright=get_settings().geizhals_playwright,
)
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url)
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
if payload.get("blocked") or not payload.get("filters"):
raise AppError(
422,
@@ -48,7 +50,8 @@ class GeizhalsAdapter:
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
from app.discovery.urlgen import generate_search_url
category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
url, internal_only = generate_search_url(
category_url=category_url,
criteria_items=criteria,
@@ -0,0 +1,38 @@
from __future__ import annotations
import re
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
from app.errors import AppError
CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE)
def parse_category_key(source_url: str) -> str:
parts = urlsplit(source_url.strip())
query = parse_qs(parts.query)
if query.get("cat"):
return str(query["cat"][0])
match = CATEGORY_PATH_RE.search(parts.path)
if match:
return match.group(1)
raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)")
def slugify(value: str) -> str:
text = value.strip().casefold().translate(str.maketrans("äöüß", "aous"))
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
return text or "category"
def with_extended_filters(source_url: str) -> str:
parts = urlsplit(source_url.strip())
query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"]
query.append(("promode", "true"))
return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment))
def _query_items(query: str) -> list[tuple[str, str]]:
from urllib.parse import parse_qsl
return parse_qsl(query, keep_blank_values=True)
+32 -11
View File
@@ -15,6 +15,8 @@ BROWSER_HEADERS = {
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
}
EXPAND_LABELS = (
"Erweiterte Filter",
"Alle Filter ausklappen",
"Immer alle Filter anzeigen",
"weitere Filter anzeigen",
)
@@ -42,19 +44,19 @@ def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, in
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
status, html = 0, ""
if playwright:
try:
status, html = _fetch_playwright(url, expand_filters=True)
if not _blocked(status, html):
return html, "live_playwright", status
except Exception:
pass
try:
status, html = _fetch_http(url)
if not _blocked(status, html):
return html, "live_http", status
except Exception:
status, html = 0, ""
if playwright:
try:
status, html = _fetch_playwright(url)
if not _blocked(status, html):
return html, "live_playwright", status
except Exception:
pass
status, html = status or 0, html or ""
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
if recorded.exists():
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
@@ -73,7 +75,7 @@ def _fetch_http(url: str) -> tuple[int, str]:
return response.status_code, response.text
def _fetch_playwright(url: str) -> tuple[int, str]:
def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]:
try:
from playwright.sync_api import sync_playwright
except ImportError as exc:
@@ -87,17 +89,36 @@ def _fetch_playwright(url: str) -> tuple[int, str]:
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
page.wait_for_timeout(4000)
if expand_filters:
_expand_filter_sections(page)
for label in EXPAND_LABELS:
button = page.get_by_text(label, exact=False)
if button.count():
button.first.click(timeout=5000)
page.wait_for_timeout(1500)
try:
button.first.click(timeout=4000)
page.wait_for_timeout(800)
except Exception:
continue
if expand_filters:
_expand_filter_sections(page)
html = page.content()
status = response.status if response is not None else 200
browser.close()
return status, html
def _expand_filter_sections(page: object) -> None:
locator = page.locator('button[aria-label*="aufklappen"]')
for _ in range(40):
if locator.count() == 0:
return
try:
locator.first.click(timeout=3000)
page.wait_for_timeout(700)
except Exception:
return
def _blocked(status: int, html: str) -> bool:
if status in {401, 403, 429}:
return True