Klappe Geizhals-Filtergruppen beim Lernen auf und nimm neue Kategorien ueber den Quellpfad auf.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
46857715df
commit
2117ca6b54
10 files changed
+203
-25
No files matched your search
@@ -11,6 +11,7 @@ from app.api.schemas import (
|
||||
DiscoverySearchPatch,
|
||||
SourceAttributeMappingIn,
|
||||
SourceAttributeMappingPatch,
|
||||
SourceCategoryIn,
|
||||
)
|
||||
from app.persistence.db import get_session
|
||||
from app.services.catalog_service import CatalogService
|
||||
@@ -30,6 +31,11 @@ def list_source_categories(source_id: str, session: Session = Depends(get_sessio
|
||||
return {"items": CatalogService(session).source_categories(source_id)}
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/categories", status_code=201)
|
||||
def register_source_category(source_id: str, payload: SourceCategoryIn, session: Session = Depends(get_session)):
|
||||
return CatalogService(session).register_source_category(source_id, payload.model_dump())
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
|
||||
def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).discover(source_id, category)
|
||||
|
||||
@@ -189,6 +189,13 @@ class ProjectCriteriaIn(StrictModel):
|
||||
items: list[dict[str, Any]]
|
||||
|
||||
|
||||
class SourceCategoryIn(StrictModel):
|
||||
source_url: str
|
||||
name: str | None = None
|
||||
product_category_id: str | None = None
|
||||
product_category_name: str | None = None
|
||||
|
||||
|
||||
class CriterionDefinitionIn(StrictModel):
|
||||
key: str
|
||||
value_type: str
|
||||
|
||||
@@ -67,6 +67,33 @@ class CatalogService:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if self.catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
from app.sources.geizhals.category_url import parse_category_key, slugify
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
source_url = str(payload["source_url"]).strip()
|
||||
category_key = parse_category_key(source_url)
|
||||
name = str(payload.get("name") or category_key)
|
||||
product_category_id = str(payload.get("product_category_id") or slugify(name))
|
||||
ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name))
|
||||
category = ensure_source_category(
|
||||
self.session,
|
||||
source_id=source_id,
|
||||
external_category_key=category_key,
|
||||
product_category_id=product_category_id,
|
||||
name=name,
|
||||
source_url=source_url,
|
||||
)
|
||||
category.source_url = source_url
|
||||
category.name = name
|
||||
category.product_category_id = product_category_id
|
||||
self.session.flush()
|
||||
TaxonomyService(self.session).discover(source_id, category_key)
|
||||
items = self.source_categories(source_id)
|
||||
return next(item for item in items if item["external_category_key"] == category_key)
|
||||
|
||||
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
|
||||
if self.catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
|
||||
@@ -100,7 +100,18 @@ class TaxonomyService:
|
||||
if catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
adapter = self.adapter or discovery_adapter(source_id)
|
||||
existing = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.external_category_key == category_key,
|
||||
)
|
||||
).first()
|
||||
try:
|
||||
payload = adapter.discover_taxonomy(
|
||||
category_key,
|
||||
source_url=None if existing is None else existing.source_url,
|
||||
)
|
||||
except TypeError:
|
||||
payload = adapter.discover_taxonomy(category_key)
|
||||
except AppError as exc:
|
||||
category = self.session.scalars(
|
||||
|
||||
@@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlparse
|
||||
from app.discovery.enums import FilterType
|
||||
from app.errors import AppError
|
||||
from app.settings import get_settings
|
||||
from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
|
||||
from app.sources.geizhals.live_page import load_category_page, load_result_page
|
||||
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
||||
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
||||
@@ -23,16 +24,17 @@ class GeizhalsAdapter:
|
||||
settings = get_settings()
|
||||
self.live = (not settings.geizhals_use_fixtures) if live is None else live
|
||||
|
||||
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
|
||||
def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
|
||||
if not self.live:
|
||||
return self._load(f"{category_ref}-taxonomy.json")
|
||||
url = f"https://geizhals.de/?cat={category_ref}&promode=true"
|
||||
url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
|
||||
category_ref = parse_category_key(url)
|
||||
html, mode, status = load_category_page(
|
||||
url,
|
||||
category_ref=category_ref,
|
||||
playwright=get_settings().geizhals_playwright,
|
||||
)
|
||||
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=url)
|
||||
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
|
||||
if payload.get("blocked") or not payload.get("filters"):
|
||||
raise AppError(
|
||||
422,
|
||||
@@ -48,7 +50,8 @@ class GeizhalsAdapter:
|
||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
from app.discovery.urlgen import generate_search_url
|
||||
|
||||
category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
||||
category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
|
||||
category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
||||
url, internal_only = generate_search_url(
|
||||
category_url=category_url,
|
||||
criteria_items=criteria,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from app.errors import AppError
|
||||
|
||||
CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_category_key(source_url: str) -> str:
|
||||
parts = urlsplit(source_url.strip())
|
||||
query = parse_qs(parts.query)
|
||||
if query.get("cat"):
|
||||
return str(query["cat"][0])
|
||||
match = CATEGORY_PATH_RE.search(parts.path)
|
||||
if match:
|
||||
return match.group(1)
|
||||
raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)")
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
text = value.strip().casefold().translate(str.maketrans("äöüß", "aous"))
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
||||
return text or "category"
|
||||
|
||||
|
||||
def with_extended_filters(source_url: str) -> str:
|
||||
parts = urlsplit(source_url.strip())
|
||||
query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"]
|
||||
query.append(("promode", "true"))
|
||||
return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment))
|
||||
|
||||
|
||||
def _query_items(query: str) -> list[tuple[str, str]]:
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
return parse_qsl(query, keep_blank_values=True)
|
||||
@@ -15,6 +15,8 @@ BROWSER_HEADERS = {
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
EXPAND_LABELS = (
|
||||
"Erweiterte Filter",
|
||||
"Alle Filter ausklappen",
|
||||
"Immer alle Filter anzeigen",
|
||||
"weitere Filter anzeigen",
|
||||
)
|
||||
@@ -42,19 +44,19 @@ def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, in
|
||||
|
||||
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url, expand_filters=True)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
status, html = _fetch_http(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_http", status
|
||||
except Exception:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status
|
||||
except Exception:
|
||||
pass
|
||||
status, html = status or 0, html or ""
|
||||
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
|
||||
if recorded.exists():
|
||||
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
|
||||
@@ -73,7 +75,7 @@ def _fetch_http(url: str) -> tuple[int, str]:
|
||||
return response.status_code, response.text
|
||||
|
||||
|
||||
def _fetch_playwright(url: str) -> tuple[int, str]:
|
||||
def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]:
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError as exc:
|
||||
@@ -87,17 +89,36 @@ def _fetch_playwright(url: str) -> tuple[int, str]:
|
||||
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
|
||||
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
page.wait_for_timeout(4000)
|
||||
if expand_filters:
|
||||
_expand_filter_sections(page)
|
||||
for label in EXPAND_LABELS:
|
||||
button = page.get_by_text(label, exact=False)
|
||||
if button.count():
|
||||
button.first.click(timeout=5000)
|
||||
page.wait_for_timeout(1500)
|
||||
try:
|
||||
button.first.click(timeout=4000)
|
||||
page.wait_for_timeout(800)
|
||||
except Exception:
|
||||
continue
|
||||
if expand_filters:
|
||||
_expand_filter_sections(page)
|
||||
html = page.content()
|
||||
status = response.status if response is not None else 200
|
||||
browser.close()
|
||||
return status, html
|
||||
|
||||
|
||||
def _expand_filter_sections(page: object) -> None:
|
||||
locator = page.locator('button[aria-label*="aufklappen"]')
|
||||
for _ in range(40):
|
||||
if locator.count() == 0:
|
||||
return
|
||||
try:
|
||||
locator.first.click(timeout=3000)
|
||||
page.wait_for_timeout(700)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _blocked(status: int, html: str) -> bool:
|
||||
if status in {401, 403, 429}:
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from app.errors import AppError
|
||||
from app.sources.geizhals.category_url import parse_category_key, slugify, with_extended_filters
|
||||
import pytest
|
||||
|
||||
|
||||
def test_parse_category_key_from_query() -> None:
|
||||
assert parse_category_key("https://geizhals.de/?cat=gra16_512&xf=1_AMD") == "gra16_512"
|
||||
|
||||
|
||||
def test_parse_category_key_rejects_unknown_url() -> None:
|
||||
with pytest.raises(AppError) as exc:
|
||||
parse_category_key("https://geizhals.de/")
|
||||
assert exc.value.code == "SOURCE_URL_INVALID"
|
||||
|
||||
|
||||
def test_slugify_and_extended_filters() -> None:
|
||||
assert slugify("Grafikkarten") == "grafikkarten"
|
||||
assert "promode=true" in with_extended_filters("https://geizhals.de/?cat=nb")
|
||||
Reference in new issue
Block a user