Lerne Taxonomie und Trefferliste live von Geizhals, damit Wizard und Kandidaten echte Quelldaten nutzen.
Deploy Development / deploy (push) Successful in 1m49s
Test Suite / pytest-backend (push) Failing after 32s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 30s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-04 14:53:08 +02:00
1 parent d784248e28
commit 46857715df
25 files changed
+1614 -27

No files matched your search

+28 -2
View File
@@ -100,7 +100,7 @@ class CatalogService:
SourceCategoryRecord.product_category_id == product_category_id,
)
).first()
if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
if source_category is not None and self._should_learn(source_category, source_id):
from app.services.discovery_service import TaxonomyService
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
@@ -172,7 +172,33 @@ class CatalogService:
if definition.id in seen:
continue
criteria.append(self._criterion_out(definition, None, None, [], None))
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
return jsonable(
{
"product_category_id": product_category_id,
"source": source_id,
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
"criteria": criteria,
}
)
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
return True
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
return False
from app.settings import get_settings
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
return False
keys = {
row.external_filter_key
for row in self.session.scalars(
select(SourceFilterDefinitionRecord).where(
SourceFilterDefinitionRecord.source_category_id == source_category.id
)
)
}
return keys <= {"17023", "price"}
def list_templates(self) -> list[dict[str, Any]]:
rows, _total = self.catalog.list_profiles(200, 0)
+107 -5
View File
@@ -14,6 +14,7 @@ from app.discovery.diff import diff_run, next_lifecycle
from app.discovery.enums import (
CandidateStatus,
DiffStatus,
FilterType,
LifecycleStatus,
MappingStatus,
MatchStatus,
@@ -99,7 +100,20 @@ class TaxonomyService:
if catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
adapter = self.adapter or discovery_adapter(source_id)
payload = adapter.discover_taxonomy(category_key)
try:
payload = adapter.discover_taxonomy(category_key)
except AppError as exc:
category = self.session.scalars(
select(SourceCategoryRecord).where(
SourceCategoryRecord.source_id == source_id,
SourceCategoryRecord.external_category_key == category_key,
)
).first()
if category is not None and exc.code == "SOURCE_BLOCKED":
category.taxonomy_status = TaxonomyStatus.BLOCKED.value
category.last_taxonomy_scan_at = now_utc()
self.session.flush()
raise
now = now_utc()
category = self.session.scalars(
select(SourceCategoryRecord).where(
@@ -148,7 +162,7 @@ class TaxonomyService:
label=str(raw.get("label") or raw["external_filter_key"]),
filter_type=filter_type,
canonical_unit=raw.get("canonical_unit"),
status=MappingStatus.VERIFIED.value,
status=MappingStatus.LEARNED_UNVERIFIED.value,
)
self.session.add(filt)
self.session.flush()
@@ -161,9 +175,14 @@ class TaxonomyService:
self._sync_options(filt, raw.get("options") or [])
if raw.get("range"):
self._sync_range(filt, raw["range"])
category.taxonomy_status = (
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
)
self._sync_learned_mapping(category, filt, raw)
requested = str(payload.get("taxonomy_status") or "")
if requested in {item.value for item in TaxonomyStatus}:
category.taxonomy_status = requested
else:
category.taxonomy_status = (
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
)
self.session.flush()
return self.filters(source_id, category.external_category_key)
@@ -275,6 +294,89 @@ class TaxonomyService:
if removed:
self._break_mappings(filt.source_category_id, filt.external_filter_key)
def _sync_learned_mapping(
self,
category: SourceCategoryRecord,
filt: SourceFilterDefinitionRecord,
raw: dict[str, Any],
) -> None:
encoding = raw.get("encoding") or {"url_param": "xf"}
definition = upsert_definition(
self.session,
{
"key": f"{category.source_id}:{category.external_category_key}:{filt.external_filter_key}",
"origin": "SOURCE",
"source": category.source_id,
"product_category_id": category.product_category_id,
"label": filt.label,
"value_type": "number"
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
else "string",
"unit": filt.canonical_unit,
"category": category.product_category_id,
},
)
mapping = self.session.scalars(
select(CriterionMappingRecord).where(
CriterionMappingRecord.source_category_id == category.id,
CriterionMappingRecord.external_filter_key == filt.external_filter_key,
)
).first()
options = raw.get("options") or []
source_value = None
if options:
source_value = str(options[0].get("query_value") or options[0].get("source_value"))
now = now_utc()
if mapping is None:
mapping = CriterionMappingRecord(
id=uuid4(),
criterion_definition_id=definition.id,
criterion_key=definition.key,
source_id=category.source_id,
source_category_id=category.id,
external_filter_key=filt.external_filter_key,
source_value=source_value,
encoding=encoding,
status=MappingStatus.LEARNED_UNVERIFIED.value,
definition_hash=sha256_text(canonical_json({"encoding": encoding, "source_value": source_value})),
created_at=now,
updated_at=now,
)
self.session.add(mapping)
self.session.flush()
else:
mapping.criterion_definition_id = definition.id
mapping.criterion_key = definition.key
mapping.source_value = source_value
mapping.encoding = encoding
mapping.definition_hash = sha256_text(canonical_json({"encoding": encoding, "source_value": source_value}))
mapping.updated_at = now
probe_value = source_value or 0
if options and options[0].get("normalized_value") is not None:
probe_value = options[0]["normalized_value"]
operator = (options[0].get("operator") if options else None) or "<="
if filt.filter_type == FilterType.NUMERIC_RANGE.value:
operator = ">="
probe_value = 1000
verified = verify_roundtrip(
category_url=category.source_url or f"https://geizhals.de/?cat={category.external_category_key}",
criterion={
"criterion": definition.key,
"operator": operator,
"value": probe_value,
},
mapping={
"criterion_key": definition.key,
"external_filter_key": filt.external_filter_key,
"source_value": source_value,
"encoding": encoding,
},
)
if mapping.status != MappingStatus.DISABLED.value and verified:
mapping.status = MappingStatus.VERIFIED.value
elif mapping.status == MappingStatus.BROKEN.value:
mapping.status = MappingStatus.LEARNED_UNVERIFIED.value
def _sync_range(self, filt: SourceFilterDefinitionRecord, raw: dict[str, Any]) -> None:
row = self.session.get(SourceFilterRangeRecord, filt.id)
if row is None: