Stelle den Frontend-MVP und die v0.3-API auf develop bereit, damit Dev API und PWA zusammen deployen.
Deploy Development / deploy (push) Successful in 1m19s
Test Suite / pytest-backend (push) Successful in 31s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 28s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-04 13:04:59 +02:00
1 parent 827e6a2a11
commit b276758ee1
152 files changed
+25846 -34

No files matched your search

+38
View File
@@ -0,0 +1,38 @@
from app.discovery.criteria import parse_project_criteria
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
from app.discovery.enums import (
CandidateStatus,
DiffStatus,
LifecycleStatus,
MappingStatus,
MatchStatus,
ProjectStatus,
RunStatus,
SearchStatus,
)
from app.discovery.identity import IdentityDecision, IdentitySignals, decide_identity, spec_fingerprint
from app.discovery.mapping import verify_roundtrip
from app.discovery.ranking import rank_candidates
from app.discovery.urlgen import generate_search_url
__all__ = [
"CandidateStatus",
"DiffStatus",
"IdentityDecision",
"IdentitySignals",
"LifecycleStatus",
"MappingStatus",
"MatchStatus",
"ProjectStatus",
"RunStatus",
"SearchStatus",
"decide_identity",
"diff_run",
"generate_search_url",
"next_lifecycle",
"observations_changed",
"parse_project_criteria",
"rank_candidates",
"spec_fingerprint",
"verify_roundtrip",
]
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
from typing import Any
from uuid import UUID
from app.domain.validation import DomainValidationError, canonical_json, sha256_text
def parse_project_criteria(definition: dict[str, Any]) -> dict[str, Any]:
items = definition.get("items")
if not isinstance(items, list) or not items:
raise DomainValidationError("criteria.items must be a non-empty list")
seen: set[str] = set()
canonical_items: list[dict[str, Any]] = []
for raw in items:
criterion_id = raw.get("criterion_definition_id")
key = raw.get("criterion")
item_id = str(raw.get("id") or criterion_id or key or "")
if not item_id:
raise DomainValidationError("each criterion needs id, criterion_definition_id or criterion")
if item_id in seen:
raise DomainValidationError("duplicate criterion id", {"id": item_id})
seen.add(item_id)
if not criterion_id and not key:
raise DomainValidationError("criterion_definition_id or criterion is required", {"id": item_id})
if not raw.get("operator"):
raise DomainValidationError("operator is required", {"id": item_id})
enabled = raw.get("enabled_for_discovery")
if enabled is None:
enabled = raw.get("mandatory", True)
canonical_items.append(
{
"id": item_id,
"criterion_definition_id": None if criterion_id is None else UUID(str(criterion_id)),
"criterion": None if key is None else str(key),
"operator": str(raw["operator"]),
"value": raw.get("value"),
"enabled_for_discovery": bool(enabled),
"mandatory": bool(enabled),
"value_type": raw.get("value_type"),
"unit": raw.get("unit"),
"semantic": raw.get("semantic"),
"category": raw.get("category"),
}
)
canonical = {
"items": [
{
"id": item["id"],
"criterion_definition_id": None
if item["criterion_definition_id"] is None
else str(item["criterion_definition_id"]),
"criterion": item["criterion"],
"operator": item["operator"],
"value": item["value"],
"enabled_for_discovery": item["enabled_for_discovery"],
}
for item in canonical_items
]
}
return {
"items": canonical_items,
"definition": canonical,
"definition_hash": sha256_text(canonical_json(canonical)),
}
def criterion_payload(row) -> dict[str, Any]:
return {
"id": row.item_id,
"criterion_definition_id": row.criterion_definition_id,
"criterion": row.criterion_key,
"operator": row.operator,
"value": row.value,
"enabled_for_discovery": row.enabled_for_discovery,
"mandatory": row.enabled_for_discovery,
}
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.persistence.discovery_models import CriterionDefinitionRecord
def get_definition_by_key(session: Session, key: str) -> CriterionDefinitionRecord | None:
return session.scalars(select(CriterionDefinitionRecord).where(CriterionDefinitionRecord.key == key)).first()
def get_definition(session: Session, definition_id: UUID) -> CriterionDefinitionRecord | None:
return session.get(CriterionDefinitionRecord, definition_id)
def upsert_definition(session: Session, payload: dict[str, Any]) -> CriterionDefinitionRecord:
key = str(payload["key"])
existing = get_definition_by_key(session, key)
if existing is not None:
if payload.get("label"):
existing.label = str(payload["label"])
if payload.get("scoring_attribute_key"):
existing.scoring_attribute_key = str(payload["scoring_attribute_key"])
if payload.get("product_category_id"):
existing.product_category_id = str(payload["product_category_id"])
if payload.get("origin"):
existing.origin = str(payload["origin"])
if payload.get("source"):
existing.source = str(payload["source"])
return existing
row = CriterionDefinitionRecord(
id=payload.get("id") or uuid4(),
key=key,
origin=str(payload.get("origin") or "SOURCE"),
source=payload.get("source"),
product_category_id=payload.get("product_category_id"),
label=str(payload.get("label") or key),
value_type=str(payload["value_type"]),
unit=payload.get("unit"),
semantic=payload.get("semantic"),
category=payload.get("category") or payload.get("product_category_id"),
description=payload.get("description"),
scoring_attribute_key=payload.get("scoring_attribute_key"),
)
session.add(row)
session.flush()
return row
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
from decimal import Decimal
from typing import Any
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
RELEVANT_ATTR_IGNORE = {"title", "html", "presentation", "raw_title"}
def observations_changed(previous: dict[str, Any] | None, current: dict[str, Any] | None) -> bool:
if previous is None or current is None:
return previous != current
if _money(previous.get("best_price")) != _money(current.get("best_price")):
return True
if previous.get("availability") != current.get("availability"):
return True
if previous.get("seller_count") != current.get("seller_count"):
return True
prev_attrs = _technical(previous.get("normalized_attributes") or {})
curr_attrs = _technical(current.get("normalized_attributes") or {})
return prev_attrs != curr_attrs
def _money(value: Any) -> Decimal | None:
if value is None:
return None
return Decimal(str(value))
def _technical(attrs: dict[str, Any]) -> dict[str, Any]:
return {key: attrs[key] for key in sorted(attrs) if key not in RELEVANT_ATTR_IGNORE}
def diff_run(
*,
previous_ids: set[Any],
current_ids: set[Any],
previous_obs: dict[Any, dict[str, Any]],
current_obs: dict[Any, dict[str, Any]],
run_status: str,
) -> dict[Any, DiffStatus]:
result: dict[Any, DiffStatus] = {}
for ref_id in current_ids:
if ref_id not in previous_ids:
result[ref_id] = DiffStatus.NEW
elif observations_changed(previous_obs.get(ref_id), current_obs.get(ref_id)):
result[ref_id] = DiffStatus.CHANGED
else:
result[ref_id] = DiffStatus.UNCHANGED
if run_status == RunStatus.SUCCESS.value:
for ref_id in previous_ids - current_ids:
result[ref_id] = DiffStatus.MISSING
return result
def next_lifecycle(
*,
current: str,
missing_in_success_run: bool,
consecutive_missing: int,
reappeared: bool,
) -> tuple[str, int]:
if current == LifecycleStatus.ARCHIVED.value:
return current, consecutive_missing
if reappeared:
return LifecycleStatus.ACTIVE.value, 0
if missing_in_success_run:
streak = consecutive_missing + 1
if streak >= 3:
return LifecycleStatus.DISAPPEARED.value, streak
return LifecycleStatus.ACTIVE.value, streak
return current, consecutive_missing
+78
View File
@@ -0,0 +1,78 @@
from enum import StrEnum
class ProjectStatus(StrEnum):
ACTIVE = "ACTIVE"
PURCHASED = "PURCHASED"
ARCHIVED = "ARCHIVED"
class MappingStatus(StrEnum):
DISCOVERED = "DISCOVERED"
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
VERIFIED = "VERIFIED"
BROKEN = "BROKEN"
DISABLED = "DISABLED"
class FilterType(StrEnum):
BOOLEAN = "BOOLEAN"
ENUM = "ENUM"
MULTI_ENUM = "MULTI_ENUM"
NUMERIC_RANGE = "NUMERIC_RANGE"
NUMERIC_THRESHOLD = "NUMERIC_THRESHOLD"
TEXT = "TEXT"
UNKNOWN = "UNKNOWN"
class CriterionOrigin(StrEnum):
SOURCE = "SOURCE"
INTERNAL = "INTERNAL"
class SearchStatus(StrEnum):
ACTIVE = "ACTIVE"
OUT_OF_SYNC = "OUT_OF_SYNC"
BLOCKED = "BLOCKED"
DISABLED = "DISABLED"
TOO_BROAD = "TOO_BROAD"
PAUSED = "PAUSED"
class RunStatus(StrEnum):
RUNNING = "RUNNING"
SUCCESS = "SUCCESS"
PARTIAL = "PARTIAL"
FAILED = "FAILED"
BLOCKED = "BLOCKED"
TOO_BROAD = "TOO_BROAD"
class RankingStatus(StrEnum):
NOT_CONFIGURED = "NOT_CONFIGURED"
CONFIGURED = "CONFIGURED"
class DiffStatus(StrEnum):
NEW = "NEW"
CHANGED = "CHANGED"
UNCHANGED = "UNCHANGED"
MISSING = "MISSING"
class LifecycleStatus(StrEnum):
ACTIVE = "ACTIVE"
DISAPPEARED = "DISAPPEARED"
ARCHIVED = "ARCHIVED"
class MatchStatus(StrEnum):
MATCHED = "MATCHED"
MATCH_UNCERTAIN = "MATCH_UNCERTAIN"
UNCERTAIN = "MATCH_UNCERTAIN"
NEW_REFERENCE = "NEW_REFERENCE"
class CandidateStatus(StrEnum):
CANDIDATE = "CANDIDATE"
SHORTLISTED = "SHORTLISTED"
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Any
from app.discovery.enums import MatchStatus
PRESENTATION_KEYS = {"title", "html", "presentation", "raw_title"}
def normalize_title(title: str) -> str:
return re.sub(r"\s+", " ", title.casefold().strip())
def spec_fingerprint(normalized_attributes: dict[str, Any], keys: tuple[str, ...] | None = None) -> str:
"""Fingerprint from whatever technical keys the observation actually has."""
use = keys if keys is not None else tuple(sorted(key for key in normalized_attributes if key not in PRESENTATION_KEYS))
parts = []
for key in use:
value = normalized_attributes.get(key)
parts.append(f"{key}={'' if value is None else str(value).casefold().strip()}")
return "|".join(parts)
@dataclass(frozen=True)
class IdentitySignals:
source_external_id: str | None
gtin_ean: str | None
manufacturer_sku: str | None
source_url: str | None
title: str
manufacturer: str | None
model: str | None
fingerprint: str
@dataclass(frozen=True)
class IdentityDecision:
match_status: MatchStatus
existing_id: Any | None
reason: str
def decide_identity(incoming: IdentitySignals, known: list[tuple[Any, IdentitySignals]]) -> IdentityDecision:
"""Conservative matching. Similar titles never merge by themselves."""
for ref_id, existing in known:
if incoming.source_external_id and incoming.source_external_id == existing.source_external_id:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_external_id")
if incoming.gtin_ean and incoming.gtin_ean == existing.gtin_ean:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "gtin_ean")
if incoming.manufacturer_sku and incoming.manufacturer_sku == existing.manufacturer_sku:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "manufacturer_sku")
if incoming.source_url and incoming.source_url == existing.source_url:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_url")
title_hits = [
(ref_id, existing)
for ref_id, existing in known
if normalize_title(incoming.title) == normalize_title(existing.title)
]
if len(title_hits) == 1 and incoming.fingerprint and incoming.fingerprint == title_hits[0][1].fingerprint:
return IdentityDecision(MatchStatus.MATCHED, title_hits[0][0], "title+fingerprint")
manufacturer_model = [
(ref_id, existing)
for ref_id, existing in known
if incoming.manufacturer
and incoming.model
and incoming.manufacturer.casefold() == (existing.manufacturer or "").casefold()
and incoming.model.casefold() == (existing.model or "").casefold()
]
if len(manufacturer_model) == 1:
existing = manufacturer_model[0][1]
if incoming.fingerprint and existing.fingerprint and incoming.fingerprint != existing.fingerprint:
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "manufacturer+model conflict with specs")
return IdentityDecision(MatchStatus.MATCHED, manufacturer_model[0][0], "manufacturer+model")
if title_hits:
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "title only")
return IdentityDecision(MatchStatus.NEW_REFERENCE, None, "no match")
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from typing import Any
from urllib.parse import parse_qsl, urlsplit
from app.discovery.enums import MappingStatus
from app.discovery.urlgen import decode_xf_roundtrip, generate_search_url
def verify_roundtrip(
*,
category_url: str,
criterion: dict[str, Any],
mapping: dict[str, Any],
) -> bool:
"""True when encoding a single mapping round-trips to the same source parameter."""
probe = {**mapping, "status": MappingStatus.VERIFIED.value}
url, internal = generate_search_url(
category_url=category_url,
criteria_items=[criterion],
mappings=[probe],
)
if internal:
return False
encoding = mapping.get("encoding") or {}
param = str(encoding.get("url_param") or "xf")
if param == "xf":
expected_key = str(mapping["external_filter_key"])
expected_value = str(mapping.get("source_value") or "")
decoded = decode_xf_roundtrip(url)
return any(key == expected_key and value == expected_value.replace("+", " ") for key, value in decoded)
expected = str(criterion.get("value"))
query = parse_qsl(urlsplit(url).query, keep_blank_values=True)
return any(name == param and value == expected for name, value in query)
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from app.clock import now_utc
from app.domain.attributes import select_attribute_snapshot
from app.domain.constraints import evaluate_constraint
from app.domain.enums import Eligibility, NormalizationStatus
from app.domain.models import AttributeStatement, Product
from app.scoring.engine import _product_eligibility
from app.scoring.mathutil import round_score
from app.scoring.product import score_product
def rank_candidates(
*,
profile,
scoring_model,
definitions,
candidates: list[dict[str, Any]],
shortlist_size: int,
as_of: datetime | None = None,
) -> list[dict[str, Any]]:
scored: list[dict[str, Any]] = []
when = as_of or now_utc()
for candidate in candidates:
attrs = _statements(candidate, when)
snapshot = select_attribute_snapshot(
attrs,
as_of=when,
snapshot_cutoff=when,
)
constraint_results = tuple(
evaluate_constraint(
constraint,
snapshot.get(constraint.attribute),
definitions[constraint.attribute],
scoring_model.minimum_confidence,
)
for constraint in profile.constraints
if constraint.attribute in definitions
)
product = Product(
id=uuid4(),
category=profile.category,
brand=str(candidate.get("manufacturer") or "unknown"),
name=str(candidate.get("source_title") or candidate.get("name") or "candidate"),
model=candidate.get("model"),
created_at=when,
)
_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
product_score, coverage, _components = score_product(
profile.preferences,
snapshot,
constraint_results,
scoring_model,
)
scored.append(
{
**candidate,
"product_eligibility": product_eligibility.value,
"product_score": round_score(product_score),
"product_coverage": coverage,
"eligible": product_eligibility == Eligibility.ELIGIBLE,
}
)
scored.sort(key=lambda item: (not item["eligible"], -(item["product_score"] or Decimal("0"))))
for index, item in enumerate(scored):
item["candidate_status"] = "SHORTLISTED" if index < shortlist_size and item["eligible"] else "CANDIDATE"
item["rank"] = index + 1
return scored
def _statements(candidate: dict[str, Any], when: datetime) -> tuple[AttributeStatement, ...]:
raw = candidate.get("normalized_attributes") or {}
now = when
items = []
variant_id = candidate.get("variant_id") or uuid4()
for key, value in raw.items():
items.append(
AttributeStatement(
id=uuid4(),
variant_id=variant_id if isinstance(variant_id, UUID) else uuid4(),
attribute=key,
raw_value=value,
normalized_value=value,
unit=None,
normalization_status=NormalizationStatus.NORMALIZED,
normalizer_version="discovery-v1",
confidence=Decimal("1"),
source_id="geizhals",
source_reference="discovery",
observed_at=now,
ingested_at=now,
)
)
return tuple(items)
+82
View File
@@ -0,0 +1,82 @@
from __future__ import annotations
from typing import Any
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlsplit, urlunsplit
from app.discovery.enums import MappingStatus
def generate_search_url(
*,
category_url: str,
criteria_items: list[dict[str, Any]],
mappings: list[dict[str, Any]],
) -> tuple[str, list[str]]:
"""Build a source URL from VERIFIED mappings only. INTERNAL_ONLY keys are skipped."""
by_criterion = {item["criterion_key"]: item for item in mappings if item.get("status") == MappingStatus.VERIFIED.value}
fragments: list[tuple[str, str]] = []
internal_only: list[str] = []
for criterion in criteria_items:
key = str(criterion["criterion"])
mapping = by_criterion.get(key)
if mapping is None:
internal_only.append(key)
continue
fragments.extend(_encode_mapping(mapping, criterion))
fragments.sort()
base = urlsplit(category_url)
query = dict(parse_qsl(base.query, keep_blank_values=True))
extra: list[tuple[str, str]] = []
for name, value in fragments:
if name in query and name != "xf":
query[name] = value
else:
extra.append((name, value))
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
return url, internal_only
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
encoding = mapping.get("encoding") or {}
param = str(encoding.get("url_param") or "xf")
if param == "xf":
filter_key = str(mapping["external_filter_key"])
source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion))
return [("xf", f"{filter_key}_{_plus(source_value)}")]
if param in {"minp", "maxp", "prmin", "prmax"}:
return [(param, str(criterion.get("value")))]
template = encoding.get("value_template")
if template:
return [(param, str(template).format(value=criterion.get("value"), **mapping))]
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
option = mapping.get("source_value")
if option:
return str(option)
value = criterion.get("value")
unit = mapping.get("unit") or ""
operator = criterion.get("operator")
if operator in {">=", ">"} and unit:
return f"ab {value}{unit}"
return str(value)
def _plus(value: str) -> str:
return value.replace(" ", "+")
def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
parts = urlsplit(url)
decoded: list[tuple[str, str]] = []
for name, value in parse_qsl(parts.query, keep_blank_values=True):
if name == "xf" and "_" in value:
key, raw = value.split("_", 1)
decoded.append((key, raw.replace("+", " ")))
return decoded
def same_origin_url(base: str, maybe_relative: str) -> str:
return urljoin(base, maybe_relative)