Stelle den Frontend-MVP und die v0.3-API auf develop bereit, damit Dev API und PWA zusammen deployen.
Deploy Development / deploy (push) Successful in 1m19s
Test Suite / pytest-backend (push) Successful in 31s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 28s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-04 13:04:59 +02:00
1 parent 827e6a2a11
commit b276758ee1
152 files changed
+25846 -34

No files matched your search

View File
Whitespace-only changes.
+224
View File
@@ -0,0 +1,224 @@
from __future__ import annotations
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.discovery.enums import FilterType, MappingStatus
from app.domain.serialize import jsonable
from app.errors import AppError
from app.persistence.discovery_models import (
CriterionDefinitionRecord,
CriterionMappingRecord,
ProductCategoryRecord,
SourceCategoryRecord,
SourceFilterDefinitionRecord,
SourceFilterOptionRecord,
SourceFilterRangeRecord,
)
from app.discovery.definitions import upsert_definition
from app.persistence.repositories import CatalogRepository
FILTER_OPERATORS = {
FilterType.BOOLEAN.value: ["=", "!="],
FilterType.ENUM.value: ["=", "!="],
FilterType.MULTI_ENUM.value: ["in", "not_in"],
FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"],
FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"],
FilterType.TEXT.value: ["=", "contains"],
}
class CatalogService:
def __init__(self, session: Session) -> None:
self.session = session
self.catalog = CatalogRepository(session)
def list_product_categories(self) -> list[dict[str, Any]]:
rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all()
return [{"id": row.id, "name": row.name, "status": row.status} for row in rows]
def get_product_category(self, category_id: str) -> ProductCategoryRecord:
row = self.session.get(ProductCategoryRecord, category_id)
if row is None:
raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id})
return row
def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]:
self.get_product_category(product_category_id)
rows = self.session.scalars(
select(SourceCategoryRecord)
.where(SourceCategoryRecord.product_category_id == product_category_id)
.order_by(SourceCategoryRecord.source_id)
).all()
sources = {item.id: item for item in self.catalog.list_sources()}
return [
{
"source": row.source_id,
"source_category_id": row.id,
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
"external_category_key": row.external_category_key,
"status": row.status,
}
for row in rows
]
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
if self.catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
rows = self.session.scalars(
select(SourceCategoryRecord)
.where(SourceCategoryRecord.source_id == source_id)
.order_by(SourceCategoryRecord.external_category_key)
).all()
return [
{
"id": row.id,
"source": row.source_id,
"source_category_id": row.id,
"external_category_key": row.external_category_key,
"name": row.name,
"product_category_id": row.product_category_id,
"status": row.status,
}
for row in rows
]
def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]:
self.get_product_category(product_category_id)
if self.catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
source_category = self.session.scalars(
select(SourceCategoryRecord).where(
SourceCategoryRecord.source_id == source_id,
SourceCategoryRecord.product_category_id == product_category_id,
)
).first()
criteria: list[dict[str, Any]] = []
if source_category is not None:
filters = self.session.scalars(
select(SourceFilterDefinitionRecord)
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
.order_by(SourceFilterDefinitionRecord.label)
).all()
mappings = {
row.external_filter_key: row
for row in self.session.scalars(
select(CriterionMappingRecord).where(
CriterionMappingRecord.source_category_id == source_category.id
)
)
}
for filt in filters:
mapping = mappings.get(filt.external_filter_key)
definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
if definition is None:
definition = upsert_definition(
self.session,
{
"key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}",
"origin": "SOURCE",
"source": source_id,
"product_category_id": product_category_id,
"label": filt.label,
"value_type": "number"
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
else "string",
"unit": filt.canonical_unit,
"category": product_category_id,
},
)
self.session.flush()
options = self.session.scalars(
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
).all()
rng = self.session.get(SourceFilterRangeRecord, filt.id)
criteria.append(
self._criterion_out(
definition,
filt,
mapping,
options,
rng,
)
)
internals = self.session.scalars(
select(CriterionDefinitionRecord)
.where(
CriterionDefinitionRecord.origin == "INTERNAL",
(CriterionDefinitionRecord.product_category_id == product_category_id)
| (CriterionDefinitionRecord.category == product_category_id),
)
.order_by(CriterionDefinitionRecord.label)
).all()
seen = {item["criterion_definition_id"] for item in criteria}
for definition in internals:
if definition.id in seen:
continue
criteria.append(self._criterion_out(definition, None, None, [], None))
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
def list_templates(self) -> list[dict[str, Any]]:
rows, _total = self.catalog.list_profiles(200, 0)
return [
{
"id": row.id,
"version": row.version,
"name": row.definition.get("name") or row.id,
"product_category_id": row.definition.get("category"),
"market": row.definition.get("market"),
"currency": row.definition.get("currency"),
"description": row.definition.get("description"),
}
for row in rows
]
def _criterion_out(
self,
definition: CriterionDefinitionRecord,
filt: SourceFilterDefinitionRecord | None,
mapping: CriterionMappingRecord | None,
options: list[SourceFilterOptionRecord],
rng: SourceFilterRangeRecord | None,
) -> dict[str, Any]:
filter_type = None if filt is None else filt.filter_type
return {
"criterion_definition_id": definition.id,
"label": definition.label,
"description": definition.description,
"origin": definition.origin,
"source": definition.source,
"product_category_id": definition.product_category_id,
"filter_type": filter_type,
"unit": definition.unit or (None if filt is None else filt.canonical_unit),
"operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]),
"options": [
{
"source_value": option.source_value,
"normalized_value": option.normalized_value,
"operator": option.operator,
"unit": option.unit,
}
for option in options
],
"range": None
if rng is None
else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit},
"mapping_status": None if mapping is None else mapping.status,
"supports_discovery": mapping is not None
and mapping.status == MappingStatus.VERIFIED.value
and filt is not None,
"scoring_attribute_key": definition.scoring_attribute_key,
}
def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord:
row = session.get(ProductCategoryRecord, category_id)
if row is not None:
return row
row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE")
session.add(row)
session.flush()
return row
File diff suppressed because it is too large. Load diff
+210
View File
@@ -0,0 +1,210 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy.orm import Session
from app.clock import now_utc
from app.domain.serialize import jsonable
from app.domain.timeutil import require_utc, to_iso_z
from app.errors import AppError
from app.persistence import models as orm
from app.persistence.db import engine
from app.persistence.repositories import CatalogRepository
from app.scoring.engine import evaluate_deal
from app.scoring.mathutil import round_score
from app.settings import get_settings
from sqlalchemy.orm import Session as SASession
from sqlalchemy.orm import sessionmaker
def evaluate_and_store(
*,
variant_id: UUID,
payload: dict[str, Any],
) -> dict[str, Any]:
now = now_utc()
as_of = payload.get("as_of")
if as_of is None:
as_of = now
else:
as_of = require_utc(as_of, "as_of")
if as_of > now:
raise AppError(422, "AS_OF_IN_FUTURE", "as_of must not be in the future")
snapshot_cutoff = now
maker = sessionmaker(bind=engine, autoflush=False, autocommit=False)
with engine.connect().execution_options(isolation_level="REPEATABLE READ") as connection:
with SASession(bind=connection) as session:
with session.begin():
repo = CatalogRepository(session)
variant = repo.get_variant(variant_id)
if variant is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
offer = repo.get_offer(payload["offer_id"])
if offer is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(payload["offer_id"])})
if offer.variant_id != variant_id:
raise AppError(422, "OFFER_VARIANT_MISMATCH", "offer does not belong to the variant")
product = repo.get_product(variant.product_id)
if product is None:
raise AppError(404, "NOT_FOUND", "product not found")
profile = repo.get_profile(payload["profile_id"], int(payload["profile_version"]))
if profile is None:
raise AppError(404, "NOT_FOUND", "profile version not found")
model = repo.get_scoring_model(profile.scoring_version)
if model is None:
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version is missing")
attributes = tuple(repo.list_attributes(variant_id))
observations, _ = repo.list_observations(offer.id, limit=None)
result = evaluate_deal(
profile=profile,
scoring_model=model,
product=product,
variant=variant,
offer=offer,
attributes=attributes,
observations=tuple(observations),
definitions=repo.all_definitions(),
as_of=as_of,
snapshot_cutoff=snapshot_cutoff,
engine_build=get_settings().engine_build,
)
response = evaluation_to_response(result, evaluation_id=uuid4(), created_at=now)
record = orm.EvaluationRecord(
id=response["id"],
variant_id=variant.id,
offer_id=offer.id,
profile_id=profile.id,
profile_version=profile.version,
scoring_version=model.version,
as_of=as_of,
snapshot_cutoff=snapshot_cutoff,
scoring_hash=model.definition_hash,
engine_build=result.engine_build,
product_eligibility=result.product_eligibility.value,
offer_eligibility=result.offer_eligibility.value,
overall_eligibility=result.overall_eligibility.value,
product_score=result.product_score,
price_score=result.price_score,
deal_score=result.deal_score,
classification=result.classification,
price_reference_scope=result.price_reference_scope.value,
input_snapshot=result.input_snapshot,
result_snapshot=response,
created_at=now,
)
repo.add_evaluation(record)
for component in result.components:
repo.add_component(
orm.EvaluationComponentRecord(
id=uuid4(),
evaluation_id=record.id,
kind=component.kind,
component_key=component.component_key,
raw_score=component.raw_score,
weight=component.weight,
contribution=component.contribution,
details=jsonable(component.details),
)
)
return response
def load_evaluation(session: Session, evaluation_id: UUID) -> dict[str, Any]:
record = CatalogRepository(session).get_evaluation(evaluation_id)
if record is None:
raise AppError(404, "NOT_FOUND", "evaluation not found", {"id": str(evaluation_id)})
return record.result_snapshot
def evaluation_to_response(result, *, evaluation_id: UUID, created_at: datetime) -> dict[str, Any]:
stats = result.price_statistics
return {
"id": str(evaluation_id),
"profile_id": result.profile_id,
"profile_version": result.profile_version,
"scoring_version": result.scoring_version,
"variant_id": str(result.variant_id),
"offer_id": str(result.offer_id),
"as_of": to_iso_z(result.as_of),
"snapshot_cutoff": to_iso_z(result.snapshot_cutoff),
"scoring_hash": result.scoring_hash,
"engine_build": result.engine_build,
"product_eligibility": result.product_eligibility.value,
"offer_eligibility": result.offer_eligibility.value,
"overall_eligibility": result.overall_eligibility.value,
"eligible": result.eligible,
"product_score": _num(result.product_score),
"price_score": _num(result.price_score),
"deal_score": _num(result.deal_score),
"classification": result.classification,
"price_basis": None if result.price_basis is None else result.price_basis.value,
"product_coverage": _num(result.product_coverage),
"failed_constraints": [_constraint(item) for item in result.failed_constraints],
"unknown_constraints": [_constraint(item) for item in result.unknown_constraints],
"constraint_results": [_constraint(item) for item in result.constraint_results],
"product_gates": [_gate(item) for item in result.product_gates],
"offer_gates": [_gate(item) for item in result.offer_gates],
"components": [_component(item) for item in result.components],
"price_statistics": {
"current_total": _money(stats.current_total),
"median_30d": _money(stats.median_30d),
"median_30d_days": stats.median_30d_days,
"median_90d": _money(stats.median_90d),
"median_90d_days": stats.median_90d_days,
"lowest_365d": _money(stats.lowest_365d),
"lowest_365d_days": stats.lowest_365d_days,
"lowest_365d_is_full_year": stats.lowest_365d_is_full_year,
"used_observation_ids": [str(item) for item in stats.used_observation_ids],
"current_observation_id": None if stats.current_observation_id is None else str(stats.current_observation_id),
},
"price_reference_scope": result.price_reference_scope.value,
"reasons": [{"code": item.code, "message": item.message} for item in result.reasons],
"warnings": list(result.warnings),
"input_snapshot": result.input_snapshot,
"created_at": to_iso_z(created_at),
}
def _num(value: Decimal | None) -> float | None:
return None if value is None else float(round_score(value))
def _money(value: Decimal | None) -> str | None:
return None if value is None else format(value.quantize(Decimal("0.01")), "f")
def _constraint(item) -> dict[str, Any]:
return {
"constraint_id": item.constraint_id,
"attribute": item.attribute,
"operator": item.operator,
"expected": jsonable(item.expected),
"actual": jsonable(item.actual),
"status": item.status,
"mandatory": item.mandatory,
"attribute_row_id": None if item.attribute_row_id is None else str(item.attribute_row_id),
"reason_code": item.reason_code,
}
def _gate(item) -> dict[str, Any]:
return {
"key": item.key,
"status": item.status,
"reason_code": item.reason_code,
"details": jsonable(item.details),
}
def _component(item) -> dict[str, Any]:
return {
"kind": item.kind,
"component_key": item.component_key,
"raw_score": None if item.raw_score is None else float(round_score(item.raw_score)),
"weight": None if item.weight is None else float(item.weight),
"contribution": None if item.contribution is None else float(round_score(item.contribution)),
"details": jsonable(item.details),
}
+94
View File
@@ -0,0 +1,94 @@
from app.errors import AppError
from app.services.ingestion_service import IngestionService
from app.sources.base import ProductSource
class ImportService:
def __init__(self, ingestion: IngestionService) -> None:
self.ingestion = ingestion
def import_product(self, source: ProductSource, external_id: str) -> None:
product = source.get_product(external_id)
try:
self.ingestion.create_product(
{
"id": product.id,
"category": product.category,
"brand": product.brand,
"name": product.name,
"model": product.model,
}
)
except AppError as exc:
if exc.status_code != 409:
raise
for variant in product.variants:
try:
self.ingestion.create_variant(
product.id,
{
"id": variant.id,
"name": variant.name,
"manufacturer_sku": variant.manufacturer_sku,
"identifiers": variant.identifiers,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for attribute in variant.attributes:
try:
self.ingestion.add_attribute(
variant.id,
{
"id": attribute.id,
"attribute": attribute.attribute,
"raw_value": attribute.raw_value,
"normalized_value": attribute.normalized_value,
"unit": attribute.unit,
"normalization_status": attribute.normalization_status,
"normalizer_version": attribute.normalizer_version,
"confidence": attribute.confidence,
"source_id": source.source_key,
"source_reference": attribute.source_reference,
"observed_at": attribute.observed_at,
"normalization_error": attribute.normalization_error,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for offer in source.get_offers(external_id):
try:
self.ingestion.create_offer(
offer.variant_id,
{
"id": offer.id,
"source_id": source.source_key,
"external_id": offer.external_id,
"seller": offer.seller,
"url": offer.url,
"market": offer.market,
"currency": offer.currency,
"condition": offer.condition,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for price in source.get_price_history(offer.external_id):
self.ingestion.add_price(
offer.id,
{
"id": price.id,
"source_id": source.source_key,
"external_observation_id": price.external_observation_id,
"observed_at": price.observed_at,
"item_price": price.item_price,
"shipping_price": price.shipping_price,
"currency": price.currency,
"availability": price.availability,
"raw_payload": price.raw_payload,
"confidence": price.confidence,
},
)
+263
View File
@@ -0,0 +1,263 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
from app.clock import now_utc
from app.domain.timeutil import require_utc
from app.domain.validation import DomainValidationError, parse_profile, validate_attribute_key, validate_currency
from app.errors import AppError
from app.normalization.manual import ManualNormalizer
from app.persistence import models as orm
from app.persistence.mapping import to_observation
from app.persistence.repositories import CatalogRepository
def _now() -> datetime:
return now_utc()
def _uuid(value: UUID | None) -> UUID:
return value or uuid4()
class IngestionService:
def __init__(self, session: Session) -> None:
self.session = session
self.repo = CatalogRepository(session)
self.normalizer = ManualNormalizer()
def create_definition(self, payload: dict[str, Any]) -> AttributeDefinition:
key = str(payload["key"])
validate_attribute_key(key)
row = orm.AttributeDefinitionRecord(
key=key,
value_type=ValueType(payload["value_type"]).value,
canonical_unit=payload.get("canonical_unit"),
minimum=payload.get("minimum"),
maximum=payload.get("maximum"),
description=payload.get("description"),
)
if self.repo.get_definition(key) is not None:
raise AppError(409, "CONFLICT", "attribute definition already exists", {"key": key})
return self.repo.add_definition(row)
def create_profile(self, payload: dict[str, Any]) -> orm.ProductProfileRecord:
try:
model_versions = self.repo.scoring_versions()
scoring_version = payload.get("scoring_version")
model = self.repo.get_scoring_model(scoring_version) if scoring_version else None
if model is None:
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version does not exist")
profile = parse_profile(
payload,
attribute_definitions=self.repo.all_definitions(),
scoring_versions=model_versions,
supported_strategies=set(model.supported_preference_strategies),
)
except DomainValidationError as exc:
raise AppError(422, "PROFILE_INVALID", str(exc), exc.details) from exc
except AppError:
raise
row = orm.ProductProfileRecord(
id=profile.id,
version=profile.version,
name=profile.name,
category=profile.category,
scoring_version=profile.scoring_version,
definition=profile.definition,
definition_hash=profile.definition_hash,
created_at=_now(),
)
if self.repo.get_profile(profile.id, profile.version) is not None:
raise AppError(409, "CONFLICT", "profile version already exists", {"id": profile.id, "version": profile.version})
return self.repo.add_profile(row)
def create_product(self, payload: dict[str, Any]) -> Product:
product_id = _uuid(payload.get("id"))
if self.repo.get_product(product_id) is not None:
raise AppError(409, "CONFLICT", "product already exists", {"id": str(product_id)})
row = orm.ProductRecord(
id=product_id,
category=str(payload["category"]),
brand=str(payload["brand"]),
name=str(payload["name"]),
model=payload.get("model"),
created_at=_now(),
)
return self.repo.add_product(row)
def create_variant(self, product_id: UUID, payload: dict[str, Any]) -> ProductVariant:
if self.repo.get_product(product_id) is None:
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
variant_id = _uuid(payload.get("id"))
if self.repo.get_variant(variant_id) is not None:
raise AppError(409, "CONFLICT", "variant already exists", {"id": str(variant_id)})
row = orm.ProductVariantRecord(
id=variant_id,
product_id=product_id,
name=str(payload["name"]),
manufacturer_sku=payload.get("manufacturer_sku"),
identifiers=payload.get("identifiers") or {},
created_at=_now(),
)
return self.repo.add_variant(row)
def add_attribute(self, variant_id: UUID, payload: dict[str, Any]) -> AttributeStatement:
variant = self.repo.get_variant(variant_id)
if variant is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
definition = self.repo.get_definition(payload["attribute"])
if definition is None:
raise AppError(422, "UNKNOWN_ATTRIBUTE", "attribute definition does not exist")
if self.repo.get_source(payload["source_id"]) is None:
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
observed_at = require_utc(payload["observed_at"], "observed_at")
ingested_at = _now()
if observed_at > ingested_at:
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
confidence = Decimal(str(payload["confidence"]))
if not Decimal("0") <= confidence <= Decimal("1"):
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
statement = AttributeStatement(
id=_uuid(payload.get("id")),
variant_id=variant_id,
attribute=definition.key,
raw_value=payload["raw_value"],
normalized_value=payload.get("normalized_value"),
unit=payload.get("unit"),
normalization_status=NormalizationStatus(payload["normalization_status"]),
normalizer_version=payload.get("normalizer_version") or self.normalizer.version,
confidence=confidence,
source_id=payload["source_id"],
source_reference=payload["source_reference"],
observed_at=observed_at,
ingested_at=ingested_at,
normalization_error=payload.get("normalization_error"),
)
statement = self.normalizer.validate(definition, statement)
if self.repo.get_attribute(statement.id) is not None:
raise AppError(409, "CONFLICT", "attribute statement already exists", {"id": str(statement.id)})
row = orm.VariantAttributeRecord(
id=statement.id,
variant_id=variant_id,
attribute_key=statement.attribute,
source_id=statement.source_id,
raw_value=statement.raw_value,
normalized_value=statement.normalized_value,
unit=statement.unit,
normalization_status=statement.normalization_status.value,
normalizer_version=statement.normalizer_version,
confidence=statement.confidence,
source_reference=statement.source_reference,
observed_at=statement.observed_at,
ingested_at=statement.ingested_at,
normalization_error=statement.normalization_error,
)
return self.repo.add_attribute(row)
def create_offer(self, variant_id: UUID, payload: dict[str, Any]) -> Offer:
if self.repo.get_variant(variant_id) is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
if self.repo.get_source(payload["source_id"]) is None:
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
offer_id = _uuid(payload.get("id"))
if self.repo.get_offer(offer_id) is not None:
raise AppError(409, "CONFLICT", "offer already exists", {"id": str(offer_id)})
if self.repo.get_offer_by_external(payload["source_id"], str(payload["external_id"])) is not None:
raise AppError(409, "CONFLICT", "offer source/external_id already exists")
row = orm.OfferRecord(
id=offer_id,
variant_id=variant_id,
source_id=payload["source_id"],
external_id=str(payload["external_id"]),
seller=str(payload["seller"]),
url=payload.get("url"),
market=str(payload["market"]),
currency=validate_currency(str(payload["currency"])),
condition=OfferCondition(payload["condition"]).value,
created_at=_now(),
)
return self.repo.add_offer(row)
def add_price(self, offer_id: UUID, payload: dict[str, Any]) -> tuple[PriceObservation, bool]:
offer = self.repo.get_offer(offer_id)
if offer is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
if payload["source_id"] != offer.source_id:
raise AppError(422, "SOURCE_MISMATCH", "source_id must match the offer")
currency = validate_currency(str(payload["currency"]))
if currency != offer.currency:
raise AppError(422, "CURRENCY_MISMATCH", "currency must match the offer")
observed_at = require_utc(payload["observed_at"], "observed_at")
ingested_at = _now()
if observed_at > ingested_at:
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
item_price = Decimal(str(payload["item_price"]))
if item_price <= 0:
raise AppError(422, "ITEM_PRICE", "item_price must be > 0")
shipping = payload.get("shipping_price")
shipping_price = None if shipping is None else Decimal(str(shipping))
if shipping_price is not None and shipping_price < 0:
raise AppError(422, "SHIPPING_PRICE", "shipping_price must be >= 0")
confidence = Decimal(str(payload["confidence"]))
if not Decimal("0") <= confidence <= Decimal("1"):
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
external_id = str(payload["external_observation_id"])
existing = self.repo.get_observation_by_external(offer_id, external_id)
incoming = {
"source_id": payload["source_id"],
"observed_at": observed_at,
"item_price": item_price,
"shipping_price": shipping_price,
"currency": currency,
"availability": Availability(payload["availability"]).value,
"raw_payload": payload["raw_payload"],
"confidence": confidence,
}
if existing is not None:
if _observation_content(existing) != incoming:
raise AppError(409, "CONFLICT", "price observation content differs")
return to_observation(existing), False
row = orm.PriceObservationRecord(
id=_uuid(payload.get("id")),
offer_id=offer_id,
source_id=payload["source_id"],
external_observation_id=external_id,
observed_at=observed_at,
ingested_at=ingested_at,
item_price=item_price,
shipping_price=shipping_price,
currency=currency,
availability=incoming["availability"],
raw_payload=payload["raw_payload"],
confidence=confidence,
)
try:
return self.repo.add_observation(row), True
except IntegrityError:
self.session.rollback()
existing = self.repo.get_observation_by_external(offer_id, external_id)
if existing is None:
raise
if _observation_content(existing) != incoming:
raise AppError(409, "CONFLICT", "price observation content differs")
return to_observation(existing), False
def _observation_content(row: orm.PriceObservationRecord) -> dict[str, Any]:
return {
"source_id": row.source_id,
"observed_at": row.observed_at,
"item_price": Decimal(str(row.item_price)),
"shipping_price": None if row.shipping_price is None else Decimal(str(row.shipping_price)),
"currency": row.currency,
"availability": row.availability,
"raw_payload": row.raw_payload,
"confidence": Decimal(str(row.confidence)),
}
+346
View File
@@ -0,0 +1,346 @@
from __future__ import annotations
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.clock import now_utc
from app.discovery.criteria import criterion_payload, parse_project_criteria
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
from app.discovery.enums import CandidateStatus, DiffStatus, ProjectStatus, RankingStatus, SearchStatus
from app.domain.serialize import jsonable
from app.domain.validation import DomainValidationError
from app.errors import AppError
from app.persistence.discovery_models import (
BuyingProjectCriteriaRecord,
BuyingProjectRecord,
DiscoverySearchRecord,
ProductCategoryRecord,
ProjectCandidateRecord,
ProjectCriterionRecord,
SearchRunItemRecord,
SearchRunRecord,
)
from app.persistence.repositories import CatalogRepository
from app.services.catalog_service import CatalogService, ensure_product_category
INACTIVE_SEARCH = {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}
def _uuid(value: UUID | None) -> UUID:
return value or uuid4()
class ProjectService:
def __init__(self, session: Session) -> None:
self.session = session
self.catalog = CatalogRepository(session)
self.categories = CatalogService(session)
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
template_id = payload.get("template_id") or payload.get("profile_id")
template_version = payload.get("template_version") or payload.get("profile_version")
profile = None
if template_id is not None:
if template_version is None:
raise AppError(422, "VALIDATION_ERROR", "template_version is required when template_id is set")
profile = self.catalog.get_profile(str(template_id), int(template_version))
if profile is None:
raise AppError(404, "NOT_FOUND", "template version not found")
product_category_id = payload.get("product_category_id") or (None if profile is None else profile.category)
if not product_category_id:
raise AppError(422, "VALIDATION_ERROR", "product_category_id is required")
if self.session.get(ProductCategoryRecord, product_category_id) is None:
ensure_product_category(self.session, str(product_category_id))
market = payload.get("market") or (None if profile is None else profile.market)
currency = payload.get("currency") or (None if profile is None else profile.currency)
if not market or not currency:
raise AppError(422, "VALIDATION_ERROR", "market and currency are required")
target_budget = payload.get("target_budget")
if target_budget is None and profile is not None and getattr(profile, "budget", None) is not None:
target_budget = getattr(profile.budget, "target", None)
criteria_payload = payload.get("criteria")
if not criteria_payload:
raise AppError(422, "CRITERIA_REQUIRED", "criteria are required")
try:
parsed = parse_project_criteria(criteria_payload)
except DomainValidationError as exc:
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
shortlist_size = int(payload.get("shortlist_size") or 5)
if shortlist_size < 1:
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
now = now_utc()
project = BuyingProjectRecord(
id=_uuid(payload.get("id")),
name=str(payload["name"]),
status=ProjectStatus.ACTIVE.value,
workspace_id=payload.get("workspace_id"),
product_category_id=str(product_category_id),
market=str(market),
currency=str(currency),
target_budget=None if target_budget is None else Decimal(str(target_budget)),
template_id=None if profile is None else profile.id,
template_version=None if profile is None else profile.version,
shortlist_size=shortlist_size,
criteria_version=1,
created_at=now,
updated_at=now,
archived_at=None,
)
self.session.add(project)
self.session.flush()
self._store_criteria(project.id, 1, parsed, str(product_category_id), now)
return self.get(project.id)
def replace_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
project = self._require_project(project_id)
try:
parsed = parse_project_criteria(payload)
except DomainValidationError as exc:
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
current = self.latest_criteria(project_id)
version = (current.version if current is not None else 0) + 1
now = now_utc()
self._store_criteria(project.id, version, parsed, project.product_category_id, now)
project.criteria_version = version
project.updated_at = now
for search in self.session.scalars(
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
):
if search.status not in INACTIVE_SEARCH:
search.status = SearchStatus.OUT_OF_SYNC.value
search.updated_at = now
self.session.flush()
return self.get(project.id)
def add_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
return self.replace_criteria(project_id, payload)
def patch(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
project = self._require_project(project_id)
allowed = {"name", "target_budget", "shortlist_size", "status"}
unknown = set(payload) - allowed
if unknown:
raise AppError(422, "VALIDATION_ERROR", "field cannot be patched", {"fields": sorted(unknown)})
if "name" in payload:
project.name = str(payload["name"])
if "target_budget" in payload:
project.target_budget = None if payload["target_budget"] is None else Decimal(str(payload["target_budget"]))
if "shortlist_size" in payload:
size = int(payload["shortlist_size"])
if size < 1:
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
project.shortlist_size = size
if "status" in payload:
status = str(payload["status"])
if status not in {item.value for item in ProjectStatus}:
raise AppError(422, "VALIDATION_ERROR", "invalid project status")
project.status = status
if status == ProjectStatus.ARCHIVED.value:
project.archived_at = now_utc()
project.updated_at = now_utc()
self.session.flush()
return self.get(project.id)
def get(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
criteria = self.latest_criteria(project_id)
return self._out(project, criteria)
def get_criteria(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
criteria = self.latest_criteria(project_id)
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
return jsonable(
{
"project_id": project.id,
"criteria_version": None if criteria is None else criteria.version,
"items": items,
}
)
def list(self, limit: int, offset: int) -> tuple[list[dict[str, Any]], int]:
total = self.session.scalar(select(func.count()).select_from(BuyingProjectRecord)) or 0
rows = self.session.scalars(
select(BuyingProjectRecord).order_by(BuyingProjectRecord.created_at).limit(limit).offset(offset)
).all()
return [self._summary(row) for row in rows], total
def latest_criteria(self, project_id: UUID) -> BuyingProjectCriteriaRecord | None:
return self.session.scalars(
select(BuyingProjectCriteriaRecord)
.where(BuyingProjectCriteriaRecord.project_id == project_id)
.order_by(BuyingProjectCriteriaRecord.version.desc())
).first()
def criteria_version(self, project_id: UUID, version: int) -> BuyingProjectCriteriaRecord | None:
return self.session.get(BuyingProjectCriteriaRecord, (project_id, version))
def criteria_items(self, project_id: UUID, version: int) -> list[dict[str, Any]]:
rows = self.session.scalars(
select(ProjectCriterionRecord)
.where(ProjectCriterionRecord.project_id == project_id, ProjectCriterionRecord.version == version)
.order_by(ProjectCriterionRecord.item_id)
).all()
return [criterion_payload(row) for row in rows]
def available_criteria(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
sources = self.categories.sources_for_category(project.product_category_id)
source = sources[0]["source"] if sources else "geizhals"
return self.categories.available_criteria(project.product_category_id, source)
def _store_criteria(
self,
project_id: UUID,
version: int,
parsed: dict[str, Any],
category: str | None,
created_at,
) -> None:
self.session.add(
BuyingProjectCriteriaRecord(
project_id=project_id,
version=version,
definition_hash=parsed["definition_hash"],
created_at=created_at,
)
)
self.session.flush()
for item in parsed["items"]:
definition = self._resolve_definition(item, category)
self.session.add(
ProjectCriterionRecord(
id=uuid4(),
project_id=project_id,
version=version,
item_id=item["id"],
criterion_definition_id=definition.id,
criterion_key=definition.key,
operator=item["operator"],
value=item["value"],
enabled_for_discovery=item["enabled_for_discovery"],
created_at=created_at,
)
)
self.session.flush()
def _resolve_definition(self, item: dict[str, Any], category: str | None):
if item.get("criterion_definition_id") is not None:
definition = get_definition(self.session, item["criterion_definition_id"])
if definition is None:
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
return definition
existing = get_definition_by_key(self.session, str(item["criterion"]))
if existing is not None:
return existing
if not item.get("value_type"):
raise AppError(
422,
"UNKNOWN_CRITERION",
"criterion definition missing; provide value_type or create it first",
{"criterion": item["criterion"]},
)
return upsert_definition(
self.session,
{
"key": item["criterion"],
"origin": "INTERNAL",
"product_category_id": item.get("category") or category,
"label": item["criterion"],
"value_type": item["value_type"],
"unit": item.get("unit"),
"semantic": item.get("semantic"),
"category": item.get("category") or category,
"scoring_attribute_key": item["criterion"],
},
)
def _require_project(self, project_id: UUID) -> BuyingProjectRecord:
project = self.session.get(BuyingProjectRecord, project_id)
if project is None:
raise AppError(404, "NOT_FOUND", "buying project not found")
return project
def _out(self, project: BuyingProjectRecord, criteria: BuyingProjectCriteriaRecord | None) -> dict[str, Any]:
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
return jsonable(
{
"id": project.id,
"name": project.name,
"status": project.status,
"workspace_id": project.workspace_id,
"product_category_id": project.product_category_id,
"market": project.market,
"currency": project.currency,
"target_budget": project.target_budget,
"template_id": project.template_id,
"template_version": project.template_version,
"profile_id": project.template_id,
"profile_version": project.template_version,
"shortlist_size": project.shortlist_size,
"created_at": project.created_at,
"updated_at": project.updated_at,
"archived_at": project.archived_at,
"criteria_version": project.criteria_version if criteria is None else criteria.version,
"criteria": None if criteria is None else {"items": items},
"criteria_hash": None if criteria is None else criteria.definition_hash,
"ranking_status": RankingStatus.NOT_CONFIGURED.value
if project.template_id is None
else RankingStatus.CONFIGURED.value,
}
)
def _summary(self, project: BuyingProjectRecord) -> dict[str, Any]:
category = self.session.get(ProductCategoryRecord, project.product_category_id)
searches = list(
self.session.scalars(
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
)
)
active = next((row for row in searches if row.status not in INACTIVE_SEARCH), searches[0] if searches else None)
last_run = None
if active is not None:
last_run = self.session.scalars(
select(SearchRunRecord)
.where(SearchRunRecord.discovery_search_id == active.id)
.order_by(SearchRunRecord.started_at.desc())
).first()
candidates = list(
self.session.scalars(
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project.id)
)
)
counts = {"NEW": 0, "CHANGED": 0, "MISSING": 0}
if last_run is not None:
for item in self.session.scalars(
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == last_run.id)
):
if item.diff_status in counts:
counts[item.diff_status] += 1
return jsonable(
{
"id": project.id,
"name": project.name,
"product_category": None
if category is None
else {"id": category.id, "name": category.name, "status": category.status},
"status": project.status,
"last_successful_run_at": None if active is None else active.last_successful_run_at,
"next_run_at": None if active is None else active.next_run_at,
"discovery_status": None if active is None else active.status,
"candidate_count": len(candidates),
"shortlist_count": sum(1 for row in candidates if row.status == CandidateStatus.SHORTLISTED.value),
"new_count": last_run.new_count if last_run is not None else counts[DiffStatus.NEW.value],
"changed_count": last_run.changed_count if last_run is not None else counts[DiffStatus.CHANGED.value],
"missing_count": last_run.missing_count if last_run is not None else counts[DiffStatus.MISSING.value],
"current_search_result_count": None if last_run is None else last_run.result_count,
"ranking_status": RankingStatus.NOT_CONFIGURED.value
if project.template_id is None
else RankingStatus.CONFIGURED.value,
}
)