Stelle den Frontend-MVP und die v0.3-API auf develop bereit, damit Dev API und PWA zusammen deployen.
Deploy Development / deploy (push) Successful in 1m19s
Test Suite / pytest-backend (push) Successful in 31s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-checks (push) Successful in 28s

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-04 13:04:59 +02:00
1 parent 827e6a2a11
commit b276758ee1
152 files changed
+25846 -34

No files matched your search

+10
View File
@@ -0,0 +1,10 @@
.git
.venv
.pytest_cache
.mypy_cache
.ruff_cache
__pycache__
*.pyc
*.pyo
*.egg-info
.tests-tmp
+40
View File
@@ -0,0 +1,40 @@
FROM ghcr.io/astral-sh/uv:0.10.11-python3.13-trixie-slim AS base
RUN apt-get update \
&& apt-get install -y --no-install-recommends libpq5 \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH="/app/.venv/bin:$PATH"
COPY pyproject.toml uv.lock ./
RUN uv sync --frozen --no-dev --no-install-project
FROM base AS runtime
COPY . .
RUN uv sync --frozen --no-dev \
&& sed -i 's/\r$//' /app/scripts/startup.sh \
&& chmod +x /app/scripts/startup.sh \
&& useradd --create-home --uid 1000 app \
&& chown -R app:app /app
USER app
EXPOSE 8000
CMD ["/app/scripts/startup.sh"]
FROM base AS test
COPY . .
RUN uv sync --frozen --extra test \
&& useradd --create-home --uid 1000 app \
&& chown -R app:app /app
USER app
CMD ["pytest", "tests", "-m", "not slow", "-ra", "--tb=short"]
+39
View File
@@ -0,0 +1,39 @@
[alembic]
script_location = migrations
prepend_sys_path = .
path_separator = os
sqlalchemy.url = driver://user:pass@localhost/dbname
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
View File
Whitespace-only changes.
View File
Whitespace-only changes.
+125
View File
@@ -0,0 +1,125 @@
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.api.schemas import (
CriterionDefinitionIn,
CriterionMappingIn,
CriterionMappingPatch,
DiscoverySearchIn,
DiscoverySearchPatch,
SourceAttributeMappingIn,
SourceAttributeMappingPatch,
)
from app.persistence.db import get_session
from app.services.catalog_service import CatalogService
from app.services.discovery_service import (
CriterionDefinitionService,
CriterionMappingService,
DiscoverySearchService,
SourceAttributeMappingService,
TaxonomyService,
)
router = APIRouter(prefix="/api/v1")
@router.get("/sources/{source_id}/categories")
def list_source_categories(source_id: str, session: Session = Depends(get_session)):
return {"items": CatalogService(session).source_categories(source_id)}
@router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
return TaxonomyService(session).discover(source_id, category)
@router.get("/sources/{source_id}/categories/{category}/filters")
def list_filters(source_id: str, category: str, session: Session = Depends(get_session)):
return TaxonomyService(session).filters(source_id, category)
@router.post("/criterion-definitions", status_code=201)
def create_criterion_definition(payload: CriterionDefinitionIn, session: Session = Depends(get_session)):
return CriterionDefinitionService(session).create(payload.model_dump())
@router.get("/criterion-definitions")
def list_criterion_definitions(category: str | None = None, session: Session = Depends(get_session)):
return {"items": CriterionDefinitionService(session).list(category)}
@router.get("/criterion-mappings")
def list_mappings(source_category_id: UUID | None = None, session: Session = Depends(get_session)):
return {"items": CriterionMappingService(session).list(source_category_id)}
@router.post("/criterion-mappings", status_code=201)
def create_mapping(payload: CriterionMappingIn, session: Session = Depends(get_session)):
return CriterionMappingService(session).create(payload.model_dump())
@router.patch("/criterion-mappings/{mapping_id}")
def patch_mapping(mapping_id: UUID, payload: CriterionMappingPatch, session: Session = Depends(get_session)):
return CriterionMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
@router.get("/source-attribute-mappings")
def list_attribute_mappings(session: Session = Depends(get_session)):
return {"items": SourceAttributeMappingService(session).list()}
@router.post("/source-attribute-mappings", status_code=201)
def create_attribute_mapping(payload: SourceAttributeMappingIn, session: Session = Depends(get_session)):
return SourceAttributeMappingService(session).create(payload.model_dump())
@router.patch("/source-attribute-mappings/{mapping_id}")
def patch_attribute_mapping(
mapping_id: UUID,
payload: SourceAttributeMappingPatch,
session: Session = Depends(get_session),
):
return SourceAttributeMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
@router.post("/discovery-searches", status_code=201)
def create_search(payload: DiscoverySearchIn, session: Session = Depends(get_session)):
return DiscoverySearchService(session).create(payload.model_dump())
@router.get("/discovery-searches")
def list_searches(due: bool = Query(default=False), session: Session = Depends(get_session)):
service = DiscoverySearchService(session)
return {"items": service.list_due() if due else service.list_all()}
@router.get("/discovery-searches/{search_id}")
def get_search(search_id: UUID, session: Session = Depends(get_session)):
return DiscoverySearchService(session).get(search_id)
@router.patch("/discovery-searches/{search_id}")
def patch_search(search_id: UUID, payload: DiscoverySearchPatch, session: Session = Depends(get_session)):
return DiscoverySearchService(session).patch(search_id, payload.model_dump(exclude_unset=True))
@router.post("/discovery-searches/{search_id}/generate-query")
def generate_query(search_id: UUID, session: Session = Depends(get_session)):
return DiscoverySearchService(session).generate_query(search_id)
@router.post("/discovery-searches/{search_id}/run", status_code=201)
def run_search(search_id: UUID, session: Session = Depends(get_session)):
return DiscoverySearchService(session).run(search_id)
@router.get("/discovery-searches/{search_id}/runs")
def list_runs(search_id: UUID, session: Session = Depends(get_session)):
return {"items": DiscoverySearchService(session).list_runs(search_id)}
@router.get("/search-runs/{run_id}")
def get_run(run_id: UUID, session: Session = Depends(get_session)):
return DiscoverySearchService(session).get_run(run_id)
+20
View File
@@ -0,0 +1,20 @@
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.api.schemas import EvaluateIn
from app.persistence.db import get_session
from app.services.evaluation_service import evaluate_and_store, load_evaluation
router = APIRouter(prefix="/api/v1")
@router.post("/evaluate/{variant_id}", status_code=201)
def evaluate(variant_id: UUID, payload: EvaluateIn):
return evaluate_and_store(variant_id=variant_id, payload=payload.model_dump())
@router.get("/evaluations/{evaluation_id}")
def get_evaluation(evaluation_id: UUID, session: Session = Depends(get_session)):
return load_evaluation(session, evaluation_id)
+45
View File
@@ -0,0 +1,45 @@
from fastapi import APIRouter
from fastapi.responses import JSONResponse
from sqlalchemy import text
from alembic.config import Config
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from app.persistence.db import engine
from app.settings import get_settings
router = APIRouter()
def _alembic_at_head() -> bool:
config = Config("alembic.ini")
script = ScriptDirectory.from_config(config)
with engine.connect() as connection:
context = MigrationContext.configure(connection)
return set(context.get_current_heads()) == set(script.get_heads())
@router.get("/health/live")
def live() -> dict[str, str]:
return {"status": "ok"}
@router.get("/health/ready", response_model=None)
def ready() -> JSONResponse | dict[str, str]:
settings = get_settings()
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
if not _alembic_at_head():
return JSONResponse(
{"status": "not_ready", "reason": "migration_behind_head"},
status_code=503,
)
except Exception:
return JSONResponse({"status": "not_ready", "reason": "database"}, status_code=503)
return {
"status": "ready",
"environment": settings.environment,
"engine_build": settings.engine_build,
}
+42
View File
@@ -0,0 +1,42 @@
from uuid import UUID
from fastapi import APIRouter, Depends, Response
from sqlalchemy.orm import Session
from app.api.schemas import OfferIn, OfferOut, PriceIn, PriceOut, page, parse_page
from app.errors import AppError
from app.persistence.db import get_session
from app.persistence.repositories import CatalogRepository
from app.services.ingestion_service import IngestionService
router = APIRouter(prefix="/api/v1")
@router.post("/variants/{variant_id}/offers", status_code=201)
def create_offer(variant_id: UUID, payload: OfferIn, session: Session = Depends(get_session)) -> OfferOut:
offer = IngestionService(session).create_offer(variant_id, payload.model_dump())
return OfferOut.model_validate(offer.__dict__)
@router.get("/offers/{offer_id}")
def get_offer(offer_id: UUID, session: Session = Depends(get_session)) -> OfferOut:
offer = CatalogRepository(session).get_offer(offer_id)
if offer is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
return OfferOut.model_validate(offer.__dict__)
@router.post("/offers/{offer_id}/prices")
def add_price(offer_id: UUID, payload: PriceIn, response: Response, session: Session = Depends(get_session)) -> PriceOut:
observation, created = IngestionService(session).add_price(offer_id, payload.model_dump())
response.status_code = 201 if created else 200
return PriceOut.model_validate(observation.__dict__)
@router.get("/offers/{offer_id}/prices")
def list_prices(offer_id: UUID, limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
if CatalogRepository(session).get_offer(offer_id) is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
limit, offset = parse_page(limit, offset)
items, total = CatalogRepository(session).list_observations(offer_id, limit, offset)
return page([PriceOut.model_validate(item.__dict__) for item in items], limit, offset, total)
+63
View File
@@ -0,0 +1,63 @@
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.api.schemas import AttributeIn, AttributeOut, ProductIn, ProductOut, VariantIn, VariantOut
from app.errors import AppError
from app.persistence.db import get_session
from app.persistence.repositories import CatalogRepository
from app.services.ingestion_service import IngestionService
router = APIRouter(prefix="/api/v1")
@router.post("/products", status_code=201)
def create_product(payload: ProductIn, session: Session = Depends(get_session)) -> ProductOut:
product = IngestionService(session).create_product(payload.model_dump())
return ProductOut.model_validate(product.__dict__)
@router.get("/products/{product_id}")
def get_product(product_id: UUID, session: Session = Depends(get_session)) -> ProductOut:
product = CatalogRepository(session).get_product(product_id)
if product is None:
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
return ProductOut.model_validate(product.__dict__)
@router.post("/products/{product_id}/variants", status_code=201)
def create_variant(product_id: UUID, payload: VariantIn, session: Session = Depends(get_session)):
variant = IngestionService(session).create_variant(product_id, payload.model_dump())
return {
"id": variant.id,
"product_id": variant.product_id,
"name": variant.name,
"manufacturer_sku": variant.manufacturer_sku,
"identifiers": variant.identifiers,
"created_at": variant.created_at,
}
@router.get("/variants/{variant_id}")
def get_variant(variant_id: UUID, session: Session = Depends(get_session)) -> VariantOut:
repo = CatalogRepository(session)
variant = repo.get_variant(variant_id)
if variant is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
attributes = [AttributeOut.model_validate(item.__dict__) for item in repo.list_attributes(variant_id)]
return VariantOut(
id=variant.id,
product_id=variant.product_id,
name=variant.name,
manufacturer_sku=variant.manufacturer_sku,
identifiers=variant.identifiers,
created_at=variant.created_at,
attributes=attributes,
)
@router.post("/variants/{variant_id}/attributes", status_code=201)
def add_attribute(variant_id: UUID, payload: AttributeIn, session: Session = Depends(get_session)) -> AttributeOut:
statement = IngestionService(session).add_attribute(variant_id, payload.model_dump())
return AttributeOut.model_validate(statement.__dict__)
+100
View File
@@ -0,0 +1,100 @@
from typing import Any
from fastapi import APIRouter, Body, Depends
from sqlalchemy.orm import Session
from app.api.schemas import AttributeDefinitionIn, page, parse_page
from app.domain.serialize import jsonable
from app.errors import AppError
from app.persistence.db import get_session
from app.persistence.repositories import CatalogRepository
from app.services.ingestion_service import IngestionService
router = APIRouter(prefix="/api/v1")
@router.post("/attribute-definitions", status_code=201)
def create_attribute_definition(payload: AttributeDefinitionIn, session: Session = Depends(get_session)):
definition = IngestionService(session).create_definition(payload.model_dump())
return jsonable(definition.__dict__)
@router.get("/attribute-definitions")
def list_attribute_definitions(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
limit, offset = parse_page(limit, offset)
items, total = CatalogRepository(session).list_definitions(limit, offset)
return page([jsonable(item.__dict__) for item in items], limit, offset, total)
@router.get("/sources")
def list_sources(session: Session = Depends(get_session)):
items = CatalogRepository(session).list_sources()
return {
"items": [{"id": item.id, "name": item.name, "adapter_type": item.adapter_type} for item in items],
"limit": len(items),
"offset": 0,
"total": len(items),
}
@router.get("/scoring-models/{version}")
def get_scoring_model(version: str, session: Session = Depends(get_session)):
model = CatalogRepository(session).get_scoring_model(version)
if model is None:
raise AppError(404, "NOT_FOUND", "scoring model not found", {"version": version})
return {
"version": model.version,
"algorithm_id": model.algorithm_id,
"definition": model.definition,
"definition_hash": model.definition_hash,
}
@router.post("/profiles", status_code=201)
def create_profile(payload: dict[str, Any] = Body(...), session: Session = Depends(get_session)):
if not isinstance(payload, dict):
raise AppError(422, "PROFILE_INVALID", "profile must be an object")
row = IngestionService(session).create_profile(payload)
return {
"id": row.id,
"version": row.version,
"name": row.name,
"category": row.category,
"scoring_version": row.scoring_version,
"definition": row.definition,
"definition_hash": row.definition_hash,
"created_at": row.created_at,
}
@router.get("/profiles")
def list_profiles(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
limit, offset = parse_page(limit, offset)
items, total = CatalogRepository(session).list_profiles(limit, offset)
return page(
[
{
"id": row.id,
"version": row.version,
"name": row.name,
"category": row.category,
"scoring_version": row.scoring_version,
"definition_hash": row.definition_hash,
"created_at": row.created_at,
}
for row in items
],
limit,
offset,
total,
)
@router.get("/profiles/{profile_id}")
def get_profile(profile_id: str, version: int | None = None, session: Session = Depends(get_session)):
if version is None:
raise AppError(422, "VERSION_REQUIRED", "query parameter version is required")
profile = CatalogRepository(session).get_profile(profile_id, version)
if profile is None:
raise AppError(404, "NOT_FOUND", "profile version not found")
return profile.definition
+125
View File
@@ -0,0 +1,125 @@
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.api.schemas import (
BuyingProjectIn,
BuyingProjectPatch,
DiscoverySearchIn,
ProjectCriteriaIn,
page,
parse_page,
)
from app.persistence.db import get_session
from app.services.catalog_service import CatalogService
from app.services.discovery_service import CandidateRankingService, DiscoverySearchService
from app.services.project_service import ProjectService
router = APIRouter(prefix="/api/v1")
@router.get("/product-categories")
def list_product_categories(session: Session = Depends(get_session)):
return {"items": CatalogService(session).list_product_categories()}
@router.get("/product-categories/{product_category_id}/sources")
def list_category_sources(product_category_id: str, session: Session = Depends(get_session)):
return {"items": CatalogService(session).sources_for_category(product_category_id)}
@router.get("/product-categories/{product_category_id}/available-criteria")
def available_criteria_for_category(
product_category_id: str,
source: str = Query(...),
session: Session = Depends(get_session),
):
return CatalogService(session).available_criteria(product_category_id, source)
@router.get("/buying-templates")
def list_buying_templates(session: Session = Depends(get_session)):
return {"items": CatalogService(session).list_templates()}
@router.post("/buying-projects", status_code=201)
def create_buying_project(payload: BuyingProjectIn, session: Session = Depends(get_session)):
return ProjectService(session).create(payload.model_dump())
@router.get("/buying-projects")
def list_buying_projects(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
limit, offset = parse_page(limit, offset)
items, total = ProjectService(session).list(limit, offset)
return page(items, limit, offset, total)
@router.get("/buying-projects/{project_id}")
def get_buying_project(project_id: UUID, session: Session = Depends(get_session)):
return ProjectService(session).get(project_id)
@router.patch("/buying-projects/{project_id}")
def patch_buying_project(project_id: UUID, payload: BuyingProjectPatch, session: Session = Depends(get_session)):
return ProjectService(session).patch(project_id, payload.model_dump(exclude_unset=True))
@router.post("/buying-projects/{project_id}/criteria", status_code=201)
def add_buying_project_criteria(
project_id: UUID,
payload: ProjectCriteriaIn,
session: Session = Depends(get_session),
):
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
@router.put("/buying-projects/{project_id}/criteria")
def put_buying_project_criteria(
project_id: UUID,
payload: ProjectCriteriaIn,
session: Session = Depends(get_session),
):
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
@router.get("/buying-projects/{project_id}/criteria")
def get_buying_project_criteria(project_id: UUID, session: Session = Depends(get_session)):
return ProjectService(session).get_criteria(project_id)
@router.get("/buying-projects/{project_id}/available-criteria")
def available_criteria(project_id: UUID, session: Session = Depends(get_session)):
return ProjectService(session).available_criteria(project_id)
@router.get("/buying-projects/{project_id}/discovery-searches")
def list_project_searches(project_id: UUID, session: Session = Depends(get_session)):
return {"items": DiscoverySearchService(session).list_for_project(project_id)}
@router.post("/buying-projects/{project_id}/discovery-searches", status_code=201)
def create_project_search(project_id: UUID, payload: DiscoverySearchIn, session: Session = Depends(get_session)):
body = payload.model_dump()
body["buying_project_id"] = project_id
return DiscoverySearchService(session).create(body)
@router.get("/buying-projects/{project_id}/candidates")
def list_candidates(project_id: UUID, session: Session = Depends(get_session)):
return {"items": CandidateRankingService(session).list_candidates(project_id)}
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}")
def get_candidate(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
return CandidateRankingService(session).get_candidate(project_id, candidate_id)
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}/observations")
def list_candidate_observations(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
return {"items": CandidateRankingService(session).list_observations(project_id, candidate_id)}
@router.get("/buying-projects/{project_id}/shortlist")
def list_shortlist(project_id: UUID, session: Session = Depends(get_session)):
return {"items": CandidateRankingService(session).list_candidates(project_id, shortlisted_only=True)}
+274
View File
@@ -0,0 +1,274 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_serializer
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class AttributeDefinitionIn(StrictModel):
key: str
value_type: str
canonical_unit: str | None = None
minimum: Decimal | None = None
maximum: Decimal | None = None
description: str | None = None
class AttributeDefinitionOut(StrictModel):
key: str
value_type: str
canonical_unit: str | None
minimum: Decimal | None
maximum: Decimal | None
description: str | None
class ProductIn(StrictModel):
id: UUID | None = None
category: str
brand: str
name: str
model: str | None = None
class ProductOut(StrictModel):
id: UUID
category: str
brand: str
name: str
model: str | None
created_at: datetime
class VariantIn(StrictModel):
id: UUID | None = None
name: str
manufacturer_sku: str | None = None
identifiers: dict[str, Any] = Field(default_factory=dict)
class AttributeIn(StrictModel):
id: UUID | None = None
attribute: str
raw_value: Any
normalized_value: Any | None = None
unit: str | None = None
normalization_status: str
normalizer_version: str | None = None
confidence: Decimal
source_id: str
source_reference: str
observed_at: datetime
normalization_error: str | None = None
class AttributeOut(StrictModel):
id: UUID
variant_id: UUID
attribute: str
raw_value: Any
normalized_value: Any | None
unit: str | None
normalization_status: str
normalizer_version: str
confidence: Decimal
source_id: str
source_reference: str
observed_at: datetime
ingested_at: datetime
normalization_error: str | None
class VariantOut(StrictModel):
id: UUID
product_id: UUID
name: str
manufacturer_sku: str | None
identifiers: dict[str, Any]
created_at: datetime
attributes: list[AttributeOut]
class OfferIn(StrictModel):
id: UUID | None = None
source_id: str
external_id: str
seller: str
url: str | None = None
market: str
currency: str
condition: str
class OfferOut(StrictModel):
id: UUID
variant_id: UUID
source_id: str
external_id: str
seller: str
url: str | None
market: str
currency: str
condition: str
created_at: datetime
class PriceIn(StrictModel):
id: UUID | None = None
source_id: str
external_observation_id: str
observed_at: datetime
item_price: Decimal
shipping_price: Decimal | None = None
currency: str
availability: str
raw_payload: Any
confidence: Decimal
@field_serializer("item_price", "shipping_price")
def serialize_money(self, value: Decimal | None) -> str | None:
return None if value is None else format(value, "f")
class PriceOut(StrictModel):
id: UUID
offer_id: UUID
source_id: str
external_observation_id: str
observed_at: datetime
ingested_at: datetime
item_price: Decimal
shipping_price: Decimal | None
currency: str
availability: str
raw_payload: Any
confidence: Decimal
@field_serializer("item_price", "shipping_price")
def serialize_money(self, value: Decimal | None) -> str | None:
return None if value is None else format(Decimal(value).quantize(Decimal("0.01")), "f")
class EvaluateIn(StrictModel):
profile_id: str
profile_version: int
offer_id: UUID
as_of: datetime | None = None
class BuyingProjectIn(StrictModel):
id: UUID | None = None
name: str
product_category_id: str | None = None
market: str | None = None
currency: str | None = None
target_budget: Decimal | None = None
template_id: str | None = None
template_version: int | None = None
profile_id: str | None = None
profile_version: int | None = None
workspace_id: str | None = None
shortlist_size: int = 5
schedule_interval_minutes: int | None = None
criteria: dict[str, Any]
class BuyingProjectPatch(StrictModel):
name: str | None = None
target_budget: Decimal | None = None
shortlist_size: int | None = None
status: str | None = None
class ProjectCriteriaIn(StrictModel):
items: list[dict[str, Any]]
class CriterionDefinitionIn(StrictModel):
key: str
value_type: str
unit: str | None = None
semantic: str | None = None
category: str | None = None
description: str | None = None
origin: str | None = None
source: str | None = None
product_category_id: str | None = None
label: str | None = None
scoring_attribute_key: str | None = None
class DiscoverySearchIn(StrictModel):
buying_project_id: UUID | None = None
source_id: str
source_category_id: UUID
schedule_interval_minutes: int = 180
broad_result_threshold: int = 300
class DiscoverySearchPatch(StrictModel):
schedule_interval_minutes: int | None = None
status: str | None = None
class SourceAttributeMappingIn(StrictModel):
source_id: str
source_category_id: UUID | None = None
raw_name: str
criterion_definition_id: UUID
status: str | None = None
class SourceAttributeMappingPatch(StrictModel):
raw_name: str | None = None
criterion_definition_id: UUID | None = None
status: str | None = None
class CriterionMappingIn(StrictModel):
criterion_definition_id: UUID | None = None
criterion_key: str | None = None
source_id: str
source_category_id: UUID
external_filter_key: str
source_value: str | None = None
encoding: dict[str, Any] = Field(default_factory=lambda: {"url_param": "xf"})
status: str | None = None
value_type: str | None = None
unit: str | None = None
semantic: str | None = None
description: str | None = None
operator: str | None = None
probe_value: Any = None
class CriterionMappingPatch(StrictModel):
source_value: str | None = None
encoding: dict[str, Any] | None = None
status: str | None = None
probe_value: Any = None
operator: str | None = None
class Page(StrictModel):
items: list[Any]
limit: int
offset: int
total: int
def page(items: list[Any], limit: int, offset: int, total: int) -> dict[str, Any]:
return {"items": items, "limit": limit, "offset": offset, "total": total}
def parse_page(limit: int = 50, offset: int = 0) -> tuple[int, int]:
from app.errors import AppError
if limit < 1 or limit > 200 or offset < 0:
raise AppError(422, "PAGINATION", "limit must be 1-200 and offset >= 0")
return limit, offset
+12
View File
@@ -0,0 +1,12 @@
from datetime import datetime, timezone
from app.settings import get_settings
def now_utc() -> datetime:
override = get_settings().clock_now
if override is not None:
if override.tzinfo is None:
return override.replace(tzinfo=timezone.utc)
return override.astimezone(timezone.utc)
return datetime.now(timezone.utc)
+38
View File
@@ -0,0 +1,38 @@
from app.discovery.criteria import parse_project_criteria
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
from app.discovery.enums import (
CandidateStatus,
DiffStatus,
LifecycleStatus,
MappingStatus,
MatchStatus,
ProjectStatus,
RunStatus,
SearchStatus,
)
from app.discovery.identity import IdentityDecision, IdentitySignals, decide_identity, spec_fingerprint
from app.discovery.mapping import verify_roundtrip
from app.discovery.ranking import rank_candidates
from app.discovery.urlgen import generate_search_url
__all__ = [
"CandidateStatus",
"DiffStatus",
"IdentityDecision",
"IdentitySignals",
"LifecycleStatus",
"MappingStatus",
"MatchStatus",
"ProjectStatus",
"RunStatus",
"SearchStatus",
"decide_identity",
"diff_run",
"generate_search_url",
"next_lifecycle",
"observations_changed",
"parse_project_criteria",
"rank_candidates",
"spec_fingerprint",
"verify_roundtrip",
]
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
from typing import Any
from uuid import UUID
from app.domain.validation import DomainValidationError, canonical_json, sha256_text
def parse_project_criteria(definition: dict[str, Any]) -> dict[str, Any]:
items = definition.get("items")
if not isinstance(items, list) or not items:
raise DomainValidationError("criteria.items must be a non-empty list")
seen: set[str] = set()
canonical_items: list[dict[str, Any]] = []
for raw in items:
criterion_id = raw.get("criterion_definition_id")
key = raw.get("criterion")
item_id = str(raw.get("id") or criterion_id or key or "")
if not item_id:
raise DomainValidationError("each criterion needs id, criterion_definition_id or criterion")
if item_id in seen:
raise DomainValidationError("duplicate criterion id", {"id": item_id})
seen.add(item_id)
if not criterion_id and not key:
raise DomainValidationError("criterion_definition_id or criterion is required", {"id": item_id})
if not raw.get("operator"):
raise DomainValidationError("operator is required", {"id": item_id})
enabled = raw.get("enabled_for_discovery")
if enabled is None:
enabled = raw.get("mandatory", True)
canonical_items.append(
{
"id": item_id,
"criterion_definition_id": None if criterion_id is None else UUID(str(criterion_id)),
"criterion": None if key is None else str(key),
"operator": str(raw["operator"]),
"value": raw.get("value"),
"enabled_for_discovery": bool(enabled),
"mandatory": bool(enabled),
"value_type": raw.get("value_type"),
"unit": raw.get("unit"),
"semantic": raw.get("semantic"),
"category": raw.get("category"),
}
)
canonical = {
"items": [
{
"id": item["id"],
"criterion_definition_id": None
if item["criterion_definition_id"] is None
else str(item["criterion_definition_id"]),
"criterion": item["criterion"],
"operator": item["operator"],
"value": item["value"],
"enabled_for_discovery": item["enabled_for_discovery"],
}
for item in canonical_items
]
}
return {
"items": canonical_items,
"definition": canonical,
"definition_hash": sha256_text(canonical_json(canonical)),
}
def criterion_payload(row) -> dict[str, Any]:
return {
"id": row.item_id,
"criterion_definition_id": row.criterion_definition_id,
"criterion": row.criterion_key,
"operator": row.operator,
"value": row.value,
"enabled_for_discovery": row.enabled_for_discovery,
"mandatory": row.enabled_for_discovery,
}
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.persistence.discovery_models import CriterionDefinitionRecord
def get_definition_by_key(session: Session, key: str) -> CriterionDefinitionRecord | None:
return session.scalars(select(CriterionDefinitionRecord).where(CriterionDefinitionRecord.key == key)).first()
def get_definition(session: Session, definition_id: UUID) -> CriterionDefinitionRecord | None:
return session.get(CriterionDefinitionRecord, definition_id)
def upsert_definition(session: Session, payload: dict[str, Any]) -> CriterionDefinitionRecord:
key = str(payload["key"])
existing = get_definition_by_key(session, key)
if existing is not None:
if payload.get("label"):
existing.label = str(payload["label"])
if payload.get("scoring_attribute_key"):
existing.scoring_attribute_key = str(payload["scoring_attribute_key"])
if payload.get("product_category_id"):
existing.product_category_id = str(payload["product_category_id"])
if payload.get("origin"):
existing.origin = str(payload["origin"])
if payload.get("source"):
existing.source = str(payload["source"])
return existing
row = CriterionDefinitionRecord(
id=payload.get("id") or uuid4(),
key=key,
origin=str(payload.get("origin") or "SOURCE"),
source=payload.get("source"),
product_category_id=payload.get("product_category_id"),
label=str(payload.get("label") or key),
value_type=str(payload["value_type"]),
unit=payload.get("unit"),
semantic=payload.get("semantic"),
category=payload.get("category") or payload.get("product_category_id"),
description=payload.get("description"),
scoring_attribute_key=payload.get("scoring_attribute_key"),
)
session.add(row)
session.flush()
return row
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
from decimal import Decimal
from typing import Any
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
RELEVANT_ATTR_IGNORE = {"title", "html", "presentation", "raw_title"}
def observations_changed(previous: dict[str, Any] | None, current: dict[str, Any] | None) -> bool:
if previous is None or current is None:
return previous != current
if _money(previous.get("best_price")) != _money(current.get("best_price")):
return True
if previous.get("availability") != current.get("availability"):
return True
if previous.get("seller_count") != current.get("seller_count"):
return True
prev_attrs = _technical(previous.get("normalized_attributes") or {})
curr_attrs = _technical(current.get("normalized_attributes") or {})
return prev_attrs != curr_attrs
def _money(value: Any) -> Decimal | None:
if value is None:
return None
return Decimal(str(value))
def _technical(attrs: dict[str, Any]) -> dict[str, Any]:
return {key: attrs[key] for key in sorted(attrs) if key not in RELEVANT_ATTR_IGNORE}
def diff_run(
*,
previous_ids: set[Any],
current_ids: set[Any],
previous_obs: dict[Any, dict[str, Any]],
current_obs: dict[Any, dict[str, Any]],
run_status: str,
) -> dict[Any, DiffStatus]:
result: dict[Any, DiffStatus] = {}
for ref_id in current_ids:
if ref_id not in previous_ids:
result[ref_id] = DiffStatus.NEW
elif observations_changed(previous_obs.get(ref_id), current_obs.get(ref_id)):
result[ref_id] = DiffStatus.CHANGED
else:
result[ref_id] = DiffStatus.UNCHANGED
if run_status == RunStatus.SUCCESS.value:
for ref_id in previous_ids - current_ids:
result[ref_id] = DiffStatus.MISSING
return result
def next_lifecycle(
*,
current: str,
missing_in_success_run: bool,
consecutive_missing: int,
reappeared: bool,
) -> tuple[str, int]:
if current == LifecycleStatus.ARCHIVED.value:
return current, consecutive_missing
if reappeared:
return LifecycleStatus.ACTIVE.value, 0
if missing_in_success_run:
streak = consecutive_missing + 1
if streak >= 3:
return LifecycleStatus.DISAPPEARED.value, streak
return LifecycleStatus.ACTIVE.value, streak
return current, consecutive_missing
+78
View File
@@ -0,0 +1,78 @@
from enum import StrEnum
class ProjectStatus(StrEnum):
ACTIVE = "ACTIVE"
PURCHASED = "PURCHASED"
ARCHIVED = "ARCHIVED"
class MappingStatus(StrEnum):
DISCOVERED = "DISCOVERED"
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
VERIFIED = "VERIFIED"
BROKEN = "BROKEN"
DISABLED = "DISABLED"
class FilterType(StrEnum):
BOOLEAN = "BOOLEAN"
ENUM = "ENUM"
MULTI_ENUM = "MULTI_ENUM"
NUMERIC_RANGE = "NUMERIC_RANGE"
NUMERIC_THRESHOLD = "NUMERIC_THRESHOLD"
TEXT = "TEXT"
UNKNOWN = "UNKNOWN"
class CriterionOrigin(StrEnum):
SOURCE = "SOURCE"
INTERNAL = "INTERNAL"
class SearchStatus(StrEnum):
ACTIVE = "ACTIVE"
OUT_OF_SYNC = "OUT_OF_SYNC"
BLOCKED = "BLOCKED"
DISABLED = "DISABLED"
TOO_BROAD = "TOO_BROAD"
PAUSED = "PAUSED"
class RunStatus(StrEnum):
RUNNING = "RUNNING"
SUCCESS = "SUCCESS"
PARTIAL = "PARTIAL"
FAILED = "FAILED"
BLOCKED = "BLOCKED"
TOO_BROAD = "TOO_BROAD"
class RankingStatus(StrEnum):
NOT_CONFIGURED = "NOT_CONFIGURED"
CONFIGURED = "CONFIGURED"
class DiffStatus(StrEnum):
NEW = "NEW"
CHANGED = "CHANGED"
UNCHANGED = "UNCHANGED"
MISSING = "MISSING"
class LifecycleStatus(StrEnum):
ACTIVE = "ACTIVE"
DISAPPEARED = "DISAPPEARED"
ARCHIVED = "ARCHIVED"
class MatchStatus(StrEnum):
MATCHED = "MATCHED"
MATCH_UNCERTAIN = "MATCH_UNCERTAIN"
UNCERTAIN = "MATCH_UNCERTAIN"
NEW_REFERENCE = "NEW_REFERENCE"
class CandidateStatus(StrEnum):
CANDIDATE = "CANDIDATE"
SHORTLISTED = "SHORTLISTED"
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Any
from app.discovery.enums import MatchStatus
PRESENTATION_KEYS = {"title", "html", "presentation", "raw_title"}
def normalize_title(title: str) -> str:
return re.sub(r"\s+", " ", title.casefold().strip())
def spec_fingerprint(normalized_attributes: dict[str, Any], keys: tuple[str, ...] | None = None) -> str:
"""Fingerprint from whatever technical keys the observation actually has."""
use = keys if keys is not None else tuple(sorted(key for key in normalized_attributes if key not in PRESENTATION_KEYS))
parts = []
for key in use:
value = normalized_attributes.get(key)
parts.append(f"{key}={'' if value is None else str(value).casefold().strip()}")
return "|".join(parts)
@dataclass(frozen=True)
class IdentitySignals:
source_external_id: str | None
gtin_ean: str | None
manufacturer_sku: str | None
source_url: str | None
title: str
manufacturer: str | None
model: str | None
fingerprint: str
@dataclass(frozen=True)
class IdentityDecision:
match_status: MatchStatus
existing_id: Any | None
reason: str
def decide_identity(incoming: IdentitySignals, known: list[tuple[Any, IdentitySignals]]) -> IdentityDecision:
"""Conservative matching. Similar titles never merge by themselves."""
for ref_id, existing in known:
if incoming.source_external_id and incoming.source_external_id == existing.source_external_id:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_external_id")
if incoming.gtin_ean and incoming.gtin_ean == existing.gtin_ean:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "gtin_ean")
if incoming.manufacturer_sku and incoming.manufacturer_sku == existing.manufacturer_sku:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "manufacturer_sku")
if incoming.source_url and incoming.source_url == existing.source_url:
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_url")
title_hits = [
(ref_id, existing)
for ref_id, existing in known
if normalize_title(incoming.title) == normalize_title(existing.title)
]
if len(title_hits) == 1 and incoming.fingerprint and incoming.fingerprint == title_hits[0][1].fingerprint:
return IdentityDecision(MatchStatus.MATCHED, title_hits[0][0], "title+fingerprint")
manufacturer_model = [
(ref_id, existing)
for ref_id, existing in known
if incoming.manufacturer
and incoming.model
and incoming.manufacturer.casefold() == (existing.manufacturer or "").casefold()
and incoming.model.casefold() == (existing.model or "").casefold()
]
if len(manufacturer_model) == 1:
existing = manufacturer_model[0][1]
if incoming.fingerprint and existing.fingerprint and incoming.fingerprint != existing.fingerprint:
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "manufacturer+model conflict with specs")
return IdentityDecision(MatchStatus.MATCHED, manufacturer_model[0][0], "manufacturer+model")
if title_hits:
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "title only")
return IdentityDecision(MatchStatus.NEW_REFERENCE, None, "no match")
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from typing import Any
from urllib.parse import parse_qsl, urlsplit
from app.discovery.enums import MappingStatus
from app.discovery.urlgen import decode_xf_roundtrip, generate_search_url
def verify_roundtrip(
*,
category_url: str,
criterion: dict[str, Any],
mapping: dict[str, Any],
) -> bool:
"""True when encoding a single mapping round-trips to the same source parameter."""
probe = {**mapping, "status": MappingStatus.VERIFIED.value}
url, internal = generate_search_url(
category_url=category_url,
criteria_items=[criterion],
mappings=[probe],
)
if internal:
return False
encoding = mapping.get("encoding") or {}
param = str(encoding.get("url_param") or "xf")
if param == "xf":
expected_key = str(mapping["external_filter_key"])
expected_value = str(mapping.get("source_value") or "")
decoded = decode_xf_roundtrip(url)
return any(key == expected_key and value == expected_value.replace("+", " ") for key, value in decoded)
expected = str(criterion.get("value"))
query = parse_qsl(urlsplit(url).query, keep_blank_values=True)
return any(name == param and value == expected for name, value in query)
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from app.clock import now_utc
from app.domain.attributes import select_attribute_snapshot
from app.domain.constraints import evaluate_constraint
from app.domain.enums import Eligibility, NormalizationStatus
from app.domain.models import AttributeStatement, Product
from app.scoring.engine import _product_eligibility
from app.scoring.mathutil import round_score
from app.scoring.product import score_product
def rank_candidates(
*,
profile,
scoring_model,
definitions,
candidates: list[dict[str, Any]],
shortlist_size: int,
as_of: datetime | None = None,
) -> list[dict[str, Any]]:
scored: list[dict[str, Any]] = []
when = as_of or now_utc()
for candidate in candidates:
attrs = _statements(candidate, when)
snapshot = select_attribute_snapshot(
attrs,
as_of=when,
snapshot_cutoff=when,
)
constraint_results = tuple(
evaluate_constraint(
constraint,
snapshot.get(constraint.attribute),
definitions[constraint.attribute],
scoring_model.minimum_confidence,
)
for constraint in profile.constraints
if constraint.attribute in definitions
)
product = Product(
id=uuid4(),
category=profile.category,
brand=str(candidate.get("manufacturer") or "unknown"),
name=str(candidate.get("source_title") or candidate.get("name") or "candidate"),
model=candidate.get("model"),
created_at=when,
)
_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
product_score, coverage, _components = score_product(
profile.preferences,
snapshot,
constraint_results,
scoring_model,
)
scored.append(
{
**candidate,
"product_eligibility": product_eligibility.value,
"product_score": round_score(product_score),
"product_coverage": coverage,
"eligible": product_eligibility == Eligibility.ELIGIBLE,
}
)
scored.sort(key=lambda item: (not item["eligible"], -(item["product_score"] or Decimal("0"))))
for index, item in enumerate(scored):
item["candidate_status"] = "SHORTLISTED" if index < shortlist_size and item["eligible"] else "CANDIDATE"
item["rank"] = index + 1
return scored
def _statements(candidate: dict[str, Any], when: datetime) -> tuple[AttributeStatement, ...]:
raw = candidate.get("normalized_attributes") or {}
now = when
items = []
variant_id = candidate.get("variant_id") or uuid4()
for key, value in raw.items():
items.append(
AttributeStatement(
id=uuid4(),
variant_id=variant_id if isinstance(variant_id, UUID) else uuid4(),
attribute=key,
raw_value=value,
normalized_value=value,
unit=None,
normalization_status=NormalizationStatus.NORMALIZED,
normalizer_version="discovery-v1",
confidence=Decimal("1"),
source_id="geizhals",
source_reference="discovery",
observed_at=now,
ingested_at=now,
)
)
return tuple(items)
+82
View File
@@ -0,0 +1,82 @@
from __future__ import annotations
from typing import Any
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlsplit, urlunsplit
from app.discovery.enums import MappingStatus
def generate_search_url(
*,
category_url: str,
criteria_items: list[dict[str, Any]],
mappings: list[dict[str, Any]],
) -> tuple[str, list[str]]:
"""Build a source URL from VERIFIED mappings only. INTERNAL_ONLY keys are skipped."""
by_criterion = {item["criterion_key"]: item for item in mappings if item.get("status") == MappingStatus.VERIFIED.value}
fragments: list[tuple[str, str]] = []
internal_only: list[str] = []
for criterion in criteria_items:
key = str(criterion["criterion"])
mapping = by_criterion.get(key)
if mapping is None:
internal_only.append(key)
continue
fragments.extend(_encode_mapping(mapping, criterion))
fragments.sort()
base = urlsplit(category_url)
query = dict(parse_qsl(base.query, keep_blank_values=True))
extra: list[tuple[str, str]] = []
for name, value in fragments:
if name in query and name != "xf":
query[name] = value
else:
extra.append((name, value))
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
return url, internal_only
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
encoding = mapping.get("encoding") or {}
param = str(encoding.get("url_param") or "xf")
if param == "xf":
filter_key = str(mapping["external_filter_key"])
source_value = str(mapping.get("source_value") or _threshold_value(mapping, criterion))
return [("xf", f"{filter_key}_{_plus(source_value)}")]
if param in {"minp", "maxp", "prmin", "prmax"}:
return [(param, str(criterion.get("value")))]
template = encoding.get("value_template")
if template:
return [(param, str(template).format(value=criterion.get("value"), **mapping))]
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
option = mapping.get("source_value")
if option:
return str(option)
value = criterion.get("value")
unit = mapping.get("unit") or ""
operator = criterion.get("operator")
if operator in {">=", ">"} and unit:
return f"ab {value}{unit}"
return str(value)
def _plus(value: str) -> str:
return value.replace(" ", "+")
def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
parts = urlsplit(url)
decoded: list[tuple[str, str]] = []
for name, value in parse_qsl(parts.query, keep_blank_values=True):
if name == "xf" and "_" in value:
key, raw = value.split("_", 1)
decoded.append((key, raw.replace("+", " ")))
return decoded
def same_origin_url(base: str, maybe_relative: str) -> str:
return urljoin(base, maybe_relative)
View File
Whitespace-only changes.
+45
View File
@@ -0,0 +1,45 @@
from datetime import datetime
from decimal import Decimal
from typing import Iterable
from app.domain.enums import NormalizationStatus
from app.domain.models import AttributeStatement
def statement_is_usable(statement: AttributeStatement | None, minimum_confidence: Decimal) -> bool:
if statement is None:
return False
if statement.normalization_status != NormalizationStatus.NORMALIZED:
return False
if statement.normalized_value is None:
return False
return statement.confidence >= minimum_confidence
def select_attribute_snapshot(
statements: Iterable[AttributeStatement],
*,
as_of: datetime,
snapshot_cutoff: datetime,
) -> dict[str, AttributeStatement]:
"""Pick the newest statement per key at or before the evaluation cutoff."""
chosen: dict[str, AttributeStatement] = {}
for statement in statements:
if statement.observed_at > as_of or statement.ingested_at > snapshot_cutoff:
continue
current = chosen.get(statement.attribute)
if current is None or _is_newer(statement, current):
chosen[statement.attribute] = statement
return chosen
def _is_newer(candidate: AttributeStatement, current: AttributeStatement) -> bool:
return (
candidate.observed_at,
candidate.ingested_at,
candidate.id,
) > (
current.observed_at,
current.ingested_at,
current.id,
)
+148
View File
@@ -0,0 +1,148 @@
from decimal import Decimal
from typing import Any
from app.domain.attributes import statement_is_usable
from app.domain.enums import ConstraintOperator, ConstraintStatus, ValueType
from app.domain.models import AttributeDefinition, AttributeStatement, Constraint, ConstraintResult
_NUMBER_OPS = {
ConstraintOperator.GE,
ConstraintOperator.LE,
ConstraintOperator.GT,
ConstraintOperator.LT,
}
_EQUALITY_OPS = {ConstraintOperator.EQ, ConstraintOperator.NE}
_SCALAR_TYPES = {ValueType.NUMBER, ValueType.STRING, ValueType.BOOLEAN}
def operator_allowed_for(value_type: ValueType, operator: ConstraintOperator) -> bool:
if operator == ConstraintOperator.EXISTS:
return True
if operator in _NUMBER_OPS:
return value_type == ValueType.NUMBER
if operator in _EQUALITY_OPS:
return value_type in _SCALAR_TYPES
if operator == ConstraintOperator.IN:
return value_type in _SCALAR_TYPES
if operator == ConstraintOperator.CONTAINS:
return value_type == ValueType.STRING_LIST
return False
def evaluate_constraint(
constraint: Constraint,
statement: AttributeStatement | None,
definition: AttributeDefinition,
minimum_confidence: Decimal,
) -> ConstraintResult:
usable = statement_is_usable(statement, minimum_confidence)
actual = statement.normalized_value if usable else None
attribute_row_id = statement.id if statement is not None else None
if not usable:
return ConstraintResult(
constraint_id=constraint.id,
attribute=constraint.attribute,
operator=constraint.operator.value,
expected=constraint.value,
actual=None,
status=ConstraintStatus.UNKNOWN.value,
mandatory=constraint.mandatory,
attribute_row_id=attribute_row_id,
reason_code="ATTRIBUTE_UNUSABLE",
)
passed = _compare(constraint.operator, actual, constraint.value, definition.value_type)
if passed is None:
status = ConstraintStatus.UNKNOWN
reason = "TYPE_MISMATCH"
elif passed:
status = ConstraintStatus.PASS
reason = "CONSTRAINT_PASS"
else:
status = ConstraintStatus.FAIL
reason = "CONSTRAINT_FAIL"
return ConstraintResult(
constraint_id=constraint.id,
attribute=constraint.attribute,
operator=constraint.operator.value,
expected=constraint.value,
actual=actual,
status=status.value,
mandatory=constraint.mandatory,
attribute_row_id=attribute_row_id,
reason_code=reason,
)
def _as_number(value: Any) -> Decimal | None:
if isinstance(value, bool) or value is None:
return None
if isinstance(value, Decimal):
return value
if isinstance(value, int):
return Decimal(value)
if isinstance(value, float):
return Decimal(str(value))
if isinstance(value, str):
return None
return None
def _same_scalar(left: Any, right: Any, value_type: ValueType) -> bool | None:
if value_type == ValueType.NUMBER:
left_n = _as_number(left)
right_n = _as_number(right)
if left_n is None or right_n is None:
return None
return left_n == right_n
if value_type == ValueType.STRING:
if not isinstance(left, str) or not isinstance(right, str):
return None
return left == right
if value_type == ValueType.BOOLEAN:
if not isinstance(left, bool) or not isinstance(right, bool):
return None
return left == right
return None
def _compare(
operator: ConstraintOperator,
actual: Any,
expected: Any,
value_type: ValueType,
) -> bool | None:
if operator == ConstraintOperator.EXISTS:
return True
if operator in _NUMBER_OPS:
actual_n = _as_number(actual)
expected_n = _as_number(expected)
if actual_n is None or expected_n is None:
return None
if operator == ConstraintOperator.GE:
return actual_n >= expected_n
if operator == ConstraintOperator.LE:
return actual_n <= expected_n
if operator == ConstraintOperator.GT:
return actual_n > expected_n
return actual_n < expected_n
if operator == ConstraintOperator.EQ:
return _same_scalar(actual, expected, value_type)
if operator == ConstraintOperator.NE:
equal = _same_scalar(actual, expected, value_type)
return None if equal is None else (not equal)
if operator == ConstraintOperator.IN:
if not isinstance(expected, list) or not expected:
return None
matches = [_same_scalar(actual, item, value_type) for item in expected]
if any(item is None for item in matches):
return None
return any(matches)
if operator == ConstraintOperator.CONTAINS:
if not isinstance(actual, list) or not isinstance(expected, str):
return None
if any(not isinstance(item, str) for item in actual):
return None
return expected in actual
return None
+81
View File
@@ -0,0 +1,81 @@
from enum import StrEnum
class ValueType(StrEnum):
NUMBER = "number"
STRING = "string"
BOOLEAN = "boolean"
STRING_LIST = "string_list"
class OfferCondition(StrEnum):
NEW = "new"
REFURBISHED = "refurbished"
USED = "used"
class Availability(StrEnum):
IN_STOCK = "in_stock"
OUT_OF_STOCK = "out_of_stock"
UNKNOWN = "unknown"
class NormalizationStatus(StrEnum):
NORMALIZED = "normalized"
UNKNOWN = "unknown"
class ConstraintStatus(StrEnum):
PASS = "PASS"
FAIL = "FAIL"
UNKNOWN = "UNKNOWN"
class Eligibility(StrEnum):
ELIGIBLE = "ELIGIBLE"
INELIGIBLE = "INELIGIBLE"
UNKNOWN = "UNKNOWN"
class PreferenceStrategy(StrEnum):
NUMERIC_LINEAR = "numeric_linear"
class PreferenceDirection(StrEnum):
HIGHER = "higher"
LOWER = "lower"
class ConstraintOperator(StrEnum):
GE = ">="
LE = "<="
GT = ">"
LT = "<"
EQ = "=="
NE = "!="
IN = "in"
CONTAINS = "contains"
EXISTS = "exists"
class Classification(StrEnum):
IGNORE = "IGNORE"
WATCH = "WATCH"
GOOD = "GOOD"
VERY_GOOD = "VERY_GOOD"
EXCEPTIONAL = "EXCEPTIONAL"
INELIGIBLE = "INELIGIBLE"
INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
class PriceBasis(StrEnum):
BUDGET_ONLY = "BUDGET_ONLY"
BUDGET_AND_HISTORY = "BUDGET_AND_HISTORY"
class PriceReferenceScope(StrEnum):
OFFER = "offer"
class AdapterType(StrEnum):
MANUAL = "manual"
+275
View File
@@ -0,0 +1,275 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID
from app.domain.enums import (
Availability,
ConstraintOperator,
Eligibility,
NormalizationStatus,
OfferCondition,
PreferenceDirection,
PreferenceStrategy,
PriceBasis,
PriceReferenceScope,
ValueType,
)
@dataclass(frozen=True)
class Budget:
target: Decimal
maximum: Decimal
@dataclass(frozen=True)
class Constraint:
id: str
attribute: str
operator: ConstraintOperator
value: Any = None
mandatory: bool = True
@dataclass(frozen=True)
class Preference:
id: str
attribute: str
strategy: PreferenceStrategy
direction: PreferenceDirection
acceptable: Decimal
ideal: Decimal
weight: Decimal
@dataclass(frozen=True)
class ClassificationBound:
upper: Decimal
inclusive: bool
label: str
@dataclass(frozen=True)
class ScoringModel:
version: str
algorithm_id: str
supported_preference_strategies: tuple[str, ...]
minimum_confidence: Decimal
maximum_price_age_hours: int
optional_constraint_penalty_points: Decimal
missing_preference_score: Decimal
product_weight: Decimal
price_weight: Decimal
budget_weight: Decimal
history_weight: Decimal
history_window_days: int
minimum_history_days: int
history_discount_for_max_score: Decimal
classification_upper_bounds: tuple[ClassificationBound, ...]
definition: dict[str, Any]
definition_hash: str
@dataclass(frozen=True)
class Profile:
id: str
version: int
name: str
category: str
currency: str
market: str
allowed_conditions: tuple[OfferCondition, ...]
budget: Budget
scoring_version: str
constraints: tuple[Constraint, ...]
preferences: tuple[Preference, ...]
definition: dict[str, Any]
definition_hash: str
@dataclass(frozen=True)
class AttributeDefinition:
key: str
value_type: ValueType
canonical_unit: str | None
minimum: Decimal | None
maximum: Decimal | None
description: str | None
@dataclass(frozen=True)
class Product:
id: UUID
category: str
brand: str
name: str
model: str | None
created_at: datetime
@dataclass(frozen=True)
class ProductVariant:
id: UUID
product_id: UUID
name: str
manufacturer_sku: str | None
identifiers: dict[str, Any]
created_at: datetime
@dataclass(frozen=True)
class AttributeStatement:
id: UUID
variant_id: UUID
attribute: str
raw_value: Any
normalized_value: Any
unit: str | None
normalization_status: NormalizationStatus
normalizer_version: str
confidence: Decimal
source_id: str
source_reference: str
observed_at: datetime
ingested_at: datetime
normalization_error: str | None = None
@dataclass(frozen=True)
class Offer:
id: UUID
variant_id: UUID
source_id: str
external_id: str
seller: str
url: str | None
market: str
currency: str
condition: OfferCondition
created_at: datetime
@dataclass(frozen=True)
class PriceObservation:
id: UUID
offer_id: UUID
source_id: str
external_observation_id: str
observed_at: datetime
ingested_at: datetime
item_price: Decimal
shipping_price: Decimal | None
currency: str
availability: Availability
raw_payload: Any
confidence: Decimal
@property
def total_price(self) -> Decimal | None:
if self.shipping_price is None:
return None
return self.item_price + self.shipping_price
@dataclass(frozen=True)
class ConstraintResult:
constraint_id: str
attribute: str
operator: str
expected: Any
actual: Any
status: str
mandatory: bool
attribute_row_id: UUID | None
reason_code: str
@dataclass(frozen=True)
class GateResult:
key: str
status: str
reason_code: str
details: dict[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class ScoreComponent:
kind: str
component_key: str
raw_score: Decimal | None
weight: Decimal | None
contribution: Decimal | None
details: dict[str, Any]
@dataclass(frozen=True)
class HistoryWindow:
days: int
values: tuple[Decimal, ...]
observation_ids: tuple[UUID, ...]
@property
def day_count(self) -> int:
return len(self.values)
@dataclass(frozen=True)
class PriceStatistics:
current_total: Decimal | None
median_30d: Decimal | None
median_30d_days: int
median_90d: Decimal | None
median_90d_days: int
lowest_365d: Decimal | None
lowest_365d_days: int
lowest_365d_is_full_year: bool
used_observation_ids: tuple[UUID, ...]
current_observation_id: UUID | None
price_reference_scope: PriceReferenceScope = PriceReferenceScope.OFFER
@dataclass(frozen=True)
class Reason:
code: str
message: str
@dataclass(frozen=True)
class EvaluationResult:
profile_id: str
profile_version: int
scoring_version: str
scoring_hash: str
engine_build: str
variant_id: UUID
offer_id: UUID
as_of: datetime
snapshot_cutoff: datetime
product_eligibility: Eligibility
offer_eligibility: Eligibility
overall_eligibility: Eligibility
eligible: bool | None
product_score: Decimal | None
product_score_raw: Decimal | None
price_score: Decimal | None
price_score_raw: Decimal | None
deal_score: Decimal | None
deal_score_raw: Decimal | None
classification: str
price_basis: PriceBasis | None
product_coverage: Decimal
constraint_results: tuple[ConstraintResult, ...]
product_gates: tuple[GateResult, ...]
offer_gates: tuple[GateResult, ...]
components: tuple[ScoreComponent, ...]
price_statistics: PriceStatistics
price_reference_scope: PriceReferenceScope
reasons: tuple[Reason, ...]
warnings: tuple[str, ...]
input_snapshot: dict[str, Any]
failed_constraints: tuple[ConstraintResult, ...]
unknown_constraints: tuple[ConstraintResult, ...]
+26
View File
@@ -0,0 +1,26 @@
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from uuid import UUID
from app.domain.timeutil import to_iso_z
def jsonable(value: Any) -> Any:
if value is None or isinstance(value, (str, int, bool)):
return value
if isinstance(value, Decimal):
return format(value, "f")
if isinstance(value, UUID):
return str(value)
if isinstance(value, datetime):
return to_iso_z(value)
if isinstance(value, date):
return value.isoformat()
if isinstance(value, dict):
return {str(key): jsonable(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [jsonable(item) for item in value]
if hasattr(value, "value"):
return jsonable(value.value)
return value
+11
View File
@@ -0,0 +1,11 @@
from datetime import datetime, timezone
def require_utc(value: datetime, field_name: str) -> datetime:
if value.tzinfo is None:
raise ValueError(f"{field_name} must include a timezone")
return value.astimezone(timezone.utc)
def to_iso_z(value: datetime) -> str:
return require_utc(value, "datetime").strftime("%Y-%m-%dT%H:%M:%SZ")
+307
View File
@@ -0,0 +1,307 @@
from __future__ import annotations
import hashlib
import json
from decimal import Decimal
from typing import Any
from app.domain.constraints import operator_allowed_for
from app.domain.enums import (
ConstraintOperator,
OfferCondition,
PreferenceDirection,
PreferenceStrategy,
ValueType,
)
from app.domain.models import (
AttributeDefinition,
Budget,
ClassificationBound,
Constraint,
Preference,
Profile,
ScoringModel,
)
SNAKE_CASE = r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$"
CURRENCY = r"^[A-Z]{3}$"
SUPPORTED_ALGORITHM = "linear-budget-history-v1"
class DomainValidationError(ValueError):
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
super().__init__(message)
self.details = details or {}
def canonical_json(data: Any) -> str:
return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def _decimal(value: Any, field: str) -> Decimal:
if isinstance(value, bool) or value is None:
raise DomainValidationError(f"{field} must be a decimal", {"field": field})
try:
return Decimal(str(value))
except Exception as exc:
raise DomainValidationError(f"{field} must be a decimal", {"field": field}) from exc
def _require(mapping: dict[str, Any], key: str) -> Any:
if key not in mapping:
raise DomainValidationError(f"missing field {key}", {"field": key})
return mapping[key]
def validate_attribute_key(key: str) -> None:
import re
if not re.fullmatch(SNAKE_CASE, key):
raise DomainValidationError("attribute key must be snake_case", {"key": key})
def validate_currency(value: str) -> str:
import re
if not re.fullmatch(CURRENCY, value):
raise DomainValidationError("currency must be a 3-letter uppercase code", {"currency": value})
return value
def parse_scoring_model(definition: dict[str, Any]) -> ScoringModel:
version = str(_require(definition, "version"))
algorithm_id = str(_require(definition, "algorithm_id"))
if algorithm_id != SUPPORTED_ALGORITHM:
raise DomainValidationError("unsupported algorithm_id", {"algorithm_id": algorithm_id})
strategies = tuple(str(item) for item in _require(definition, "supported_preference_strategies"))
if not strategies:
raise DomainValidationError("supported_preference_strategies must not be empty")
if PreferenceStrategy.NUMERIC_LINEAR.value not in strategies:
raise DomainValidationError("generic-v1 must support numeric_linear")
min_conf = _decimal(_require(definition, "minimum_confidence"), "minimum_confidence")
if not Decimal("0") <= min_conf <= Decimal("1"):
raise DomainValidationError("minimum_confidence must be between 0 and 1")
max_age = int(_require(definition, "maximum_price_age_hours"))
history_window = int(_require(definition, "history_window_days"))
min_history = int(_require(definition, "minimum_history_days"))
if max_age <= 0 or history_window <= 0 or min_history <= 0:
raise DomainValidationError("time windows must be positive")
product_weight = _decimal(_require(definition, "product_weight"), "product_weight")
price_weight = _decimal(_require(definition, "price_weight"), "price_weight")
budget_weight = _decimal(_require(definition, "budget_weight"), "budget_weight")
history_weight = _decimal(_require(definition, "history_weight"), "history_weight")
for name, weight in (
("product_weight", product_weight),
("price_weight", price_weight),
("budget_weight", budget_weight),
("history_weight", history_weight),
):
if weight < 0:
raise DomainValidationError(f"{name} must be non-negative")
if product_weight + price_weight != Decimal("1"):
raise DomainValidationError("product_weight + price_weight must equal 1")
if budget_weight + history_weight != Decimal("1"):
raise DomainValidationError("budget_weight + history_weight must equal 1")
bounds_raw = _require(definition, "classification_upper_bounds")
bounds: list[ClassificationBound] = []
previous = Decimal("0")
for index, raw in enumerate(bounds_raw):
upper = _decimal(raw["upper"], f"classification_upper_bounds[{index}].upper")
inclusive = bool(raw["inclusive"])
label = str(raw["label"])
if upper < previous or (index and upper == previous):
raise DomainValidationError("classification bounds must be strictly ascending")
if index == 0 and upper <= 0:
raise DomainValidationError("classification bounds must start above 0")
bounds.append(ClassificationBound(upper=upper, inclusive=inclusive, label=label))
previous = upper
if not bounds or bounds[-1].upper != Decimal("100"):
raise DomainValidationError("classification bounds must end at 100")
canonical = json.loads(canonical_json(definition))
return ScoringModel(
version=version,
algorithm_id=algorithm_id,
supported_preference_strategies=strategies,
minimum_confidence=min_conf,
maximum_price_age_hours=max_age,
optional_constraint_penalty_points=_decimal(
_require(definition, "optional_constraint_penalty_points"),
"optional_constraint_penalty_points",
),
missing_preference_score=_decimal(
_require(definition, "missing_preference_score"),
"missing_preference_score",
),
product_weight=product_weight,
price_weight=price_weight,
budget_weight=budget_weight,
history_weight=history_weight,
history_window_days=history_window,
minimum_history_days=min_history,
history_discount_for_max_score=_decimal(
_require(definition, "history_discount_for_max_score"),
"history_discount_for_max_score",
),
classification_upper_bounds=tuple(bounds),
definition=canonical,
definition_hash=sha256_text(canonical_json(canonical)),
)
def parse_profile(
definition: dict[str, Any],
*,
attribute_definitions: dict[str, AttributeDefinition],
scoring_versions: set[str],
supported_strategies: set[str],
) -> Profile:
profile_id = str(_require(definition, "id"))
version = int(_require(definition, "version"))
if version <= 0:
raise DomainValidationError("profile version must be > 0")
name = str(_require(definition, "name"))
category = str(_require(definition, "category"))
if not category:
raise DomainValidationError("category is required")
currency = validate_currency(str(_require(definition, "currency")))
market = str(_require(definition, "market"))
if not market:
raise DomainValidationError("market is required")
raw_conditions = _require(definition, "allowed_conditions")
if not raw_conditions:
raise DomainValidationError("allowed_conditions is required")
conditions = tuple(OfferCondition(item) for item in raw_conditions)
scoring_version = str(_require(definition, "scoring_version"))
if scoring_version not in scoring_versions:
raise DomainValidationError("unknown scoring_version", {"scoring_version": scoring_version})
budget_raw = _require(definition, "budget")
target = _decimal(_require(budget_raw, "target"), "budget.target")
maximum = _decimal(_require(budget_raw, "maximum"), "budget.maximum")
if not (Decimal("0") < target < maximum):
raise DomainValidationError("budget must satisfy 0 < target < maximum")
constraints = _parse_constraints(_require(definition, "constraints"), attribute_definitions)
preferences = _parse_preferences(
_require(definition, "preferences"),
attribute_definitions,
supported_strategies,
)
canonical = json.loads(canonical_json(definition))
return Profile(
id=profile_id,
version=version,
name=name,
category=category,
currency=currency,
market=market,
allowed_conditions=conditions,
budget=Budget(target=target, maximum=maximum),
scoring_version=scoring_version,
constraints=constraints,
preferences=preferences,
definition=canonical,
definition_hash=sha256_text(canonical_json(canonical)),
)
def _parse_constraints(
raw_items: list[dict[str, Any]],
attribute_definitions: dict[str, AttributeDefinition],
) -> tuple[Constraint, ...]:
seen: set[str] = set()
items: list[Constraint] = []
for raw in raw_items:
constraint_id = str(_require(raw, "id"))
if constraint_id in seen:
raise DomainValidationError("duplicate constraint id", {"id": constraint_id})
seen.add(constraint_id)
attribute = str(_require(raw, "attribute"))
definition = attribute_definitions.get(attribute)
if definition is None:
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
operator = ConstraintOperator(str(_require(raw, "operator")))
if not operator_allowed_for(definition.value_type, operator):
raise DomainValidationError(
"operator is not valid for attribute type",
{"attribute": attribute, "operator": operator.value, "value_type": definition.value_type.value},
)
value = raw.get("value")
if operator == ConstraintOperator.EXISTS:
if "value" in raw and raw["value"] is not None:
raise DomainValidationError("exists must not have a value", {"id": constraint_id})
value = None
elif operator == ConstraintOperator.IN:
if not isinstance(value, list) or not value:
raise DomainValidationError("in requires a non-empty list", {"id": constraint_id})
elif operator == ConstraintOperator.CONTAINS:
if not isinstance(value, str):
raise DomainValidationError("contains requires a string", {"id": constraint_id})
items.append(
Constraint(
id=constraint_id,
attribute=attribute,
operator=operator,
value=value,
mandatory=bool(raw.get("mandatory", True)),
)
)
return tuple(items)
def _parse_preferences(
raw_items: list[dict[str, Any]],
attribute_definitions: dict[str, AttributeDefinition],
supported_strategies: set[str],
) -> tuple[Preference, ...]:
if not raw_items:
raise DomainValidationError("at least one preference is required")
seen: set[str] = set()
items: list[Preference] = []
for raw in raw_items:
preference_id = str(_require(raw, "id"))
if preference_id in seen:
raise DomainValidationError("duplicate preference id", {"id": preference_id})
seen.add(preference_id)
attribute = str(_require(raw, "attribute"))
definition = attribute_definitions.get(attribute)
if definition is None:
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
strategy = str(_require(raw, "strategy"))
if strategy not in supported_strategies:
raise DomainValidationError("unsupported preference strategy", {"strategy": strategy})
if strategy != PreferenceStrategy.NUMERIC_LINEAR.value:
raise DomainValidationError("v0.1 only supports numeric_linear", {"strategy": strategy})
if definition.value_type != ValueType.NUMBER:
raise DomainValidationError("numeric_linear requires a number attribute", {"attribute": attribute})
direction = PreferenceDirection(str(_require(raw, "direction")))
acceptable = _decimal(_require(raw, "acceptable"), "acceptable")
ideal = _decimal(_require(raw, "ideal"), "ideal")
weight = _decimal(_require(raw, "weight"), "weight")
if weight <= 0:
raise DomainValidationError("preference weight must be positive", {"id": preference_id})
if direction == PreferenceDirection.HIGHER and not ideal > acceptable:
raise DomainValidationError("higher requires ideal > acceptable", {"id": preference_id})
if direction == PreferenceDirection.LOWER and not ideal < acceptable:
raise DomainValidationError("lower requires ideal < acceptable", {"id": preference_id})
items.append(
Preference(
id=preference_id,
attribute=attribute,
strategy=PreferenceStrategy.NUMERIC_LINEAR,
direction=direction,
acceptable=acceptable,
ideal=ideal,
weight=weight,
)
)
return tuple(items)
+20
View File
@@ -0,0 +1,20 @@
from typing import Any
class AppError(Exception):
def __init__(
self,
status_code: int,
code: str,
message: str,
details: dict[str, Any] | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.details = details or {}
def error_body(code: str, message: str, details: dict[str, Any] | None = None) -> dict[str, Any]:
return {"error": {"code": code, "message": message, "details": details or {}}}
+48
View File
@@ -0,0 +1,48 @@
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api.discovery import router as discovery_router
from app.api.evaluations import router as evaluations_router
from app.api.health import router as health_router
from app.api.offers import router as offers_router
from app.api.products import router as products_router
from app.api.profiles import router as profiles_router
from app.api.projects import router as projects_router
from app.errors import AppError, error_body
from app.settings import get_settings
settings = get_settings()
app = FastAPI(
title="Buying Agent",
version="0.1.0",
description="Generic buying and deal evaluation API. The backend is the system of record; no frontend or n8n is required.",
)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(health_router)
app.include_router(profiles_router)
app.include_router(products_router)
app.include_router(offers_router)
app.include_router(evaluations_router)
app.include_router(projects_router)
app.include_router(discovery_router)
@app.exception_handler(AppError)
def handle_app_error(_request: Request, exc: AppError) -> JSONResponse:
return JSONResponse(error_body(exc.code, exc.message, exc.details), status_code=exc.status_code)
@app.exception_handler(RequestValidationError)
def handle_validation_error(_request: Request, exc: RequestValidationError) -> JSONResponse:
return JSONResponse(
error_body("VALIDATION_ERROR", "request validation failed", {"errors": exc.errors()}),
status_code=422,
)
Whitespace-only changes.
+11
View File
@@ -0,0 +1,11 @@
from typing import Protocol
from app.domain.models import AttributeDefinition, AttributeStatement
class Normalizer(Protocol):
version: str
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
"""Validate an already normalized statement. Must not parse free text."""
...
+61
View File
@@ -0,0 +1,61 @@
from decimal import Decimal
from typing import Any
from app.domain.enums import NormalizationStatus, ValueType
from app.domain.models import AttributeDefinition, AttributeStatement
from app.errors import AppError
class ManualNormalizer:
version = "manual-v1"
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
if statement.normalization_status == NormalizationStatus.UNKNOWN:
if statement.normalized_value is not None:
raise AppError(
422,
"NORMALIZATION_INCONSISTENT",
"unknown statements must not have a normalized_value",
)
return statement
if statement.normalization_status != NormalizationStatus.NORMALIZED:
raise AppError(422, "NORMALIZATION_STATUS", "unsupported normalization_status")
if statement.normalized_value is None:
raise AppError(422, "NORMALIZATION_MISSING", "normalized_value is required")
_assert_type(definition, statement.normalized_value)
if definition.value_type == ValueType.NUMBER:
number = Decimal(str(statement.normalized_value))
if definition.key == "weight_kg" and number <= 0:
raise AppError(422, "VALUE_RANGE", "weight_kg must be > 0")
if definition.minimum is not None and number < definition.minimum:
raise AppError(422, "VALUE_RANGE", f"{definition.key} is below the minimum")
if definition.maximum is not None and number > definition.maximum:
raise AppError(422, "VALUE_RANGE", f"{definition.key} is above the maximum")
if definition.canonical_unit and statement.unit != definition.canonical_unit:
raise AppError(
422,
"UNIT_MISMATCH",
"unit must match the canonical unit",
{"expected": definition.canonical_unit, "actual": statement.unit},
)
if definition.canonical_unit is None and statement.unit not in {None, ""}:
raise AppError(422, "UNIT_MISMATCH", "this attribute has no canonical unit")
return statement
def _assert_type(definition: AttributeDefinition, value: Any) -> None:
if definition.value_type == ValueType.NUMBER:
if isinstance(value, bool) or not isinstance(value, (int, float, Decimal)):
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a number")
return
if definition.value_type == ValueType.STRING:
if not isinstance(value, str):
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a string")
return
if definition.value_type == ValueType.BOOLEAN:
if not isinstance(value, bool):
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a boolean")
return
if definition.value_type == ValueType.STRING_LIST:
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a list of strings")
View File
Whitespace-only changes.
+22
View File
@@ -0,0 +1,22 @@
from collections.abc import Generator
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from app.settings import get_settings
settings = get_settings()
engine = create_engine(settings.database_url, pool_pre_ping=True)
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
def get_session() -> Generator[Session, None, None]:
session = SessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
+321
View File
@@ -0,0 +1,321 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID
from sqlalchemy import Boolean, DateTime, ForeignKey, ForeignKeyConstraint, Integer, Numeric, String, Text, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
from app.persistence.models import Base
class ProductCategoryRecord(Base):
__tablename__ = "product_categories"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="ACTIVE")
class BuyingProjectRecord(Base):
__tablename__ = "buying_projects"
__table_args__ = (
ForeignKeyConstraint(
["template_id", "template_version"],
["product_profiles.id", "product_profiles.version"],
name="fk_buying_projects_template",
),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
workspace_id: Mapped[str | None] = mapped_column(String(64))
product_category_id: Mapped[str] = mapped_column(ForeignKey("product_categories.id"), nullable=False)
market: Mapped[str] = mapped_column(String(8), nullable=False)
currency: Mapped[str] = mapped_column(String(3), nullable=False)
target_budget: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
template_id: Mapped[str | None] = mapped_column(String(128))
template_version: Mapped[int | None] = mapped_column(Integer)
shortlist_size: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
archived_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@property
def profile_id(self) -> str | None:
return self.template_id
@property
def profile_version(self) -> int | None:
return self.template_version
class BuyingProjectCriteriaRecord(Base):
__tablename__ = "buying_project_criteria"
project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), primary_key=True)
version: Mapped[int] = mapped_column(Integer, primary_key=True)
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ProjectCriterionRecord(Base):
__tablename__ = "project_criteria"
__table_args__ = (
ForeignKeyConstraint(
["project_id", "version"],
["buying_project_criteria.project_id", "buying_project_criteria.version"],
name="fk_project_criteria_version",
),
UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
project_id: Mapped[UUID] = mapped_column(nullable=False)
version: Mapped[int] = mapped_column(Integer, nullable=False)
item_id: Mapped[str] = mapped_column(String(128), nullable=False)
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
criterion_key: Mapped[str | None] = mapped_column(String(128))
operator: Mapped[str] = mapped_column(String(16), nullable=False)
value: Mapped[Any] = mapped_column(JSONB)
enabled_for_discovery: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
@property
def mandatory(self) -> bool:
return self.enabled_for_discovery
@property
def criteria_version(self) -> int:
return self.version
class SourceCategoryRecord(Base):
__tablename__ = "source_categories"
__table_args__ = (UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
external_category_key: Mapped[str] = mapped_column(String(64), nullable=False)
name: Mapped[str] = mapped_column(String(256), nullable=False)
source_url: Mapped[str] = mapped_column(Text, nullable=False)
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
canonical_category: Mapped[str | None] = mapped_column(String(128))
status: Mapped[str] = mapped_column(String(32), nullable=False)
last_verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class SourceFilterDefinitionRecord(Base):
__tablename__ = "source_filter_definitions"
__table_args__ = (UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
label: Mapped[str] = mapped_column(String(256), nullable=False)
filter_type: Mapped[str] = mapped_column(String(32), nullable=False)
canonical_unit: Mapped[str | None] = mapped_column(String(32))
status: Mapped[str] = mapped_column(String(32), nullable=False)
class SourceFilterOptionRecord(Base):
__tablename__ = "source_filter_options"
__table_args__ = (UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), nullable=False)
source_value: Mapped[str] = mapped_column(String(256), nullable=False)
normalized_value: Mapped[Any] = mapped_column(JSONB)
operator: Mapped[str | None] = mapped_column(String(16))
unit: Mapped[str | None] = mapped_column(String(32))
class SourceFilterRangeRecord(Base):
__tablename__ = "source_filter_ranges"
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), primary_key=True)
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
step: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
unit: Mapped[str | None] = mapped_column(String(32))
class CriterionDefinitionRecord(Base):
__tablename__ = "criterion_definitions"
__table_args__ = (UniqueConstraint("key", name="uq_criterion_definition_key"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
key: Mapped[str] = mapped_column(String(128), nullable=False)
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="SOURCE")
source: Mapped[str | None] = mapped_column(String(64))
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
label: Mapped[str] = mapped_column(String(256), nullable=False)
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
unit: Mapped[str | None] = mapped_column(String(32))
semantic: Mapped[str | None] = mapped_column(String(128))
category: Mapped[str | None] = mapped_column(String(128))
description: Mapped[str | None] = mapped_column(Text)
scoring_attribute_key: Mapped[str | None] = mapped_column(String(128))
class SourceAttributeMappingRecord(Base):
__tablename__ = "source_attribute_mappings"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
source_category_id: Mapped[UUID | None] = mapped_column(ForeignKey("source_categories.id"))
raw_name: Mapped[str] = mapped_column(String(256), nullable=False)
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class CriterionMappingRecord(Base):
__tablename__ = "criterion_mappings"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
criterion_key: Mapped[str] = mapped_column(String(128), nullable=False)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
source_value: Mapped[str | None] = mapped_column(String(256))
encoding: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class DiscoverySearchRecord(Base):
__tablename__ = "discovery_searches"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
schedule_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, default=180)
broad_result_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=300)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_successful_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class SearchRunRecord(Base):
__tablename__ = "search_runs"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
discovery_search_id: Mapped[UUID] = mapped_column(ForeignKey("discovery_searches.id"), nullable=False)
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status: Mapped[str] = mapped_column(String(32), nullable=False)
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
new_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
changed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
unchanged_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
missing_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
http_status_summary: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
error_code: Mapped[str | None] = mapped_column(String(64))
class SourceProductReferenceRecord(Base):
__tablename__ = "source_product_references"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
source_external_id: Mapped[str | None] = mapped_column(String(256))
source_url: Mapped[str | None] = mapped_column(Text)
source_title: Mapped[str] = mapped_column(String(512), nullable=False)
manufacturer: Mapped[str | None] = mapped_column(String(256))
model: Mapped[str | None] = mapped_column(String(256))
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
gtin_ean: Mapped[str | None] = mapped_column(String(32))
normalized_spec_fingerprint: Mapped[str | None] = mapped_column(Text)
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
lifecycle_status: Mapped[str] = mapped_column(String(32), nullable=False)
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
consecutive_missing_success_runs: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
class ProductObservationRecord(Base):
__tablename__ = "product_observations"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
raw_title: Mapped[str] = mapped_column(String(512), nullable=False)
best_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
currency: Mapped[str | None] = mapped_column(String(3))
availability: Mapped[str | None] = mapped_column(String(32))
seller_count: Mapped[int | None] = mapped_column(Integer)
class ProductObservationAttributeRecord(Base):
__tablename__ = "product_observation_attributes"
__table_args__ = (UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
observation_id: Mapped[UUID] = mapped_column(ForeignKey("product_observations.id"), nullable=False)
criterion_definition_id: Mapped[UUID | None] = mapped_column(ForeignKey("criterion_definitions.id"))
attribute_key: Mapped[str] = mapped_column(String(128), nullable=False)
raw_name: Mapped[str | None] = mapped_column(String(256))
raw_value: Mapped[Any] = mapped_column(JSONB)
normalized_value: Mapped[Any] = mapped_column(JSONB)
unit: Mapped[str | None] = mapped_column(String(32))
class SearchRunItemRecord(Base):
__tablename__ = "search_run_items"
__table_args__ = (UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
observation_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_observations.id"))
diff_status: Mapped[str] = mapped_column(String(32), nullable=False)
class ProjectCandidateRecord(Base):
__tablename__ = "project_candidates"
__table_args__ = (
UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
status: Mapped[str] = mapped_column(String(32), nullable=False)
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
product_eligibility: Mapped[str | None] = mapped_column(String(16))
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class CandidateStatusHistoryRecord(Base):
__tablename__ = "candidate_status_history"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
candidate_id: Mapped[UUID] = mapped_column(ForeignKey("project_candidates.id"), nullable=False)
from_status: Mapped[str | None] = mapped_column(String(32))
to_status: Mapped[str] = mapped_column(String(32), nullable=False)
reason: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+105
View File
@@ -0,0 +1,105 @@
from decimal import Decimal
from app.domain.enums import (
Availability,
NormalizationStatus,
OfferCondition,
ValueType,
)
from app.domain.models import (
AttributeDefinition,
AttributeStatement,
Offer,
PriceObservation,
Product,
ProductVariant,
)
from app.persistence import models as orm
def decimal_or_none(value) -> Decimal | None:
return None if value is None else Decimal(str(value))
def to_definition(row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
return AttributeDefinition(
key=row.key,
value_type=ValueType(row.value_type),
canonical_unit=row.canonical_unit,
minimum=decimal_or_none(row.minimum),
maximum=decimal_or_none(row.maximum),
description=row.description,
)
def to_product(row: orm.ProductRecord) -> Product:
return Product(
id=row.id,
category=row.category,
brand=row.brand,
name=row.name,
model=row.model,
created_at=row.created_at,
)
def to_variant(row: orm.ProductVariantRecord) -> ProductVariant:
return ProductVariant(
id=row.id,
product_id=row.product_id,
name=row.name,
manufacturer_sku=row.manufacturer_sku,
identifiers=row.identifiers or {},
created_at=row.created_at,
)
def to_attribute(row: orm.VariantAttributeRecord) -> AttributeStatement:
return AttributeStatement(
id=row.id,
variant_id=row.variant_id,
attribute=row.attribute_key,
raw_value=row.raw_value,
normalized_value=row.normalized_value,
unit=row.unit,
normalization_status=NormalizationStatus(row.normalization_status),
normalizer_version=row.normalizer_version,
confidence=Decimal(str(row.confidence)),
source_id=row.source_id,
source_reference=row.source_reference,
observed_at=row.observed_at,
ingested_at=row.ingested_at,
normalization_error=row.normalization_error,
)
def to_offer(row: orm.OfferRecord) -> Offer:
return Offer(
id=row.id,
variant_id=row.variant_id,
source_id=row.source_id,
external_id=row.external_id,
seller=row.seller,
url=row.url,
market=row.market,
currency=row.currency,
condition=OfferCondition(row.condition),
created_at=row.created_at,
)
def to_observation(row: orm.PriceObservationRecord) -> PriceObservation:
return PriceObservation(
id=row.id,
offer_id=row.offer_id,
source_id=row.source_id,
external_observation_id=row.external_observation_id,
observed_at=row.observed_at,
ingested_at=row.ingested_at,
item_price=Decimal(str(row.item_price)),
shipping_price=decimal_or_none(row.shipping_price),
currency=row.currency,
availability=Availability(row.availability),
raw_payload=row.raw_payload,
confidence=Decimal(str(row.confidence)),
)
+249
View File
@@ -0,0 +1,249 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Numeric,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
pass
class AttributeDefinitionRecord(Base):
__tablename__ = "attribute_definitions"
__table_args__ = (
CheckConstraint(
"value_type IN ('number', 'string', 'boolean', 'string_list')",
name="ck_attribute_definitions_value_type",
),
)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
canonical_unit: Mapped[str | None] = mapped_column(String(32))
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
description: Mapped[str | None] = mapped_column(Text)
class SourceRecord(Base):
__tablename__ = "sources"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(128), nullable=False)
adapter_type: Mapped[str] = mapped_column(String(64), nullable=False)
class ScoringModelRecord(Base):
__tablename__ = "scoring_models"
version: Mapped[str] = mapped_column(String(64), primary_key=True)
algorithm_id: Mapped[str] = mapped_column(String(128), nullable=False)
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ProductProfileRecord(Base):
__tablename__ = "product_profiles"
__table_args__ = (
CheckConstraint("version > 0", name="ck_product_profiles_version"),
)
id: Mapped[str] = mapped_column(String(128), primary_key=True)
version: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
category: Mapped[str] = mapped_column(String(128), nullable=False)
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ProductRecord(Base):
__tablename__ = "products"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
category: Mapped[str] = mapped_column(String(128), nullable=False)
brand: Mapped[str] = mapped_column(String(128), nullable=False)
name: Mapped[str] = mapped_column(String(256), nullable=False)
model: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
variants: Mapped[list["ProductVariantRecord"]] = relationship(back_populates="product")
class ProductVariantRecord(Base):
__tablename__ = "product_variants"
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
product_id: Mapped[UUID] = mapped_column(ForeignKey("products.id"), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
identifiers: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
product: Mapped[ProductRecord] = relationship(back_populates="variants")
class VariantAttributeRecord(Base):
__tablename__ = "variant_attributes"
__table_args__ = (
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
CheckConstraint(
"normalization_status IN ('normalized', 'unknown')",
name="ck_variant_attributes_status",
),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False)
attribute_key: Mapped[str] = mapped_column(ForeignKey("attribute_definitions.key"), nullable=False)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
raw_value: Mapped[Any] = mapped_column(JSONB, nullable=False)
normalized_value: Mapped[Any] = mapped_column(JSONB)
unit: Mapped[str | None] = mapped_column(String(32))
normalization_status: Mapped[str] = mapped_column(String(32), nullable=False)
normalizer_version: Mapped[str] = mapped_column(String(64), nullable=False)
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
source_reference: Mapped[str] = mapped_column(String(256), nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
normalization_error: Mapped[str | None] = mapped_column(Text)
class OfferRecord(Base):
__tablename__ = "offers"
__table_args__ = (
UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
external_id: Mapped[str] = mapped_column(String(256), nullable=False)
seller: Mapped[str] = mapped_column(String(256), nullable=False)
url: Mapped[str | None] = mapped_column(Text)
market: Mapped[str] = mapped_column(String(16), nullable=False)
currency: Mapped[str] = mapped_column(String(3), nullable=False)
condition: Mapped[str] = mapped_column(String(32), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class PriceObservationRecord(Base):
__tablename__ = "price_observations"
__table_args__ = (
UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
CheckConstraint(
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
name="ck_price_obs_availability",
),
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
external_observation_id: Mapped[str] = mapped_column(String(256), nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
item_price: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
shipping_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
currency: Mapped[str] = mapped_column(String(3), nullable=False)
availability: Mapped[str] = mapped_column(String(32), nullable=False)
raw_payload: Mapped[Any] = mapped_column(JSONB, nullable=False)
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
class EvaluationRecord(Base):
__tablename__ = "evaluations"
__table_args__ = (
ForeignKeyConstraint(
["profile_id", "profile_version"],
["product_profiles.id", "product_profiles.version"],
name="fk_evaluations_profile",
),
CheckConstraint(
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_product_eligibility",
),
CheckConstraint(
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_offer_eligibility",
),
CheckConstraint(
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_overall_eligibility",
),
CheckConstraint(
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
name="ck_eval_product_score",
),
CheckConstraint(
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
name="ck_eval_price_score",
),
CheckConstraint(
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
name="ck_eval_deal_score",
),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
profile_id: Mapped[str] = mapped_column(String(128), nullable=False)
profile_version: Mapped[int] = mapped_column(nullable=False)
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
as_of: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
snapshot_cutoff: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
scoring_hash: Mapped[str] = mapped_column(String(64), nullable=False)
engine_build: Mapped[str] = mapped_column(String(64), nullable=False)
product_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
offer_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
overall_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
price_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
deal_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
classification: Mapped[str] = mapped_column(String(32), nullable=False)
price_reference_scope: Mapped[str] = mapped_column(String(32), nullable=False)
input_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
result_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
components: Mapped[list["EvaluationComponentRecord"]] = relationship(back_populates="evaluation")
class EvaluationComponentRecord(Base):
__tablename__ = "evaluation_components"
__table_args__ = (
UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
)
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"), nullable=False)
kind: Mapped[str] = mapped_column(String(64), nullable=False)
component_key: Mapped[str] = mapped_column(String(128), nullable=False)
raw_score: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
weight: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
contribution: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
details: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
evaluation: Mapped[EvaluationRecord] = relationship(back_populates="components")
+183
View File
@@ -0,0 +1,183 @@
from datetime import datetime
from typing import Any
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
from app.domain.validation import parse_profile, parse_scoring_model
from app.persistence import models as orm
from app.persistence.mapping import to_attribute, to_definition, to_observation, to_offer, to_product, to_variant
class CatalogRepository:
def __init__(self, session: Session) -> None:
self.session = session
def get_definition(self, key: str) -> AttributeDefinition | None:
row = self.session.get(orm.AttributeDefinitionRecord, key)
return None if row is None else to_definition(row)
def list_definitions(self, limit: int, offset: int) -> tuple[list[AttributeDefinition], int]:
total = self.session.scalar(select(func.count()).select_from(orm.AttributeDefinitionRecord)) or 0
rows = self.session.scalars(
select(orm.AttributeDefinitionRecord).order_by(orm.AttributeDefinitionRecord.key).limit(limit).offset(offset)
).all()
return [to_definition(row) for row in rows], total
def add_definition(self, row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
self.session.add(row)
self.session.flush()
return to_definition(row)
def list_sources(self) -> list[orm.SourceRecord]:
return list(self.session.scalars(select(orm.SourceRecord).order_by(orm.SourceRecord.id)).all())
def get_source(self, source_id: str) -> orm.SourceRecord | None:
return self.session.get(orm.SourceRecord, source_id)
def get_scoring_model(self, version: str):
row = self.session.get(orm.ScoringModelRecord, version)
return None if row is None else parse_scoring_model(row.definition)
def scoring_versions(self) -> set[str]:
return set(self.session.scalars(select(orm.ScoringModelRecord.version)).all())
def all_definitions(self) -> dict[str, AttributeDefinition]:
rows = self.session.scalars(select(orm.AttributeDefinitionRecord)).all()
return {row.key: to_definition(row) for row in rows}
def get_profile(self, profile_id: str, version: int):
row = self.session.get(orm.ProductProfileRecord, (profile_id, version))
if row is None:
return None
model = self.get_scoring_model(row.scoring_version)
if model is None:
return None
return parse_profile(
row.definition,
attribute_definitions=self.all_definitions(),
scoring_versions=self.scoring_versions(),
supported_strategies=set(model.supported_preference_strategies),
)
def add_profile(self, row: orm.ProductProfileRecord):
self.session.add(row)
self.session.flush()
return row
def list_profiles(self, limit: int, offset: int) -> tuple[list[orm.ProductProfileRecord], int]:
total = self.session.scalar(select(func.count()).select_from(orm.ProductProfileRecord)) or 0
rows = self.session.scalars(
select(orm.ProductProfileRecord)
.order_by(orm.ProductProfileRecord.id, orm.ProductProfileRecord.version)
.limit(limit)
.offset(offset)
).all()
return list(rows), total
def get_product(self, product_id: UUID) -> Product | None:
row = self.session.get(orm.ProductRecord, product_id)
return None if row is None else to_product(row)
def add_product(self, row: orm.ProductRecord) -> Product:
self.session.add(row)
self.session.flush()
return to_product(row)
def get_variant(self, variant_id: UUID) -> ProductVariant | None:
row = self.session.get(orm.ProductVariantRecord, variant_id)
return None if row is None else to_variant(row)
def add_variant(self, row: orm.ProductVariantRecord) -> ProductVariant:
self.session.add(row)
self.session.flush()
return to_variant(row)
def list_attributes(self, variant_id: UUID) -> list[AttributeStatement]:
rows = self.session.scalars(
select(orm.VariantAttributeRecord)
.where(orm.VariantAttributeRecord.variant_id == variant_id)
.order_by(
orm.VariantAttributeRecord.observed_at,
orm.VariantAttributeRecord.ingested_at,
orm.VariantAttributeRecord.id,
)
).all()
return [to_attribute(row) for row in rows]
def get_attribute(self, attribute_id: UUID) -> AttributeStatement | None:
row = self.session.get(orm.VariantAttributeRecord, attribute_id)
return None if row is None else to_attribute(row)
def add_attribute(self, row: orm.VariantAttributeRecord) -> AttributeStatement:
self.session.add(row)
self.session.flush()
return to_attribute(row)
def get_offer(self, offer_id: UUID) -> Offer | None:
row = self.session.get(orm.OfferRecord, offer_id)
return None if row is None else to_offer(row)
def get_offer_by_external(self, source_id: str, external_id: str) -> Offer | None:
row = self.session.scalar(
select(orm.OfferRecord).where(
orm.OfferRecord.source_id == source_id,
orm.OfferRecord.external_id == external_id,
)
)
return None if row is None else to_offer(row)
def add_offer(self, row: orm.OfferRecord) -> Offer:
self.session.add(row)
self.session.flush()
return to_offer(row)
def list_observations(self, offer_id: UUID, limit: int | None = None, offset: int = 0) -> tuple[list[PriceObservation], int]:
total = (
self.session.scalar(
select(func.count())
.select_from(orm.PriceObservationRecord)
.where(orm.PriceObservationRecord.offer_id == offer_id)
)
or 0
)
query = (
select(orm.PriceObservationRecord)
.where(orm.PriceObservationRecord.offer_id == offer_id)
.order_by(
orm.PriceObservationRecord.observed_at,
orm.PriceObservationRecord.ingested_at,
orm.PriceObservationRecord.id,
)
.offset(offset)
)
if limit is not None:
query = query.limit(limit)
rows = self.session.scalars(query).all()
return [to_observation(row) for row in rows], total
def get_observation_by_external(self, offer_id: UUID, external_id: str) -> orm.PriceObservationRecord | None:
return self.session.scalar(
select(orm.PriceObservationRecord).where(
orm.PriceObservationRecord.offer_id == offer_id,
orm.PriceObservationRecord.external_observation_id == external_id,
)
)
def add_observation(self, row: orm.PriceObservationRecord) -> PriceObservation:
self.session.add(row)
self.session.flush()
return to_observation(row)
def get_evaluation(self, evaluation_id: UUID) -> orm.EvaluationRecord | None:
return self.session.get(orm.EvaluationRecord, evaluation_id)
def add_evaluation(self, row: orm.EvaluationRecord) -> orm.EvaluationRecord:
self.session.add(row)
self.session.flush()
return row
def add_component(self, row: orm.EvaluationComponentRecord) -> None:
self.session.add(row)
View File
Whitespace-only changes.
+42
View File
@@ -0,0 +1,42 @@
from decimal import Decimal
from app.domain.enums import Classification, Eligibility
from app.domain.models import ScoringModel
from app.scoring.mathutil import clamp
def classify_deal_score(score: Decimal, model: ScoringModel) -> str:
for bound in model.classification_upper_bounds:
if bound.inclusive and score <= bound.upper:
return bound.label
if not bound.inclusive and score < bound.upper:
return bound.label
return Classification.EXCEPTIONAL.value
def score_deal(product_score: Decimal, price_score: Decimal, model: ScoringModel) -> Decimal:
return clamp(model.product_weight * product_score + model.price_weight * price_score)
def eligibility_from_parts(product: Eligibility, offer: Eligibility) -> Eligibility:
if Eligibility.INELIGIBLE in {product, offer}:
return Eligibility.INELIGIBLE
if Eligibility.UNKNOWN in {product, offer}:
return Eligibility.UNKNOWN
return Eligibility.ELIGIBLE
def derive_eligible(overall: Eligibility) -> bool | None:
if overall == Eligibility.ELIGIBLE:
return True
if overall == Eligibility.INELIGIBLE:
return False
return None
def classification_for(overall: Eligibility, deal_score: Decimal | None, model: ScoringModel) -> str:
if overall == Eligibility.INELIGIBLE:
return Classification.INELIGIBLE.value
if overall == Eligibility.UNKNOWN or deal_score is None:
return Classification.INSUFFICIENT_DATA.value
return classify_deal_score(deal_score, model)
+545
View File
@@ -0,0 +1,545 @@
from collections.abc import Sequence
from datetime import datetime, timedelta
from decimal import Decimal
from typing import Any
from app.domain.attributes import select_attribute_snapshot
from app.domain.constraints import evaluate_constraint
from app.domain.enums import (
Availability,
ConstraintStatus,
Eligibility,
PriceReferenceScope,
)
from app.domain.models import (
AttributeDefinition,
AttributeStatement,
ConstraintResult,
EvaluationResult,
GateResult,
Offer,
PriceObservation,
Product,
ProductVariant,
Profile,
Reason,
ScoreComponent,
ScoringModel,
)
from app.domain.serialize import jsonable
from app.scoring.deal import classification_for, derive_eligible, eligibility_from_parts, score_deal
from app.scoring.mathutil import round_score
from app.scoring.price import (
build_offer_reference_history,
price_statistics_from_windows,
score_price,
select_current_observation,
)
from app.scoring.product import score_product
def evaluate_deal(
*,
profile: Profile,
scoring_model: ScoringModel,
product: Product,
variant: ProductVariant,
offer: Offer,
attributes: tuple[AttributeStatement, ...],
observations: tuple[PriceObservation, ...],
definitions: dict[str, AttributeDefinition],
as_of: datetime,
snapshot_cutoff: datetime,
engine_build: str,
) -> EvaluationResult:
observations = tuple(item for item in observations if item.offer_id == offer.id)
snapshot = select_attribute_snapshot(attributes, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
constraint_results = tuple(
evaluate_constraint(
constraint,
snapshot.get(constraint.attribute),
definitions[constraint.attribute],
scoring_model.minimum_confidence,
)
for constraint in profile.constraints
)
product_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
product_score_raw, coverage, preference_components = score_product(
profile.preferences,
snapshot,
constraint_results,
scoring_model,
)
current = select_current_observation(observations, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
offer_gates, offer_eligibility = _offer_eligibility(profile, offer, current, scoring_model, as_of)
overall = eligibility_from_parts(product_eligibility, offer_eligibility)
windows: dict[int, Any] = {}
price_score_raw: Decimal | None = None
budget_score: Decimal | None = None
history_score: Decimal | None = None
price_basis = None
history_warning: str | None = None
if _can_score_price(current, profile, scoring_model, as_of):
windows = build_offer_reference_history(observations, current, scoring_model)
price_score_raw, budget_score, history_score, price_basis, history_warning = score_price(
current_price=current.total_price,
reference_history=windows[90].values,
budget=profile.budget,
model=scoring_model,
)
stats = price_statistics_from_windows(current, windows)
deal_score_raw = None
if overall == Eligibility.ELIGIBLE and price_score_raw is not None:
deal_score_raw = score_deal(product_score_raw, price_score_raw, scoring_model)
classification = classification_for(overall, deal_score_raw, scoring_model)
reasons, warnings = _reasons(
constraint_results,
product_gates,
offer_gates,
stats,
current,
history_warning,
price_score_raw,
)
components = _all_components(
preference_components,
product_gates,
offer_gates,
budget_score,
history_score,
scoring_model,
)
input_snapshot = {
"profile": profile.definition,
"scoring_model": scoring_model.definition,
"product": jsonable(_product_dict(product)),
"variant": jsonable(_variant_dict(variant)),
"offer": jsonable(_offer_dict(offer)),
"attribute_snapshot": {
key: jsonable(_attribute_dict(value))
for key, value in snapshot.items()
},
"current_observation": None if current is None else jsonable(_observation_dict(current)),
"history_observation_ids": [str(item) for item in stats.used_observation_ids],
"as_of": jsonable(as_of),
"snapshot_cutoff": jsonable(snapshot_cutoff),
"price_reference_scope": PriceReferenceScope.OFFER.value,
"unrounded": {
"product_score": jsonable(product_score_raw),
"price_score": jsonable(price_score_raw),
"deal_score": jsonable(deal_score_raw),
"budget_score": jsonable(budget_score),
"history_score": jsonable(history_score),
},
}
return EvaluationResult(
profile_id=profile.id,
profile_version=profile.version,
scoring_version=scoring_model.version,
scoring_hash=scoring_model.definition_hash,
engine_build=engine_build,
variant_id=variant.id,
offer_id=offer.id,
as_of=as_of,
snapshot_cutoff=snapshot_cutoff,
product_eligibility=product_eligibility,
offer_eligibility=offer_eligibility,
overall_eligibility=overall,
eligible=derive_eligible(overall),
product_score=round_score(product_score_raw),
product_score_raw=product_score_raw,
price_score=None if price_score_raw is None else round_score(price_score_raw),
price_score_raw=price_score_raw,
deal_score=None if deal_score_raw is None else round_score(deal_score_raw),
deal_score_raw=deal_score_raw,
classification=classification,
price_basis=price_basis,
product_coverage=coverage,
constraint_results=constraint_results,
product_gates=product_gates,
offer_gates=offer_gates,
components=components,
price_statistics=stats,
price_reference_scope=PriceReferenceScope.OFFER,
reasons=reasons,
warnings=warnings,
input_snapshot=input_snapshot,
failed_constraints=tuple(
item for item in constraint_results if item.status == ConstraintStatus.FAIL.value
),
unknown_constraints=tuple(
item for item in constraint_results if item.status == ConstraintStatus.UNKNOWN.value
),
)
def statement_price_confidence(observation: PriceObservation, model: ScoringModel) -> bool:
return observation.confidence >= model.minimum_confidence
def _fresh_enough(observation: PriceObservation, as_of: datetime, model: ScoringModel) -> bool:
return as_of - observation.observed_at <= timedelta(hours=model.maximum_price_age_hours)
def _product_eligibility(
profile: Profile,
product: Product,
constraint_results: tuple[ConstraintResult, ...],
) -> tuple[tuple[GateResult, ...], Eligibility]:
category_ok = product.category == profile.category
category_gate = GateResult(
key="category",
status="PASS" if category_ok else "FAIL",
reason_code="CATEGORY_MATCH" if category_ok else "CATEGORY_MISMATCH",
details={"expected": profile.category, "actual": product.category},
)
mandatory = [item for item in constraint_results if item.mandatory]
if not category_ok or any(item.status == ConstraintStatus.FAIL.value for item in mandatory):
eligibility = Eligibility.INELIGIBLE
elif any(item.status == ConstraintStatus.UNKNOWN.value for item in mandatory):
eligibility = Eligibility.UNKNOWN
else:
eligibility = Eligibility.ELIGIBLE
return (category_gate,), eligibility
def _offer_eligibility(
profile: Profile,
offer: Offer,
current: PriceObservation | None,
model: ScoringModel,
as_of: datetime,
) -> tuple[tuple[GateResult, ...], Eligibility]:
gates: list[GateResult] = []
market_ok = offer.market == profile.market
gates.append(_gate("market", market_ok, "MARKET_MATCH", "MARKET_MISMATCH", offer.market, profile.market))
condition_ok = offer.condition in profile.allowed_conditions
gates.append(
_gate(
"condition",
condition_ok,
"CONDITION_ALLOWED",
"CONDITION_NOT_ALLOWED",
offer.condition.value,
[item.value for item in profile.allowed_conditions],
)
)
currency_ok = offer.currency == profile.currency
gates.append(_gate("currency", currency_ok, "CURRENCY_MATCH", "CURRENCY_MISMATCH", offer.currency, profile.currency))
if current is None:
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_MISSING", {}))
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_MISSING", {}))
gates.append(GateResult("price_confidence", "UNKNOWN", "PRICE_MISSING", {}))
gates.append(GateResult("shipping", "UNKNOWN", "SHIPPING_UNKNOWN", {}))
gates.append(GateResult("budget", "UNKNOWN", "PRICE_MISSING", {}))
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
return tuple(gates), _combine_offer_gates(gates)
fresh = _fresh_enough(current, as_of, model)
if not fresh:
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_STALE", {"observed_at": current.observed_at.isoformat()}))
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_STALE", {}))
else:
gates.append(GateResult("price_present", "PASS", "PRICE_PRESENT", {"observation_id": str(current.id)}))
gates.append(GateResult("price_fresh", "PASS", "PRICE_FRESH", {}))
confident = statement_price_confidence(current, model)
gates.append(
GateResult(
"price_confidence",
"PASS" if confident else "UNKNOWN",
"PRICE_CONFIDENCE_OK" if confident else "PRICE_LOW_CONFIDENCE",
{"confidence": str(current.confidence)},
)
)
shipping_known = current.shipping_price is not None
gates.append(
GateResult(
"shipping",
"PASS" if shipping_known else "UNKNOWN",
"SHIPPING_KNOWN" if shipping_known else "SHIPPING_UNKNOWN",
{},
)
)
total = current.total_price
if total is None or not currency_ok:
gates.append(GateResult("budget", "UNKNOWN", "PRICE_INCOMPLETE", {}))
elif total <= profile.budget.maximum:
gates.append(GateResult("budget", "PASS", "WITHIN_BUDGET", {"total": str(total)}))
else:
gates.append(GateResult("budget", "FAIL", "OVER_BUDGET", {"total": str(total)}))
if current.availability == Availability.IN_STOCK:
gates.append(GateResult("availability", "PASS", "IN_STOCK", {}))
elif current.availability == Availability.OUT_OF_STOCK:
gates.append(GateResult("availability", "FAIL", "OUT_OF_STOCK", {}))
else:
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
if current.currency != offer.currency:
gates.append(GateResult("observation_currency", "FAIL", "CURRENCY_MISMATCH", {}))
if current.source_id != offer.source_id:
gates.append(GateResult("observation_source", "FAIL", "SOURCE_MISMATCH", {}))
return tuple(gates), _combine_offer_gates(gates)
def _gate(key: str, ok: bool, pass_code: str, fail_code: str, actual: Any, expected: Any) -> GateResult:
return GateResult(
key=key,
status="PASS" if ok else "FAIL",
reason_code=pass_code if ok else fail_code,
details={"actual": actual, "expected": expected},
)
def _can_score_price(
current: PriceObservation | None,
profile: Profile,
model: ScoringModel,
as_of: datetime,
) -> bool:
if current is None:
return False
if current.currency != profile.currency:
return False
if current.total_price is None:
return False
if not _fresh_enough(current, as_of, model):
return False
return statement_price_confidence(current, model)
def _combine_offer_gates(gates: Sequence[GateResult]) -> Eligibility:
statuses = [gate.status for gate in gates]
if "FAIL" in statuses:
return Eligibility.INELIGIBLE
if "UNKNOWN" in statuses:
return Eligibility.UNKNOWN
return Eligibility.ELIGIBLE
def _reasons(
constraints: tuple[ConstraintResult, ...],
product_gates: tuple[GateResult, ...],
offer_gates: tuple[GateResult, ...],
stats,
current: PriceObservation | None,
history_warning: str | None,
price_score_raw: Decimal | None,
) -> tuple[tuple[Reason, ...], tuple[str, ...]]:
reasons: list[Reason] = []
warnings: list[str] = []
for gate in (*product_gates, *offer_gates):
if gate.status != "PASS":
reasons.append(Reason(gate.reason_code, _gate_message(gate)))
for item in constraints:
if item.status != ConstraintStatus.PASS.value:
reasons.append(Reason(item.reason_code, _constraint_message(item)))
if history_warning:
warnings.append(history_warning)
reasons.append(Reason(history_warning, "Zu wenige Historientage für den 90-Tage-Median."))
if stats.lowest_365d is not None and not stats.lowest_365d_is_full_year:
warnings.append("LOWEST_365D_PARTIAL")
reasons.append(
Reason(
"LOWEST_365D_PARTIAL",
f"lowest_365d {stats.lowest_365d} ist das Minimum der vorhandenen {stats.lowest_365d_days} Tage, kein belegtes Jahrestief.",
)
)
if (
price_score_raw is not None
and current is not None
and current.total_price is not None
and stats.median_90d is not None
and stats.median_90d != 0
):
delta = (stats.median_90d - current.total_price) / stats.median_90d * Decimal("100")
percent = round_score(delta)
if current.total_price < stats.median_90d:
reasons.append(
Reason(
"PRICE_BELOW_MEDIAN",
f"Gesamtpreis {_de_decimal(percent)} % unter dem Median der beobachteten 90-Tage-Historie.",
)
)
elif current.total_price > stats.median_90d:
reasons.append(
Reason(
"PRICE_ABOVE_MEDIAN",
f"Gesamtpreis {_de_decimal(percent.copy_abs())} % über dem Median der beobachteten 90-Tage-Historie.",
)
)
else:
reasons.append(Reason("PRICE_AT_MEDIAN", "Gesamtpreis entspricht dem Median der 90-Tage-Historie."))
return tuple(reasons), tuple(warnings)
def _de_decimal(value: Decimal) -> str:
return format(value, "f").replace(".", ",")
def _gate_message(gate: GateResult) -> str:
messages = {
"CATEGORY_MISMATCH": "Die Produktkategorie entspricht nicht dem Profil.",
"MARKET_MISMATCH": "Der Angebotsmarkt entspricht nicht dem Profil.",
"CONDITION_NOT_ALLOWED": "Der Angebotszustand ist im Profil nicht erlaubt.",
"CURRENCY_MISMATCH": "Die Währung entspricht nicht dem Profil.",
"PRICE_MISSING": "Es liegt keine aktuelle Preisbeobachtung vor.",
"PRICE_STALE": "Die aktuelle Preisbeobachtung ist nicht frisch genug.",
"PRICE_LOW_CONFIDENCE": "Die Preis-Confidence liegt unter der Mindestschwelle.",
"SHIPPING_UNKNOWN": "Obligatorische Versandkosten sind unbekannt.",
"OVER_BUDGET": "Der Gesamtpreis liegt über dem Budgetmaximum.",
"OUT_OF_STOCK": "Das Angebot ist nicht lieferbar.",
"AVAILABILITY_UNKNOWN": "Die Verfügbarkeit ist unbekannt.",
"SOURCE_MISMATCH": "Die Beobachtungsquelle weicht vom Angebot ab.",
}
return messages.get(gate.reason_code, gate.reason_code)
def _constraint_message(item: ConstraintResult) -> str:
if item.status == ConstraintStatus.FAIL.value:
return f"Constraint {item.constraint_id} ist nicht erfüllt."
return f"Constraint {item.constraint_id} kann nicht bewertet werden."
def _all_components(
preference_components: tuple[ScoreComponent, ...],
product_gates: tuple[GateResult, ...],
offer_gates: tuple[GateResult, ...],
budget_score: Decimal | None,
history_score: Decimal | None,
model: ScoringModel,
) -> tuple[ScoreComponent, ...]:
extra: list[ScoreComponent] = list(preference_components)
if budget_score is not None:
extra.append(
ScoreComponent(
kind="budget",
component_key="budget",
raw_score=budget_score,
weight=model.budget_weight,
contribution=model.budget_weight * budget_score,
details={},
)
)
if history_score is not None:
extra.append(
ScoreComponent(
kind="history",
component_key="history",
raw_score=history_score,
weight=model.history_weight,
contribution=model.history_weight * history_score,
details={},
)
)
for gate in product_gates:
extra.append(
ScoreComponent(
kind="product_gate",
component_key=gate.key,
raw_score=None,
weight=None,
contribution=None,
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
)
)
for gate in offer_gates:
extra.append(
ScoreComponent(
kind="offer_gate",
component_key=gate.key,
raw_score=None,
weight=None,
contribution=None,
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
)
)
return tuple(extra)
def _product_dict(product: Product) -> dict[str, Any]:
return {
"id": product.id,
"category": product.category,
"brand": product.brand,
"name": product.name,
"model": product.model,
"created_at": product.created_at,
}
def _variant_dict(variant: ProductVariant) -> dict[str, Any]:
return {
"id": variant.id,
"product_id": variant.product_id,
"name": variant.name,
"manufacturer_sku": variant.manufacturer_sku,
"identifiers": variant.identifiers,
"created_at": variant.created_at,
}
def _offer_dict(offer: Offer) -> dict[str, Any]:
return {
"id": offer.id,
"variant_id": offer.variant_id,
"source_id": offer.source_id,
"external_id": offer.external_id,
"seller": offer.seller,
"url": offer.url,
"market": offer.market,
"currency": offer.currency,
"condition": offer.condition,
"created_at": offer.created_at,
}
def _attribute_dict(statement: AttributeStatement) -> dict[str, Any]:
return {
"id": statement.id,
"variant_id": statement.variant_id,
"attribute": statement.attribute,
"raw_value": statement.raw_value,
"normalized_value": statement.normalized_value,
"unit": statement.unit,
"normalization_status": statement.normalization_status,
"normalizer_version": statement.normalizer_version,
"confidence": statement.confidence,
"source_id": statement.source_id,
"source_reference": statement.source_reference,
"observed_at": statement.observed_at,
"ingested_at": statement.ingested_at,
"normalization_error": statement.normalization_error,
}
def _observation_dict(observation: PriceObservation) -> dict[str, Any]:
return {
"id": observation.id,
"offer_id": observation.offer_id,
"source_id": observation.source_id,
"external_observation_id": observation.external_observation_id,
"observed_at": observation.observed_at,
"ingested_at": observation.ingested_at,
"item_price": observation.item_price,
"shipping_price": observation.shipping_price,
"currency": observation.currency,
"availability": observation.availability,
"raw_payload": observation.raw_payload,
"confidence": observation.confidence,
"total_price": observation.total_price,
}
+21
View File
@@ -0,0 +1,21 @@
from decimal import ROUND_HALF_UP, Decimal
HUNDRED = Decimal("100")
ZERO = Decimal("0")
TWO_PLACES = Decimal("0.01")
def clamp(value: Decimal) -> Decimal:
return min(HUNDRED, max(ZERO, value))
def round_score(value: Decimal) -> Decimal:
return value.quantize(TWO_PLACES, rounding=ROUND_HALF_UP)
def as_decimal(value: object) -> Decimal:
if isinstance(value, Decimal):
return value
if isinstance(value, bool) or value is None:
raise TypeError("value is not numeric")
return Decimal(str(value))
+148
View File
@@ -0,0 +1,148 @@
from collections.abc import Sequence
from datetime import date, datetime, timedelta
from decimal import Decimal
from uuid import UUID
from app.domain.enums import Availability, PriceBasis
from app.domain.models import Budget, HistoryWindow, PriceObservation, PriceStatistics, ScoringModel
from app.scoring.mathutil import clamp
def total_or_none(observation: PriceObservation) -> Decimal | None:
return observation.total_price
def select_current_observation(
observations: Sequence[PriceObservation],
*,
as_of: datetime,
snapshot_cutoff: datetime,
) -> PriceObservation | None:
eligible = [
item
for item in observations
if item.observed_at <= as_of and item.ingested_at <= snapshot_cutoff
]
if not eligible:
return None
return max(eligible, key=lambda item: (item.observed_at, item.ingested_at, item.id))
def median(values: Sequence[Decimal]) -> Decimal | None:
if not values:
return None
ordered = sorted(values)
mid = len(ordered) // 2
if len(ordered) % 2:
return ordered[mid]
return (ordered[mid - 1] + ordered[mid]) / Decimal("2")
def _quality_ok(observation: PriceObservation, minimum_confidence: Decimal) -> bool:
return (
observation.availability == Availability.IN_STOCK
and observation.total_price is not None
and observation.confidence >= minimum_confidence
)
def daily_series(
observations: Sequence[PriceObservation],
*,
window_start: date,
window_end_exclusive: date,
minimum_confidence: Decimal,
) -> tuple[tuple[Decimal, ...], tuple[UUID, ...]]:
by_day: dict[date, PriceObservation] = {}
for item in observations:
day = item.observed_at.date()
if day < window_start or day >= window_end_exclusive:
continue
current = by_day.get(day)
if current is None or (item.observed_at, item.ingested_at, item.id) > (
current.observed_at,
current.ingested_at,
current.id,
):
by_day[day] = item
values: list[Decimal] = []
ids: list[UUID] = []
for day in sorted(by_day):
observation = by_day[day]
if _quality_ok(observation, minimum_confidence):
values.append(observation.total_price or Decimal("0"))
ids.append(observation.id)
return tuple(values), tuple(ids)
def build_offer_reference_history(
observations: Sequence[PriceObservation],
current: PriceObservation,
model: ScoringModel,
) -> dict[int, HistoryWindow]:
current_day = current.observed_at.date()
windows: dict[int, HistoryWindow] = {}
for days in (30, 90, 365):
values, ids = daily_series(
observations,
window_start=current_day - timedelta(days=days),
window_end_exclusive=current_day,
minimum_confidence=model.minimum_confidence,
)
windows[days] = HistoryWindow(days=days, values=values, observation_ids=ids)
return windows
def score_price(
*,
current_price: Decimal,
reference_history: Sequence[Decimal],
budget: Budget,
model: ScoringModel,
) -> tuple[Decimal, Decimal, Decimal | None, PriceBasis, str | None]:
"""Pure price score over current_price, reference_history and budget."""
if current_price <= budget.target:
budget_score = Decimal("100")
else:
budget_score = clamp(
Decimal("100") * (budget.maximum - current_price) / (budget.maximum - budget.target)
)
if len(reference_history) < model.minimum_history_days:
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
reference = median(reference_history)
if reference is None or reference == 0:
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
discount = (reference - current_price) / reference
history_score = clamp(
Decimal("50") + Decimal("50") * discount / model.history_discount_for_max_score
)
price_score = model.budget_weight * budget_score + model.history_weight * history_score
return price_score, budget_score, history_score, PriceBasis.BUDGET_AND_HISTORY, None
def price_statistics_from_windows(
current: PriceObservation | None,
windows: dict[int, HistoryWindow],
) -> PriceStatistics:
window_30 = windows.get(30, HistoryWindow(30, (), ()))
window_90 = windows.get(90, HistoryWindow(90, (), ()))
window_365 = windows.get(365, HistoryWindow(365, (), ()))
used = window_30.observation_ids + window_90.observation_ids + window_365.observation_ids
if current is not None:
used = (current.id,) + used
lowest = min(window_365.values) if window_365.values else None
return PriceStatistics(
current_total=None if current is None else current.total_price,
median_30d=median(window_30.values),
median_30d_days=window_30.day_count,
median_90d=median(window_90.values),
median_90d_days=window_90.day_count,
lowest_365d=lowest,
lowest_365d_days=window_365.day_count,
lowest_365d_is_full_year=window_365.day_count >= 365,
used_observation_ids=tuple(dict.fromkeys(used)),
current_observation_id=None if current is None else current.id,
)
+86
View File
@@ -0,0 +1,86 @@
from decimal import Decimal
from app.domain.attributes import statement_is_usable
from app.domain.enums import ConstraintStatus, PreferenceDirection
from app.domain.models import (
AttributeStatement,
ConstraintResult,
Preference,
ScoreComponent,
ScoringModel,
)
from app.scoring.mathutil import as_decimal, clamp
def score_numeric_linear(value: Decimal, acceptable: Decimal, ideal: Decimal, direction: PreferenceDirection) -> Decimal:
if direction == PreferenceDirection.HIGHER:
return clamp(Decimal("50") + Decimal("50") * (value - acceptable) / (ideal - acceptable))
return clamp(Decimal("50") + Decimal("50") * (acceptable - value) / (acceptable - ideal))
def score_product(
preferences: tuple[Preference, ...],
snapshot: dict[str, AttributeStatement],
constraint_results: tuple[ConstraintResult, ...],
model: ScoringModel,
) -> tuple[Decimal, Decimal, tuple[ScoreComponent, ...]]:
components: list[ScoreComponent] = []
weighted = Decimal("0")
total_weight = sum((pref.weight for pref in preferences), Decimal("0"))
usable_weight = Decimal("0")
for preference in preferences:
statement = snapshot.get(preference.attribute)
usable = statement_is_usable(statement, model.minimum_confidence)
if usable:
raw = score_numeric_linear(
as_decimal(statement.normalized_value),
preference.acceptable,
preference.ideal,
preference.direction,
)
usable_weight += preference.weight
else:
raw = model.missing_preference_score
contribution = preference.weight * raw
weighted += contribution
components.append(
ScoreComponent(
kind="preference",
component_key=preference.id,
raw_score=raw,
weight=preference.weight,
contribution=contribution,
details={
"attribute": preference.attribute,
"strategy": preference.strategy.value,
"direction": preference.direction.value,
"acceptable": preference.acceptable,
"ideal": preference.ideal,
"usable": usable,
"attribute_row_id": None if statement is None else str(statement.id),
},
)
)
base = weighted / total_weight if total_weight else Decimal("0")
optional_issues = [
result
for result in constraint_results
if not result.mandatory and result.status in {ConstraintStatus.FAIL.value, ConstraintStatus.UNKNOWN.value}
]
penalty = model.optional_constraint_penalty_points * Decimal(len(optional_issues))
product_score = clamp(base - penalty)
if optional_issues:
components.append(
ScoreComponent(
kind="optional_penalty",
component_key="optional_constraints",
raw_score=penalty,
weight=None,
contribution=-penalty,
details={"count": len(optional_issues), "constraint_ids": [item.constraint_id for item in optional_issues]},
)
)
coverage = (usable_weight / total_weight) if total_weight else Decimal("0")
return product_score, coverage, tuple(components)
View File
Whitespace-only changes.
+224
View File
@@ -0,0 +1,224 @@
from __future__ import annotations
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.discovery.enums import FilterType, MappingStatus
from app.domain.serialize import jsonable
from app.errors import AppError
from app.persistence.discovery_models import (
CriterionDefinitionRecord,
CriterionMappingRecord,
ProductCategoryRecord,
SourceCategoryRecord,
SourceFilterDefinitionRecord,
SourceFilterOptionRecord,
SourceFilterRangeRecord,
)
from app.discovery.definitions import upsert_definition
from app.persistence.repositories import CatalogRepository
FILTER_OPERATORS = {
FilterType.BOOLEAN.value: ["=", "!="],
FilterType.ENUM.value: ["=", "!="],
FilterType.MULTI_ENUM.value: ["in", "not_in"],
FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"],
FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"],
FilterType.TEXT.value: ["=", "contains"],
}
class CatalogService:
def __init__(self, session: Session) -> None:
self.session = session
self.catalog = CatalogRepository(session)
def list_product_categories(self) -> list[dict[str, Any]]:
rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all()
return [{"id": row.id, "name": row.name, "status": row.status} for row in rows]
def get_product_category(self, category_id: str) -> ProductCategoryRecord:
row = self.session.get(ProductCategoryRecord, category_id)
if row is None:
raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id})
return row
def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]:
self.get_product_category(product_category_id)
rows = self.session.scalars(
select(SourceCategoryRecord)
.where(SourceCategoryRecord.product_category_id == product_category_id)
.order_by(SourceCategoryRecord.source_id)
).all()
sources = {item.id: item for item in self.catalog.list_sources()}
return [
{
"source": row.source_id,
"source_category_id": row.id,
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
"external_category_key": row.external_category_key,
"status": row.status,
}
for row in rows
]
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
if self.catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
rows = self.session.scalars(
select(SourceCategoryRecord)
.where(SourceCategoryRecord.source_id == source_id)
.order_by(SourceCategoryRecord.external_category_key)
).all()
return [
{
"id": row.id,
"source": row.source_id,
"source_category_id": row.id,
"external_category_key": row.external_category_key,
"name": row.name,
"product_category_id": row.product_category_id,
"status": row.status,
}
for row in rows
]
def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]:
self.get_product_category(product_category_id)
if self.catalog.get_source(source_id) is None:
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
source_category = self.session.scalars(
select(SourceCategoryRecord).where(
SourceCategoryRecord.source_id == source_id,
SourceCategoryRecord.product_category_id == product_category_id,
)
).first()
criteria: list[dict[str, Any]] = []
if source_category is not None:
filters = self.session.scalars(
select(SourceFilterDefinitionRecord)
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
.order_by(SourceFilterDefinitionRecord.label)
).all()
mappings = {
row.external_filter_key: row
for row in self.session.scalars(
select(CriterionMappingRecord).where(
CriterionMappingRecord.source_category_id == source_category.id
)
)
}
for filt in filters:
mapping = mappings.get(filt.external_filter_key)
definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
if definition is None:
definition = upsert_definition(
self.session,
{
"key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}",
"origin": "SOURCE",
"source": source_id,
"product_category_id": product_category_id,
"label": filt.label,
"value_type": "number"
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
else "string",
"unit": filt.canonical_unit,
"category": product_category_id,
},
)
self.session.flush()
options = self.session.scalars(
select(SourceFilterOptionRecord).where(SourceFilterOptionRecord.filter_id == filt.id)
).all()
rng = self.session.get(SourceFilterRangeRecord, filt.id)
criteria.append(
self._criterion_out(
definition,
filt,
mapping,
options,
rng,
)
)
internals = self.session.scalars(
select(CriterionDefinitionRecord)
.where(
CriterionDefinitionRecord.origin == "INTERNAL",
(CriterionDefinitionRecord.product_category_id == product_category_id)
| (CriterionDefinitionRecord.category == product_category_id),
)
.order_by(CriterionDefinitionRecord.label)
).all()
seen = {item["criterion_definition_id"] for item in criteria}
for definition in internals:
if definition.id in seen:
continue
criteria.append(self._criterion_out(definition, None, None, [], None))
return jsonable({"product_category_id": product_category_id, "source": source_id, "criteria": criteria})
def list_templates(self) -> list[dict[str, Any]]:
rows, _total = self.catalog.list_profiles(200, 0)
return [
{
"id": row.id,
"version": row.version,
"name": row.definition.get("name") or row.id,
"product_category_id": row.definition.get("category"),
"market": row.definition.get("market"),
"currency": row.definition.get("currency"),
"description": row.definition.get("description"),
}
for row in rows
]
def _criterion_out(
self,
definition: CriterionDefinitionRecord,
filt: SourceFilterDefinitionRecord | None,
mapping: CriterionMappingRecord | None,
options: list[SourceFilterOptionRecord],
rng: SourceFilterRangeRecord | None,
) -> dict[str, Any]:
filter_type = None if filt is None else filt.filter_type
return {
"criterion_definition_id": definition.id,
"label": definition.label,
"description": definition.description,
"origin": definition.origin,
"source": definition.source,
"product_category_id": definition.product_category_id,
"filter_type": filter_type,
"unit": definition.unit or (None if filt is None else filt.canonical_unit),
"operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]),
"options": [
{
"source_value": option.source_value,
"normalized_value": option.normalized_value,
"operator": option.operator,
"unit": option.unit,
}
for option in options
],
"range": None
if rng is None
else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit},
"mapping_status": None if mapping is None else mapping.status,
"supports_discovery": mapping is not None
and mapping.status == MappingStatus.VERIFIED.value
and filt is not None,
"scoring_attribute_key": definition.scoring_attribute_key,
}
def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord:
row = session.get(ProductCategoryRecord, category_id)
if row is not None:
return row
row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE")
session.add(row)
session.flush()
return row
File diff suppressed because it is too large. Load diff
+210
View File
@@ -0,0 +1,210 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy.orm import Session
from app.clock import now_utc
from app.domain.serialize import jsonable
from app.domain.timeutil import require_utc, to_iso_z
from app.errors import AppError
from app.persistence import models as orm
from app.persistence.db import engine
from app.persistence.repositories import CatalogRepository
from app.scoring.engine import evaluate_deal
from app.scoring.mathutil import round_score
from app.settings import get_settings
from sqlalchemy.orm import Session as SASession
from sqlalchemy.orm import sessionmaker
def evaluate_and_store(
*,
variant_id: UUID,
payload: dict[str, Any],
) -> dict[str, Any]:
now = now_utc()
as_of = payload.get("as_of")
if as_of is None:
as_of = now
else:
as_of = require_utc(as_of, "as_of")
if as_of > now:
raise AppError(422, "AS_OF_IN_FUTURE", "as_of must not be in the future")
snapshot_cutoff = now
maker = sessionmaker(bind=engine, autoflush=False, autocommit=False)
with engine.connect().execution_options(isolation_level="REPEATABLE READ") as connection:
with SASession(bind=connection) as session:
with session.begin():
repo = CatalogRepository(session)
variant = repo.get_variant(variant_id)
if variant is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
offer = repo.get_offer(payload["offer_id"])
if offer is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(payload["offer_id"])})
if offer.variant_id != variant_id:
raise AppError(422, "OFFER_VARIANT_MISMATCH", "offer does not belong to the variant")
product = repo.get_product(variant.product_id)
if product is None:
raise AppError(404, "NOT_FOUND", "product not found")
profile = repo.get_profile(payload["profile_id"], int(payload["profile_version"]))
if profile is None:
raise AppError(404, "NOT_FOUND", "profile version not found")
model = repo.get_scoring_model(profile.scoring_version)
if model is None:
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version is missing")
attributes = tuple(repo.list_attributes(variant_id))
observations, _ = repo.list_observations(offer.id, limit=None)
result = evaluate_deal(
profile=profile,
scoring_model=model,
product=product,
variant=variant,
offer=offer,
attributes=attributes,
observations=tuple(observations),
definitions=repo.all_definitions(),
as_of=as_of,
snapshot_cutoff=snapshot_cutoff,
engine_build=get_settings().engine_build,
)
response = evaluation_to_response(result, evaluation_id=uuid4(), created_at=now)
record = orm.EvaluationRecord(
id=response["id"],
variant_id=variant.id,
offer_id=offer.id,
profile_id=profile.id,
profile_version=profile.version,
scoring_version=model.version,
as_of=as_of,
snapshot_cutoff=snapshot_cutoff,
scoring_hash=model.definition_hash,
engine_build=result.engine_build,
product_eligibility=result.product_eligibility.value,
offer_eligibility=result.offer_eligibility.value,
overall_eligibility=result.overall_eligibility.value,
product_score=result.product_score,
price_score=result.price_score,
deal_score=result.deal_score,
classification=result.classification,
price_reference_scope=result.price_reference_scope.value,
input_snapshot=result.input_snapshot,
result_snapshot=response,
created_at=now,
)
repo.add_evaluation(record)
for component in result.components:
repo.add_component(
orm.EvaluationComponentRecord(
id=uuid4(),
evaluation_id=record.id,
kind=component.kind,
component_key=component.component_key,
raw_score=component.raw_score,
weight=component.weight,
contribution=component.contribution,
details=jsonable(component.details),
)
)
return response
def load_evaluation(session: Session, evaluation_id: UUID) -> dict[str, Any]:
record = CatalogRepository(session).get_evaluation(evaluation_id)
if record is None:
raise AppError(404, "NOT_FOUND", "evaluation not found", {"id": str(evaluation_id)})
return record.result_snapshot
def evaluation_to_response(result, *, evaluation_id: UUID, created_at: datetime) -> dict[str, Any]:
stats = result.price_statistics
return {
"id": str(evaluation_id),
"profile_id": result.profile_id,
"profile_version": result.profile_version,
"scoring_version": result.scoring_version,
"variant_id": str(result.variant_id),
"offer_id": str(result.offer_id),
"as_of": to_iso_z(result.as_of),
"snapshot_cutoff": to_iso_z(result.snapshot_cutoff),
"scoring_hash": result.scoring_hash,
"engine_build": result.engine_build,
"product_eligibility": result.product_eligibility.value,
"offer_eligibility": result.offer_eligibility.value,
"overall_eligibility": result.overall_eligibility.value,
"eligible": result.eligible,
"product_score": _num(result.product_score),
"price_score": _num(result.price_score),
"deal_score": _num(result.deal_score),
"classification": result.classification,
"price_basis": None if result.price_basis is None else result.price_basis.value,
"product_coverage": _num(result.product_coverage),
"failed_constraints": [_constraint(item) for item in result.failed_constraints],
"unknown_constraints": [_constraint(item) for item in result.unknown_constraints],
"constraint_results": [_constraint(item) for item in result.constraint_results],
"product_gates": [_gate(item) for item in result.product_gates],
"offer_gates": [_gate(item) for item in result.offer_gates],
"components": [_component(item) for item in result.components],
"price_statistics": {
"current_total": _money(stats.current_total),
"median_30d": _money(stats.median_30d),
"median_30d_days": stats.median_30d_days,
"median_90d": _money(stats.median_90d),
"median_90d_days": stats.median_90d_days,
"lowest_365d": _money(stats.lowest_365d),
"lowest_365d_days": stats.lowest_365d_days,
"lowest_365d_is_full_year": stats.lowest_365d_is_full_year,
"used_observation_ids": [str(item) for item in stats.used_observation_ids],
"current_observation_id": None if stats.current_observation_id is None else str(stats.current_observation_id),
},
"price_reference_scope": result.price_reference_scope.value,
"reasons": [{"code": item.code, "message": item.message} for item in result.reasons],
"warnings": list(result.warnings),
"input_snapshot": result.input_snapshot,
"created_at": to_iso_z(created_at),
}
def _num(value: Decimal | None) -> float | None:
return None if value is None else float(round_score(value))
def _money(value: Decimal | None) -> str | None:
return None if value is None else format(value.quantize(Decimal("0.01")), "f")
def _constraint(item) -> dict[str, Any]:
return {
"constraint_id": item.constraint_id,
"attribute": item.attribute,
"operator": item.operator,
"expected": jsonable(item.expected),
"actual": jsonable(item.actual),
"status": item.status,
"mandatory": item.mandatory,
"attribute_row_id": None if item.attribute_row_id is None else str(item.attribute_row_id),
"reason_code": item.reason_code,
}
def _gate(item) -> dict[str, Any]:
return {
"key": item.key,
"status": item.status,
"reason_code": item.reason_code,
"details": jsonable(item.details),
}
def _component(item) -> dict[str, Any]:
return {
"kind": item.kind,
"component_key": item.component_key,
"raw_score": None if item.raw_score is None else float(round_score(item.raw_score)),
"weight": None if item.weight is None else float(item.weight),
"contribution": None if item.contribution is None else float(round_score(item.contribution)),
"details": jsonable(item.details),
}
+94
View File
@@ -0,0 +1,94 @@
from app.errors import AppError
from app.services.ingestion_service import IngestionService
from app.sources.base import ProductSource
class ImportService:
def __init__(self, ingestion: IngestionService) -> None:
self.ingestion = ingestion
def import_product(self, source: ProductSource, external_id: str) -> None:
product = source.get_product(external_id)
try:
self.ingestion.create_product(
{
"id": product.id,
"category": product.category,
"brand": product.brand,
"name": product.name,
"model": product.model,
}
)
except AppError as exc:
if exc.status_code != 409:
raise
for variant in product.variants:
try:
self.ingestion.create_variant(
product.id,
{
"id": variant.id,
"name": variant.name,
"manufacturer_sku": variant.manufacturer_sku,
"identifiers": variant.identifiers,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for attribute in variant.attributes:
try:
self.ingestion.add_attribute(
variant.id,
{
"id": attribute.id,
"attribute": attribute.attribute,
"raw_value": attribute.raw_value,
"normalized_value": attribute.normalized_value,
"unit": attribute.unit,
"normalization_status": attribute.normalization_status,
"normalizer_version": attribute.normalizer_version,
"confidence": attribute.confidence,
"source_id": source.source_key,
"source_reference": attribute.source_reference,
"observed_at": attribute.observed_at,
"normalization_error": attribute.normalization_error,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for offer in source.get_offers(external_id):
try:
self.ingestion.create_offer(
offer.variant_id,
{
"id": offer.id,
"source_id": source.source_key,
"external_id": offer.external_id,
"seller": offer.seller,
"url": offer.url,
"market": offer.market,
"currency": offer.currency,
"condition": offer.condition,
},
)
except AppError as exc:
if exc.status_code != 409:
raise
for price in source.get_price_history(offer.external_id):
self.ingestion.add_price(
offer.id,
{
"id": price.id,
"source_id": source.source_key,
"external_observation_id": price.external_observation_id,
"observed_at": price.observed_at,
"item_price": price.item_price,
"shipping_price": price.shipping_price,
"currency": price.currency,
"availability": price.availability,
"raw_payload": price.raw_payload,
"confidence": price.confidence,
},
)
+263
View File
@@ -0,0 +1,263 @@
from datetime import datetime
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
from app.clock import now_utc
from app.domain.timeutil import require_utc
from app.domain.validation import DomainValidationError, parse_profile, validate_attribute_key, validate_currency
from app.errors import AppError
from app.normalization.manual import ManualNormalizer
from app.persistence import models as orm
from app.persistence.mapping import to_observation
from app.persistence.repositories import CatalogRepository
def _now() -> datetime:
return now_utc()
def _uuid(value: UUID | None) -> UUID:
return value or uuid4()
class IngestionService:
def __init__(self, session: Session) -> None:
self.session = session
self.repo = CatalogRepository(session)
self.normalizer = ManualNormalizer()
def create_definition(self, payload: dict[str, Any]) -> AttributeDefinition:
key = str(payload["key"])
validate_attribute_key(key)
row = orm.AttributeDefinitionRecord(
key=key,
value_type=ValueType(payload["value_type"]).value,
canonical_unit=payload.get("canonical_unit"),
minimum=payload.get("minimum"),
maximum=payload.get("maximum"),
description=payload.get("description"),
)
if self.repo.get_definition(key) is not None:
raise AppError(409, "CONFLICT", "attribute definition already exists", {"key": key})
return self.repo.add_definition(row)
def create_profile(self, payload: dict[str, Any]) -> orm.ProductProfileRecord:
try:
model_versions = self.repo.scoring_versions()
scoring_version = payload.get("scoring_version")
model = self.repo.get_scoring_model(scoring_version) if scoring_version else None
if model is None:
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version does not exist")
profile = parse_profile(
payload,
attribute_definitions=self.repo.all_definitions(),
scoring_versions=model_versions,
supported_strategies=set(model.supported_preference_strategies),
)
except DomainValidationError as exc:
raise AppError(422, "PROFILE_INVALID", str(exc), exc.details) from exc
except AppError:
raise
row = orm.ProductProfileRecord(
id=profile.id,
version=profile.version,
name=profile.name,
category=profile.category,
scoring_version=profile.scoring_version,
definition=profile.definition,
definition_hash=profile.definition_hash,
created_at=_now(),
)
if self.repo.get_profile(profile.id, profile.version) is not None:
raise AppError(409, "CONFLICT", "profile version already exists", {"id": profile.id, "version": profile.version})
return self.repo.add_profile(row)
def create_product(self, payload: dict[str, Any]) -> Product:
product_id = _uuid(payload.get("id"))
if self.repo.get_product(product_id) is not None:
raise AppError(409, "CONFLICT", "product already exists", {"id": str(product_id)})
row = orm.ProductRecord(
id=product_id,
category=str(payload["category"]),
brand=str(payload["brand"]),
name=str(payload["name"]),
model=payload.get("model"),
created_at=_now(),
)
return self.repo.add_product(row)
def create_variant(self, product_id: UUID, payload: dict[str, Any]) -> ProductVariant:
if self.repo.get_product(product_id) is None:
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
variant_id = _uuid(payload.get("id"))
if self.repo.get_variant(variant_id) is not None:
raise AppError(409, "CONFLICT", "variant already exists", {"id": str(variant_id)})
row = orm.ProductVariantRecord(
id=variant_id,
product_id=product_id,
name=str(payload["name"]),
manufacturer_sku=payload.get("manufacturer_sku"),
identifiers=payload.get("identifiers") or {},
created_at=_now(),
)
return self.repo.add_variant(row)
def add_attribute(self, variant_id: UUID, payload: dict[str, Any]) -> AttributeStatement:
variant = self.repo.get_variant(variant_id)
if variant is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
definition = self.repo.get_definition(payload["attribute"])
if definition is None:
raise AppError(422, "UNKNOWN_ATTRIBUTE", "attribute definition does not exist")
if self.repo.get_source(payload["source_id"]) is None:
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
observed_at = require_utc(payload["observed_at"], "observed_at")
ingested_at = _now()
if observed_at > ingested_at:
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
confidence = Decimal(str(payload["confidence"]))
if not Decimal("0") <= confidence <= Decimal("1"):
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
statement = AttributeStatement(
id=_uuid(payload.get("id")),
variant_id=variant_id,
attribute=definition.key,
raw_value=payload["raw_value"],
normalized_value=payload.get("normalized_value"),
unit=payload.get("unit"),
normalization_status=NormalizationStatus(payload["normalization_status"]),
normalizer_version=payload.get("normalizer_version") or self.normalizer.version,
confidence=confidence,
source_id=payload["source_id"],
source_reference=payload["source_reference"],
observed_at=observed_at,
ingested_at=ingested_at,
normalization_error=payload.get("normalization_error"),
)
statement = self.normalizer.validate(definition, statement)
if self.repo.get_attribute(statement.id) is not None:
raise AppError(409, "CONFLICT", "attribute statement already exists", {"id": str(statement.id)})
row = orm.VariantAttributeRecord(
id=statement.id,
variant_id=variant_id,
attribute_key=statement.attribute,
source_id=statement.source_id,
raw_value=statement.raw_value,
normalized_value=statement.normalized_value,
unit=statement.unit,
normalization_status=statement.normalization_status.value,
normalizer_version=statement.normalizer_version,
confidence=statement.confidence,
source_reference=statement.source_reference,
observed_at=statement.observed_at,
ingested_at=statement.ingested_at,
normalization_error=statement.normalization_error,
)
return self.repo.add_attribute(row)
def create_offer(self, variant_id: UUID, payload: dict[str, Any]) -> Offer:
if self.repo.get_variant(variant_id) is None:
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
if self.repo.get_source(payload["source_id"]) is None:
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
offer_id = _uuid(payload.get("id"))
if self.repo.get_offer(offer_id) is not None:
raise AppError(409, "CONFLICT", "offer already exists", {"id": str(offer_id)})
if self.repo.get_offer_by_external(payload["source_id"], str(payload["external_id"])) is not None:
raise AppError(409, "CONFLICT", "offer source/external_id already exists")
row = orm.OfferRecord(
id=offer_id,
variant_id=variant_id,
source_id=payload["source_id"],
external_id=str(payload["external_id"]),
seller=str(payload["seller"]),
url=payload.get("url"),
market=str(payload["market"]),
currency=validate_currency(str(payload["currency"])),
condition=OfferCondition(payload["condition"]).value,
created_at=_now(),
)
return self.repo.add_offer(row)
def add_price(self, offer_id: UUID, payload: dict[str, Any]) -> tuple[PriceObservation, bool]:
offer = self.repo.get_offer(offer_id)
if offer is None:
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
if payload["source_id"] != offer.source_id:
raise AppError(422, "SOURCE_MISMATCH", "source_id must match the offer")
currency = validate_currency(str(payload["currency"]))
if currency != offer.currency:
raise AppError(422, "CURRENCY_MISMATCH", "currency must match the offer")
observed_at = require_utc(payload["observed_at"], "observed_at")
ingested_at = _now()
if observed_at > ingested_at:
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
item_price = Decimal(str(payload["item_price"]))
if item_price <= 0:
raise AppError(422, "ITEM_PRICE", "item_price must be > 0")
shipping = payload.get("shipping_price")
shipping_price = None if shipping is None else Decimal(str(shipping))
if shipping_price is not None and shipping_price < 0:
raise AppError(422, "SHIPPING_PRICE", "shipping_price must be >= 0")
confidence = Decimal(str(payload["confidence"]))
if not Decimal("0") <= confidence <= Decimal("1"):
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
external_id = str(payload["external_observation_id"])
existing = self.repo.get_observation_by_external(offer_id, external_id)
incoming = {
"source_id": payload["source_id"],
"observed_at": observed_at,
"item_price": item_price,
"shipping_price": shipping_price,
"currency": currency,
"availability": Availability(payload["availability"]).value,
"raw_payload": payload["raw_payload"],
"confidence": confidence,
}
if existing is not None:
if _observation_content(existing) != incoming:
raise AppError(409, "CONFLICT", "price observation content differs")
return to_observation(existing), False
row = orm.PriceObservationRecord(
id=_uuid(payload.get("id")),
offer_id=offer_id,
source_id=payload["source_id"],
external_observation_id=external_id,
observed_at=observed_at,
ingested_at=ingested_at,
item_price=item_price,
shipping_price=shipping_price,
currency=currency,
availability=incoming["availability"],
raw_payload=payload["raw_payload"],
confidence=confidence,
)
try:
return self.repo.add_observation(row), True
except IntegrityError:
self.session.rollback()
existing = self.repo.get_observation_by_external(offer_id, external_id)
if existing is None:
raise
if _observation_content(existing) != incoming:
raise AppError(409, "CONFLICT", "price observation content differs")
return to_observation(existing), False
def _observation_content(row: orm.PriceObservationRecord) -> dict[str, Any]:
return {
"source_id": row.source_id,
"observed_at": row.observed_at,
"item_price": Decimal(str(row.item_price)),
"shipping_price": None if row.shipping_price is None else Decimal(str(row.shipping_price)),
"currency": row.currency,
"availability": row.availability,
"raw_payload": row.raw_payload,
"confidence": Decimal(str(row.confidence)),
}
+346
View File
@@ -0,0 +1,346 @@
from __future__ import annotations
from decimal import Decimal
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.clock import now_utc
from app.discovery.criteria import criterion_payload, parse_project_criteria
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
from app.discovery.enums import CandidateStatus, DiffStatus, ProjectStatus, RankingStatus, SearchStatus
from app.domain.serialize import jsonable
from app.domain.validation import DomainValidationError
from app.errors import AppError
from app.persistence.discovery_models import (
BuyingProjectCriteriaRecord,
BuyingProjectRecord,
DiscoverySearchRecord,
ProductCategoryRecord,
ProjectCandidateRecord,
ProjectCriterionRecord,
SearchRunItemRecord,
SearchRunRecord,
)
from app.persistence.repositories import CatalogRepository
from app.services.catalog_service import CatalogService, ensure_product_category
INACTIVE_SEARCH = {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}
def _uuid(value: UUID | None) -> UUID:
return value or uuid4()
class ProjectService:
def __init__(self, session: Session) -> None:
self.session = session
self.catalog = CatalogRepository(session)
self.categories = CatalogService(session)
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
template_id = payload.get("template_id") or payload.get("profile_id")
template_version = payload.get("template_version") or payload.get("profile_version")
profile = None
if template_id is not None:
if template_version is None:
raise AppError(422, "VALIDATION_ERROR", "template_version is required when template_id is set")
profile = self.catalog.get_profile(str(template_id), int(template_version))
if profile is None:
raise AppError(404, "NOT_FOUND", "template version not found")
product_category_id = payload.get("product_category_id") or (None if profile is None else profile.category)
if not product_category_id:
raise AppError(422, "VALIDATION_ERROR", "product_category_id is required")
if self.session.get(ProductCategoryRecord, product_category_id) is None:
ensure_product_category(self.session, str(product_category_id))
market = payload.get("market") or (None if profile is None else profile.market)
currency = payload.get("currency") or (None if profile is None else profile.currency)
if not market or not currency:
raise AppError(422, "VALIDATION_ERROR", "market and currency are required")
target_budget = payload.get("target_budget")
if target_budget is None and profile is not None and getattr(profile, "budget", None) is not None:
target_budget = getattr(profile.budget, "target", None)
criteria_payload = payload.get("criteria")
if not criteria_payload:
raise AppError(422, "CRITERIA_REQUIRED", "criteria are required")
try:
parsed = parse_project_criteria(criteria_payload)
except DomainValidationError as exc:
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
shortlist_size = int(payload.get("shortlist_size") or 5)
if shortlist_size < 1:
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
now = now_utc()
project = BuyingProjectRecord(
id=_uuid(payload.get("id")),
name=str(payload["name"]),
status=ProjectStatus.ACTIVE.value,
workspace_id=payload.get("workspace_id"),
product_category_id=str(product_category_id),
market=str(market),
currency=str(currency),
target_budget=None if target_budget is None else Decimal(str(target_budget)),
template_id=None if profile is None else profile.id,
template_version=None if profile is None else profile.version,
shortlist_size=shortlist_size,
criteria_version=1,
created_at=now,
updated_at=now,
archived_at=None,
)
self.session.add(project)
self.session.flush()
self._store_criteria(project.id, 1, parsed, str(product_category_id), now)
return self.get(project.id)
def replace_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
project = self._require_project(project_id)
try:
parsed = parse_project_criteria(payload)
except DomainValidationError as exc:
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
current = self.latest_criteria(project_id)
version = (current.version if current is not None else 0) + 1
now = now_utc()
self._store_criteria(project.id, version, parsed, project.product_category_id, now)
project.criteria_version = version
project.updated_at = now
for search in self.session.scalars(
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
):
if search.status not in INACTIVE_SEARCH:
search.status = SearchStatus.OUT_OF_SYNC.value
search.updated_at = now
self.session.flush()
return self.get(project.id)
def add_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
return self.replace_criteria(project_id, payload)
def patch(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
project = self._require_project(project_id)
allowed = {"name", "target_budget", "shortlist_size", "status"}
unknown = set(payload) - allowed
if unknown:
raise AppError(422, "VALIDATION_ERROR", "field cannot be patched", {"fields": sorted(unknown)})
if "name" in payload:
project.name = str(payload["name"])
if "target_budget" in payload:
project.target_budget = None if payload["target_budget"] is None else Decimal(str(payload["target_budget"]))
if "shortlist_size" in payload:
size = int(payload["shortlist_size"])
if size < 1:
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
project.shortlist_size = size
if "status" in payload:
status = str(payload["status"])
if status not in {item.value for item in ProjectStatus}:
raise AppError(422, "VALIDATION_ERROR", "invalid project status")
project.status = status
if status == ProjectStatus.ARCHIVED.value:
project.archived_at = now_utc()
project.updated_at = now_utc()
self.session.flush()
return self.get(project.id)
def get(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
criteria = self.latest_criteria(project_id)
return self._out(project, criteria)
def get_criteria(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
criteria = self.latest_criteria(project_id)
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
return jsonable(
{
"project_id": project.id,
"criteria_version": None if criteria is None else criteria.version,
"items": items,
}
)
def list(self, limit: int, offset: int) -> tuple[list[dict[str, Any]], int]:
total = self.session.scalar(select(func.count()).select_from(BuyingProjectRecord)) or 0
rows = self.session.scalars(
select(BuyingProjectRecord).order_by(BuyingProjectRecord.created_at).limit(limit).offset(offset)
).all()
return [self._summary(row) for row in rows], total
def latest_criteria(self, project_id: UUID) -> BuyingProjectCriteriaRecord | None:
return self.session.scalars(
select(BuyingProjectCriteriaRecord)
.where(BuyingProjectCriteriaRecord.project_id == project_id)
.order_by(BuyingProjectCriteriaRecord.version.desc())
).first()
def criteria_version(self, project_id: UUID, version: int) -> BuyingProjectCriteriaRecord | None:
return self.session.get(BuyingProjectCriteriaRecord, (project_id, version))
def criteria_items(self, project_id: UUID, version: int) -> list[dict[str, Any]]:
rows = self.session.scalars(
select(ProjectCriterionRecord)
.where(ProjectCriterionRecord.project_id == project_id, ProjectCriterionRecord.version == version)
.order_by(ProjectCriterionRecord.item_id)
).all()
return [criterion_payload(row) for row in rows]
def available_criteria(self, project_id: UUID) -> dict[str, Any]:
project = self._require_project(project_id)
sources = self.categories.sources_for_category(project.product_category_id)
source = sources[0]["source"] if sources else "geizhals"
return self.categories.available_criteria(project.product_category_id, source)
def _store_criteria(
self,
project_id: UUID,
version: int,
parsed: dict[str, Any],
category: str | None,
created_at,
) -> None:
self.session.add(
BuyingProjectCriteriaRecord(
project_id=project_id,
version=version,
definition_hash=parsed["definition_hash"],
created_at=created_at,
)
)
self.session.flush()
for item in parsed["items"]:
definition = self._resolve_definition(item, category)
self.session.add(
ProjectCriterionRecord(
id=uuid4(),
project_id=project_id,
version=version,
item_id=item["id"],
criterion_definition_id=definition.id,
criterion_key=definition.key,
operator=item["operator"],
value=item["value"],
enabled_for_discovery=item["enabled_for_discovery"],
created_at=created_at,
)
)
self.session.flush()
def _resolve_definition(self, item: dict[str, Any], category: str | None):
if item.get("criterion_definition_id") is not None:
definition = get_definition(self.session, item["criterion_definition_id"])
if definition is None:
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
return definition
existing = get_definition_by_key(self.session, str(item["criterion"]))
if existing is not None:
return existing
if not item.get("value_type"):
raise AppError(
422,
"UNKNOWN_CRITERION",
"criterion definition missing; provide value_type or create it first",
{"criterion": item["criterion"]},
)
return upsert_definition(
self.session,
{
"key": item["criterion"],
"origin": "INTERNAL",
"product_category_id": item.get("category") or category,
"label": item["criterion"],
"value_type": item["value_type"],
"unit": item.get("unit"),
"semantic": item.get("semantic"),
"category": item.get("category") or category,
"scoring_attribute_key": item["criterion"],
},
)
def _require_project(self, project_id: UUID) -> BuyingProjectRecord:
project = self.session.get(BuyingProjectRecord, project_id)
if project is None:
raise AppError(404, "NOT_FOUND", "buying project not found")
return project
def _out(self, project: BuyingProjectRecord, criteria: BuyingProjectCriteriaRecord | None) -> dict[str, Any]:
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
return jsonable(
{
"id": project.id,
"name": project.name,
"status": project.status,
"workspace_id": project.workspace_id,
"product_category_id": project.product_category_id,
"market": project.market,
"currency": project.currency,
"target_budget": project.target_budget,
"template_id": project.template_id,
"template_version": project.template_version,
"profile_id": project.template_id,
"profile_version": project.template_version,
"shortlist_size": project.shortlist_size,
"created_at": project.created_at,
"updated_at": project.updated_at,
"archived_at": project.archived_at,
"criteria_version": project.criteria_version if criteria is None else criteria.version,
"criteria": None if criteria is None else {"items": items},
"criteria_hash": None if criteria is None else criteria.definition_hash,
"ranking_status": RankingStatus.NOT_CONFIGURED.value
if project.template_id is None
else RankingStatus.CONFIGURED.value,
}
)
def _summary(self, project: BuyingProjectRecord) -> dict[str, Any]:
category = self.session.get(ProductCategoryRecord, project.product_category_id)
searches = list(
self.session.scalars(
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
)
)
active = next((row for row in searches if row.status not in INACTIVE_SEARCH), searches[0] if searches else None)
last_run = None
if active is not None:
last_run = self.session.scalars(
select(SearchRunRecord)
.where(SearchRunRecord.discovery_search_id == active.id)
.order_by(SearchRunRecord.started_at.desc())
).first()
candidates = list(
self.session.scalars(
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project.id)
)
)
counts = {"NEW": 0, "CHANGED": 0, "MISSING": 0}
if last_run is not None:
for item in self.session.scalars(
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == last_run.id)
):
if item.diff_status in counts:
counts[item.diff_status] += 1
return jsonable(
{
"id": project.id,
"name": project.name,
"product_category": None
if category is None
else {"id": category.id, "name": category.name, "status": category.status},
"status": project.status,
"last_successful_run_at": None if active is None else active.last_successful_run_at,
"next_run_at": None if active is None else active.next_run_at,
"discovery_status": None if active is None else active.status,
"candidate_count": len(candidates),
"shortlist_count": sum(1 for row in candidates if row.status == CandidateStatus.SHORTLISTED.value),
"new_count": last_run.new_count if last_run is not None else counts[DiffStatus.NEW.value],
"changed_count": last_run.changed_count if last_run is not None else counts[DiffStatus.CHANGED.value],
"missing_count": last_run.missing_count if last_run is not None else counts[DiffStatus.MISSING.value],
"current_search_result_count": None if last_run is None else last_run.result_count,
"ranking_status": RankingStatus.NOT_CONFIGURED.value
if project.template_id is None
else RankingStatus.CONFIGURED.value,
}
)
+34
View File
@@ -0,0 +1,34 @@
from datetime import datetime
from functools import lru_cache
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
db_host: str = "postgres"
db_port: int = 5432
db_name: str = "buyingagent"
db_user: str = "buyingagent"
db_password: str = "change_me"
environment: str = "development"
engine_build: str = "dev"
clock_now: datetime | None = None
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
@property
def cors_origins(self) -> list[str]:
return [item.strip() for item in self.cors_allowed_origins.split(",") if item.strip()]
@property
def database_url(self) -> str:
return (
f"postgresql+psycopg://{self.db_user}:{self.db_password}"
f"@{self.db_host}:{self.db_port}/{self.db_name}"
)
@lru_cache
def get_settings() -> Settings:
return Settings()
View File
Whitespace-only changes.
+85
View File
@@ -0,0 +1,85 @@
from dataclasses import dataclass
from datetime import datetime
from decimal import Decimal
from typing import Any, Protocol
from uuid import UUID
@dataclass(frozen=True)
class SourceProduct:
source_key: str
external_id: str
id: UUID
category: str
brand: str
name: str
model: str | None
variants: tuple["SourceVariant", ...]
@dataclass(frozen=True)
class SourceVariant:
id: UUID
name: str
manufacturer_sku: str | None
identifiers: dict[str, Any]
attributes: tuple["SourceAttribute", ...]
@dataclass(frozen=True)
class SourceAttribute:
id: UUID
attribute: str
raw_value: Any
normalized_value: Any
unit: str | None
normalization_status: str
normalizer_version: str
confidence: Decimal
source_reference: str
observed_at: datetime
normalization_error: str | None = None
@dataclass(frozen=True)
class SourceOffer:
id: UUID
variant_id: UUID
external_id: str
seller: str
url: str | None
market: str
currency: str
condition: str
@dataclass(frozen=True)
class SourcePriceObservation:
id: UUID | None
external_offer_id: str
external_observation_id: str
observed_at: datetime
item_price: Decimal
shipping_price: Decimal | None
currency: str
availability: str
raw_payload: Any
confidence: Decimal
class ProductSource(Protocol):
source_key: str
def get_product(self, external_id: str) -> SourceProduct: ...
def get_offers(self, external_product_id: str) -> list[SourceOffer]: ...
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]: ...
class DiscoverySourceAdapter(Protocol):
source_key: str
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: ...
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: ...
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: ...
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: ...
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: ...
+3
View File
@@ -0,0 +1,3 @@
from app.sources.geizhals.adapter import GeizhalsAdapter
__all__ = ["GeizhalsAdapter"]
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlparse
from app.discovery.enums import FilterType
from app.errors import AppError
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
class GeizhalsAdapter:
source_key = "geizhals"
def __init__(self, fixture_dir: Path | None = None, live: bool = False) -> None:
self.fixture_dir = fixture_dir or FIXTURE_DIR
self.live = live
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]:
payload = self._load(f"{category_ref}-taxonomy.json")
return payload
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
from app.discovery.urlgen import generate_search_url
category_url = f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
url, internal_only = generate_search_url(
category_url=category_url,
criteria_items=criteria,
mappings=mappings,
)
return {"url": url, "internal_only": internal_only}
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
if self.live:
raise AppError(422, "LIVE_FETCH_DISABLED", "live Geizhals fetch is not enabled in this process")
page = int(pagination.get("page") or 1)
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
return self._load(f"{category}-search-page{page}.json")
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
return list(response.get("items") or [])
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
attrs = result.get("normalized_attributes") or {}
return {
"source_external_id": result.get("source_external_id"),
"source_url": result.get("source_url"),
"source_title": result.get("source_title"),
"manufacturer": result.get("manufacturer"),
"model": result.get("model"),
"manufacturer_sku": result.get("manufacturer_sku"),
"gtin_ean": result.get("gtin_ean"),
"best_price": result.get("best_price"),
"currency": result.get("currency") or "EUR",
"availability": result.get("availability") or "unknown",
"seller_count": result.get("seller_count"),
"raw_attributes": result.get("raw_attributes") or {},
"normalized_attributes": attrs,
}
def _load(self, name: str) -> dict[str, Any]:
path = self.fixture_dir / name
if not path.exists():
raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}")
return json.loads(path.read_text(encoding="utf-8"))
def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str:
for mapping in mappings:
if mapping.get("category_key"):
return str(mapping["category_key"])
return "nb"
def filter_type(raw: str) -> str:
try:
return FilterType(raw).value
except ValueError:
return FilterType.UNKNOWN.value
+110
View File
@@ -0,0 +1,110 @@
import json
from datetime import datetime
from decimal import Decimal
from pathlib import Path
from uuid import UUID
from app.errors import AppError
from app.sources.base import (
SourceAttribute,
SourceOffer,
SourcePriceObservation,
SourceProduct,
SourceVariant,
)
def _parse_dt(value: str) -> datetime:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
class ManualSource:
source_key = "manual"
def __init__(self, path: Path) -> None:
self.path = path
self._data = json.loads(path.read_text(encoding="utf-8"))
self._products = {item["external_id"]: item for item in self._data["products"]}
self._offers = {item["external_id"]: item for item in self._data.get("offers", [])}
self._prices = self._data.get("prices", [])
def list_external_product_ids(self) -> list[str]:
return list(self._products)
def get_product(self, external_id: str) -> SourceProduct:
raw = self._products.get(external_id)
if raw is None:
raise AppError(404, "NOT_FOUND", "manual product not found", {"external_id": external_id})
variants = []
for variant in raw["variants"]:
attributes = tuple(
SourceAttribute(
id=UUID(item["id"]),
attribute=item["attribute"],
raw_value=item["raw_value"],
normalized_value=item.get("normalized_value"),
unit=item.get("unit"),
normalization_status=item["normalization_status"],
normalizer_version=item.get("normalizer_version", "manual-v1"),
confidence=Decimal(str(item["confidence"])),
source_reference=item["source_reference"],
observed_at=_parse_dt(item["observed_at"]),
normalization_error=item.get("normalization_error"),
)
for item in variant["attributes"]
)
variants.append(
SourceVariant(
id=UUID(variant["id"]),
name=variant["name"],
manufacturer_sku=variant.get("manufacturer_sku"),
identifiers=variant.get("identifiers") or {},
attributes=attributes,
)
)
return SourceProduct(
source_key=self.source_key,
external_id=external_id,
id=UUID(raw["id"]),
category=raw["category"],
brand=raw["brand"],
name=raw["name"],
model=raw.get("model"),
variants=tuple(variants),
)
def get_offers(self, external_product_id: str) -> list[SourceOffer]:
product = self.get_product(external_product_id)
variant_ids = {variant.id for variant in product.variants}
return [
SourceOffer(
id=UUID(item["id"]),
variant_id=UUID(item["variant_id"]),
external_id=item["external_id"],
seller=item["seller"],
url=item.get("url"),
market=item["market"],
currency=item["currency"],
condition=item["condition"],
)
for item in self._offers.values()
if UUID(item["variant_id"]) in variant_ids
]
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]:
return [
SourcePriceObservation(
id=UUID(item["id"]) if item.get("id") else None,
external_offer_id=external_offer_id,
external_observation_id=item["external_observation_id"],
observed_at=_parse_dt(item["observed_at"]),
item_price=Decimal(str(item["item_price"])),
shipping_price=None if item.get("shipping_price") is None else Decimal(str(item["shipping_price"])),
currency=item["currency"],
availability=item["availability"],
raw_payload=item["raw_payload"],
confidence=Decimal(str(item["confidence"])),
)
for item in self._prices
if item["external_offer_id"] == external_offer_id
]
@@ -0,0 +1,38 @@
{
"page": 1,
"page_count": 2,
"result_count": 3,
"http_status": 200,
"items": [
{
"source_external_id": "gh-nb-a",
"source_url": "https://geizhals.de/example-mobile-13-a.html",
"source_title": "Example Mobile 13 32GB",
"manufacturer": "Example",
"model": "Mobile 13",
"manufacturer_sku": "EM13-32-1T",
"gtin_ean": "4000000000001",
"best_price": "1298.00",
"currency": "EUR",
"availability": "in_stock",
"seller_count": 8,
"raw_attributes": {"RAM": "32GB"},
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
},
{
"source_external_id": "gh-nb-b",
"source_url": "https://geizhals.de/example-mobile-13-b.html",
"source_title": "Example Mobile 13 8GB",
"manufacturer": "Example",
"model": "Mobile 13",
"manufacturer_sku": "EM13-8-1T",
"gtin_ean": "4000000000002",
"best_price": "1298.00",
"currency": "EUR",
"availability": "in_stock",
"seller_count": 5,
"raw_attributes": {"RAM": "8GB"},
"normalized_attributes": {"ram_gb": 8, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
}
]
}
@@ -0,0 +1,23 @@
{
"page": 2,
"page_count": 2,
"result_count": 3,
"http_status": 200,
"items": [
{
"source_external_id": "gh-nb-c",
"source_url": "https://geizhals.de/example-mobile-13-c.html",
"source_title": "Example Mobile 13 unknown weight",
"manufacturer": "Example",
"model": "Mobile 13 UW",
"manufacturer_sku": "EM13-32-UW",
"gtin_ean": "4000000000003",
"best_price": "1298.00",
"currency": "EUR",
"availability": "in_stock",
"seller_count": 3,
"raw_attributes": {"RAM": "32GB"},
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "battery_score": 90, "display_score": 90, "cpu_score": 90}
}
]
}
@@ -0,0 +1,27 @@
{
"source": "geizhals",
"external_category_key": "nb",
"name": "Notebooks",
"canonical_category": "notebook",
"source_url": "https://geizhals.de/?cat=nb",
"filters": [
{
"external_filter_key": "17023",
"label": "Arbeitsspeicher",
"filter_type": "NUMERIC_THRESHOLD",
"canonical_unit": "GB",
"options": [
{"source_value": "ab 16GB", "normalized_value": 16, "operator": ">=", "unit": "GB"},
{"source_value": "ab 24GB", "normalized_value": 24, "operator": ">=", "unit": "GB"},
{"source_value": "ab 32GB", "normalized_value": 32, "operator": ">=", "unit": "GB"}
]
},
{
"external_filter_key": "price",
"label": "Preis",
"filter_type": "NUMERIC_RANGE",
"canonical_unit": "EUR",
"range": {"minimum": 0, "maximum": null, "step": 1, "unit": "EUR"}
}
]
}
+80
View File
@@ -0,0 +1,80 @@
{
"products": [
{
"id": "11111111-1111-4111-8111-111111111111",
"external_id": "example-mobile-13",
"category": "notebook",
"brand": "Example",
"name": "Example Mobile 13",
"model": "EM13",
"variants": [
{
"id": "22222222-2222-4222-8222-222222222222",
"name": "32 GB / 1 TB",
"manufacturer_sku": "EM13-32-1T",
"identifiers": {"sku": "EM13-32-1T"},
"attributes": [
{"id": "aaaaaaa1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/ram", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "aaaaaaa1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/storage", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "aaaaaaa1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/weight", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "aaaaaaa1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/battery", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "aaaaaaa1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/display", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "aaaaaaa1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/cpu", "observed_at": "2026-10-03T09:00:00Z"}
]
},
{
"id": "22222222-2222-4222-8222-22222222222b",
"name": "8 GB / 1 TB",
"manufacturer_sku": "EM13-8-1T",
"identifiers": {"sku": "EM13-8-1T"},
"attributes": [
{"id": "bbbbbbb1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "8GB RAM", "normalized_value": 8, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/ram", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "bbbbbbb1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/storage", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "bbbbbbb1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/weight", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "bbbbbbb1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/battery", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "bbbbbbb1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/display", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "bbbbbbb1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/cpu", "observed_at": "2026-10-03T09:00:00Z"}
]
},
{
"id": "22222222-2222-4222-8222-22222222222c",
"name": "32 GB / unknown weight",
"manufacturer_sku": "EM13-32-UW",
"identifiers": {},
"attributes": [
{"id": "ccccccc1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/ram", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "ccccccc1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/storage", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "ccccccc1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "weight unknown", "normalized_value": null, "unit": "kg", "normalization_status": "unknown", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/weight", "observed_at": "2026-10-03T09:00:00Z", "normalization_error": "manual unknown"},
{"id": "ccccccc1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/battery", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "ccccccc1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/display", "observed_at": "2026-10-03T09:00:00Z"},
{"id": "ccccccc1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/cpu", "observed_at": "2026-10-03T09:00:00Z"}
]
}
]
}
],
"offers": [
{"id": "33333333-3333-4333-8333-333333333333", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-a", "seller": "DemoShop", "url": "https://example.test/a", "market": "DE", "currency": "EUR", "condition": "new"},
{"id": "33333333-3333-4333-8333-33333333333b", "variant_id": "22222222-2222-4222-8222-22222222222b", "external_id": "demo-offer-b", "seller": "DemoShop", "url": "https://example.test/b", "market": "DE", "currency": "EUR", "condition": "new"},
{"id": "33333333-3333-4333-8333-33333333333c", "variant_id": "22222222-2222-4222-8222-22222222222c", "external_id": "demo-offer-c", "seller": "DemoShop", "url": "https://example.test/c", "market": "DE", "currency": "EUR", "condition": "new"},
{"id": "33333333-3333-4333-8333-33333333333d", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-d", "seller": "OtherShop", "url": "https://example.test/d", "market": "DE", "currency": "EUR", "condition": "new"}
],
"prices": [
{"id": "ddddddd1-0000-4000-8000-0000000000a1", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000a2", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000a3", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000a4", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000b1", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000b2", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000b3", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000b4", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000c1", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000c2", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000c3", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000c4", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000d1", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000d2", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000d3", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
{"id": "ddddddd1-0000-4000-8000-0000000000d4", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "2049.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "2049.00"}}
]
}
+38
View File
@@ -0,0 +1,38 @@
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
from app.settings import get_settings
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
config.set_main_option("sqlalchemy.url", get_settings().database_url)
target_metadata = None
def run_migrations_offline() -> None:
url = config.get_main_option("sqlalchemy.url")
context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+25
View File
@@ -0,0 +1,25 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: str | None = ${repr(down_revision)}
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,21 @@
"""Deploy skeleton — domain schema follows in 0002.
Revision ID: 0001_deploy_skeleton
Revises:
Create Date: 2026-10-03
"""
from collections.abc import Sequence
revision: str = "0001_deploy_skeleton"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,233 @@
"""Global product/price schema for v0.1. No user or workspace columns.
Revision ID: 0002_domain_schema
Revises: 0001_deploy_skeleton
Create Date: 2026-10-03
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0002_domain_schema"
down_revision: str | None = "0001_deploy_skeleton"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"attribute_definitions",
sa.Column("key", sa.String(128), primary_key=True),
sa.Column("value_type", sa.String(32), nullable=False),
sa.Column("canonical_unit", sa.String(32)),
sa.Column("minimum", sa.Numeric(12, 4)),
sa.Column("maximum", sa.Numeric(12, 4)),
sa.Column("description", sa.Text()),
sa.CheckConstraint(
"value_type IN ('number', 'string', 'boolean', 'string_list')",
name="ck_attribute_definitions_value_type",
),
)
op.create_table(
"sources",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(128), nullable=False),
sa.Column("adapter_type", sa.String(64), nullable=False),
)
op.create_table(
"scoring_models",
sa.Column("version", sa.String(64), primary_key=True),
sa.Column("algorithm_id", sa.String(128), nullable=False),
sa.Column("definition", postgresql.JSONB, nullable=False),
sa.Column("definition_hash", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"product_profiles",
sa.Column("id", sa.String(128), primary_key=True),
sa.Column("version", sa.Integer(), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("category", sa.String(128), nullable=False),
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
sa.Column("definition", postgresql.JSONB, nullable=False),
sa.Column("definition_hash", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("version > 0", name="ck_product_profiles_version"),
)
op.create_table(
"products",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("category", sa.String(128), nullable=False),
sa.Column("brand", sa.String(128), nullable=False),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("model", sa.String(128)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"product_variants",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("product_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("products.id"), nullable=False),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("manufacturer_sku", sa.String(128)),
sa.Column("identifiers", postgresql.JSONB, nullable=False, server_default=sa.text("'{}'::jsonb")),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_product_variants_product_id", "product_variants", ["product_id"])
op.create_table(
"variant_attributes",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
sa.Column("attribute_key", sa.String(128), sa.ForeignKey("attribute_definitions.key"), nullable=False),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("raw_value", postgresql.JSONB, nullable=False),
sa.Column("normalized_value", postgresql.JSONB),
sa.Column("unit", sa.String(32)),
sa.Column("normalization_status", sa.String(32), nullable=False),
sa.Column("normalizer_version", sa.String(64), nullable=False),
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
sa.Column("source_reference", sa.String(256), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("normalization_error", sa.Text()),
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
sa.CheckConstraint(
"normalization_status IN ('normalized', 'unknown')",
name="ck_variant_attributes_status",
),
)
op.execute(
"CREATE INDEX ix_variant_attributes_lookup ON variant_attributes "
"(variant_id, attribute_key, observed_at DESC, ingested_at DESC, id DESC)"
)
op.create_table(
"offers",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("external_id", sa.String(256), nullable=False),
sa.Column("seller", sa.String(256), nullable=False),
sa.Column("url", sa.Text()),
sa.Column("market", sa.String(16), nullable=False),
sa.Column("currency", sa.String(3), nullable=False),
sa.Column("condition", sa.String(32), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
sa.CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
)
op.create_index("ix_offers_variant_id", "offers", ["variant_id"])
op.create_table(
"price_observations",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("external_observation_id", sa.String(256), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("item_price", sa.Numeric(12, 2), nullable=False),
sa.Column("shipping_price", sa.Numeric(12, 2)),
sa.Column("currency", sa.String(3), nullable=False),
sa.Column("availability", sa.String(32), nullable=False),
sa.Column("raw_payload", postgresql.JSONB, nullable=False),
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
sa.UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
sa.CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
sa.CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
sa.CheckConstraint(
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
name="ck_price_obs_availability",
),
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
)
op.execute(
"CREATE INDEX ix_price_observations_lookup ON price_observations "
"(offer_id, observed_at DESC, ingested_at DESC, id DESC)"
)
op.create_table(
"evaluations",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
sa.Column("profile_id", sa.String(128), nullable=False),
sa.Column("profile_version", sa.Integer(), nullable=False),
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
sa.Column("snapshot_cutoff", sa.DateTime(timezone=True), nullable=False),
sa.Column("scoring_hash", sa.String(64), nullable=False),
sa.Column("engine_build", sa.String(64), nullable=False),
sa.Column("product_eligibility", sa.String(16), nullable=False),
sa.Column("offer_eligibility", sa.String(16), nullable=False),
sa.Column("overall_eligibility", sa.String(16), nullable=False),
sa.Column("product_score", sa.Numeric(6, 2)),
sa.Column("price_score", sa.Numeric(6, 2)),
sa.Column("deal_score", sa.Numeric(6, 2)),
sa.Column("classification", sa.String(32), nullable=False),
sa.Column("price_reference_scope", sa.String(32), nullable=False),
sa.Column("input_snapshot", postgresql.JSONB, nullable=False),
sa.Column("result_snapshot", postgresql.JSONB, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id", "profile_version"],
["product_profiles.id", "product_profiles.version"],
name="fk_evaluations_profile",
),
sa.CheckConstraint(
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_product_eligibility",
),
sa.CheckConstraint(
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_offer_eligibility",
),
sa.CheckConstraint(
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
name="ck_eval_overall_eligibility",
),
sa.CheckConstraint(
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
name="ck_eval_product_score",
),
sa.CheckConstraint(
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
name="ck_eval_price_score",
),
sa.CheckConstraint(
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
name="ck_eval_deal_score",
),
)
op.create_index("ix_evaluations_variant_id", "evaluations", ["variant_id"])
op.execute(
"CREATE INDEX ix_evaluations_profile_created ON evaluations "
"(profile_id, profile_version, created_at DESC)"
)
op.create_table(
"evaluation_components",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("evaluation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("evaluations.id"), nullable=False),
sa.Column("kind", sa.String(64), nullable=False),
sa.Column("component_key", sa.String(128), nullable=False),
sa.Column("raw_score", sa.Numeric(12, 6)),
sa.Column("weight", sa.Numeric(12, 6)),
sa.Column("contribution", sa.Numeric(12, 6)),
sa.Column("details", postgresql.JSONB, nullable=False),
sa.UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
)
def downgrade() -> None:
op.drop_table("evaluation_components")
op.drop_table("evaluations")
op.drop_table("price_observations")
op.drop_table("offers")
op.drop_table("variant_attributes")
op.drop_table("product_variants")
op.drop_table("products")
op.drop_table("product_profiles")
op.drop_table("scoring_models")
op.drop_table("sources")
op.drop_table("attribute_definitions")
@@ -0,0 +1,229 @@
"""Discovery, taxonomy, search runs, and project candidates.
Revision ID: 0003_discovery_schema
Revises: 0002_domain_schema
Create Date: 2026-10-04
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0003_discovery_schema"
down_revision: str | None = "0002_domain_schema"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"buying_projects",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("workspace_id", sa.String(64)),
sa.Column("profile_id", sa.String(128), nullable=False),
sa.Column("profile_version", sa.Integer(), nullable=False),
sa.Column("shortlist_size", sa.Integer(), nullable=False, server_default="5"),
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("archived_at", sa.DateTime(timezone=True)),
sa.ForeignKeyConstraint(
["profile_id", "profile_version"],
["product_profiles.id", "product_profiles.version"],
name="fk_buying_projects_profile",
),
sa.CheckConstraint("status IN ('ACTIVE', 'PURCHASED', 'ARCHIVED')", name="ck_buying_projects_status"),
sa.CheckConstraint("shortlist_size > 0", name="ck_buying_projects_shortlist"),
sa.CheckConstraint("schedule_interval_minutes > 0", name="ck_buying_projects_interval"),
)
op.create_table(
"buying_project_criteria",
sa.Column("project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), primary_key=True),
sa.Column("version", sa.Integer(), primary_key=True),
sa.Column("definition", postgresql.JSONB, nullable=False),
sa.Column("definition_hash", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("version > 0", name="ck_project_criteria_version"),
)
op.create_table(
"source_categories",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("external_category_key", sa.String(64), nullable=False),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("source_url", sa.Text(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("last_verified_at", sa.DateTime(timezone=True)),
sa.UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),
)
op.create_table(
"source_filter_definitions",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
sa.Column("external_filter_key", sa.String(64), nullable=False),
sa.Column("label", sa.String(256), nullable=False),
sa.Column("filter_type", sa.String(32), nullable=False),
sa.Column("canonical_unit", sa.String(32)),
sa.Column("status", sa.String(32), nullable=False),
sa.UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),
)
op.create_table(
"source_filter_options",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), nullable=False),
sa.Column("source_value", sa.String(256), nullable=False),
sa.Column("normalized_value", postgresql.JSONB),
sa.Column("operator", sa.String(16)),
sa.Column("unit", sa.String(32)),
sa.UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),
)
op.create_table(
"source_filter_ranges",
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), primary_key=True),
sa.Column("minimum", sa.Numeric(12, 2)),
sa.Column("maximum", sa.Numeric(12, 2)),
sa.Column("step", sa.Numeric(12, 2)),
sa.Column("unit", sa.String(32)),
)
op.create_table(
"criterion_definitions",
sa.Column("key", sa.String(128), primary_key=True),
sa.Column("value_type", sa.String(32), nullable=False),
sa.Column("unit", sa.String(32)),
sa.Column("semantic", sa.String(128)),
sa.Column("description", sa.Text()),
)
op.create_table(
"criterion_mappings",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
sa.Column("external_filter_key", sa.String(64), nullable=False),
sa.Column("source_value", sa.String(256)),
sa.Column("encoding", postgresql.JSONB, nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("definition_hash", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"discovery_searches",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
sa.Column("criteria_version", sa.Integer(), nullable=False),
sa.Column("mapping_version", sa.String(64), nullable=False),
sa.Column("generated_url", sa.Text(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("broad_result_threshold", sa.Integer(), nullable=False, server_default="300"),
sa.Column("next_run_at", sa.DateTime(timezone=True)),
sa.Column("last_successful_run_at", sa.DateTime(timezone=True)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"search_runs",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("discovery_search_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("discovery_searches.id"), nullable=False),
sa.Column("criteria_version", sa.Integer(), nullable=False),
sa.Column("mapping_version", sa.String(64), nullable=False),
sa.Column("generated_url", sa.Text(), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("result_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("http_status_summary", postgresql.JSONB, nullable=False),
sa.Column("error_code", sa.String(64)),
)
op.create_table(
"source_product_references",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
sa.Column("source_external_id", sa.String(256)),
sa.Column("source_url", sa.Text()),
sa.Column("source_title", sa.String(512), nullable=False),
sa.Column("manufacturer", sa.String(256)),
sa.Column("model", sa.String(256)),
sa.Column("manufacturer_sku", sa.String(128)),
sa.Column("gtin_ean", sa.String(32)),
sa.Column("normalized_spec_fingerprint", sa.Text()),
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("lifecycle_status", sa.String(32), nullable=False),
sa.Column("match_status", sa.String(32), nullable=False),
sa.Column("consecutive_missing_success_runs", sa.Integer(), nullable=False, server_default="0"),
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
)
op.create_table(
"product_observations",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("raw_title", sa.String(512), nullable=False),
sa.Column("best_price", sa.Numeric(12, 2)),
sa.Column("currency", sa.String(3)),
sa.Column("raw_attributes", postgresql.JSONB, nullable=False),
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False),
sa.Column("availability", sa.String(32)),
sa.Column("seller_count", sa.Integer()),
)
op.create_table(
"search_run_items",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
sa.Column("observation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_observations.id")),
sa.Column("diff_status", sa.String(32), nullable=False),
sa.UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),
)
op.create_table(
"project_candidates",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("match_status", sa.String(32), nullable=False),
sa.Column("product_eligibility", sa.String(16)),
sa.Column("product_score", sa.Numeric(6, 2)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
)
op.create_table(
"candidate_status_history",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("candidate_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("project_candidates.id"), nullable=False),
sa.Column("from_status", sa.String(32)),
sa.Column("to_status", sa.String(32), nullable=False),
sa.Column("reason", sa.String(128)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_discovery_searches_next_run", "discovery_searches", ["next_run_at"])
op.create_index("ix_search_runs_search", "search_runs", ["discovery_search_id", "started_at"])
def downgrade() -> None:
op.drop_table("candidate_status_history")
op.drop_table("project_candidates")
op.drop_table("search_run_items")
op.drop_table("product_observations")
op.drop_table("source_product_references")
op.drop_table("search_runs")
op.drop_table("discovery_searches")
op.drop_table("criterion_mappings")
op.drop_table("criterion_definitions")
op.drop_table("source_filter_ranges")
op.drop_table("source_filter_options")
op.drop_table("source_filter_definitions")
op.drop_table("source_categories")
op.drop_table("buying_project_criteria")
op.drop_table("buying_projects")
@@ -0,0 +1,127 @@
"""Dynamic attribute and criterion value records.
Revision ID: 0004_dynamic_attributes
Revises: 0003_discovery_schema
Create Date: 2026-10-04
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0004_dynamic_attributes"
down_revision: str | None = "0003_discovery_schema"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column("criterion_definitions", sa.Column("category", sa.String(128)))
op.add_column("source_categories", sa.Column("canonical_category", sa.String(128)))
op.create_table(
"project_criteria",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("version", sa.Integer(), nullable=False),
sa.Column("item_id", sa.String(128), nullable=False),
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
sa.Column("operator", sa.String(16), nullable=False),
sa.Column("value", postgresql.JSONB),
sa.Column("mandatory", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.ForeignKeyConstraint(
["project_id", "version"],
["buying_project_criteria.project_id", "buying_project_criteria.version"],
name="fk_project_criteria_version",
),
sa.UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
)
op.create_table(
"product_observation_attributes",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column(
"observation_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("product_observations.id"),
nullable=False,
),
sa.Column("attribute_key", sa.String(128), nullable=False),
sa.Column("raw_value", postgresql.JSONB),
sa.Column("normalized_value", postgresql.JSONB),
sa.Column("unit", sa.String(32)),
sa.UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),
)
_migrate_criteria_json()
_migrate_observation_json()
op.drop_column("buying_project_criteria", "definition")
op.drop_column("product_observations", "raw_attributes")
op.drop_column("product_observations", "normalized_attributes")
def _migrate_criteria_json() -> None:
bind = op.get_bind()
rows = bind.execute(sa.text("SELECT project_id, version, definition FROM buying_project_criteria")).mappings()
for row in rows:
items = (row["definition"] or {}).get("items") or []
for raw in items:
bind.execute(
sa.text(
"""
INSERT INTO project_criteria
(id, project_id, version, item_id, criterion_key, operator, value, mandatory)
VALUES
(gen_random_uuid(), :project_id, :version, :item_id, :criterion_key, :operator, :value, :mandatory)
"""
),
{
"project_id": row["project_id"],
"version": row["version"],
"item_id": str(raw.get("id") or raw.get("criterion")),
"criterion_key": str(raw.get("criterion")),
"operator": str(raw.get("operator")),
"value": raw.get("value"),
"mandatory": bool(raw.get("mandatory", True)),
},
)
def _migrate_observation_json() -> None:
bind = op.get_bind()
rows = bind.execute(
sa.text("SELECT id, raw_attributes, normalized_attributes FROM product_observations")
).mappings()
for row in rows:
raw = row["raw_attributes"] or {}
normalized = row["normalized_attributes"] or {}
keys = set(raw) | set(normalized)
for key in keys:
bind.execute(
sa.text(
"""
INSERT INTO product_observation_attributes
(id, observation_id, attribute_key, raw_value, normalized_value, unit)
VALUES
(gen_random_uuid(), :observation_id, :attribute_key, :raw_value, :normalized_value, NULL)
"""
),
{
"observation_id": row["id"],
"attribute_key": key,
"raw_value": raw.get(key),
"normalized_value": normalized.get(key),
},
)
def downgrade() -> None:
op.add_column("product_observations", sa.Column("raw_attributes", postgresql.JSONB, nullable=False, server_default="{}"))
op.add_column(
"product_observations",
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False, server_default="{}"),
)
op.add_column("buying_project_criteria", sa.Column("definition", postgresql.JSONB, nullable=False, server_default='{"items":[]}'))
op.drop_table("product_observation_attributes")
op.drop_table("project_criteria")
op.drop_column("source_categories", "canonical_category")
op.drop_column("criterion_definitions", "category")
@@ -0,0 +1,261 @@
"""Frontend MVP v0.3 contracts: categories, optional templates, criterion UUIDs.
Revision ID: 0005_frontend_mvp_v03
Revises: 0004_dynamic_attributes
Create Date: 2026-10-04
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0005_frontend_mvp_v03"
down_revision: str | None = "0004_dynamic_attributes"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"product_categories",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("status", sa.String(32), nullable=False, server_default="ACTIVE"),
)
op.execute("INSERT INTO product_categories (id, name, status) VALUES ('notebook', 'Notebooks', 'ACTIVE')")
op.add_column("source_categories", sa.Column("product_category_id", sa.String(64)))
op.create_foreign_key(
"fk_source_categories_product_category",
"source_categories",
"product_categories",
["product_category_id"],
["id"],
)
op.execute(
"""
UPDATE source_categories
SET product_category_id = canonical_category
WHERE canonical_category IS NOT NULL
AND EXISTS (SELECT 1 FROM product_categories WHERE id = source_categories.canonical_category)
"""
)
op.add_column("criterion_definitions", sa.Column("id", postgresql.UUID(as_uuid=True)))
op.execute("UPDATE criterion_definitions SET id = gen_random_uuid() WHERE id IS NULL")
op.alter_column("criterion_definitions", "id", nullable=False)
op.add_column(
"criterion_definitions",
sa.Column("origin", sa.String(16), nullable=False, server_default="SOURCE"),
)
op.add_column("criterion_definitions", sa.Column("source", sa.String(64)))
op.add_column("criterion_definitions", sa.Column("product_category_id", sa.String(64)))
op.add_column("criterion_definitions", sa.Column("label", sa.String(256)))
op.add_column("criterion_definitions", sa.Column("scoring_attribute_key", sa.String(128)))
op.execute("UPDATE criterion_definitions SET label = key WHERE label IS NULL")
op.execute("UPDATE criterion_definitions SET scoring_attribute_key = key")
op.execute(
"""
UPDATE criterion_definitions
SET product_category_id = category
WHERE category IS NOT NULL
AND EXISTS (SELECT 1 FROM product_categories WHERE id = criterion_definitions.category)
"""
)
op.alter_column("criterion_definitions", "label", nullable=False)
op.create_foreign_key(
"fk_criterion_definitions_product_category",
"criterion_definitions",
"product_categories",
["product_category_id"],
["id"],
)
op.add_column("project_criteria", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
op.add_column(
"project_criteria",
sa.Column("enabled_for_discovery", sa.Boolean(), nullable=False, server_default=sa.true()),
)
op.add_column("project_criteria", sa.Column("created_at", sa.DateTime(timezone=True)))
op.execute(
"""
UPDATE project_criteria
SET enabled_for_discovery = mandatory,
created_at = timezone('utc', now())
"""
)
op.execute(
"""
UPDATE project_criteria AS pc
SET criterion_definition_id = cd.id
FROM criterion_definitions AS cd
WHERE pc.criterion_key = cd.key
"""
)
op.alter_column("project_criteria", "criterion_definition_id", nullable=False)
op.alter_column("project_criteria", "created_at", nullable=False)
op.drop_constraint("project_criteria_criterion_key_fkey", "project_criteria", type_="foreignkey")
op.add_column("criterion_mappings", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
op.execute(
"""
UPDATE criterion_mappings AS cm
SET criterion_definition_id = cd.id
FROM criterion_definitions AS cd
WHERE cm.criterion_key = cd.key
"""
)
op.alter_column("criterion_mappings", "criterion_definition_id", nullable=False)
op.drop_constraint("criterion_mappings_criterion_key_fkey", "criterion_mappings", type_="foreignkey")
op.drop_constraint("criterion_definitions_pkey", "criterion_definitions", type_="primary")
op.create_primary_key("criterion_definitions_pkey", "criterion_definitions", ["id"])
op.create_unique_constraint("uq_criterion_definition_key", "criterion_definitions", ["key"])
op.create_foreign_key(
"fk_project_criteria_definition",
"project_criteria",
"criterion_definitions",
["criterion_definition_id"],
["id"],
)
op.create_foreign_key(
"fk_criterion_mappings_definition",
"criterion_mappings",
"criterion_definitions",
["criterion_definition_id"],
["id"],
)
op.create_table(
"source_attribute_mappings",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id")),
sa.Column("raw_name", sa.String(256), nullable=False),
sa.Column(
"criterion_definition_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("criterion_definitions.id"),
nullable=False,
),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.add_column("buying_projects", sa.Column("product_category_id", sa.String(64)))
op.add_column("buying_projects", sa.Column("market", sa.String(8)))
op.add_column("buying_projects", sa.Column("currency", sa.String(3)))
op.add_column("buying_projects", sa.Column("target_budget", sa.Numeric(12, 2)))
op.add_column("buying_projects", sa.Column("template_id", sa.String(128)))
op.add_column("buying_projects", sa.Column("template_version", sa.Integer()))
op.add_column("buying_projects", sa.Column("criteria_version", sa.Integer(), nullable=False, server_default="1"))
op.add_column("buying_projects", sa.Column("updated_at", sa.DateTime(timezone=True)))
op.execute(
"""
UPDATE buying_projects
SET template_id = profile_id,
template_version = profile_version,
product_category_id = 'notebook',
market = 'DE',
currency = 'EUR',
updated_at = created_at
"""
)
op.execute(
"""
UPDATE buying_projects AS bp
SET criteria_version = COALESCE((
SELECT MAX(version) FROM buying_project_criteria WHERE project_id = bp.id
), 1)
"""
)
op.alter_column("buying_projects", "product_category_id", nullable=False)
op.alter_column("buying_projects", "market", nullable=False)
op.alter_column("buying_projects", "currency", nullable=False)
op.alter_column("buying_projects", "updated_at", nullable=False)
op.create_foreign_key(
"fk_buying_projects_product_category",
"buying_projects",
"product_categories",
["product_category_id"],
["id"],
)
op.drop_constraint("fk_buying_projects_profile", "buying_projects", type_="foreignkey")
op.drop_constraint("ck_buying_projects_interval", "buying_projects", type_="check")
op.drop_column("buying_projects", "profile_id")
op.drop_column("buying_projects", "profile_version")
op.drop_column("buying_projects", "schedule_interval_minutes")
op.create_foreign_key(
"fk_buying_projects_template",
"buying_projects",
"product_profiles",
["template_id", "template_version"],
["id", "version"],
)
op.add_column(
"discovery_searches",
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
)
op.add_column("search_runs", sa.Column("new_count", sa.Integer(), nullable=False, server_default="0"))
op.add_column("search_runs", sa.Column("changed_count", sa.Integer(), nullable=False, server_default="0"))
op.add_column("search_runs", sa.Column("unchanged_count", sa.Integer(), nullable=False, server_default="0"))
op.add_column("search_runs", sa.Column("missing_count", sa.Integer(), nullable=False, server_default="0"))
op.add_column(
"product_observation_attributes",
sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)),
)
op.add_column("product_observation_attributes", sa.Column("raw_name", sa.String(256)))
op.create_foreign_key(
"fk_observation_attributes_definition",
"product_observation_attributes",
"criterion_definitions",
["criterion_definition_id"],
["id"],
)
op.execute(
"""
UPDATE product_observation_attributes AS poa
SET criterion_definition_id = cd.id,
raw_name = poa.attribute_key
FROM criterion_definitions AS cd
WHERE poa.attribute_key = cd.scoring_attribute_key
OR poa.attribute_key = cd.key
"""
)
def downgrade() -> None:
op.drop_constraint("fk_observation_attributes_definition", "product_observation_attributes", type_="foreignkey")
op.drop_column("product_observation_attributes", "raw_name")
op.drop_column("product_observation_attributes", "criterion_definition_id")
op.drop_column("search_runs", "missing_count")
op.drop_column("search_runs", "unchanged_count")
op.drop_column("search_runs", "changed_count")
op.drop_column("search_runs", "new_count")
op.drop_column("discovery_searches", "schedule_interval_minutes")
op.drop_constraint("fk_buying_projects_template", "buying_projects", type_="foreignkey")
op.add_column("buying_projects", sa.Column("profile_id", sa.String(128)))
op.add_column("buying_projects", sa.Column("profile_version", sa.Integer()))
op.add_column(
"buying_projects",
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
)
op.execute("UPDATE buying_projects SET profile_id = template_id, profile_version = template_version")
op.alter_column("buying_projects", "profile_id", nullable=False)
op.alter_column("buying_projects", "profile_version", nullable=False)
op.drop_constraint("fk_buying_projects_product_category", "buying_projects", type_="foreignkey")
op.drop_column("buying_projects", "updated_at")
op.drop_column("buying_projects", "criteria_version")
op.drop_column("buying_projects", "template_version")
op.drop_column("buying_projects", "template_id")
op.drop_column("buying_projects", "target_budget")
op.drop_column("buying_projects", "currency")
op.drop_column("buying_projects", "market")
op.drop_column("buying_projects", "product_category_id")
op.drop_table("source_attribute_mappings")
raise NotImplementedError("full downgrade of criterion UUID primary key is not supported")
@@ -0,0 +1,23 @@
{
"id": "mobile-development-notebook",
"version": 1,
"name": "Mobiler Entwicklungsrechner",
"category": "notebook",
"currency": "EUR",
"market": "DE",
"allowed_conditions": ["new"],
"budget": {"target": "1500.00", "maximum": "2000.00"},
"scoring_version": "generic-v1",
"constraints": [
{"id": "min_ram", "attribute": "ram_gb", "operator": ">=", "value": 16, "mandatory": true},
{"id": "min_storage", "attribute": "storage_gb", "operator": ">=", "value": 512, "mandatory": true},
{"id": "max_weight", "attribute": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": true}
],
"preferences": [
{"id": "ram", "attribute": "ram_gb", "strategy": "numeric_linear", "direction": "higher", "acceptable": 16, "ideal": 32, "weight": 0.30},
{"id": "weight", "attribute": "weight_kg", "strategy": "numeric_linear", "direction": "lower", "acceptable": 1.6, "ideal": 1.0, "weight": 0.25},
{"id": "battery", "attribute": "battery_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.20},
{"id": "display", "attribute": "display_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.15},
{"id": "cpu", "attribute": "cpu_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.10}
]
}
+36
View File
@@ -0,0 +1,36 @@
[project]
name = "buying-agent"
version = "0.1.0"
description = "Generic buying and deal evaluation agent"
requires-python = ">=3.13"
dependencies = [
"alembic>=1.16",
"fastapi>=0.118",
"psycopg[binary]>=3.2",
"pydantic>=2.11",
"pydantic-settings>=2.11",
"sqlalchemy>=2.0",
"uvicorn[standard]>=0.37",
]
[project.optional-dependencies]
test = [
"httpx>=0.28",
"pytest>=8.4",
]
[build-system]
requires = ["setuptools>=75"]
build-backend = "setuptools.build_meta"
[tool.setuptools.packages.find]
include = ["app*", "scripts*"]
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-ra"
markers = [
"slow: long-running tests skipped in the default CI suite",
"integration: needs PostgreSQL 17",
"e2e: HTTP end-to-end against PostgreSQL 17",
]
+23
View File
@@ -0,0 +1,23 @@
{
"version": "generic-v1",
"algorithm_id": "linear-budget-history-v1",
"supported_preference_strategies": ["numeric_linear"],
"minimum_confidence": 0.8,
"maximum_price_age_hours": 48,
"optional_constraint_penalty_points": 5,
"missing_preference_score": 0,
"product_weight": 0.6,
"price_weight": 0.4,
"budget_weight": 0.4,
"history_weight": 0.6,
"history_window_days": 90,
"minimum_history_days": 3,
"history_discount_for_max_score": 0.2,
"classification_upper_bounds": [
{"upper": 60, "inclusive": false, "label": "IGNORE"},
{"upper": 75, "inclusive": false, "label": "WATCH"},
{"upper": 85, "inclusive": false, "label": "GOOD"},
{"upper": 92, "inclusive": true, "label": "VERY_GOOD"},
{"upper": 100, "inclusive": true, "label": "EXCEPTIONAL"}
]
}
View File
Whitespace-only changes.
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy.orm import Session
from app.domain.validation import parse_scoring_model
from app.persistence.db import SessionLocal
from app.persistence.models import ScoringModelRecord, SourceRecord
from app.services.catalog_service import ensure_product_category
ROOT = Path(__file__).resolve().parent.parent
RULES = ROOT / "rules" / "generic-v1.json"
def run_bootstrap(session: Session) -> None:
_seed_source(session)
_seed_scoring_model(session)
ensure_product_category(session, "notebook", "Notebooks")
session.commit()
def _seed_source(session: Session) -> None:
_upsert_source(session, "manual", "Manual entry", "manual")
_upsert_source(session, "geizhals", "Geizhals", "geizhals")
def _upsert_source(session: Session, source_id: str, name: str, adapter_type: str) -> None:
existing = session.get(SourceRecord, source_id)
if existing is None:
session.add(SourceRecord(id=source_id, name=name, adapter_type=adapter_type))
return
if existing.adapter_type != adapter_type:
raise SystemExit(f"bootstrap conflict: source '{source_id}' has a different adapter_type")
def _seed_scoring_model(session: Session) -> None:
definition = json.loads(RULES.read_text(encoding="utf-8"))
model = parse_scoring_model(definition)
existing = session.get(ScoringModelRecord, model.version)
if existing is None:
session.add(
ScoringModelRecord(
version=model.version,
algorithm_id=model.algorithm_id,
definition=model.definition,
definition_hash=model.definition_hash,
created_at=datetime.now(timezone.utc),
)
)
return
if existing.definition_hash != model.definition_hash:
raise SystemExit(
f"bootstrap conflict: scoring model {model.version} hash "
f"{existing.definition_hash} != {model.definition_hash}"
)
def main() -> None:
session = SessionLocal()
try:
run_bootstrap(session)
print("bootstrap ok")
finally:
session.close()
if __name__ == "__main__":
main()
+138
View File
@@ -0,0 +1,138 @@
from __future__ import annotations
import json
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
FIXTURE = json.loads((ROOT / "fixtures" / "manual-demo.json").read_text(encoding="utf-8"))
BASE = "http://127.0.0.1:8000"
AS_OF = "2026-10-03T12:00:00Z"
DEFINITIONS = [
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "RAM in GB"},
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0, "description": "Storage in GB"},
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0, "description": "Weight in kg"},
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual battery score"},
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual display score"},
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100, "description": "Manual CPU score"},
{"key": "ports", "value_type": "string_list", "description": "Ports"},
]
def request(method: str, path: str, body=None, expected=(200, 201, 409)):
data = None if body is None else json.dumps(body, default=str).encode()
req = urllib.request.Request(
f"{BASE}{path}",
data=data,
method=method,
headers={"Content-Type": "application/json"} if body is not None else {},
)
try:
with urllib.request.urlopen(req) as response:
payload = json.loads(response.read().decode())
if response.status not in expected:
raise SystemExit(f"{method} {path} -> {response.status}: {payload}")
return response.status, payload
except urllib.error.HTTPError as exc:
payload = json.loads(exc.read().decode())
if exc.code not in expected:
raise SystemExit(f"{method} {path} -> {exc.code}: {payload}") from exc
return exc.code, payload
def main() -> None:
for definition in DEFINITIONS:
request("POST", "/api/v1/attribute-definitions", definition)
request("POST", "/api/v1/profiles", PROFILE)
product = FIXTURE["products"][0]
request(
"POST",
"/api/v1/products",
{
"id": product["id"],
"category": product["category"],
"brand": product["brand"],
"name": product["name"],
"model": product.get("model"),
},
)
for variant in product["variants"]:
request(
"POST",
f"/api/v1/products/{product['id']}/variants",
{
"id": variant["id"],
"name": variant["name"],
"manufacturer_sku": variant.get("manufacturer_sku"),
"identifiers": variant.get("identifiers") or {},
},
)
for attribute in variant["attributes"]:
request("POST", f"/api/v1/variants/{variant['id']}/attributes", {**attribute, "source_id": "manual"})
for offer in FIXTURE["offers"]:
request(
"POST",
f"/api/v1/variants/{offer['variant_id']}/offers",
{
"id": offer["id"],
"source_id": "manual",
"external_id": offer["external_id"],
"seller": offer["seller"],
"url": offer.get("url"),
"market": offer["market"],
"currency": offer["currency"],
"condition": offer["condition"],
},
)
for price in FIXTURE["prices"]:
offer = next(item for item in FIXTURE["offers"] if item["external_id"] == price["external_offer_id"])
request(
"POST",
f"/api/v1/offers/{offer['id']}/prices",
{
"id": price["id"],
"source_id": "manual",
"external_observation_id": price["external_observation_id"],
"observed_at": price["observed_at"],
"item_price": price["item_price"],
"shipping_price": price["shipping_price"],
"currency": price["currency"],
"availability": price["availability"],
"raw_payload": price["raw_payload"],
"confidence": price["confidence"],
},
expected=(200, 201),
)
cases = {
"A": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-333333333333"),
"B": ("22222222-2222-4222-8222-22222222222b", "33333333-3333-4333-8333-33333333333b"),
"C": ("22222222-2222-4222-8222-22222222222c", "33333333-3333-4333-8333-33333333333c"),
"D": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-33333333333d"),
}
for label, (variant_id, offer_id) in cases.items():
status, result = request(
"POST",
f"/api/v1/evaluate/{variant_id}",
{
"profile_id": "mobile-development-notebook",
"profile_version": 1,
"offer_id": offer_id,
"as_of": AS_OF,
},
expected=(201,),
)
print(
f"case {label}: {status} overall={result['overall_eligibility']} "
f"product={result['product_score']} price={result['price_score']} "
f"deal={result['deal_score']} class={result['classification']}"
)
if __name__ == "__main__":
main()
+81
View File
@@ -0,0 +1,81 @@
"""Notebook-pilot metadata only. Not part of the core schema or bootstrap."""
from __future__ import annotations
from sqlalchemy.orm import Session
from app.discovery.definitions import get_definition_by_key, upsert_definition
from app.persistence.models import AttributeDefinitionRecord
from app.services.catalog_service import ensure_product_category
NOTEBOOK_ATTRIBUTE_SEEDS = (
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
)
NOTEBOOK_CRITERION_SEEDS = (
{
"key": "ram_gb",
"origin": "SOURCE",
"source": "geizhals",
"product_category_id": "notebook",
"label": "Arbeitsspeicher",
"value_type": "number",
"unit": "GB",
"semantic": "working_memory",
"category": "notebook",
"scoring_attribute_key": "ram_gb",
},
{
"key": "storage_gb",
"origin": "SOURCE",
"source": "geizhals",
"product_category_id": "notebook",
"label": "SSD",
"value_type": "number",
"unit": "GB",
"semantic": "storage",
"category": "notebook",
"scoring_attribute_key": "storage_gb",
},
{
"key": "weight_kg",
"origin": "SOURCE",
"source": "geizhals",
"product_category_id": "notebook",
"label": "Gewicht",
"value_type": "number",
"unit": "kg",
"semantic": "weight",
"category": "notebook",
"scoring_attribute_key": "weight_kg",
},
{
"key": "price_eur",
"origin": "SOURCE",
"source": "geizhals",
"product_category_id": "notebook",
"label": "Preis",
"value_type": "number",
"unit": "EUR",
"semantic": "price",
"category": "notebook",
"scoring_attribute_key": "price_eur",
},
)
def seed_notebook_pilot(session: Session) -> None:
ensure_product_category(session, "notebook", "Notebooks")
for item in NOTEBOOK_ATTRIBUTE_SEEDS:
if session.get(AttributeDefinitionRecord, item["key"]) is None:
session.add(AttributeDefinitionRecord(**item, description=None))
for item in NOTEBOOK_CRITERION_SEEDS:
if get_definition_by_key(session, item["key"]) is None:
upsert_definition(session, item)
session.flush()
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
echo "=== BuyingAgent API startup ==="
alembic upgrade head
python -m scripts.bootstrap
echo "=== starting uvicorn ==="
exec uvicorn app.main:app --host 0.0.0.0 --port 8000
View File
Whitespace-only changes.
+150
View File
@@ -0,0 +1,150 @@
import json
from datetime import datetime, timezone
from decimal import Decimal
from pathlib import Path
from uuid import UUID, uuid4
import pytest
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
from app.domain.models import (
AttributeDefinition,
AttributeStatement,
Offer,
PriceObservation,
Product,
ProductVariant,
)
from app.domain.validation import parse_profile, parse_scoring_model
ROOT = Path(__file__).resolve().parents[1]
AS_OF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
CUTOFF = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)
@pytest.fixture
def scoring_model():
definition = json.loads((ROOT / "rules" / "generic-v1.json").read_text(encoding="utf-8"))
return parse_scoring_model(definition)
@pytest.fixture
def definitions() -> dict[str, AttributeDefinition]:
return {
"ram_gb": AttributeDefinition("ram_gb", ValueType.NUMBER, "GB", Decimal("0"), None, None),
"storage_gb": AttributeDefinition("storage_gb", ValueType.NUMBER, "GB", Decimal("0"), None, None),
"weight_kg": AttributeDefinition("weight_kg", ValueType.NUMBER, "kg", Decimal("0"), None, None),
"battery_score": AttributeDefinition("battery_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
"display_score": AttributeDefinition("display_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
"cpu_score": AttributeDefinition("cpu_score", ValueType.NUMBER, None, Decimal("0"), Decimal("100"), None),
"ports": AttributeDefinition("ports", ValueType.STRING_LIST, None, None, None, None),
"color": AttributeDefinition("color", ValueType.STRING, None, None, None, None),
"has_lte": AttributeDefinition("has_lte", ValueType.BOOLEAN, None, None, None, None),
}
@pytest.fixture
def profile(scoring_model, definitions):
raw = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
return parse_profile(
raw,
attribute_definitions=definitions,
scoring_versions={scoring_model.version},
supported_strategies=set(scoring_model.supported_preference_strategies),
)
def utc(year, month, day, hour=10, minute=0) -> datetime:
return datetime(year, month, day, hour, minute, tzinfo=timezone.utc)
def statement(attribute: str, value, *, confidence="1.0", status="normalized", observed=None, ingested=None, row_id=None):
return AttributeStatement(
id=row_id or uuid4(),
variant_id=UUID("22222222-2222-4222-8222-222222222222"),
attribute=attribute,
raw_value=value,
normalized_value=None if status != "normalized" else value,
unit="GB" if attribute.endswith("_gb") else ("kg" if attribute.endswith("_kg") else None),
normalization_status=NormalizationStatus(status),
normalizer_version="manual-v1",
confidence=Decimal(str(confidence)),
source_id="manual",
source_reference=f"test/{attribute}",
observed_at=observed or utc(2026, 10, 3, 9),
ingested_at=ingested or utc(2026, 10, 3, 9, 1),
)
def product_a() -> Product:
return Product(
id=UUID("11111111-1111-4111-8111-111111111111"),
category="notebook",
brand="Example",
name="Example Mobile 13",
model="EM13",
created_at=utc(2026, 10, 1, 8),
)
def variant_a() -> ProductVariant:
return ProductVariant(
id=UUID("22222222-2222-4222-8222-222222222222"),
product_id=UUID("11111111-1111-4111-8111-111111111111"),
name="32/1024",
manufacturer_sku="EM13-32-1T",
identifiers={},
created_at=utc(2026, 10, 1, 8),
)
def offer_a() -> Offer:
return Offer(
id=UUID("33333333-3333-4333-8333-333333333333"),
variant_id=UUID("22222222-2222-4222-8222-222222222222"),
source_id="manual",
external_id="demo-offer-a",
seller="DemoShop",
url=None,
market="DE",
currency="EUR",
condition=OfferCondition.NEW,
created_at=utc(2026, 10, 1, 8),
)
def observation(day, total, *, offer_id=None, obs_id=None, availability="in_stock", confidence="1.0", hour=10):
return PriceObservation(
id=obs_id or uuid4(),
offer_id=offer_id or UUID("33333333-3333-4333-8333-333333333333"),
source_id="manual",
external_observation_id=str(obs_id or uuid4()),
observed_at=utc(2026, day[0], day[1], hour) if isinstance(day, tuple) else day,
ingested_at=utc(2026, 10, 3, 11),
item_price=Decimal(str(total)),
shipping_price=Decimal("0.00"),
currency="EUR",
availability=Availability(availability),
raw_payload={"total": str(total)},
confidence=Decimal(str(confidence)),
)
def case_a_attributes():
return (
statement("ram_gb", 32),
statement("storage_gb", 1024),
statement("weight_kg", Decimal("1.2")),
statement("battery_score", 90),
statement("display_score", 90),
statement("cpu_score", 90),
)
def case_a_prices():
return (
observation((9, 1), "1599.00"),
observation((9, 15), "1599.00"),
observation((10, 1), "1599.00"),
observation((10, 3), "1298.00"),
)
View File
Whitespace-only changes.
@@ -0,0 +1,199 @@
import json
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from app.main import app
from app.persistence.db import SessionLocal, engine
from scripts.bootstrap import run_bootstrap
from scripts.seed_notebook_pilot import seed_notebook_pilot
pytestmark = [pytest.mark.e2e, pytest.mark.integration]
ROOT = Path(__file__).resolve().parents[2]
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
DEFINITIONS = [
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
]
CRITERIA = {
"items": [
{"id": "ram", "criterion": "ram_gb", "operator": ">=", "value": 24, "mandatory": True},
{"id": "price", "criterion": "price_eur", "operator": "<=", "value": 2000, "mandatory": True},
{"id": "weight", "criterion": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": True},
]
}
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
session = SessionLocal()
try:
run_bootstrap(session)
seed_notebook_pilot(session)
session.commit()
finally:
session.close()
@pytest.fixture
def client() -> TestClient:
return TestClient(app)
def _ok(response, allowed=(200, 201, 409)) -> None:
assert response.status_code in allowed, response.text
def _seed_profile(client: TestClient) -> None:
for definition in DEFINITIONS:
_ok(client.post("/api/v1/attribute-definitions", json=definition))
_ok(client.post("/api/v1/profiles", json=PROFILE))
def test_notebook_discovery_pilot(client: TestClient) -> None:
_seed_profile(client)
taxonomy = client.post("/api/v1/sources/geizhals/categories/nb/discover-taxonomy")
assert taxonomy.status_code == 201, taxonomy.text
category_id = taxonomy.json()["id"]
filters = client.get("/api/v1/sources/geizhals/categories/nb/filters")
assert filters.status_code == 200
assert {item["external_filter_key"] for item in filters.json()["filters"]} >= {"17023", "price"}
ram = client.post(
"/api/v1/criterion-mappings",
json={
"criterion_key": "ram_gb",
"source_id": "geizhals",
"source_category_id": category_id,
"external_filter_key": "17023",
"source_value": "ab 24GB",
"encoding": {"url_param": "xf"},
},
)
assert ram.status_code == 201, ram.text
assert ram.json()["status"] == "VERIFIED"
price = client.post(
"/api/v1/criterion-mappings",
json={
"criterion_key": "price_eur",
"source_id": "geizhals",
"source_category_id": category_id,
"external_filter_key": "price",
"source_value": None,
"encoding": {"url_param": "maxp"},
"probe_value": 2000,
"operator": "<=",
},
)
assert price.status_code == 201, price.text
assert price.json()["status"] == "VERIFIED"
project = client.post(
"/api/v1/buying-projects",
json={
"name": "Notebook-Pilot",
"profile_id": "mobile-development-notebook",
"profile_version": 1,
"shortlist_size": 5,
"schedule_interval_minutes": 180,
"criteria": CRITERIA,
},
)
assert project.status_code == 201, project.text
project_id = project.json()["id"]
assert project.json()["workspace_id"] is None
assert project.json()["criteria_version"] == 1
available = client.get(f"/api/v1/buying-projects/{project_id}/available-criteria")
assert available.status_code == 200, available.text
assert available.json()["product_category_id"] == "notebook"
scoring_keys = {item.get("scoring_attribute_key") for item in available.json()["criteria"]}
assert {"ram_gb", "price_eur"} <= scoring_keys
ram = next(item for item in available.json()["criteria"] if item.get("scoring_attribute_key") == "ram_gb")
assert ram["mapping_status"] == "VERIFIED"
search = client.post(
"/api/v1/discovery-searches",
json={
"buying_project_id": project_id,
"source_id": "geizhals",
"source_category_id": category_id,
"broad_result_threshold": 300,
},
)
assert search.status_code == 201, search.text
search_id = search.json()["id"]
assert "xf=17023_ab+24GB" in search.json()["generated_url"]
assert "maxp=2000" in search.json()["generated_url"]
assert search.json()["schedule_interval_minutes"] == 180
due = client.get("/api/v1/discovery-searches?due=true")
assert due.status_code == 200
assert any(item["id"] == search_id for item in due.json()["items"])
run = client.post(f"/api/v1/discovery-searches/{search_id}/run")
assert run.status_code == 201, run.text
assert run.json()["status"] == "SUCCESS"
assert run.json()["page_count"] == 2
assert run.json()["result_count"] == 3
assert {item["diff_status"] for item in run.json()["items"]} == {"NEW"}
shortlist = client.get(f"/api/v1/buying-projects/{project_id}/shortlist")
assert shortlist.status_code == 200
assert len(shortlist.json()["items"]) == 1
assert shortlist.json()["items"][0]["product_eligibility"] == "ELIGIBLE"
assert shortlist.json()["items"][0]["status"] == "SHORTLISTED"
candidates = client.get(f"/api/v1/buying-projects/{project_id}/candidates")
assert candidates.status_code == 200
eligibilities = {item["product_eligibility"] for item in candidates.json()["items"]}
assert eligibilities == {"ELIGIBLE", "INELIGIBLE", "UNKNOWN"}
assert all(item.get("deal_score") is None for item in candidates.json()["items"])
updated = client.post(
f"/api/v1/buying-projects/{project_id}/criteria",
json={"items": CRITERIA["items"][:2]},
)
assert updated.status_code == 201
assert updated.json()["criteria_version"] == 2
stale = client.get(f"/api/v1/discovery-searches/{search_id}")
assert stale.json()["status"] == "OUT_OF_SYNC"
blocked_run = client.post(f"/api/v1/discovery-searches/{search_id}/run")
assert blocked_run.status_code == 409
refreshed = client.post(f"/api/v1/discovery-searches/{search_id}/generate-query")
assert refreshed.status_code == 200
assert refreshed.json()["status"] == "ACTIVE"
assert "weight_kg" not in refreshed.json().get("internal_only", [])
assert "weight_kg" not in refreshed.json()["generated_url"]
client.patch(f"/api/v1/discovery-searches/{search_id}", json={"status": "DISABLED"})
broad = client.post(
"/api/v1/discovery-searches",
json={
"buying_project_id": project_id,
"source_id": "geizhals",
"source_category_id": category_id,
"broad_result_threshold": 2,
},
)
assert broad.status_code == 201, broad.text
broad_run = client.post(f"/api/v1/discovery-searches/{broad.json()['id']}/run")
assert broad_run.status_code == 201, broad_run.text
assert broad_run.json()["status"] == "TOO_BROAD"
assert broad_run.json()["result_count"] == 3
stored = client.get(f"/api/v1/discovery-searches/{broad.json()['id']}")
assert stored.json()["status"] == "TOO_BROAD"
+204
View File
@@ -0,0 +1,204 @@
import json
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from app.main import app
from app.persistence.db import SessionLocal, engine
from scripts.bootstrap import run_bootstrap
pytestmark = [pytest.mark.e2e, pytest.mark.integration]
ROOT = Path(__file__).resolve().parents[2]
PROFILE = json.loads((ROOT / "profiles" / "mobile-development-notebook.json").read_text(encoding="utf-8"))
FIXTURE = json.loads((ROOT / "fixtures" / "manual-demo.json").read_text(encoding="utf-8"))
AS_OF = "2026-10-03T12:00:00Z"
DEFINITIONS = [
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
]
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
session = SessionLocal()
try:
run_bootstrap(session)
finally:
session.close()
@pytest.fixture
def client() -> TestClient:
return TestClient(app)
def _ok(response, allowed=(200, 201, 409)) -> None:
assert response.status_code in allowed, response.text
def _seed(client: TestClient) -> None:
for definition in DEFINITIONS:
_ok(client.post("/api/v1/attribute-definitions", json=definition))
_ok(client.post("/api/v1/profiles", json=PROFILE))
product = FIXTURE["products"][0]
_ok(
client.post(
"/api/v1/products",
json={
"id": product["id"],
"category": product["category"],
"brand": product["brand"],
"name": product["name"],
"model": product.get("model"),
},
)
)
for variant in product["variants"]:
_ok(
client.post(
f"/api/v1/products/{product['id']}/variants",
json={
"id": variant["id"],
"name": variant["name"],
"manufacturer_sku": variant.get("manufacturer_sku"),
"identifiers": variant.get("identifiers") or {},
},
)
)
for attribute in variant["attributes"]:
_ok(client.post(f"/api/v1/variants/{variant['id']}/attributes", json={**attribute, "source_id": "manual"}))
for offer in FIXTURE["offers"]:
_ok(
client.post(
f"/api/v1/variants/{offer['variant_id']}/offers",
json={
"id": offer["id"],
"source_id": "manual",
"external_id": offer["external_id"],
"seller": offer["seller"],
"url": offer.get("url"),
"market": offer["market"],
"currency": offer["currency"],
"condition": offer["condition"],
},
)
)
for price in FIXTURE["prices"]:
offer = next(item for item in FIXTURE["offers"] if item["external_id"] == price["external_offer_id"])
_ok(
client.post(
f"/api/v1/offers/{offer['id']}/prices",
json={
"id": price["id"],
"source_id": "manual",
"external_observation_id": price["external_observation_id"],
"observed_at": price["observed_at"],
"item_price": price["item_price"],
"shipping_price": price["shipping_price"],
"currency": price["currency"],
"availability": price["availability"],
"raw_payload": price["raw_payload"],
"confidence": price["confidence"],
},
),
allowed=(200, 201),
)
def test_http_cases_and_replay(client: TestClient) -> None:
_seed(client)
cases = {
"A": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-333333333333"),
"B": ("22222222-2222-4222-8222-22222222222b", "33333333-3333-4333-8333-33333333333b"),
"C": ("22222222-2222-4222-8222-22222222222c", "33333333-3333-4333-8333-33333333333c"),
"D": ("22222222-2222-4222-8222-222222222222", "33333333-3333-4333-8333-33333333333d"),
}
created = {}
for label, (variant_id, offer_id) in cases.items():
response = client.post(
f"/api/v1/evaluate/{variant_id}",
json={
"profile_id": "mobile-development-notebook",
"profile_version": 1,
"offer_id": offer_id,
"as_of": AS_OF,
},
)
assert response.status_code == 201, response.text
created[label] = response.json()
assert created["A"]["product_score"] == 90.21
assert created["A"]["price_score"] == 98.24
assert created["A"]["deal_score"] == 93.42
assert created["A"]["classification"] == "EXCEPTIONAL"
assert created["A"]["eligible"] is True
assert created["A"]["price_reference_scope"] == "offer"
assert created["B"]["product_eligibility"] == "INELIGIBLE"
assert created["B"]["offer_eligibility"] == "ELIGIBLE"
assert created["B"]["deal_score"] is None
assert created["B"]["product_score"] is not None
assert created["C"]["product_eligibility"] == "UNKNOWN"
assert created["C"]["classification"] == "INSUFFICIENT_DATA"
assert created["D"]["product_eligibility"] == "ELIGIBLE"
assert created["D"]["offer_eligibility"] == "INELIGIBLE"
assert created["D"]["product_score"] == 90.21
assert created["D"]["price_score"] is not None
assert created["D"]["deal_score"] is None
stored = client.get(f"/api/v1/evaluations/{created['A']['id']}")
assert stored.status_code == 200
assert stored.json()["deal_score"] == created["A"]["deal_score"]
assert stored.json()["input_snapshot"] == created["A"]["input_snapshot"]
extra = client.post(
"/api/v1/offers/33333333-3333-4333-8333-333333333333/prices",
json={
"source_id": "manual",
"external_observation_id": "a-later",
"observed_at": "2026-10-03T11:00:00Z",
"item_price": "999.00",
"shipping_price": "0.00",
"currency": "EUR",
"availability": "in_stock",
"raw_payload": {"total": "999.00"},
"confidence": 1.0,
},
)
assert extra.status_code == 201
replay = client.get(f"/api/v1/evaluations/{created['A']['id']}")
assert replay.json()["deal_score"] == created["A"]["deal_score"]
assert replay.json()["price_statistics"]["current_total"] == created["A"]["price_statistics"]["current_total"]
def test_error_contracts(client: TestClient) -> None:
missing = client.get("/api/v1/products/00000000-0000-4000-8000-000000000000")
assert missing.status_code == 404
assert "error" in missing.json()
assert missing.json()["error"]["code"] == "NOT_FOUND"
bad = client.post("/api/v1/products", json={"category": "notebook"})
assert bad.status_code == 422
assert "error" in bad.json()
client.post("/api/v1/attribute-definitions", json={"key": "ports", "value_type": "string_list"})
again = client.post("/api/v1/attribute-definitions", json={"key": "ports", "value_type": "string_list"})
assert again.status_code == 409
no_version = client.get("/api/v1/profiles/mobile-development-notebook")
assert no_version.status_code == 422
Whitespace-only changes.
@@ -0,0 +1,249 @@
from copy import deepcopy
from decimal import Decimal
import pytest
from sqlalchemy import text
from app.discovery.enums import DiffStatus, LifecycleStatus, MatchStatus, RunStatus
from app.errors import AppError
from app.persistence.db import SessionLocal, engine
from app.persistence.discovery_models import (
ProductObservationRecord,
ProjectCandidateRecord,
SourceProductReferenceRecord,
)
from app.services.discovery_service import (
CriterionMappingService,
DiscoverySearchService,
SearchExecutionService,
TaxonomyService,
)
from app.services.ingestion_service import IngestionService
from app.services.project_service import ProjectService
from app.sources.geizhals.adapter import GeizhalsAdapter
from scripts.bootstrap import run_bootstrap
from scripts.seed_notebook_pilot import seed_notebook_pilot
from tests.e2e.test_http_walkthrough import DEFINITIONS, PROFILE
pytestmark = pytest.mark.integration
ITEM_A = {
"source_external_id": "gh-nb-a",
"source_url": "https://geizhals.de/example-mobile-13-a.html",
"source_title": "Example Mobile 13 32GB",
"manufacturer": "Example",
"model": "Mobile 13",
"manufacturer_sku": "EM13-32-1T",
"gtin_ean": "4000000000001",
"best_price": "1298.00",
"currency": "EUR",
"availability": "in_stock",
"seller_count": 8,
"raw_attributes": {"RAM": "32GB"},
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2},
}
def _item(suffix: str, **overrides) -> dict:
item = deepcopy(ITEM_A)
item["source_external_id"] = f"gh-nb-{suffix}"
item["source_url"] = f"https://geizhals.de/example-mobile-13-{suffix}.html"
item["manufacturer_sku"] = f"EM13-{suffix}"
item["gtin_ean"] = None
item["source_title"] = f"Example Mobile 13 {suffix}"
item.update(overrides)
return item
class ScriptedAdapter:
source_key = "geizhals"
def __init__(self, pages: list[dict]) -> None:
self.pages = pages
self._inner = GeizhalsAdapter()
def discover_taxonomy(self, category_ref: str) -> dict:
return self._inner.discover_taxonomy(category_ref)
def build_query(self, criteria, mappings) -> dict:
return self._inner.build_query(criteria, mappings)
def fetch_results(self, query, pagination) -> dict:
page = int(pagination.get("page") or 1)
return self.pages[page - 1]
def parse_results(self, response) -> list:
return list(response.get("items") or [])
def normalize_result(self, result) -> dict:
return self._inner.normalize_result(result)
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
def _session():
session = SessionLocal()
run_bootstrap(session)
seed_notebook_pilot(session)
ingestion = IngestionService(session)
for definition in DEFINITIONS:
try:
ingestion.create_definition(definition)
except AppError as exc:
if exc.status_code != 409:
raise
try:
ingestion.create_profile(PROFILE)
except AppError as exc:
if exc.status_code != 409:
raise
return session
def _prepare_search(session, adapter, threshold=300):
taxonomy = TaxonomyService(session, adapter).discover("geizhals", "nb")
mappings = CriterionMappingService(session)
mappings.create(
{
"criterion_key": "ram_gb",
"source_id": "geizhals",
"source_category_id": taxonomy["id"],
"external_filter_key": "17023",
"source_value": "ab 24GB",
"encoding": {"url_param": "xf"},
}
)
project = ProjectService(session).create(
{
"name": "Lifecycle",
"profile_id": "mobile-development-notebook",
"profile_version": 1,
"criteria": {"items": [{"id": "ram", "criterion": "ram_gb", "operator": ">=", "value": 24}]},
}
)
search = DiscoverySearchService(session, adapter).create(
{
"buying_project_id": project["id"],
"source_id": "geizhals",
"source_category_id": taxonomy["id"],
"broad_result_threshold": threshold,
}
)
return search["id"]
def _page(items, status=200, page_count=1):
return {"page": 1, "page_count": page_count, "result_count": len(items), "http_status": status, "items": items}
def test_partial_does_not_mark_missing_or_disappeared():
session = _session()
try:
adapter = ScriptedAdapter([_page([_item("partial")])])
search_id = _prepare_search(session, adapter)
first = SearchExecutionService(session, adapter).execute(search_id)
assert first["status"] == RunStatus.SUCCESS.value
ref_id = first["items"][0]["source_product_reference_id"]
adapter.pages = [_page([], status=429)]
second = SearchExecutionService(session, adapter).execute(search_id)
assert second["status"] == RunStatus.PARTIAL.value
assert all(item["diff_status"] != DiffStatus.MISSING.value for item in second["items"])
ref = session.get(SourceProductReferenceRecord, ref_id)
assert ref is not None
assert ref.lifecycle_status == LifecycleStatus.ACTIVE.value
assert ref.consecutive_missing_success_runs == 0
finally:
session.close()
def test_three_success_misses_disappear_and_reappear():
session = _session()
try:
item = _item("gone")
adapter = ScriptedAdapter([_page([item])])
search_id = _prepare_search(session, adapter)
first = SearchExecutionService(session, adapter).execute(search_id)
ref_id = first["items"][0]["source_product_reference_id"]
adapter.pages = [_page([])]
for _ in range(3):
SearchExecutionService(session, adapter).execute(search_id)
ref = session.get(SourceProductReferenceRecord, ref_id)
assert ref is not None
assert ref.lifecycle_status == LifecycleStatus.DISAPPEARED.value
changed = deepcopy(item)
changed["best_price"] = "999.00"
adapter.pages = [_page([changed])]
run = SearchExecutionService(session, adapter).execute(search_id)
session.refresh(ref)
assert ref.lifecycle_status == LifecycleStatus.ACTIVE.value
assert any(item["diff_status"] == DiffStatus.CHANGED.value for item in run["items"])
assert all(item["diff_status"] != DiffStatus.NEW.value for item in run["items"])
prices = [
row.best_price
for row in session.query(ProductObservationRecord).filter(
ProductObservationRecord.source_product_reference_id == ref_id
)
]
assert Decimal("1298.00") in prices
assert Decimal("999.00") in prices
finally:
session.close()
def test_uncertain_match_stays_separate():
session = _session()
try:
first = _item("matched", manufacturer="Example", model="Uncertain 13")
second = _item(
"uncertain",
source_external_id=None,
manufacturer_sku=None,
source_url="https://geizhals.de/other-uncertain.html",
source_title=first["source_title"],
manufacturer="Example",
model="Uncertain 13",
normalized_attributes={"ram_gb": 16, "storage_gb": 512, "weight_kg": 2.0},
)
adapter = ScriptedAdapter([_page([first, second])])
search_id = _prepare_search(session, adapter)
project_id = DiscoverySearchService(session, adapter).get(search_id)["buying_project_id"]
run = SearchExecutionService(session, adapter).execute(search_id)
assert run["status"] == RunStatus.SUCCESS.value
ref_ids = {item["source_product_reference_id"] for item in run["items"]}
assert len(ref_ids) == 2
refs = [session.get(SourceProductReferenceRecord, ref_id) for ref_id in ref_ids]
assert MatchStatus.MATCH_UNCERTAIN.value in {ref.match_status for ref in refs if ref is not None}
candidates = (
session.query(ProjectCandidateRecord)
.filter(ProjectCandidateRecord.buying_project_id == project_id)
.all()
)
assert len(candidates) == 2
assert MatchStatus.MATCH_UNCERTAIN.value in {row.match_status for row in candidates}
finally:
session.close()
def test_backoff_and_blocked():
session = _session()
try:
adapter = ScriptedAdapter([_page([ITEM_A], status=429)])
search_id = _prepare_search(session, adapter)
backoff = SearchExecutionService(session, adapter).execute(search_id)
assert backoff["status"] == RunStatus.PARTIAL.value
assert backoff["error_code"] == "BACKOFF"
adapter.pages = [_page([ITEM_A], status=403)]
blocked = SearchExecutionService(session, adapter).execute(search_id)
assert blocked["status"] == RunStatus.BLOCKED.value
stored = DiscoverySearchService(session, adapter).get(search_id)
assert stored["status"] == "BLOCKED"
assert stored["next_run_at"] is None
finally:
session.close()
@@ -0,0 +1,59 @@
from pathlib import Path
import pytest
from sqlalchemy import text
from app.persistence.db import SessionLocal, engine
from app.services.import_service import ImportService
from app.services.ingestion_service import IngestionService
from app.sources.manual import ManualSource
from scripts.bootstrap import run_bootstrap
pytestmark = pytest.mark.integration
ROOT = Path(__file__).resolve().parents[2]
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
def test_manual_source_import_is_idempotent() -> None:
session = SessionLocal()
try:
run_bootstrap(session)
ingestion = IngestionService(session)
for item in (
{"key": "ram_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "storage_gb", "value_type": "number", "canonical_unit": "GB", "minimum": 0},
{"key": "weight_kg", "value_type": "number", "canonical_unit": "kg", "minimum": 0},
{"key": "battery_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "display_score", "value_type": "number", "minimum": 0, "maximum": 100},
{"key": "cpu_score", "value_type": "number", "minimum": 0, "maximum": 100},
):
try:
ingestion.create_definition(item)
except Exception:
session.rollback()
source = ManualSource(ROOT / "fixtures" / "manual-demo.json")
importer = ImportService(ingestion)
importer.import_product(source, "example-mobile-13")
session.commit()
importer.import_product(source, "example-mobile-13")
session.commit()
from sqlalchemy import func, select
from app.persistence.models import OfferRecord, PriceObservationRecord, ProductRecord
products = session.scalar(select(func.count()).select_from(ProductRecord))
offers = session.scalar(select(func.count()).select_from(OfferRecord))
prices = session.scalar(select(func.count()).select_from(PriceObservationRecord))
assert products >= 1
assert offers >= 4
assert prices >= 16
finally:
session.close()
+40
View File
@@ -0,0 +1,40 @@
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from alembic.config import Config
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from app.main import app
from app.persistence.db import engine
pytestmark = pytest.mark.integration
client = TestClient(app)
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
def test_health_ready_when_migrated() -> None:
response = client.get("/health/ready")
assert response.status_code == 200
body = response.json()
assert body["status"] == "ready"
assert "environment" in body
assert "engine_build" in body
def test_alembic_is_at_head() -> None:
config = Config("alembic.ini")
script = ScriptDirectory.from_config(config)
with engine.connect() as connection:
context = MigrationContext.configure(connection)
assert set(context.get_current_heads()) == set(script.get_heads())
@@ -0,0 +1,53 @@
import pytest
from sqlalchemy import text
from alembic import command
from alembic.config import Config
from app.domain.validation import parse_scoring_model
from app.persistence.db import SessionLocal, engine
from app.persistence.models import ScoringModelRecord
from scripts.bootstrap import run_bootstrap
pytestmark = pytest.mark.integration
@pytest.fixture(scope="module", autouse=True)
def require_postgres() -> None:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
except Exception as exc:
pytest.skip(f"PostgreSQL not available: {exc}")
def test_upgrade_is_idempotent() -> None:
config = Config("alembic.ini")
command.upgrade(config, "head")
command.upgrade(config, "head")
def test_bootstrap_is_idempotent_and_rejects_hash_change(tmp_path, monkeypatch) -> None:
session = SessionLocal()
try:
run_bootstrap(session)
run_bootstrap(session)
row = session.get(ScoringModelRecord, "generic-v1")
assert row is not None
model = parse_scoring_model(row.definition)
assert row.definition_hash == model.definition_hash
row.definition_hash = "0" * 64
session.commit()
with pytest.raises(SystemExit):
run_bootstrap(session)
row.definition_hash = model.definition_hash
session.commit()
finally:
session.close()
@pytest.mark.slow
def test_downgrade_and_upgrade_on_disposable_db() -> None:
config = Config("alembic.ini")
command.downgrade(config, "base")
command.upgrade(config, "head")
View File
Whitespace-only changes.
+59
View File
@@ -0,0 +1,59 @@
from decimal import Decimal
from app.domain.constraints import evaluate_constraint
from app.domain.enums import ConstraintOperator
from app.domain.models import Constraint
from tests.conftest import statement
def _eval(definitions, attribute, operator, value, stmt, confidence="0.8"):
return evaluate_constraint(
Constraint("c1", attribute, ConstraintOperator(operator), value, True),
stmt,
definitions[attribute],
Decimal(confidence),
)
def test_number_operators_pass_fail_unknown(definitions):
assert _eval(definitions, "ram_gb", ">=", 16, statement("ram_gb", 16)).status == "PASS"
assert _eval(definitions, "ram_gb", ">", 16, statement("ram_gb", 16)).status == "FAIL"
assert _eval(definitions, "ram_gb", "<", 16, statement("ram_gb", 8)).status == "PASS"
assert _eval(definitions, "ram_gb", "<=", 16, statement("ram_gb", 16)).status == "PASS"
assert _eval(definitions, "ram_gb", ">=", 16, None).status == "UNKNOWN"
def test_equality_and_in(definitions):
assert _eval(definitions, "color", "==", "black", statement("color", "black")).status == "PASS"
assert _eval(definitions, "color", "!=", "silver", statement("color", "black")).status == "PASS"
assert _eval(definitions, "has_lte", "==", True, statement("has_lte", True)).status == "PASS"
assert _eval(definitions, "ram_gb", "in", [16, 32], statement("ram_gb", 32)).status == "PASS"
assert _eval(definitions, "ram_gb", "in", [16], statement("ram_gb", 8)).status == "FAIL"
def test_contains_and_empty_list(definitions):
empty = statement("ports", [])
assert _eval(definitions, "ports", "exists", None, empty).status == "PASS"
assert _eval(definitions, "ports", "contains", "hdmi", empty).status == "FAIL"
present = statement("ports", ["hdmi", "usb-c"])
assert _eval(definitions, "ports", "contains", "hdmi", present).status == "PASS"
def test_exists_unknown_when_unusable(definitions):
assert _eval(definitions, "ram_gb", "exists", None, None).status == "UNKNOWN"
unknown = statement("ram_gb", None, status="unknown")
assert _eval(definitions, "ram_gb", "exists", None, unknown).status == "UNKNOWN"
assert _eval(definitions, "ram_gb", "!=", 8, None).status == "UNKNOWN"
def test_confidence_boundary(definitions):
exact = statement("ram_gb", 32, confidence="0.8")
below = statement("ram_gb", 32, confidence="0.799")
assert _eval(definitions, "ram_gb", ">=", 16, exact).status == "PASS"
assert _eval(definitions, "ram_gb", ">=", 16, below).status == "UNKNOWN"
def test_no_type_coercion(definitions):
text_number = statement("ram_gb", "32")
text_number = text_number.__class__(**{**text_number.__dict__, "normalized_value": "32"})
assert _eval(definitions, "ram_gb", ">=", 16, text_number).status == "UNKNOWN"
+74
View File
@@ -0,0 +1,74 @@
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
PREV = {"best_price": "1298.00", "availability": "in_stock", "seller_count": 8, "normalized_attributes": {"ram_gb": 32}}
CURR = {"best_price": "1199.00", "availability": "in_stock", "seller_count": 8, "normalized_attributes": {"ram_gb": 32}}
def test_changed_only_for_relevant_fields():
assert observations_changed(PREV, CURR) is True
title_only = {**PREV, "normalized_attributes": {"ram_gb": 32, "title": "new title", "html": "<b>"}}
assert observations_changed(PREV, title_only) is False
assert observations_changed(PREV, {**PREV, "seller_count": 12}) is True
assert observations_changed(PREV, {**PREV, "availability": "out_of_stock"}) is True
def test_diff_statuses_and_partial_skips_missing():
previous_ids = {"a", "b"}
current_ids = {"b", "c"}
previous_obs = {"a": PREV, "b": PREV}
current_obs = {"b": CURR, "c": PREV}
success = diff_run(
previous_ids=previous_ids,
current_ids=current_ids,
previous_obs=previous_obs,
current_obs=current_obs,
run_status=RunStatus.SUCCESS.value,
)
assert success["c"] == DiffStatus.NEW
assert success["b"] == DiffStatus.CHANGED
assert success["a"] == DiffStatus.MISSING
partial = diff_run(
previous_ids=previous_ids,
current_ids=current_ids,
previous_obs=previous_obs,
current_obs=current_obs,
run_status=RunStatus.PARTIAL.value,
)
assert "a" not in partial
assert DiffStatus.MISSING not in partial.values()
def test_unchanged_and_disappeared_lifecycle():
same = diff_run(
previous_ids={"a"},
current_ids={"a"},
previous_obs={"a": PREV},
current_obs={"a": PREV},
run_status=RunStatus.SUCCESS.value,
)
assert same["a"] == DiffStatus.UNCHANGED
status, streak = next_lifecycle(
current=LifecycleStatus.ACTIVE.value,
missing_in_success_run=True,
consecutive_missing=2,
reappeared=False,
)
assert status == LifecycleStatus.DISAPPEARED.value
assert streak == 3
status, streak = next_lifecycle(
current=LifecycleStatus.DISAPPEARED.value,
missing_in_success_run=False,
consecutive_missing=3,
reappeared=True,
)
assert status == LifecycleStatus.ACTIVE.value
assert streak == 0
archived, _ = next_lifecycle(
current=LifecycleStatus.ARCHIVED.value,
missing_in_success_run=True,
consecutive_missing=5,
reappeared=True,
)
assert archived == LifecycleStatus.ARCHIVED.value
@@ -0,0 +1,83 @@
from app.discovery.enums import MatchStatus
from app.discovery.identity import IdentitySignals, decide_identity, spec_fingerprint
def _sig(**overrides) -> IdentitySignals:
base = IdentitySignals(
source_external_id="gh-1",
gtin_ean="4000000000001",
manufacturer_sku="EM13-32",
source_url="https://geizhals.de/a.html",
title="Example Mobile 13",
manufacturer="Example",
model="Mobile 13",
fingerprint=spec_fingerprint({"manufacturer": "Example", "model": "Mobile 13", "ram": 32}),
)
return IdentitySignals(**{**base.__dict__, **overrides})
def test_sku_and_ean_and_url_match():
known = [("a", _sig())]
assert decide_identity(_sig(), known).reason == "source_external_id"
assert decide_identity(_sig(source_external_id="other"), known).reason == "gtin_ean"
assert decide_identity(_sig(source_external_id="other", gtin_ean="x"), known).reason == "manufacturer_sku"
assert decide_identity(
_sig(source_external_id="other", gtin_ean="x", manufacturer_sku="y"),
known,
).reason == "source_url"
def test_title_plus_fingerprint_matches_stable_specs():
known = [("a", _sig(source_external_id=None, gtin_ean=None, manufacturer_sku=None, source_url=None))]
incoming = _sig(
source_external_id=None,
gtin_ean=None,
manufacturer_sku=None,
source_url=None,
title="Example Mobile 13",
)
decision = decide_identity(incoming, known)
assert decision.match_status == MatchStatus.MATCHED
assert decision.reason == "title+fingerprint"
def test_title_only_is_uncertain_and_never_merges():
known = [("a", _sig(source_external_id=None, gtin_ean=None, manufacturer_sku=None, source_url=None))]
incoming = _sig(
source_external_id=None,
gtin_ean=None,
manufacturer_sku=None,
source_url=None,
fingerprint="different",
manufacturer="Other",
model="Other",
)
decision = decide_identity(incoming, known)
assert decision.match_status == MatchStatus.MATCH_UNCERTAIN
assert decision.existing_id is None
assert decision.reason == "title only"
def test_fingerprint_uses_observed_keys_only():
left = spec_fingerprint({"capacity_l": 2, "color": "black"})
right = spec_fingerprint({"capacity_l": 5, "color": "black"})
assert left != right
assert "capacity_l=2" in left
assert "ram_gb" not in left
assert spec_fingerprint({"title": "ignored", "html": "<b>", "noise": 1}) == "noise=1"
def test_similar_names_do_not_merge():
known = [("a", _sig(source_external_id=None, gtin_ean=None, manufacturer_sku=None, source_url=None))]
incoming = _sig(
source_external_id=None,
gtin_ean=None,
manufacturer_sku=None,
source_url=None,
title="Example Mobile 14",
fingerprint="other",
manufacturer="Example",
model="Mobile 14",
)
decision = decide_identity(incoming, known)
assert decision.match_status == MatchStatus.NEW_REFERENCE
@@ -0,0 +1,48 @@
from decimal import Decimal
from app.discovery.ranking import rank_candidates
def test_shortlist_uses_product_score_only(profile, scoring_model, definitions):
ranked = rank_candidates(
profile=profile,
scoring_model=scoring_model,
definitions=definitions,
shortlist_size=1,
candidates=[
{
"id": "a",
"manufacturer": "Example",
"source_title": "Example Mobile 13 32GB",
"normalized_attributes": {
"ram_gb": 32,
"storage_gb": 1024,
"weight_kg": 1.2,
"battery_score": 90,
"display_score": 90,
"cpu_score": 90,
},
},
{
"id": "b",
"manufacturer": "Example",
"source_title": "Example Mobile 13 8GB",
"normalized_attributes": {
"ram_gb": 8,
"storage_gb": 1024,
"weight_kg": 1.2,
"battery_score": 90,
"display_score": 90,
"cpu_score": 90,
},
},
],
)
assert ranked[0]["id"] == "a"
assert ranked[0]["eligible"] is True
assert ranked[0]["candidate_status"] == "SHORTLISTED"
assert ranked[0]["product_score"] == Decimal("90.21")
assert ranked[1]["eligible"] is False
assert ranked[1]["candidate_status"] == "CANDIDATE"
assert "deal_score" not in ranked[0]
assert "price_score" not in ranked[0]
@@ -0,0 +1,81 @@
from app.discovery.mapping import verify_roundtrip
from app.discovery.urlgen import generate_search_url
CRITERIA = [
{"id": "ram", "criterion": "ram_gb", "operator": ">=", "value": 24, "mandatory": True},
{"id": "price", "criterion": "price_eur", "operator": "<=", "value": 2000, "mandatory": True},
{"id": "weight", "criterion": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": True},
]
MAPPINGS = [
{
"criterion_key": "ram_gb",
"external_filter_key": "17023",
"source_value": "ab 24GB",
"encoding": {"url_param": "xf"},
"status": "VERIFIED",
},
{
"criterion_key": "price_eur",
"external_filter_key": "price",
"source_value": None,
"encoding": {"url_param": "maxp"},
"status": "VERIFIED",
},
]
def test_ram_and_price_url_is_reproducible():
first, internal = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=CRITERIA,
mappings=MAPPINGS,
)
second, _ = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=CRITERIA,
mappings=MAPPINGS,
)
assert first == second
assert "xf=17023_ab+24GB" in first
assert "maxp=2000" in first
assert internal == ["weight_kg"]
def test_internal_only_does_not_change_url():
without_weight, _ = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=CRITERIA[:2],
mappings=MAPPINGS,
)
with_weight, internal = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=CRITERIA,
mappings=MAPPINGS,
)
assert without_weight == with_weight
assert "weight_kg" in internal
def test_broken_mapping_is_not_used():
url, internal = generate_search_url(
category_url="https://geizhals.de/?cat=nb",
criteria_items=CRITERIA,
mappings=[{**MAPPINGS[0], "status": "BROKEN"}, MAPPINGS[1]],
)
assert "17023" not in url
assert "ram_gb" in internal
def test_roundtrip_verifies_xf_and_maxp():
assert verify_roundtrip(
category_url="https://geizhals.de/?cat=nb",
criterion=CRITERIA[0],
mapping=MAPPINGS[0],
)
assert verify_roundtrip(
category_url="https://geizhals.de/?cat=nb",
criterion=CRITERIA[1],
mapping=MAPPINGS[1],
)
+165
View File
@@ -0,0 +1,165 @@
from datetime import timedelta
from decimal import Decimal
from uuid import uuid4
from app.domain.enums import Eligibility, PriceReferenceScope
from app.scoring.deal import classify_deal_score
from app.scoring.engine import evaluate_deal
from app.scoring.mathutil import round_score
from tests.conftest import (
AS_OF,
CUTOFF,
case_a_attributes,
case_a_prices,
offer_a,
observation,
product_a,
statement,
variant_a,
)
def _run(profile, scoring_model, definitions, attributes=None, observations=None, product=None, offer=None):
return evaluate_deal(
profile=profile,
scoring_model=scoring_model,
product=product or product_a(),
variant=variant_a(),
offer=offer or offer_a(),
attributes=attributes or case_a_attributes(),
observations=observations or case_a_prices(),
definitions=definitions,
as_of=AS_OF,
snapshot_cutoff=CUTOFF,
engine_build="test",
)
def test_reference_case_a(profile, scoring_model, definitions):
result = _run(profile, scoring_model, definitions)
assert result.product_eligibility == Eligibility.ELIGIBLE
assert result.offer_eligibility == Eligibility.ELIGIBLE
assert result.overall_eligibility == Eligibility.ELIGIBLE
assert result.eligible is True
assert result.product_score == Decimal("90.21")
assert result.price_statistics.median_30d == Decimal("1599.00")
assert result.price_statistics.median_30d_days == 2
assert result.price_statistics.median_90d == Decimal("1599.00")
assert result.price_statistics.median_90d_days == 3
assert result.price_statistics.lowest_365d == Decimal("1599.00")
assert result.price_statistics.lowest_365d_days == 3
assert result.price_score == Decimal("98.24")
assert result.deal_score == Decimal("93.42")
assert result.classification == "EXCEPTIONAL"
assert result.product_coverage == Decimal("1")
assert result.price_reference_scope == PriceReferenceScope.OFFER
assert result.price_basis.value == "BUDGET_AND_HISTORY"
assert any(reason.code == "PRICE_BELOW_MEDIAN" for reason in result.reasons)
def test_case_b_ineligible_product(profile, scoring_model, definitions):
attrs = (
statement("ram_gb", 8),
statement("storage_gb", 1024),
statement("weight_kg", Decimal("1.2")),
statement("battery_score", 90),
statement("display_score", 90),
statement("cpu_score", 90),
)
result = _run(profile, scoring_model, definitions, attributes=attrs)
assert result.product_eligibility == Eligibility.INELIGIBLE
assert result.offer_eligibility == Eligibility.ELIGIBLE
assert result.overall_eligibility == Eligibility.INELIGIBLE
assert result.eligible is False
assert result.product_score is not None
assert result.deal_score is None
assert result.classification == "INELIGIBLE"
assert any(item.constraint_id == "min_ram" and item.status == "FAIL" for item in result.constraint_results)
def test_case_c_unknown_weight(profile, scoring_model, definitions):
attrs = (
statement("ram_gb", 32),
statement("storage_gb", 1024),
statement("weight_kg", None, status="unknown"),
statement("battery_score", 90),
statement("display_score", 90),
statement("cpu_score", 90),
)
result = _run(profile, scoring_model, definitions, attributes=attrs)
assert result.product_eligibility == Eligibility.UNKNOWN
assert result.offer_eligibility == Eligibility.ELIGIBLE
assert result.overall_eligibility == Eligibility.UNKNOWN
assert result.eligible is None
assert result.product_score is not None
assert result.deal_score is None
assert result.classification == "INSUFFICIENT_DATA"
def test_case_d_over_budget_keeps_product_fit(profile, scoring_model, definitions):
prices = (
observation((9, 1), "1599.00"),
observation((9, 15), "1599.00"),
observation((10, 1), "1599.00"),
observation((10, 3), "2049.00"),
)
result = _run(profile, scoring_model, definitions, observations=prices)
assert result.product_eligibility == Eligibility.ELIGIBLE
assert result.offer_eligibility == Eligibility.INELIGIBLE
assert result.overall_eligibility == Eligibility.INELIGIBLE
assert result.product_score == Decimal("90.21")
assert result.price_score is not None
assert result.deal_score is None
def test_freshness_boundary(profile, scoring_model, definitions):
fresh = observation((10, 3), "1298.00")
fresh = fresh.__class__(**{**fresh.__dict__, "observed_at": AS_OF - timedelta(hours=48)})
result = _run(profile, scoring_model, definitions, observations=(fresh,))
assert result.offer_eligibility in {Eligibility.ELIGIBLE, Eligibility.UNKNOWN}
stale = fresh.__class__(**{**fresh.__dict__, "observed_at": AS_OF - timedelta(hours=48, seconds=1)})
result_stale = _run(profile, scoring_model, definitions, observations=(stale,))
assert result_stale.offer_eligibility == Eligibility.UNKNOWN
assert result_stale.price_score is None
def test_unknown_shipping(profile, scoring_model, definitions):
price = observation((10, 3), "1298.00")
price = price.__class__(**{**price.__dict__, "shipping_price": None})
result = _run(profile, scoring_model, definitions, observations=(price,))
assert result.offer_eligibility == Eligibility.UNKNOWN
assert result.price_score is None
def test_classification_boundaries(scoring_model):
assert classify_deal_score(Decimal("59.999"), scoring_model) == "IGNORE"
assert classify_deal_score(Decimal("60"), scoring_model) == "WATCH"
assert classify_deal_score(Decimal("74.999"), scoring_model) == "WATCH"
assert classify_deal_score(Decimal("75"), scoring_model) == "GOOD"
assert classify_deal_score(Decimal("85"), scoring_model) == "VERY_GOOD"
assert classify_deal_score(Decimal("92"), scoring_model) == "VERY_GOOD"
assert classify_deal_score(Decimal("92.01"), scoring_model) == "EXCEPTIONAL"
def test_rounding_does_not_shift_class(scoring_model):
raw = Decimal("59.996")
assert classify_deal_score(raw, scoring_model) == "IGNORE"
assert round_score(raw) == Decimal("60.00")
def test_other_category_without_engine_change(profile, scoring_model, definitions):
camera = product_a().__class__(**{**product_a().__dict__, "category": "camera"})
result = _run(profile, scoring_model, definitions, product=camera)
assert result.product_eligibility == Eligibility.INELIGIBLE
assert any(gate.reason_code == "CATEGORY_MISMATCH" for gate in result.product_gates)
def test_does_not_mix_other_offer_history(profile, scoring_model, definitions):
other = observation((9, 1), "100.00", offer_id=uuid4())
mixed = case_a_prices() + (other,)
result = _run(profile, scoring_model, definitions, observations=mixed)
# Engine is called with the provided list; the assembler in v0.1 should only
# receive one offer. Guard the function by using only same-offer rows.
same_offer = tuple(item for item in mixed if item.offer_id == offer_a().id)
result = _run(profile, scoring_model, definitions, observations=same_offer)
assert result.price_statistics.median_90d == Decimal("1599.00")
+11
View File
@@ -0,0 +1,11 @@
from fastapi.testclient import TestClient
from app.main import app
client = TestClient(app)
def test_health_live_does_not_touch_the_database() -> None:
response = client.get("/health/live")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
+103
View File
@@ -0,0 +1,103 @@
from datetime import datetime, timezone
from decimal import Decimal
from uuid import uuid4
from app.domain.models import Budget
from app.scoring.price import daily_series, median, score_price, select_current_observation
from tests.conftest import observation, utc
def test_budget_and_history_points(scoring_model):
budget = Budget(Decimal("1500.00"), Decimal("2000.00"))
score, budget_score, history_score, basis, warning = score_price(
current_price=Decimal("1500.00"),
reference_history=(Decimal("1500"), Decimal("1500"), Decimal("1500")),
budget=budget,
model=scoring_model,
)
assert budget_score == Decimal("100")
assert history_score == Decimal("50")
assert basis.value == "BUDGET_AND_HISTORY"
assert warning is None
over, over_budget, _, _, _ = score_price(
current_price=Decimal("2000.00"),
reference_history=(Decimal("1000"), Decimal("1000"), Decimal("1000")),
budget=budget,
model=scoring_model,
)
assert over_budget == Decimal("0")
cheap, _, hist, _, _ = score_price(
current_price=Decimal("800.00"),
reference_history=(Decimal("1000"), Decimal("1000"), Decimal("1000")),
budget=budget,
model=scoring_model,
)
assert hist == Decimal("100")
expensive, _, hist2, _, _ = score_price(
current_price=Decimal("1200.00"),
reference_history=(Decimal("1000"), Decimal("1000"), Decimal("1000")),
budget=budget,
model=scoring_model,
)
assert hist2 == Decimal("0")
def test_budget_only_with_short_history(scoring_model):
budget = Budget(Decimal("1500.00"), Decimal("2000.00"))
for days in ((), (Decimal("10"),), (Decimal("10"), Decimal("11"))):
score, budget_score, history_score, basis, warning = score_price(
current_price=Decimal("1298.00"),
reference_history=days,
budget=budget,
model=scoring_model,
)
assert score == budget_score == Decimal("100")
assert history_score is None
assert basis.value == "BUDGET_ONLY"
assert warning == "INSUFFICIENT_HISTORY"
def test_median_even_and_odd():
assert median((Decimal("1"), Decimal("3"), Decimal("5"))) == Decimal("3")
assert median((Decimal("1"), Decimal("2"), Decimal("3"), Decimal("4"))) == Decimal("2.5")
def test_daily_series_latest_then_quality_filter(scoring_model):
offer = uuid4()
morning = observation((10, 1), "10.00", offer_id=offer, hour=8)
later_unknown = observation((10, 1), "9.00", offer_id=offer, hour=12, availability="unknown")
values, _ = daily_series(
(morning, later_unknown),
window_start=datetime(2026, 9, 1, tzinfo=timezone.utc).date(),
window_end_exclusive=datetime(2026, 10, 3, tzinfo=timezone.utc).date(),
minimum_confidence=scoring_model.minimum_confidence,
)
assert values == ()
def test_current_observation_does_not_fall_back():
cheap_old = observation((10, 2), "10.00")
unknown_new = observation((10, 3), "99.00", availability="unknown")
current = select_current_observation(
(cheap_old, unknown_new),
as_of=utc(2026, 10, 3, 12),
snapshot_cutoff=utc(2026, 10, 3, 12),
)
assert current is not None
assert current.availability.value == "unknown"
assert current.item_price == Decimal("99.00")
def test_no_offer_object_required_for_price_function(scoring_model):
score, _, _, basis, _ = score_price(
current_price=Decimal("1298.00"),
reference_history=(Decimal("1599"), Decimal("1599"), Decimal("1599")),
budget=Budget(Decimal("1500"), Decimal("2000")),
model=scoring_model,
)
from app.scoring.mathutil import round_score
assert basis.value == "BUDGET_AND_HISTORY"
assert round_score(score) == Decimal("98.24")
+91
View File
@@ -0,0 +1,91 @@
from decimal import Decimal
import pytest
from app.domain.enums import ConstraintStatus, PreferenceDirection, PreferenceStrategy
from app.domain.models import ConstraintResult, Preference
from app.domain.validation import DomainValidationError, parse_profile
from app.scoring.mathutil import round_score
from app.scoring.product import score_numeric_linear, score_product
from tests.conftest import statement
def test_numeric_linear_acceptable_and_ideal():
assert score_numeric_linear(Decimal("16"), Decimal("16"), Decimal("32"), PreferenceDirection.HIGHER) == Decimal("50")
assert score_numeric_linear(Decimal("32"), Decimal("16"), Decimal("32"), PreferenceDirection.HIGHER) == Decimal("100")
assert score_numeric_linear(Decimal("1.6"), Decimal("1.6"), Decimal("1.0"), PreferenceDirection.LOWER) == Decimal("50")
assert score_numeric_linear(Decimal("1.0"), Decimal("1.6"), Decimal("1.0"), PreferenceDirection.LOWER) == Decimal("100")
def test_numeric_linear_clamps():
assert score_numeric_linear(Decimal("8"), Decimal("16"), Decimal("32"), PreferenceDirection.HIGHER) == Decimal("25")
assert score_numeric_linear(Decimal("64"), Decimal("16"), Decimal("32"), PreferenceDirection.HIGHER) == Decimal("100")
assert score_numeric_linear(Decimal("0.5"), Decimal("1.6"), Decimal("1.0"), PreferenceDirection.LOWER) == Decimal("100")
def test_weights_need_not_sum_to_one(scoring_model):
prefs = (
Preference("a", "ram_gb", PreferenceStrategy.NUMERIC_LINEAR, PreferenceDirection.HIGHER, Decimal("16"), Decimal("32"), Decimal("2")),
Preference("b", "cpu_score", PreferenceStrategy.NUMERIC_LINEAR, PreferenceDirection.HIGHER, Decimal("60"), Decimal("100"), Decimal("2")),
)
snapshot = {"ram_gb": statement("ram_gb", 32), "cpu_score": statement("cpu_score", 100)}
score, coverage, _ = score_product(prefs, snapshot, (), scoring_model)
assert score == Decimal("100")
assert coverage == Decimal("1")
def test_missing_preference_scores_zero(scoring_model):
prefs = (
Preference("ram", "ram_gb", PreferenceStrategy.NUMERIC_LINEAR, PreferenceDirection.HIGHER, Decimal("16"), Decimal("32"), Decimal("1")),
)
score, coverage, _ = score_product(prefs, {}, (), scoring_model)
assert score == Decimal("0")
assert coverage == Decimal("0")
def test_optional_penalty(scoring_model):
prefs = (
Preference("ram", "ram_gb", PreferenceStrategy.NUMERIC_LINEAR, PreferenceDirection.HIGHER, Decimal("16"), Decimal("32"), Decimal("1")),
)
snapshot = {"ram_gb": statement("ram_gb", 32)}
optional = ConstraintResult("opt", "ports", "contains", "hdmi", None, ConstraintStatus.FAIL.value, False, None, "CONSTRAINT_FAIL")
score, _, _ = score_product(prefs, snapshot, (optional,), scoring_model)
assert score == Decimal("95")
def test_unknown_strategy_rejected(definitions, scoring_model):
raw = {
"id": "x",
"version": 1,
"name": "x",
"category": "notebook",
"currency": "EUR",
"market": "DE",
"allowed_conditions": ["new"],
"budget": {"target": "100.00", "maximum": "200.00"},
"scoring_version": "generic-v1",
"constraints": [],
"preferences": [
{
"id": "ram",
"attribute": "ram_gb",
"strategy": "boolean_preferred",
"direction": "higher",
"acceptable": 16,
"ideal": 32,
"weight": 1,
}
],
}
with pytest.raises(DomainValidationError):
parse_profile(
raw,
attribute_definitions=definitions,
scoring_versions={scoring_model.version},
supported_strategies=set(scoring_model.supported_preference_strategies),
)
def test_reference_preference_scores():
assert round_score(score_numeric_linear(Decimal("1.2"), Decimal("1.6"), Decimal("1.0"), PreferenceDirection.LOWER)) == Decimal("83.33")
assert score_numeric_linear(Decimal("90"), Decimal("60"), Decimal("100"), PreferenceDirection.HIGHER) == Decimal("87.5")
Loaded 100 of 152 files, more files were not shown because too many files have changed in this diff. Show more