Compare commits
18
Commits
94053d7fbe
...
develop
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6d53e0eeb | ||
|
|
33336bcb6e | ||
|
|
68a8e3d945 | ||
|
|
2117ca6b54 | ||
|
|
46857715df | ||
|
|
d784248e28 | ||
|
|
b276758ee1 | ||
|
|
827e6a2a11 | ||
|
|
c68bc8d09f | ||
|
|
b0779859ab | ||
|
|
e59997e208 | ||
|
|
c9c0ff96fc | ||
|
|
e1800e7b0f | ||
|
|
0b3af2dee4 | ||
|
|
cbfe374508 | ||
|
|
4b73e5fb53 | ||
|
|
07dcdf5cd8 | ||
|
|
3ac42645ef |
No files matched your search
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.gitea
|
||||
.specs
|
||||
docs
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.venv
|
||||
.pytest_cache
|
||||
.mypy_cache
|
||||
.ruff_cache
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.md
|
||||
!README.md
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copy to .env next to docker-compose*.yml. Never commit .env.
|
||||
# Dev server: ~/docker/buyingagent-dev/.env
|
||||
# Prod server: ~/docker/buyingagent/.env
|
||||
|
||||
# ─── local / production (docker-compose.yml) ────────────────────────────────
|
||||
DB_NAME=buyingagent
|
||||
DB_USER=buyingagent
|
||||
DB_PASSWORD=change_me
|
||||
ENVIRONMENT=production
|
||||
ENGINE_BUILD=dev
|
||||
BUYING_AGENT_API_BIND=127.0.0.1
|
||||
BUYING_AGENT_API_PORT=8000
|
||||
BUYING_AGENT_FRONTEND_BIND=127.0.0.1
|
||||
BUYING_AGENT_FRONTEND_PORT=4173
|
||||
CORS_ALLOWED_ORIGINS=http://localhost:4173,http://127.0.0.1:4173,http://localhost:5173,http://127.0.0.1:5173
|
||||
|
||||
# Production on the Raspberry Pi (192.168.2.49):
|
||||
# BUYING_AGENT_API_BIND=0.0.0.0
|
||||
# BUYING_AGENT_API_PORT=8006
|
||||
|
||||
# ─── development server (docker-compose.dev-env.yml) ────────────────────────
|
||||
# DB_NAME=buyingagent_dev
|
||||
# DB_USER=buyingagent_dev
|
||||
# DB_PASSWORD=dev_password_change_me
|
||||
# ENVIRONMENT=development
|
||||
# BUYING_AGENT_API_PORT=8095
|
||||
@@ -0,0 +1 @@
|
||||
*.sh text eol=lf
|
||||
@@ -0,0 +1,70 @@
|
||||
name: Deploy Development
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy BuyingAgent to development
|
||||
run: |
|
||||
set -e
|
||||
echo "=== Deploying BuyingAgent to DEVELOPMENT ==="
|
||||
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
||||
TARGET="$HOME/docker/buyingagent-dev"
|
||||
mkdir -p "$TARGET"
|
||||
cd "$TARGET"
|
||||
if [ ! -d .git ]; then
|
||||
git clone -b develop "$REPO" .
|
||||
else
|
||||
git fetch origin develop
|
||||
git checkout develop
|
||||
git reset --hard origin/develop
|
||||
fi
|
||||
if [ ! -f .env ]; then
|
||||
printf '%s\n' \
|
||||
'DB_NAME=buyingagent_dev' \
|
||||
'DB_USER=buyingagent_dev' \
|
||||
'DB_PASSWORD=dev_password_change_me' \
|
||||
'ENVIRONMENT=development' \
|
||||
'BUYING_AGENT_API_PORT=8095' > .env
|
||||
echo "created development .env"
|
||||
fi
|
||||
if grep -q '^BUYING_AGENT_API_PORT=' .env; then
|
||||
sed -i 's/^BUYING_AGENT_API_PORT=.*/BUYING_AGENT_API_PORT=8095/' .env
|
||||
else
|
||||
echo 'BUYING_AGENT_API_PORT=8095' >> .env
|
||||
fi
|
||||
export ENGINE_BUILD BUYING_AGENT_API_PORT
|
||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||
BUYING_AGENT_API_PORT=8095
|
||||
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
||||
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:5174,http://localhost:5173' >> .env
|
||||
fi
|
||||
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
||||
echo 'BUYING_AGENT_FRONTEND_PORT=5174' >> .env
|
||||
fi
|
||||
COMPOSE_BAKE=0 docker compose -f docker-compose.dev-env.yml build --pull=false api frontend
|
||||
if ! ENGINE_BUILD="$ENGINE_BUILD" BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||
docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
||||
echo "compose up --wait failed — api logs:"
|
||||
docker compose -f docker-compose.dev-env.yml logs api --tail 150 || true
|
||||
docker compose -f docker-compose.dev-env.yml ps || true
|
||||
exit 1
|
||||
fi
|
||||
if ! curl -sf http://localhost:8095/health/live; then
|
||||
echo "DEV /health/live not reachable — api logs:"
|
||||
docker compose -f docker-compose.dev-env.yml logs api --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "DEV /health/live OK"
|
||||
if ! curl -sf http://localhost:8095/health/ready; then
|
||||
echo "DEV /health/ready failed — api logs:"
|
||||
docker compose -f docker-compose.dev-env.yml logs api --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "DEV /health/ready OK"
|
||||
echo "=== BuyingAgent DEV deploy complete (${ENGINE_BUILD}) ==="
|
||||
@@ -0,0 +1,79 @@
|
||||
name: Deploy Production
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy BuyingAgent to production
|
||||
run: |
|
||||
set -e
|
||||
echo "=== Deploying BuyingAgent to PRODUCTION ==="
|
||||
REPO="http://192.168.2.144:3000/Lars/BuyingAgent.git"
|
||||
TARGET="$HOME/docker/buyingagent"
|
||||
mkdir -p "$TARGET"
|
||||
cd "$TARGET"
|
||||
if [ ! -d .git ]; then
|
||||
git clone -b main "$REPO" .
|
||||
else
|
||||
git fetch origin main
|
||||
git checkout main
|
||||
git reset --hard origin/main
|
||||
fi
|
||||
if [ ! -f .env ]; then
|
||||
printf '%s\n' \
|
||||
'DB_NAME=buyingagent' \
|
||||
'DB_USER=buyingagent' \
|
||||
'DB_PASSWORD=change_me' \
|
||||
'ENVIRONMENT=production' \
|
||||
'BUYING_AGENT_API_BIND=0.0.0.0' \
|
||||
'BUYING_AGENT_API_PORT=8006' > .env
|
||||
echo "created production .env — set a unique DB_PASSWORD after first start"
|
||||
fi
|
||||
if grep -q '^BUYING_AGENT_API_BIND=' .env; then
|
||||
sed -i 's/^BUYING_AGENT_API_BIND=.*/BUYING_AGENT_API_BIND=0.0.0.0/' .env
|
||||
else
|
||||
echo 'BUYING_AGENT_API_BIND=0.0.0.0' >> .env
|
||||
fi
|
||||
if grep -q '^BUYING_AGENT_API_PORT=' .env; then
|
||||
sed -i 's/^BUYING_AGENT_API_PORT=.*/BUYING_AGENT_API_PORT=8006/' .env
|
||||
else
|
||||
echo 'BUYING_AGENT_API_PORT=8006' >> .env
|
||||
fi
|
||||
export ENGINE_BUILD BUYING_AGENT_API_BIND BUYING_AGENT_API_PORT
|
||||
ENGINE_BUILD="$(git rev-parse --short HEAD)"
|
||||
BUYING_AGENT_API_BIND=0.0.0.0
|
||||
BUYING_AGENT_API_PORT=8006
|
||||
if ! grep -q '^CORS_ALLOWED_ORIGINS=' .env; then
|
||||
echo 'CORS_ALLOWED_ORIGINS=http://192.168.2.49:4173,http://localhost:4173' >> .env
|
||||
fi
|
||||
if ! grep -q '^BUYING_AGENT_FRONTEND_PORT=' .env; then
|
||||
echo 'BUYING_AGENT_FRONTEND_PORT=4173' >> .env
|
||||
fi
|
||||
COMPOSE_BAKE=0 docker compose build --pull=false api frontend
|
||||
if ! ENGINE_BUILD="$ENGINE_BUILD" \
|
||||
BUYING_AGENT_API_BIND="$BUYING_AGENT_API_BIND" \
|
||||
BUYING_AGENT_API_PORT="$BUYING_AGENT_API_PORT" \
|
||||
docker compose up -d --wait; then
|
||||
echo "compose up --wait failed — api logs:"
|
||||
docker compose logs api --tail 150 || true
|
||||
docker compose ps || true
|
||||
exit 1
|
||||
fi
|
||||
if ! curl -sf http://localhost:8006/health/live; then
|
||||
echo "PROD /health/live not reachable — api logs:"
|
||||
docker compose logs api --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PROD /health/live OK"
|
||||
if ! curl -sf http://localhost:8006/health/ready; then
|
||||
echo "PROD /health/ready failed — api logs:"
|
||||
docker compose logs api --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PROD /health/ready OK"
|
||||
echo "=== BuyingAgent PROD deploy complete (${ENGINE_BUILD}) ==="
|
||||
@@ -0,0 +1,74 @@
|
||||
name: Test Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
workflow_run:
|
||||
workflows: ["Deploy Development"]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
pytest-backend:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: pytest against Dev Postgres (buyingagent_test)
|
||||
run: |
|
||||
set -e
|
||||
APP_DIR="$HOME/docker/buyingagent-dev"
|
||||
echo "pytest uses Dev Postgres (${APP_DIR}); event=${{ github.event_name }}"
|
||||
cd "$APP_DIR"
|
||||
for i in $(seq 1 60); do
|
||||
if [ -f scripts/ci-pytest.sh ]; then
|
||||
echo "ci-pytest.sh ready (attempt $i)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo "timeout waiting for scripts/ci-pytest.sh after Dev deploy"
|
||||
ls -la scripts || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
sh scripts/ci-pytest.sh
|
||||
|
||||
smoke-dev:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Non-destructive smoke against the running Dev API
|
||||
run: |
|
||||
set -e
|
||||
echo "live Dev API (buyingagent-dev), read-only health"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8095/health/live >/dev/null; then
|
||||
echo "DEV /health/live ready (attempt $i)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo "timeout waiting for DEV /health/live"
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
curl -sf http://localhost:8095/health/live
|
||||
echo
|
||||
curl -sf http://localhost:8095/health/ready
|
||||
echo
|
||||
echo "LAN smoke OK: http://192.168.2.49:8095"
|
||||
|
||||
frontend-checks:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Frontend typecheck, lint, test, build
|
||||
run: |
|
||||
set -e
|
||||
APP_DIR="$HOME/docker/buyingagent-dev"
|
||||
cd "$APP_DIR/frontend"
|
||||
npm ci || npm install
|
||||
npm run typecheck
|
||||
npm run lint
|
||||
npm test
|
||||
npm run build
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
# Environment and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Python
|
||||
.venv/
|
||||
**/__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Frontend
|
||||
node_modules/
|
||||
frontend/dev-dist/
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
*.swp
|
||||
.idea/
|
||||
.vscode/*
|
||||
@@ -0,0 +1,377 @@
|
||||
# Buying Agent – Architecture Addendum v1
|
||||
|
||||
**Status:** ergänzende Architekturvorgabe
|
||||
**Gültig zusammen mit:** `buying-agent-v0.1-spec-v1.1.md`
|
||||
**Stand:** 2026-10-03
|
||||
|
||||
## 1. Zweck
|
||||
|
||||
Dieses Addendum ergänzt die bestehende Spezifikation um übergreifende Architekturentscheidungen für das spätere Zielsystem.
|
||||
|
||||
Es ersetzt **nicht** die fachliche Spezifikation `buying-agent-v0.1-spec-v1.1.md`.
|
||||
|
||||
Bei Konflikten zwischen beiden Dokumenten hat dieses Addendum für die hier behandelten Architekturthemen Vorrang.
|
||||
|
||||
Ziel ist, den aktuellen v0.1-Slice schlank zu halten, ohne spätere Erweiterungen auf Frontend, mehrere Kaufprojekte und mehrere Benutzer unnötig zu erschweren.
|
||||
|
||||
---
|
||||
|
||||
## 2. Zielstruktur des Repositories
|
||||
|
||||
Das Projekt wird als Monorepo strukturiert:
|
||||
|
||||
```text
|
||||
buying-agent/
|
||||
├── README.md
|
||||
├── docker-compose.yml
|
||||
├── .env.example
|
||||
├── .gitignore
|
||||
├── docs/
|
||||
│ ├── buying-agent-v0.1-spec-v1.1.md
|
||||
│ └── buying-agent-architecture-addendum-v1.md
|
||||
├── backend/
|
||||
│ ├── Dockerfile
|
||||
│ ├── pyproject.toml
|
||||
│ ├── uv.lock
|
||||
│ ├── alembic.ini
|
||||
│ ├── migrations/
|
||||
│ ├── app/
|
||||
│ ├── rules/
|
||||
│ ├── profiles/
|
||||
│ ├── fixtures/
|
||||
│ ├── scripts/
|
||||
│ └── tests/
|
||||
└── frontend/
|
||||
└── README.md
|
||||
```
|
||||
|
||||
### Regeln
|
||||
|
||||
- `docker-compose.yml` bleibt im Repository-Root.
|
||||
- Das Backend ist der aktuell implementierte v0.1-Scope.
|
||||
- `frontend/` wird in v0.1 **nicht implementiert**.
|
||||
- `frontend/README.md` dokumentiert lediglich, dass dort später die Benutzeroberfläche entsteht.
|
||||
- Keine zusätzliche `infra/`-Struktur in v0.1, solange sie nicht konkret benötigt wird.
|
||||
|
||||
---
|
||||
|
||||
## 3. Backend-/Frontend-Grenze
|
||||
|
||||
Das Backend bleibt das **System of Record** und besitzt:
|
||||
|
||||
- Domain-Modell
|
||||
- Constraints
|
||||
- Scoring
|
||||
- Evaluation
|
||||
- Persistenz
|
||||
- Preis- und Produktlogik
|
||||
- zukünftige Kaufentscheidungslogik
|
||||
|
||||
Das spätere Frontend konsumiert ausschließlich die versionierte Backend-API, primär unter:
|
||||
|
||||
```text
|
||||
/api/v1
|
||||
```
|
||||
|
||||
Das Frontend enthält keine fachlich verbindliche Business-Logik.
|
||||
|
||||
---
|
||||
|
||||
## 4. Zukünftige Multi-User-Architektur
|
||||
|
||||
Das Zielsystem soll später von Familie und gegebenenfalls engen Freunden genutzt werden können.
|
||||
|
||||
Die Architektur darf daher nicht von einem impliziten Single-User-Modell ausgehen.
|
||||
|
||||
Zielmodell:
|
||||
|
||||
```text
|
||||
User
|
||||
↕
|
||||
WorkspaceMember
|
||||
↕
|
||||
Workspace
|
||||
↓
|
||||
BuyingProject
|
||||
↓
|
||||
ProjectCandidate
|
||||
↓
|
||||
ProductVariant
|
||||
```
|
||||
|
||||
Ein Benutzer kann Mitglied in mehreren Workspaces sein.
|
||||
|
||||
Beispiele:
|
||||
|
||||
```text
|
||||
Lars
|
||||
├── Privat
|
||||
├── Familie
|
||||
└── Freunde
|
||||
```
|
||||
|
||||
### Noch nicht Bestandteil von v0.1
|
||||
|
||||
Nicht implementieren:
|
||||
|
||||
- Login
|
||||
- Authentifizierung
|
||||
- Passwortverwaltung
|
||||
- Einladungen
|
||||
- RBAC
|
||||
- komplexe ACLs
|
||||
- Sharing-UI
|
||||
- Workspace-Administration
|
||||
|
||||
Die aktuelle Architektur darf deren spätere Einführung aber nicht unnötig erschweren.
|
||||
|
||||
---
|
||||
|
||||
## 5. Globale versus Workspace-bezogene Daten
|
||||
|
||||
### Global wiederverwendbare Daten
|
||||
|
||||
Diese Entitäten bleiben systemweit gemeinsam nutzbar:
|
||||
|
||||
- Product
|
||||
- ProductVariant
|
||||
- Offer
|
||||
- PriceObservation
|
||||
- AttributeDefinition
|
||||
- Source
|
||||
|
||||
Sie erhalten **keine** `user_id` oder `workspace_id`.
|
||||
|
||||
Ein Produkt oder Angebot darf nicht pro Benutzer oder Workspace dupliziert werden.
|
||||
|
||||
### Workspace-bezogene Daten
|
||||
|
||||
Spätere benutzerbezogene Beschaffungskontexte gehören zu einem Workspace:
|
||||
|
||||
- BuyingProject
|
||||
- ProjectCandidate
|
||||
- projektspezifische Profil-/Anforderungsauswahl
|
||||
- Notification Preferences
|
||||
- Beschaffungsdeadline
|
||||
- Zielpreis
|
||||
- Kaufentscheidungsstatus
|
||||
- gegebenenfalls spätere Warenkorb-/Beschaffungsinformationen
|
||||
|
||||
Diese Objekte werden erst in späteren Slices implementiert.
|
||||
|
||||
---
|
||||
|
||||
## 6. BuyingProject als zukünftiges Kernobjekt
|
||||
|
||||
Ein `BuyingProject` repräsentiert ein konkretes Beschaffungsvorhaben.
|
||||
|
||||
Beispiele:
|
||||
|
||||
- Mobiler Entwicklungsrechner
|
||||
- Lokaler LLM-Server
|
||||
- Fernseher
|
||||
- Kamera
|
||||
|
||||
Ein Workspace kann mehrere BuyingProjects parallel besitzen.
|
||||
|
||||
Ein BuyingProject kann mehrere ProjectCandidates beobachten.
|
||||
|
||||
```text
|
||||
BuyingProject
|
||||
├── Candidate A → ProductVariant
|
||||
├── Candidate B → ProductVariant
|
||||
└── Candidate C → ProductVariant
|
||||
```
|
||||
|
||||
### Wichtig für v0.1
|
||||
|
||||
`BuyingProject` und `ProjectCandidate` müssen in v0.1 noch nicht implementiert werden.
|
||||
|
||||
Die aktuelle Implementierung darf jedoch nicht annehmen:
|
||||
|
||||
- es existiert genau ein aktives Profil,
|
||||
- es existiert genau ein Kaufvorhaben,
|
||||
- es gibt globale Singleton-Konfiguration für einen aktuellen Kauf,
|
||||
- alle Daten gehören automatisch demselben Benutzer,
|
||||
- es existiert nur eine gleichzeitig relevante Evaluation.
|
||||
|
||||
Mehrere unabhängige Profile und Evaluationen müssen bereits parallel koexistieren können.
|
||||
|
||||
---
|
||||
|
||||
## 7. Rolle von n8n
|
||||
|
||||
n8n bleibt eine externe Orchestrierungsschicht.
|
||||
|
||||
Spätere mögliche Aufgaben:
|
||||
|
||||
- Scheduler
|
||||
- Preisabfragen
|
||||
- Datenimport
|
||||
- Benachrichtigungen
|
||||
- externe Integrationen
|
||||
- volatile oder häufig veränderte Workflows
|
||||
|
||||
Kern-Business-Logik bleibt im Programmcode, insbesondere wenn sie:
|
||||
|
||||
- deterministisch sein soll,
|
||||
- versioniert werden muss,
|
||||
- getestet werden muss,
|
||||
- fachliche Entscheidungen reproduzierbar machen soll.
|
||||
|
||||
n8n darf für veränderliche Prozesslogik eingesetzt werden, aber nicht zur Voraussetzung für das Funktionieren der Buying Engine werden.
|
||||
|
||||
---
|
||||
|
||||
## 8. Zukünftiges Frontend
|
||||
|
||||
Das Zielsystem soll später eine einfach bedienbare Oberfläche erhalten.
|
||||
|
||||
Die Standardoberfläche soll auch für nichttechnische Familienmitglieder verständlich bleiben.
|
||||
|
||||
Die Engine darf gleichzeitig deutlich leistungsfähiger sein als die Oberfläche.
|
||||
|
||||
Voraussichtliche spätere Kernbereiche:
|
||||
|
||||
- Übersicht laufender Kaufprojekte
|
||||
- Kandidaten je Projekt
|
||||
- aktueller bester Deal
|
||||
- Preisverlauf
|
||||
- Produkt-Fit
|
||||
- Offer-Fit
|
||||
- DealScore
|
||||
- verständliche Gründe
|
||||
- Beobachtungsstatus
|
||||
- konkrete Kaufempfehlung
|
||||
- ggf. spätere Benachrichtigungs- und Warenkorbfunktionen
|
||||
|
||||
Keine dieser UI-Funktionen ist Bestandteil von v0.1.
|
||||
|
||||
---
|
||||
|
||||
## 9. Kaufentscheidungslogik
|
||||
|
||||
Langfristig soll das System nicht nur Scores liefern, sondern auch konfigurierbare Kaufentscheidungen unterstützen.
|
||||
|
||||
Beispiele möglicher Einflussfaktoren:
|
||||
|
||||
- Zielpreis
|
||||
- Maximalpreis
|
||||
- spätestes Beschaffungsdatum
|
||||
- aktueller Preis
|
||||
- Preisentwicklung der letzten Tage/Wochen/Monate
|
||||
- Abstand zu historischem Median
|
||||
- Produkt-Fit
|
||||
- Offer-Fit
|
||||
- Verfügbarkeit
|
||||
- Lieferzeit
|
||||
- Händlerqualität
|
||||
- alternative Kandidaten
|
||||
- Beschaffungsdringlichkeit
|
||||
|
||||
Die Kaufentscheidung soll nachvollziehbar und möglichst deterministisch bleiben.
|
||||
|
||||
LLMs dürfen später Erklärungen, Normalisierung oder qualitative Interpretation unterstützen, sollen aber keine nicht nachvollziehbare Blackbox-Entscheidung erzwingen.
|
||||
|
||||
---
|
||||
|
||||
## 10. Datenquellen
|
||||
|
||||
Das Zielsystem muss mehrere Produkt- und Preisquellen unterstützen können.
|
||||
|
||||
Mögliche spätere Quellen:
|
||||
|
||||
- Amazon / Keepa
|
||||
- Preisvergleichsportale
|
||||
- Hersteller
|
||||
- Händler
|
||||
- weitere APIs
|
||||
|
||||
v0.1 implementiert weiterhin **keine reale externe Quelle**.
|
||||
|
||||
Die bestehende Source-Adapter-Grenze bleibt die vorgesehene Erweiterungsstelle.
|
||||
|
||||
---
|
||||
|
||||
## 11. Scope-Grenze für die laufende v0.1-Implementierung
|
||||
|
||||
Dieses Addendum darf nicht als Anlass dienen, v0.1 unnötig zu vergrößern.
|
||||
|
||||
Insbesondere **nicht jetzt implementieren**:
|
||||
|
||||
- echtes Frontend
|
||||
- Authentifizierung
|
||||
- User Management
|
||||
- Workspace Management
|
||||
- RBAC
|
||||
- Einladungen
|
||||
- BuyingProject
|
||||
- ProjectCandidate
|
||||
- Notifications
|
||||
- n8n
|
||||
- externe Preisquellen
|
||||
- automatische Produktsuche
|
||||
- Warenkorbmanagement
|
||||
|
||||
Ziel ist Architekturkompatibilität, nicht vorgezogene Feature-Entwicklung.
|
||||
|
||||
---
|
||||
|
||||
## 12. Arbeitsauftrag für die laufende Implementierung
|
||||
|
||||
Der Coding-Agent soll vor Fortsetzung des laufenden Builds einen kurzen Architecture-Alignment-Check durchführen.
|
||||
|
||||
### Prüfen
|
||||
|
||||
1. Entspricht die Repository-Struktur bereits der Monorepo-Struktur mit:
|
||||
- `backend/`
|
||||
- `frontend/`
|
||||
- `docs/`
|
||||
- Root-`docker-compose.yml`
|
||||
|
||||
2. Gibt es Annahmen über:
|
||||
- genau einen Benutzer,
|
||||
- genau ein aktives Profil,
|
||||
- genau ein Kaufprojekt,
|
||||
- globale Singleton-Zustände?
|
||||
|
||||
3. Sind Product, ProductVariant, Offer und PriceObservation weiterhin global und unabhängig von späteren Workspaces modelliert?
|
||||
|
||||
4. Ist das Backend klar von zukünftiger Frontend- und n8n-Logik getrennt?
|
||||
|
||||
### Änderungen
|
||||
|
||||
Nur notwendige strukturelle Anpassungen durchführen.
|
||||
|
||||
Keine vorgezogene Multi-User- oder Frontend-Funktionalität implementieren.
|
||||
|
||||
Die fachlichen Anforderungen aus `buying-agent-v0.1-spec-v1.1.md` bleiben vollständig gültig.
|
||||
|
||||
### Abschlussbericht
|
||||
|
||||
Dokumentieren:
|
||||
|
||||
- geänderte oder verschobene Dateien,
|
||||
- notwendige Architekturänderungen,
|
||||
- Änderungen an Datenbank/Migrationen,
|
||||
- ausgeführte Tests,
|
||||
- Testergebnisse,
|
||||
- verbleibende Risiken für spätere Workspace-/BuyingProject-Erweiterung.
|
||||
|
||||
---
|
||||
|
||||
## 13. Vorrang und Weiterentwicklung
|
||||
|
||||
Dieses Dokument ist ein Architektur-Addendum zur v0.1-Spezifikation.
|
||||
|
||||
Nach Abschluss von v0.1 sollen die Erkenntnisse aus:
|
||||
|
||||
- `buying-agent-v0.1-spec-v1.1.md`
|
||||
- diesem Addendum
|
||||
- der realen Implementierung
|
||||
|
||||
in eine konsolidierte Folgespezifikation für den nächsten Slice übernommen werden.
|
||||
|
||||
Die Kernregel lautet:
|
||||
|
||||
> Nicht vorzeitig bauen, aber bekannte Zielanforderungen nicht architektonisch verbauen.
|
||||
@@ -0,0 +1,987 @@
|
||||
# Buying Agent – Frontend MVP Specification v0.1
|
||||
|
||||
**Status:** implementierungsbereit
|
||||
**Spezifikationsversion:** 0.1
|
||||
**Stand:** 2026-10-04
|
||||
|
||||
**Ergänzt verbindlich:**
|
||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||
2. `buying-agent-architecture-addendum-v1.md`
|
||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||
|
||||
Diese Frontend-Spezifikation hebt für den nutzbaren MVP die frühere Aussage „kein Frontend in diesem Slice“ auf.
|
||||
Sie ändert keine fachliche Backend-Logik; sie definiert die Benutzeroberfläche und die dafür notwendigen API-Verträge.
|
||||
|
||||
---
|
||||
|
||||
## 1. Ziel
|
||||
|
||||
Der Buying Agent soll als **nutzbare PWA** einen vollständigen vertikalen Ablauf unterstützen:
|
||||
|
||||
```text
|
||||
BuyingProject anlegen
|
||||
↓
|
||||
Produktkategorie wählen
|
||||
↓
|
||||
dynamische Kriterien aus Source-Taxonomie auswählen
|
||||
↓
|
||||
Kriterien konfigurieren
|
||||
↓
|
||||
Geizhals-Suche generieren
|
||||
↓
|
||||
SearchRun starten / Status verfolgen
|
||||
↓
|
||||
Kandidaten sehen
|
||||
↓
|
||||
Ranking + Shortlist sehen
|
||||
↓
|
||||
Kandidat öffnen
|
||||
↓
|
||||
Status, Historie, bisherige Ergebnisse und Empfehlungen sehen
|
||||
↓
|
||||
Kriterien bei Bedarf anpassen
|
||||
```
|
||||
|
||||
Der MVP muss diesen Flow über die Oberfläche vollständig nutzbar machen.
|
||||
|
||||
---
|
||||
|
||||
## 2. Technologiestack
|
||||
|
||||
Frontend:
|
||||
|
||||
- React
|
||||
- TypeScript
|
||||
- Vite
|
||||
- PWA
|
||||
- TanStack Query für Server-State / Data Fetching
|
||||
- React Router oder äquivalente schlanke Routing-Lösung
|
||||
- schlanke komponentenbasierte UI
|
||||
- kein Next.js
|
||||
- kein SSR
|
||||
- keine SEO-Anforderungen
|
||||
|
||||
Das Frontend liegt im bestehenden Monorepo:
|
||||
|
||||
```text
|
||||
buying-agent/
|
||||
├── backend/
|
||||
├── frontend/
|
||||
├── docs/
|
||||
└── docker-compose.yml
|
||||
```
|
||||
|
||||
Backend bleibt System of Record.
|
||||
|
||||
---
|
||||
|
||||
## 3. Gestaltungsprinzipien
|
||||
|
||||
1. **Einfach für Standardnutzer, leistungsfähig im Hintergrund.**
|
||||
2. Keine technische Datenbank-/Engine-Sicht in der Standardoberfläche.
|
||||
3. Dynamische Kriterien statt fest verdrahteter Notebook-Felder.
|
||||
4. Responsive von Anfang an.
|
||||
5. Zwei bewusst gestaltete Hauptlayouts:
|
||||
- Desktop
|
||||
- Mobile
|
||||
6. Nüchternes, klares Design.
|
||||
7. Bedienbarkeit vor visueller Komplexität.
|
||||
8. Admin-/Source-Themen von der normalen Nutzung trennen.
|
||||
9. Keine Businesslogik im Frontend.
|
||||
10. Frontend konsumiert ausschließlich die versionierte Backend-API.
|
||||
|
||||
---
|
||||
|
||||
## 4. PWA und Responsive Layout
|
||||
|
||||
### Desktop
|
||||
|
||||
Desktop verwendet primär:
|
||||
|
||||
```text
|
||||
Sidebar
|
||||
+ Hauptinhalt
|
||||
+ optionaler Detailbereich
|
||||
```
|
||||
|
||||
Beispiel:
|
||||
|
||||
```text
|
||||
┌───────────────┬──────────────────────────────────────┐
|
||||
│ Buying Agent │ Projekt: Mobiler Entwicklungsrechner│
|
||||
│ │ │
|
||||
│ Projekte │ Status / Suche / Ranking / Shortlist│
|
||||
│ Admin │ │
|
||||
│ Einstellungen │ │
|
||||
└───────────────┴──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Mobile
|
||||
|
||||
Mobile verwendet primär:
|
||||
|
||||
- Bottom Navigation oder kompakte Hauptnavigation
|
||||
- Vollbild-Detailseiten
|
||||
- Tabs innerhalb eines BuyingProjects
|
||||
- keine verkleinerte Desktop-Sidebar
|
||||
|
||||
Beispiel:
|
||||
|
||||
```text
|
||||
Projekt
|
||||
────────────────
|
||||
Übersicht
|
||||
Kandidaten
|
||||
Shortlist
|
||||
Kriterien
|
||||
Historie
|
||||
|
||||
[ Projekte ] [ Suche ] [ Admin ]
|
||||
```
|
||||
|
||||
### Mitai-Bezug
|
||||
|
||||
Falls im bestehenden Mitai-Code wiederverwendbare Layout-, PWA-, Responsive- oder Navigationsmuster vorhanden sind, sollen diese bevorzugt übernommen oder konsistent nachgebildet werden.
|
||||
|
||||
Keine fachliche Kopplung an Mitai.
|
||||
|
||||
---
|
||||
|
||||
## 5. Hauptnavigation
|
||||
|
||||
Der MVP benötigt mindestens:
|
||||
|
||||
```text
|
||||
Projekte
|
||||
Admin
|
||||
```
|
||||
|
||||
Optional:
|
||||
|
||||
```text
|
||||
Systemstatus
|
||||
```
|
||||
|
||||
Noch nicht erforderlich:
|
||||
|
||||
- Benutzerverwaltung
|
||||
- Warenkorb
|
||||
- Benachrichtigungseinstellungen
|
||||
- Account-/Profilverwaltung
|
||||
|
||||
---
|
||||
|
||||
## 6. Screen 1 – BuyingProject-Liste
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects
|
||||
```
|
||||
|
||||
Funktionen:
|
||||
|
||||
- aktive BuyingProjects anzeigen
|
||||
- Projektname
|
||||
- Kategorie
|
||||
- Status
|
||||
- letzter erfolgreicher SearchRun
|
||||
- Anzahl Kandidaten
|
||||
- Anzahl Shortlist-Kandidaten
|
||||
- aktueller Discovery-Status
|
||||
- Button „Neues Projekt“
|
||||
|
||||
Mögliche Status:
|
||||
|
||||
```text
|
||||
ACTIVE
|
||||
PURCHASED
|
||||
ARCHIVED
|
||||
OUT_OF_SYNC
|
||||
BLOCKED
|
||||
TOO_BROAD
|
||||
```
|
||||
|
||||
Die Liste darf keine produktkategoriespezifischen Felder voraussetzen.
|
||||
|
||||
---
|
||||
|
||||
## 7. Screen 2 – BuyingProject anlegen
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/new
|
||||
```
|
||||
|
||||
### Schritt 1 – Basisdaten
|
||||
|
||||
Mindestens:
|
||||
|
||||
```text
|
||||
name
|
||||
source
|
||||
source_category
|
||||
market
|
||||
currency
|
||||
shortlist_size
|
||||
schedule_interval
|
||||
```
|
||||
|
||||
Default:
|
||||
|
||||
```text
|
||||
shortlist_size = 5
|
||||
schedule_interval_minutes = 180
|
||||
```
|
||||
|
||||
### Schritt 2 – Kriterien
|
||||
|
||||
Nach Auswahl der Source-Kategorie lädt das Frontend dynamisch:
|
||||
|
||||
- Canonical Criteria
|
||||
- Source Filter Definitions
|
||||
- CriterionMappings
|
||||
- Mapping-Status
|
||||
|
||||
Das Frontend erzeugt daraus dynamisch die Eingabefelder.
|
||||
|
||||
Es gibt **keine fest codierten Notebook-Felder**.
|
||||
|
||||
---
|
||||
|
||||
## 8. Dynamische Kriterien
|
||||
|
||||
Verfügbare Kriterien entstehen aus:
|
||||
|
||||
```text
|
||||
SourceCategory
|
||||
↓
|
||||
SourceFilterDefinitions
|
||||
↓
|
||||
CriterionDefinitions
|
||||
↓
|
||||
CriterionMappings
|
||||
```
|
||||
|
||||
Das Frontend rendert Controls anhand des Typs.
|
||||
|
||||
### Unterstützte Control-Typen
|
||||
|
||||
```text
|
||||
BOOLEAN
|
||||
ENUM
|
||||
MULTI_ENUM
|
||||
NUMERIC_RANGE
|
||||
NUMERIC_THRESHOLD
|
||||
TEXT
|
||||
```
|
||||
|
||||
Für ein Kriterium sollen mindestens konfigurierbar sein:
|
||||
|
||||
```text
|
||||
operator
|
||||
value
|
||||
mandatory / optional
|
||||
```
|
||||
|
||||
Wenn vom Scoring unterstützt zusätzlich:
|
||||
|
||||
```text
|
||||
weight
|
||||
preference strategy
|
||||
acceptable
|
||||
ideal
|
||||
direction
|
||||
```
|
||||
|
||||
### Sichtbarer Mapping-Status
|
||||
|
||||
Pro Kriterium:
|
||||
|
||||
```text
|
||||
VERIFIED
|
||||
LEARNED_UNVERIFIED
|
||||
INTERNAL_ONLY
|
||||
BROKEN
|
||||
DISABLED
|
||||
```
|
||||
|
||||
Standardnutzer sollen primär verständliche Hinweise sehen:
|
||||
|
||||
- „Wird direkt bei Geizhals gefiltert“
|
||||
- „Wird erst intern bewertet“
|
||||
- „Filter derzeit nicht verfügbar“
|
||||
|
||||
Technische IDs bleiben im Admin-Bereich.
|
||||
|
||||
---
|
||||
|
||||
## 9. Kriterienänderungen
|
||||
|
||||
Wenn Kriterien geändert werden:
|
||||
|
||||
```text
|
||||
criteria_version++
|
||||
```
|
||||
|
||||
Die bestehende DiscoverySearch wird:
|
||||
|
||||
```text
|
||||
OUT_OF_SYNC
|
||||
```
|
||||
|
||||
Danach muss das Backend eine neue Query aus den aktuellen:
|
||||
|
||||
```text
|
||||
Criteria
|
||||
+
|
||||
VERIFIED Mappings
|
||||
```
|
||||
|
||||
erzeugen.
|
||||
|
||||
Das Frontend zeigt klar:
|
||||
|
||||
```text
|
||||
Suchkriterien wurden geändert.
|
||||
Die externe Suche muss aktualisiert werden.
|
||||
```
|
||||
|
||||
Aktion:
|
||||
|
||||
```text
|
||||
Suche aktualisieren
|
||||
```
|
||||
|
||||
Danach:
|
||||
|
||||
```text
|
||||
generate-query
|
||||
→ optional run
|
||||
→ neuer SearchRun
|
||||
→ neues Ranking
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Screen 3 – BuyingProject-Detail
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/:projectId
|
||||
```
|
||||
|
||||
Desktop: strukturierte Detailansicht.
|
||||
|
||||
Mobile: Tabs.
|
||||
|
||||
Mindestens folgende Bereiche:
|
||||
|
||||
### Übersicht
|
||||
|
||||
- Projektstatus
|
||||
- Kategorie
|
||||
- letzter SearchRun
|
||||
- Discovery-Status
|
||||
- Trefferzahl
|
||||
- Kandidatenzahl
|
||||
- Shortlist-Größe
|
||||
- Zeitpunkt nächste geplante Suche
|
||||
- wichtige Warnungen
|
||||
|
||||
### Kandidaten
|
||||
|
||||
- aktuelle Kandidaten
|
||||
- ProductScore
|
||||
- product_eligibility
|
||||
- aktueller best_price, sofern vorhanden
|
||||
- Diff-Status
|
||||
- Lifecycle-Status
|
||||
- Shortlist-Markierung
|
||||
|
||||
### Shortlist
|
||||
|
||||
Top-N gemäß Projektkonfiguration.
|
||||
|
||||
### Kriterien
|
||||
|
||||
Dynamische Kriterien anzeigen und bearbeiten.
|
||||
|
||||
### Historie
|
||||
|
||||
- SearchRuns
|
||||
- wichtige Statusänderungen
|
||||
- bisherige Rankings
|
||||
- soweit vorhanden frühere Bewertungen / Empfehlungen
|
||||
|
||||
---
|
||||
|
||||
## 11. Screen 4 – Kandidatenliste
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/:projectId/candidates
|
||||
```
|
||||
|
||||
Darstellung:
|
||||
|
||||
- sortierbare Liste/Tabelle auf Desktop
|
||||
- Cards/List Items auf Mobile
|
||||
|
||||
Mindestens:
|
||||
|
||||
```text
|
||||
Produktname
|
||||
Hersteller
|
||||
wichtige dynamische Attribute
|
||||
best_price
|
||||
ProductScore
|
||||
product_eligibility
|
||||
diff_status
|
||||
lifecycle_status
|
||||
shortlist_status
|
||||
last_seen
|
||||
```
|
||||
|
||||
### Dynamische Attribute
|
||||
|
||||
Welche Eigenschaften sichtbar sind, darf nicht notebook-spezifisch hart codiert sein.
|
||||
|
||||
Mögliche Strategie:
|
||||
|
||||
1. Source-/Category-Definition liefert relevante display attributes.
|
||||
2. Projekt kann später bevorzugte sichtbare Attribute konfigurieren.
|
||||
3. MVP darf zunächst die wichtigsten vorhandenen normalisierten Attribute anzeigen.
|
||||
|
||||
---
|
||||
|
||||
## 12. Screen 5 – Kandidatendetail
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/:projectId/candidates/:candidateId
|
||||
```
|
||||
|
||||
Der Nutzer soll in ein Produkt „hineinspringen“ können.
|
||||
|
||||
Mindestens:
|
||||
|
||||
### Aktueller Status
|
||||
|
||||
- Name
|
||||
- Hersteller
|
||||
- Source-Link
|
||||
- ProductScore
|
||||
- product_eligibility
|
||||
- Shortlist-Status
|
||||
- aktueller best_price
|
||||
- letzte Sichtung
|
||||
|
||||
### Attribute
|
||||
|
||||
Alle verfügbaren normalisierten dynamischen Produktattribute.
|
||||
|
||||
Keine notebook-spezifische Feldliste.
|
||||
|
||||
### Historie
|
||||
|
||||
- Preisentwicklung aus ProductObservations
|
||||
- relevante Attributänderungen
|
||||
- NEW / CHANGED / MISSING / DISAPPEARED
|
||||
- SearchRun-Bezug
|
||||
|
||||
### Bewertungen / Ergebnisse
|
||||
|
||||
- aktuelle Bewertung
|
||||
- frühere gespeicherte Bewertungen, soweit Backend vorhanden
|
||||
- Gründe / Constraint-Ergebnisse
|
||||
- Coverage / fehlende Daten
|
||||
|
||||
### Spätere Erweiterbarkeit
|
||||
|
||||
Dieser Screen wird später Zielort für:
|
||||
|
||||
- Deep Enrichment
|
||||
- Tests
|
||||
- Reviews
|
||||
- Benchmarks
|
||||
- Händlervergleich
|
||||
- Kaufempfehlung
|
||||
|
||||
Diese Funktionen sind noch nicht Bestandteil dieses MVP-Slices.
|
||||
|
||||
---
|
||||
|
||||
## 13. Screen 6 – Shortlist
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/:projectId/shortlist
|
||||
```
|
||||
|
||||
Zweck:
|
||||
|
||||
Nur die aktuell relevantesten Produkte zeigen.
|
||||
|
||||
Mindestens:
|
||||
|
||||
- Rang
|
||||
- Produkt
|
||||
- ProductScore
|
||||
- Product Eligibility
|
||||
- aktueller Preis
|
||||
- relevante Stärken
|
||||
- relevante Schwächen / fehlende Daten
|
||||
- letzter Änderungsstatus
|
||||
|
||||
Default:
|
||||
|
||||
```text
|
||||
Top 5
|
||||
```
|
||||
|
||||
konfigurierbar am BuyingProject.
|
||||
|
||||
---
|
||||
|
||||
## 14. Screen 7 – Search Runs / Historie
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/projects/:projectId/history
|
||||
```
|
||||
|
||||
Anzeige:
|
||||
|
||||
```text
|
||||
Zeitpunkt
|
||||
Status
|
||||
Trefferzahl
|
||||
Page Count
|
||||
NEW
|
||||
CHANGED
|
||||
MISSING
|
||||
DISAPPEARED
|
||||
TOO_BROAD
|
||||
Fehler / BLOCKED
|
||||
```
|
||||
|
||||
Ein SearchRun muss geöffnet werden können, um Detailinformationen zu sehen.
|
||||
|
||||
Kein HTML anzeigen oder archivieren.
|
||||
|
||||
---
|
||||
|
||||
## 15. Screen 8 – Admin: Taxonomie & Mapping
|
||||
|
||||
Route:
|
||||
|
||||
```text
|
||||
/admin/sources
|
||||
/admin/sources/:source/categories/:category
|
||||
```
|
||||
|
||||
Dieser Bereich darf technischer sein.
|
||||
|
||||
Funktionen:
|
||||
|
||||
- gelernte Kategorien sehen
|
||||
- SourceFilterDefinitions sehen
|
||||
- SourceFilterOptions/Ranges sehen
|
||||
- Canonical Criteria sehen
|
||||
- CriterionMappings sehen
|
||||
- Mapping-Status ändern
|
||||
- fehlerhafte kleine Mapping-Änderungen direkt korrigieren
|
||||
- BROKEN / DISABLED erkennen
|
||||
- Taxonomie neu einlesen
|
||||
- Query-Mapping nachvollziehen
|
||||
|
||||
### Wichtig
|
||||
|
||||
Das Frontend darf kein URL-Encoding selbst erfinden.
|
||||
|
||||
Es editiert/pflegt nur persistierte Mapping-Daten.
|
||||
|
||||
Backend bleibt für Validierung und Query-Generation verantwortlich.
|
||||
|
||||
---
|
||||
|
||||
## 16. Server State und Data Fetching
|
||||
|
||||
Empfehlung:
|
||||
|
||||
```text
|
||||
TanStack Query
|
||||
```
|
||||
|
||||
für:
|
||||
|
||||
- Projekte
|
||||
- Kriterien
|
||||
- Taxonomie
|
||||
- Mappings
|
||||
- SearchRuns
|
||||
- Kandidaten
|
||||
- Shortlist
|
||||
- Produktdetails
|
||||
|
||||
Client-State nur für:
|
||||
|
||||
- UI-Zustände
|
||||
- Formulare
|
||||
- Tabs
|
||||
- lokale Filter
|
||||
|
||||
Keine fachlich relevante Server-Wahrheit ausschließlich im Browser halten.
|
||||
|
||||
---
|
||||
|
||||
## 17. API-Verträge
|
||||
|
||||
Das Frontend nutzt mindestens:
|
||||
|
||||
### Projekte
|
||||
|
||||
```text
|
||||
GET /api/v1/buying-projects
|
||||
POST /api/v1/buying-projects
|
||||
GET /api/v1/buying-projects/{id}
|
||||
PATCH /api/v1/buying-projects/{id}
|
||||
```
|
||||
|
||||
### Projektkriterien
|
||||
|
||||
```text
|
||||
GET /api/v1/buying-projects/{id}/criteria
|
||||
PUT /api/v1/buying-projects/{id}/criteria
|
||||
```
|
||||
|
||||
Falls Backend intern versionierte Ressourcen bevorzugt, kann PUT eine neue Version erzeugen.
|
||||
|
||||
### Taxonomie
|
||||
|
||||
```text
|
||||
POST /api/v1/sources/{source}/categories/{category}/discover-taxonomy
|
||||
GET /api/v1/sources/{source}/categories/{category}/filters
|
||||
```
|
||||
|
||||
### Criterion Definitions / Mappings
|
||||
|
||||
```text
|
||||
GET /api/v1/criterion-definitions
|
||||
GET /api/v1/criterion-mappings
|
||||
POST /api/v1/criterion-mappings
|
||||
PATCH /api/v1/criterion-mappings/{id}
|
||||
```
|
||||
|
||||
### Discovery Search
|
||||
|
||||
```text
|
||||
GET /api/v1/buying-projects/{id}/discovery-search
|
||||
POST /api/v1/discovery-searches/{id}/generate-query
|
||||
POST /api/v1/discovery-searches/{id}/run
|
||||
```
|
||||
|
||||
### Search Runs
|
||||
|
||||
```text
|
||||
GET /api/v1/discovery-searches/{id}/runs
|
||||
GET /api/v1/search-runs/{id}
|
||||
```
|
||||
|
||||
### Candidates
|
||||
|
||||
```text
|
||||
GET /api/v1/buying-projects/{id}/candidates
|
||||
GET /api/v1/buying-projects/{id}/shortlist
|
||||
GET /api/v1/buying-projects/{id}/candidates/{candidateId}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 18. Status- und Fehlerdarstellung
|
||||
|
||||
Frontend muss mindestens folgende Zustände sichtbar machen:
|
||||
|
||||
### Discovery
|
||||
|
||||
```text
|
||||
SUCCESS
|
||||
PARTIAL
|
||||
FAILED
|
||||
BLOCKED
|
||||
TOO_BROAD
|
||||
OUT_OF_SYNC
|
||||
```
|
||||
|
||||
### Candidate Diff
|
||||
|
||||
```text
|
||||
NEW
|
||||
CHANGED
|
||||
UNCHANGED
|
||||
MISSING
|
||||
DISAPPEARED
|
||||
```
|
||||
|
||||
### Matching
|
||||
|
||||
```text
|
||||
MATCHED
|
||||
MATCH_UNCERTAIN
|
||||
```
|
||||
|
||||
### Mapping
|
||||
|
||||
```text
|
||||
DISCOVERED
|
||||
LEARNED_UNVERIFIED
|
||||
VERIFIED
|
||||
BROKEN
|
||||
DISABLED
|
||||
INTERNAL_ONLY
|
||||
```
|
||||
|
||||
Fehlertexte für Standardnutzer verständlich formulieren.
|
||||
|
||||
Technische Details optional aufklappbar.
|
||||
|
||||
---
|
||||
|
||||
## 19. Loading / Empty / Error States
|
||||
|
||||
Jeder Screen braucht explizit:
|
||||
|
||||
- Loading
|
||||
- Empty
|
||||
- Error
|
||||
- Retry
|
||||
|
||||
Beispiele:
|
||||
|
||||
```text
|
||||
Noch keine Kandidaten gefunden.
|
||||
Noch kein SearchRun ausgeführt.
|
||||
Taxonomie wurde noch nicht geladen.
|
||||
Diese Suche ist derzeit BLOCKED.
|
||||
Die Kriterien wurden geändert; Suche ist OUT_OF_SYNC.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 20. PWA-Grundgerüst
|
||||
|
||||
Mindestens:
|
||||
|
||||
- installierbare PWA
|
||||
- Manifest
|
||||
- Icons / App-Metadaten
|
||||
- responsive shell
|
||||
- app-like navigation
|
||||
- sinnvolle Offline-Fallback-Seite
|
||||
|
||||
Keine Offline-Businesslogik erforderlich.
|
||||
|
||||
Keine komplexe Synchronisationslogik in v0.1.
|
||||
|
||||
---
|
||||
|
||||
## 21. Tests
|
||||
|
||||
### Unit / Component
|
||||
|
||||
- dynamischer Criterion Renderer je Filtertyp
|
||||
- Mapping-Status-Darstellung
|
||||
- Candidate Status-Darstellung
|
||||
- responsive Komponenten
|
||||
- Formvalidierung
|
||||
|
||||
### API Integration
|
||||
|
||||
- Projektliste
|
||||
- Projektanlage
|
||||
- Kriterien laden/speichern
|
||||
- SearchRun starten
|
||||
- Kandidaten laden
|
||||
- Shortlist laden
|
||||
- Admin-Mapping aktualisieren
|
||||
|
||||
### E2E
|
||||
|
||||
Mindestens ein vollständiger Notebook-Pilot:
|
||||
|
||||
```text
|
||||
Projekt anlegen
|
||||
→ Kategorie Notebooks
|
||||
→ dynamische Kriterien laden
|
||||
→ Kriterien setzen
|
||||
→ Geizhals Query generieren
|
||||
→ SearchRun starten
|
||||
→ Kandidaten erscheinen
|
||||
→ Ranking sichtbar
|
||||
→ Shortlist sichtbar
|
||||
→ Kandidat öffnen
|
||||
→ Historie sichtbar
|
||||
→ Kriterium ändern
|
||||
→ OUT_OF_SYNC sichtbar
|
||||
→ Query neu erzeugen
|
||||
```
|
||||
|
||||
### Responsive E2E
|
||||
|
||||
Mindestens:
|
||||
|
||||
- Desktop-Viewport
|
||||
- Mobile-Viewport
|
||||
|
||||
---
|
||||
|
||||
## 22. Nicht implementieren
|
||||
|
||||
In diesem Slice weiterhin nicht:
|
||||
|
||||
- Login/Auth
|
||||
- RBAC
|
||||
- Einladungen
|
||||
- Multi-User-UI
|
||||
- Keepa
|
||||
- Idealo
|
||||
- WhatsApp
|
||||
- Warenkorb
|
||||
- Kaufautomatisierung
|
||||
- LLM-Automation
|
||||
- Deep Enrichment
|
||||
- finale Decision Engine
|
||||
|
||||
Die Architektur darf diese Erweiterungen nicht verbauen.
|
||||
|
||||
---
|
||||
|
||||
## 23. Definition of Done
|
||||
|
||||
- [ ] React + TypeScript + Vite Frontend existiert im Monorepo.
|
||||
- [ ] PWA ist installierbar.
|
||||
- [ ] Desktop- und Mobile-Layouts sind bewusst getrennt gestaltet.
|
||||
- [ ] BuyingProjects können über die UI angelegt und geöffnet werden.
|
||||
- [ ] Produktkategorie kann ausgewählt werden.
|
||||
- [ ] Kriterien werden dynamisch aus Taxonomie / CriterionDefinitions erzeugt.
|
||||
- [ ] Keine notebook-spezifischen Criteria-Felder im Frontend-Core.
|
||||
- [ ] Kriterien können bearbeitet und gespeichert werden.
|
||||
- [ ] Mapping-Status ist sichtbar.
|
||||
- [ ] INTERNAL_ONLY-Kriterien können gespeichert werden.
|
||||
- [ ] Discovery Search kann generiert werden.
|
||||
- [ ] SearchRun kann aus der UI gestartet werden.
|
||||
- [ ] SearchRun-Status ist sichtbar.
|
||||
- [ ] Kandidatenliste funktioniert.
|
||||
- [ ] Ranking ist sichtbar.
|
||||
- [ ] Shortlist funktioniert.
|
||||
- [ ] Kandidatendetail zeigt dynamische Attribute und Historie.
|
||||
- [ ] Projekt-Historie zeigt SearchRuns und relevante Änderungen.
|
||||
- [ ] Kriterienänderung setzt/zeigt OUT_OF_SYNC und ermöglicht Regeneration.
|
||||
- [ ] Admin-Bereich erlaubt Prüfung/Korrektur von Source-Taxonomie und Mappings.
|
||||
- [ ] Loading-/Empty-/Error-Zustände existieren.
|
||||
- [ ] E2E-Test des vollständigen Notebook-Piloten ist grün.
|
||||
- [ ] Desktop- und Mobile-E2E sind grün.
|
||||
- [ ] Backend bleibt alleinige Quelle fachlicher Wahrheit.
|
||||
|
||||
---
|
||||
|
||||
# Coding-Agent-Arbeitsauftrag
|
||||
|
||||
Verwende als verbindliche Anforderungen:
|
||||
|
||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||
2. `buying-agent-architecture-addendum-v1.md`
|
||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||
4. `buying-agent-frontend-mvp-spec-v0.1.md`
|
||||
|
||||
Die Frontend-MVP-Spec hebt für den nutzbaren MVP die frühere Scope-Aussage „kein Frontend“ auf.
|
||||
|
||||
## 1. Vor Implementierung
|
||||
|
||||
Prüfe den aktuellen Repository-Stand gegen alle vier Dokumente.
|
||||
|
||||
Berichte kurz:
|
||||
|
||||
- welche Backend-APIs für den Frontend-Flow bereits existieren,
|
||||
- welche Endpunkte fehlen,
|
||||
- ob aktuelle DTOs dynamische Kriterien und Produktattribute vollständig tragen,
|
||||
- ob bestehender Code notebook-spezifische Felder enthält,
|
||||
- welche Änderungen rein additiv möglich sind.
|
||||
|
||||
Keine größere strukturelle Abweichung ohne Rückfrage.
|
||||
|
||||
## 2. Vertikalen MVP implementieren
|
||||
|
||||
Nicht Backend und Frontend getrennt in langen Phasen bauen.
|
||||
|
||||
Stattdessen vertikal:
|
||||
|
||||
```text
|
||||
Projektliste
|
||||
→ Projektanlage
|
||||
→ dynamische Kriterien
|
||||
→ Query Generation
|
||||
→ SearchRun
|
||||
→ Kandidaten
|
||||
→ Ranking
|
||||
→ Shortlist
|
||||
→ Kandidatendetail
|
||||
```
|
||||
|
||||
Jeder Schritt soll nach Implementierung unmittelbar über die UI prüfbar sein.
|
||||
|
||||
## 3. Dynamik sicherstellen
|
||||
|
||||
Keine festen Felder wie:
|
||||
|
||||
```text
|
||||
ram_gb
|
||||
storage_gb
|
||||
weight_kg
|
||||
display_inches
|
||||
```
|
||||
|
||||
in generischen Frontend-Komponenten hardcoden.
|
||||
|
||||
Controls und sichtbare Produktattribute werden metadata-driven erzeugt.
|
||||
|
||||
Notebook-spezifische Werte dürfen nur als gelernte/gespeicherte Daten vorkommen.
|
||||
|
||||
## 4. UI
|
||||
|
||||
- React + TypeScript + Vite
|
||||
- PWA
|
||||
- TanStack Query
|
||||
- einfache, nüchterne Bedienung
|
||||
- Desktop: Sidebar + Content/Detail
|
||||
- Mobile: kompakte Navigation / Bottom Navigation + Vollbild-Details
|
||||
- vorhandene Mitai-Muster wiederverwenden, falls im zugänglichen Repo vorhanden und passend
|
||||
|
||||
## 5. Tests
|
||||
|
||||
Führe aus:
|
||||
|
||||
- Frontend Unit/Component Tests
|
||||
- API Integration Tests
|
||||
- E2E vollständiger Notebook-Pilot
|
||||
- Desktop E2E
|
||||
- Mobile E2E
|
||||
|
||||
## 6. Abschlussbericht
|
||||
|
||||
Dokumentiere:
|
||||
|
||||
- implementierte Screens
|
||||
- neue/angepasste Backend-Endpunkte
|
||||
- verwendete Frontend-Bibliotheken
|
||||
- responsive Verhalten
|
||||
- ausgeführte Tests und Ergebnisse
|
||||
- offene Abweichungen
|
||||
- Risiken für spätere Multi-User-, Deep-Enrichment- und Decision-Erweiterungen
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,986 @@
|
||||
# Buying Agent – Source & Discovery Specification v0.1
|
||||
|
||||
**Status:** implementierungsbereit
|
||||
**Spezifikationsversion:** 0.1
|
||||
**Stand:** 2026-10-04
|
||||
**Ergänzt:** `buying-agent-v0.1-spec-v1.1.md` und `buying-agent-architecture-addendum-v1.md`
|
||||
|
||||
## 1. Zweck
|
||||
|
||||
Diese Spezifikation definiert den ersten real nutzbaren Discovery-Slice des Buying Agents.
|
||||
|
||||
Ziel ist, externe Produktsuchen automatisiert aus den Kriterien eines BuyingProjects abzuleiten, regelmäßig auszuführen, Ergebnisse dauerhaft zu verwalten, Veränderungen zu erkennen und die Treffer anschließend durch die eigene Buying-Engine zu bewerten.
|
||||
|
||||
Der erste Pilot erfolgt mit **Geizhals, Kategorie Notebooks**. Die Architektur darf jedoch nicht notebook-spezifisch sein.
|
||||
|
||||
```text
|
||||
BuyingProject
|
||||
↓
|
||||
Canonical Criteria
|
||||
↓
|
||||
Source Taxonomy + Mapping
|
||||
↓
|
||||
generierte Geizhals-Suche
|
||||
↓
|
||||
periodischer SearchRun
|
||||
↓
|
||||
persistierte Produkte + Beobachtungen
|
||||
↓
|
||||
Diff zum vorherigen Lauf
|
||||
↓
|
||||
NEW / CHANGED / MISSING / DISAPPEARED
|
||||
↓
|
||||
interne Constraints + Ranking
|
||||
↓
|
||||
Shortlist
|
||||
↓
|
||||
später Deep Enrichment
|
||||
↓
|
||||
später Decision
|
||||
```
|
||||
|
||||
## 2. Fachliche Leitprinzipien
|
||||
|
||||
1. Geizhals ist Discovery-Quelle, nicht Decision Engine.
|
||||
2. Die eigenen BuyingProject-Kriterien bleiben die fachliche Quelle der Wahrheit.
|
||||
3. Geizhals wird als möglichst starker Vorfilter genutzt.
|
||||
4. Nicht auf Geizhals abbildbare Kriterien bleiben interne Kriterien.
|
||||
5. Discovery, Ranking und Decision bleiben getrennte fachliche Stufen.
|
||||
6. Externe Filtertaxonomien werden **on demand** gelernt, nicht vollständig vorab katalogisiert.
|
||||
7. URL-Encoding und Filtersyntax dürfen nicht vom LLM erfunden werden.
|
||||
8. Produktidentität wird zunächst konservativ und pragmatisch behandelt.
|
||||
9. Bei Unsicherheit erfolgt keine automatische Zusammenführung.
|
||||
10. Suchläufe speichern strukturierte Zustände, keine HTML-Archive.
|
||||
|
||||
## 3. Scope v0.1
|
||||
|
||||
### Enthalten
|
||||
|
||||
- Geizhals als erste Discovery-Quelle
|
||||
- Pilotkategorie Notebooks
|
||||
- On-Demand-Erfassung der Geizhals-Filtertaxonomie
|
||||
- Speicherung von Source-Kategorien, Filterdefinitionen, Optionen und Wertebereichen
|
||||
- Mapping zwischen Canonical Criteria und Source-Filtern
|
||||
- reproduzierbare Generierung von Geizhals-Such-URLs
|
||||
- periodische SearchRuns
|
||||
- vollständige Ergebnisaufnahme einer aktiven Suche
|
||||
- persistierte SourceProductReferences
|
||||
- Historisierung sichtbarer Attribute und Preise
|
||||
- Diffing zwischen Suchläufen
|
||||
- Status `NEW`, `CHANGED`, `UNCHANGED`, `MISSING`, `DISAPPEARED`
|
||||
- interne Bewertung der Kandidaten
|
||||
- konfigurierbare Shortlist, Default 5
|
||||
- defensives Fehler-/Backoff-Verhalten
|
||||
- Tests
|
||||
|
||||
### Nicht enthalten
|
||||
|
||||
- Frontend-Implementierung
|
||||
- Keepa
|
||||
- Idealo
|
||||
- LLM-Automation
|
||||
- automatische Kaufentscheidung
|
||||
- Kaufautomatisierung
|
||||
- Warenkorbmanagement
|
||||
- Authentifizierung / RBAC
|
||||
- Multi-User-UI
|
||||
- automatische Deep-Enrichment-Quellen
|
||||
- Umgehung von Schutzmechanismen externer Websites
|
||||
|
||||
## 4. Source Taxonomy
|
||||
|
||||
### 4.1 SourceCategory
|
||||
|
||||
Repräsentiert eine Kategorie einer externen Quelle.
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "geizhals",
|
||||
"external_category_key": "nb",
|
||||
"name": "Notebooks",
|
||||
"source_url": "https://geizhals.de/?cat=nb",
|
||||
"status": "ACTIVE",
|
||||
"last_verified_at": "2026-10-04T10:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 SourceFilterDefinition
|
||||
|
||||
Beschreibt einen von Geizhals angebotenen Filter.
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "geizhals",
|
||||
"category_key": "nb",
|
||||
"external_filter_key": "17023",
|
||||
"label": "Arbeitsspeicher",
|
||||
"filter_type": "NUMERIC_THRESHOLD",
|
||||
"canonical_unit": "GB",
|
||||
"status": "VERIFIED"
|
||||
}
|
||||
```
|
||||
|
||||
Mögliche `filter_type`-Werte:
|
||||
|
||||
```text
|
||||
BOOLEAN
|
||||
ENUM
|
||||
MULTI_ENUM
|
||||
NUMERIC_RANGE
|
||||
NUMERIC_THRESHOLD
|
||||
TEXT
|
||||
UNKNOWN
|
||||
```
|
||||
|
||||
### 4.3 SourceFilterOption
|
||||
|
||||
Für diskrete Filterwerte:
|
||||
|
||||
```json
|
||||
{
|
||||
"filter_key": "17023",
|
||||
"source_value": "ab 24GB",
|
||||
"normalized_value": 24,
|
||||
"operator": ">=",
|
||||
"unit": "GB"
|
||||
}
|
||||
```
|
||||
|
||||
### 4.4 SourceFilterRange
|
||||
|
||||
Für Bereiche bzw. frei parametrisierbare Werte:
|
||||
|
||||
```json
|
||||
{
|
||||
"filter_key": "price",
|
||||
"minimum": 0,
|
||||
"maximum": null,
|
||||
"step": 1,
|
||||
"unit": "EUR"
|
||||
}
|
||||
```
|
||||
|
||||
## 5. On-Demand Taxonomy Learning
|
||||
|
||||
Die Taxonomie wird nur für tatsächlich verwendete Kategorien gelernt.
|
||||
|
||||
Initial:
|
||||
|
||||
```text
|
||||
Geizhals → Notebooks
|
||||
```
|
||||
|
||||
Später z. B.:
|
||||
|
||||
```text
|
||||
Monitore
|
||||
Fernseher
|
||||
Kameras
|
||||
Rasenmäher
|
||||
...
|
||||
```
|
||||
|
||||
### Ablauf
|
||||
|
||||
```text
|
||||
Kategorie öffnen
|
||||
↓
|
||||
verfügbare Filter erkennen
|
||||
↓
|
||||
Filterdefinitionen speichern
|
||||
↓
|
||||
Optionen / Bereiche erfassen
|
||||
↓
|
||||
URL-Verhalten beobachten
|
||||
↓
|
||||
Mapping erzeugen
|
||||
↓
|
||||
technisch verifizieren
|
||||
```
|
||||
|
||||
### Mapping-Status
|
||||
|
||||
```text
|
||||
DISCOVERED
|
||||
LEARNED_UNVERIFIED
|
||||
VERIFIED
|
||||
BROKEN
|
||||
DISABLED
|
||||
```
|
||||
|
||||
Nur `VERIFIED` darf für automatische produktive Suchläufe verwendet werden.
|
||||
|
||||
### Änderbarkeit
|
||||
|
||||
Die gelernten Mappings sollen später im Frontend administrierbar sein.
|
||||
|
||||
v0.1 benötigt dafür noch keine UI, aber die Persistenz muss Änderungen ermöglichen, ohne Source-Code anzupassen.
|
||||
|
||||
## 6. Canonical Criteria
|
||||
|
||||
Canonical Criteria sind die interne fachliche Darstellung eines Suchkriteriums.
|
||||
|
||||
Sie sollen sich stark an der Source-Taxonomie orientieren, aber nicht fest an Geizhals gekoppelt sein.
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "ram_gb",
|
||||
"value_type": "number",
|
||||
"unit": "GB",
|
||||
"semantic": "working_memory"
|
||||
}
|
||||
```
|
||||
|
||||
Ein BuyingProject verwendet daraus konkrete Anforderungen:
|
||||
|
||||
```json
|
||||
{
|
||||
"criterion": "ram_gb",
|
||||
"operator": ">=",
|
||||
"value": 24,
|
||||
"mandatory": true
|
||||
}
|
||||
```
|
||||
|
||||
### Source Mapping
|
||||
|
||||
```text
|
||||
Canonical Criterion
|
||||
ram_gb >= 24
|
||||
|
||||
↓
|
||||
|
||||
Geizhals Mapping
|
||||
filter_key = 17023
|
||||
source_value = "ab 24GB"
|
||||
|
||||
↓
|
||||
|
||||
URL fragment
|
||||
xf=17023_ab+24GB
|
||||
```
|
||||
|
||||
Nicht jedes Canonical Criterion muss ein Source Mapping besitzen.
|
||||
|
||||
Fehlt ein Mapping:
|
||||
|
||||
```text
|
||||
Criterion → INTERNAL_ONLY
|
||||
```
|
||||
|
||||
Das Kriterium wird dann erst nach Discovery intern bewertet.
|
||||
|
||||
## 7. DiscoverySearch
|
||||
|
||||
Eine `DiscoverySearch` gehört zu einem BuyingProject und einer Source.
|
||||
|
||||
Sie enthält mindestens:
|
||||
|
||||
```text
|
||||
id
|
||||
buying_project_id
|
||||
source_id
|
||||
source_category_id
|
||||
criteria_version
|
||||
mapping_version
|
||||
generated_url
|
||||
status
|
||||
schedule_interval_minutes
|
||||
broad_result_threshold
|
||||
created_at
|
||||
updated_at
|
||||
last_successful_run_at
|
||||
```
|
||||
|
||||
Defaults:
|
||||
|
||||
```text
|
||||
schedule_interval_minutes = 180
|
||||
shortlist_size = 5
|
||||
broad_result_threshold = 300
|
||||
```
|
||||
|
||||
## 8. Search URL Generation
|
||||
|
||||
Die URL wird ausschließlich aus:
|
||||
|
||||
```text
|
||||
BuyingProject Criteria
|
||||
+
|
||||
VERIFIED CriterionMappings
|
||||
+
|
||||
SourceCategory
|
||||
+
|
||||
konfigurierbaren Source-Optionen
|
||||
```
|
||||
|
||||
generiert.
|
||||
|
||||
Die URL ist abgeleitet, nicht die fachliche Quelle der Wahrheit.
|
||||
|
||||
Eine Änderung der BuyingProject-Kriterien erzeugt eine neue `criteria_version`.
|
||||
|
||||
Wenn die gespeicherte DiscoverySearch nicht mehr zur aktuellen Criteria-/Mapping-Version passt:
|
||||
|
||||
```text
|
||||
status = OUT_OF_SYNC
|
||||
```
|
||||
|
||||
Danach muss die URL neu generiert werden.
|
||||
|
||||
## 9. Ergebnisumfang
|
||||
|
||||
Aktive Suchläufe werden grundsätzlich vollständig eingelesen.
|
||||
|
||||
Keine stille Begrenzung auf Top-N.
|
||||
|
||||
Nur vollständige Ergebnismengen erlauben belastbare Erkennung von:
|
||||
|
||||
```text
|
||||
NEW
|
||||
CHANGED
|
||||
MISSING
|
||||
DISAPPEARED
|
||||
```
|
||||
|
||||
### TOO_BROAD
|
||||
|
||||
Überschreitet die Treffermenge den konfigurierten Schwellwert:
|
||||
|
||||
```text
|
||||
result_count > broad_result_threshold
|
||||
```
|
||||
|
||||
wird die Suche markiert:
|
||||
|
||||
```text
|
||||
TOO_BROAD
|
||||
```
|
||||
|
||||
Die Suche darf technisch weiter ausgeführt werden, erzeugt aber einen Hinweis, dass die Source-Kriterien geschärft werden sollten.
|
||||
|
||||
Spätere Versionen dürfen breite Suchen differenziert behandeln, etwa seltenere Full-Reconciliation.
|
||||
|
||||
## 10. SearchRun
|
||||
|
||||
Jeder Abruf erzeugt einen unveränderlichen `SearchRun`.
|
||||
|
||||
Felder:
|
||||
|
||||
```text
|
||||
id
|
||||
discovery_search_id
|
||||
criteria_version
|
||||
mapping_version
|
||||
generated_url
|
||||
started_at
|
||||
finished_at
|
||||
status
|
||||
result_count
|
||||
page_count
|
||||
http_status_summary
|
||||
error_code
|
||||
```
|
||||
|
||||
Status:
|
||||
|
||||
```text
|
||||
RUNNING
|
||||
SUCCESS
|
||||
PARTIAL
|
||||
FAILED
|
||||
BLOCKED
|
||||
```
|
||||
|
||||
Es wird kein HTML-Snapshot gespeichert.
|
||||
|
||||
## 11. SourceProductReference
|
||||
|
||||
Ein Treffer einer externen Quelle wird als `SourceProductReference` dauerhaft angelegt.
|
||||
|
||||
Mindestens verfügbare Felder:
|
||||
|
||||
```text
|
||||
id
|
||||
source_id
|
||||
source_category_id
|
||||
source_url
|
||||
source_title
|
||||
manufacturer
|
||||
model
|
||||
manufacturer_sku
|
||||
gtin_ean
|
||||
normalized_spec_fingerprint
|
||||
first_seen_at
|
||||
last_seen_at
|
||||
status
|
||||
```
|
||||
|
||||
Keines dieser Felder außer der internen UUID wird pauschal als garantiert stabil betrachtet.
|
||||
|
||||
## 12. Produktidentität – Pilotansatz
|
||||
|
||||
v0.1 implementiert bewusst keine komplexe globale Identity Engine.
|
||||
|
||||
Für den Notebook-Pilot werden folgende Signale verwendet:
|
||||
|
||||
1. vorhandene stabile Source-ID, falls entdeckt
|
||||
2. GTIN/EAN, falls vorhanden
|
||||
3. Hersteller-SKU, falls vorhanden
|
||||
4. kanonische Source-URL
|
||||
5. normalisierter Titel
|
||||
6. Hersteller + Modell
|
||||
7. differenzierende technische Specs
|
||||
|
||||
Für Notebooks kann der initiale Fingerprint beispielsweise enthalten:
|
||||
|
||||
```text
|
||||
manufacturer
|
||||
model
|
||||
cpu
|
||||
ram
|
||||
storage
|
||||
display_size
|
||||
gpu
|
||||
```
|
||||
|
||||
### Matching-Regeln
|
||||
|
||||
- Hohe Sicherheit → existierende SourceProductReference verwenden
|
||||
- Unsicherheit → `MATCH_UNCERTAIN`
|
||||
- Keine sichere Übereinstimmung → neue SourceProductReference
|
||||
- niemals zwei Produkte allein aufgrund ähnlicher Namen automatisch verschmelzen
|
||||
|
||||
Die tatsächliche Stabilität der Identitätsmerkmale wird im Notebook-Pilot beobachtet und danach verfeinert.
|
||||
|
||||
## 13. Produktbeobachtungen und Historie
|
||||
|
||||
Technische Eigenschaften und Preise werden nicht einfach überschrieben.
|
||||
|
||||
Jeder SearchRun kann neue Beobachtungen erzeugen.
|
||||
|
||||
### ProductObservation
|
||||
|
||||
```text
|
||||
source_product_reference_id
|
||||
search_run_id
|
||||
observed_at
|
||||
raw_title
|
||||
best_price
|
||||
currency
|
||||
raw_attributes
|
||||
normalized_attributes
|
||||
availability
|
||||
seller_count
|
||||
```
|
||||
|
||||
Veränderte Werte bleiben historisch nachvollziehbar.
|
||||
|
||||
Beispiele:
|
||||
|
||||
```text
|
||||
Preis: 1499 → 1299
|
||||
RAM: 16 → 32
|
||||
Anbieterzahl: 8 → 12
|
||||
Verfügbarkeit geändert
|
||||
```
|
||||
|
||||
Bei technischen Attributänderungen muss das System berücksichtigen, dass auch fehlerhaftes Matching vorliegen könnte.
|
||||
|
||||
## 14. Diffing
|
||||
|
||||
Jeder erfolgreiche Full SearchRun wird mit dem vorherigen erfolgreichen Full SearchRun derselben DiscoverySearch verglichen.
|
||||
|
||||
### Status
|
||||
|
||||
```text
|
||||
NEW
|
||||
CHANGED
|
||||
UNCHANGED
|
||||
MISSING
|
||||
DISAPPEARED
|
||||
```
|
||||
|
||||
### Regeln
|
||||
|
||||
**NEW**
|
||||
Produkt war im vorherigen vollständigen Lauf nicht vorhanden.
|
||||
|
||||
**CHANGED**
|
||||
Produkt vorhanden, aber mindestens ein relevantes beobachtetes Feld hat sich verändert.
|
||||
|
||||
**UNCHANGED**
|
||||
Produkt vorhanden und relevante Felder unverändert.
|
||||
|
||||
**MISSING**
|
||||
Produkt fehlt in einem vollständigen Lauf.
|
||||
|
||||
**DISAPPEARED**
|
||||
Produkt fehlt in drei aufeinanderfolgenden vollständigen Läufen.
|
||||
|
||||
Produkte werden nicht gelöscht.
|
||||
|
||||
Historie bleibt erhalten.
|
||||
|
||||
Ein später erneut auftauchendes Produkt wird reaktiviert und nicht als neues Produkt dupliziert, sofern das Matching ausreichend sicher ist.
|
||||
|
||||
## 15. Lifecycle eines BuyingProjects
|
||||
|
||||
Während ein BuyingProject aktiv ist:
|
||||
|
||||
```text
|
||||
ACTIVE
|
||||
↓
|
||||
DiscoverySearches laufen
|
||||
↓
|
||||
Produkte + Historie wachsen
|
||||
↓
|
||||
Ranking + Shortlist
|
||||
```
|
||||
|
||||
Nach finalem Kauf:
|
||||
|
||||
```text
|
||||
PURCHASED
|
||||
↓
|
||||
BuyingProject archivieren
|
||||
↓
|
||||
Discovery stoppen
|
||||
```
|
||||
|
||||
Produkte, Beobachtungen und Evaluationen können mit dem Projekt archiviert bzw. aus aktiven Ansichten entfernt werden.
|
||||
|
||||
Keine physische Löschung historischer Daten als Standardverhalten.
|
||||
|
||||
## 16. Ranking
|
||||
|
||||
Nach Discovery übernimmt ausschließlich die interne Buying Engine.
|
||||
|
||||
```text
|
||||
Source Results
|
||||
↓
|
||||
interne Hard Constraints
|
||||
↓
|
||||
ProductScore
|
||||
↓
|
||||
Ranking
|
||||
```
|
||||
|
||||
Geizhals-Sortierung darf für die technische Ausgabe genutzt werden, ist aber keine fachliche Ranking-Entscheidung.
|
||||
|
||||
## 17. Shortlist
|
||||
|
||||
Nach internem Ranking wird eine Shortlist gebildet.
|
||||
|
||||
Default:
|
||||
|
||||
```text
|
||||
shortlist_size = 5
|
||||
```
|
||||
|
||||
Der Wert ist pro BuyingProject konfigurierbar.
|
||||
|
||||
Shortlist-Kandidaten erhalten:
|
||||
|
||||
```text
|
||||
SHORTLISTED
|
||||
```
|
||||
|
||||
Andere gültige Kandidaten:
|
||||
|
||||
```text
|
||||
CANDIDATE
|
||||
```
|
||||
|
||||
Die spätere Deep-Enrichment-Stufe arbeitet primär auf der Shortlist.
|
||||
|
||||
## 18. Deep Enrichment
|
||||
|
||||
Nicht Bestandteil dieses Implementierungsslices.
|
||||
|
||||
Zielbild:
|
||||
|
||||
```text
|
||||
Shortlist
|
||||
↓
|
||||
Herstellerdaten
|
||||
Tests
|
||||
Benchmarks
|
||||
Reviews
|
||||
weitere Händler
|
||||
Preisportale
|
||||
Qualitätsinformationen
|
||||
↓
|
||||
erneute Bewertung
|
||||
```
|
||||
|
||||
Discovery darf daher nicht voraussetzen, dass Geizhals alle für die finale Kaufentscheidung notwendigen Daten liefert.
|
||||
|
||||
## 19. Discovery, Ranking und Decision
|
||||
|
||||
Die drei fachlichen Stufen müssen getrennt bleiben.
|
||||
|
||||
### Discovery
|
||||
|
||||
```text
|
||||
Welche Produkte kommen potenziell infrage?
|
||||
```
|
||||
|
||||
### Ranking
|
||||
|
||||
```text
|
||||
Welche dieser Produkte passen am besten zum BuyingProject?
|
||||
```
|
||||
|
||||
### Decision
|
||||
|
||||
```text
|
||||
Ist jetzt der richtige Zeitpunkt, Kandidat X zu kaufen?
|
||||
```
|
||||
|
||||
Decision wird später u. a. berücksichtigen können:
|
||||
|
||||
- Zielpreis
|
||||
- Deadline
|
||||
- historische Preisentwicklung
|
||||
- aktuelle Verfügbarkeit
|
||||
- Product Fit
|
||||
- Offer Fit
|
||||
- Alternativen
|
||||
- Beschaffungsdringlichkeit
|
||||
|
||||
Decision ist nicht Bestandteil dieses Slices.
|
||||
|
||||
## 20. Scheduling und defensiver Abruf
|
||||
|
||||
Default:
|
||||
|
||||
```text
|
||||
alle 180 Minuten
|
||||
```
|
||||
|
||||
pro DiscoverySearch konfigurierbar.
|
||||
|
||||
### Anforderungen
|
||||
|
||||
- keine parallelen Massenabrufe
|
||||
- Pagination sequenziell oder stark begrenzt parallel
|
||||
- Caching, wo sinnvoll
|
||||
- Timeout
|
||||
- Retry mit exponentiellem Backoff
|
||||
- Jitter
|
||||
- Stop bei wiederholten Fehlern
|
||||
- keine Umgehung von Captchas oder Bot-Schutz
|
||||
|
||||
### Reaktion
|
||||
|
||||
**HTTP 429**
|
||||
|
||||
```text
|
||||
BACKOFF
|
||||
```
|
||||
|
||||
**HTTP 403**
|
||||
|
||||
```text
|
||||
BLOCKED
|
||||
```
|
||||
|
||||
**Captcha / Challenge erkannt**
|
||||
|
||||
```text
|
||||
BLOCKED
|
||||
```
|
||||
|
||||
Automatische weitere Abrufe der betroffenen DiscoverySearch werden pausiert.
|
||||
|
||||
Keine Schutzmechanismen umgehen.
|
||||
|
||||
## 21. Source Adapter Contract
|
||||
|
||||
Ein Discovery-Adapter soll mindestens folgende Fähigkeiten kapseln:
|
||||
|
||||
```python
|
||||
class DiscoverySourceAdapter(Protocol):
|
||||
def discover_taxonomy(category_ref): ...
|
||||
def build_query(criteria, mappings): ...
|
||||
def fetch_results(query, pagination): ...
|
||||
def parse_results(response): ...
|
||||
def normalize_result(result): ...
|
||||
```
|
||||
|
||||
Taxonomy Learning und Result Retrieval dürfen intern getrennte Services sein.
|
||||
|
||||
Der Rest des Systems darf keine Geizhals-spezifische URL- oder HTML-Logik kennen.
|
||||
|
||||
## 22. Services
|
||||
|
||||
Mindestens:
|
||||
|
||||
```text
|
||||
TaxonomyService
|
||||
CriterionMappingService
|
||||
DiscoverySearchService
|
||||
SearchExecutionService
|
||||
SourceProductIdentityService
|
||||
SearchDiffService
|
||||
CandidateRankingService
|
||||
ShortlistService
|
||||
```
|
||||
|
||||
Keine Businesslogik in n8n.
|
||||
|
||||
n8n kann später den Scheduler bzw. externe Events orchestrieren.
|
||||
|
||||
## 23. Persistenz
|
||||
|
||||
Neue bzw. ergänzende Tabellen können sein:
|
||||
|
||||
```text
|
||||
source_categories
|
||||
source_filter_definitions
|
||||
source_filter_options
|
||||
source_filter_ranges
|
||||
criterion_definitions
|
||||
criterion_mappings
|
||||
discovery_searches
|
||||
search_runs
|
||||
source_product_references
|
||||
product_observations
|
||||
project_candidates
|
||||
candidate_status_history
|
||||
```
|
||||
|
||||
Bestehende globale Entitäten aus der Hauptspezifikation bleiben bestehen:
|
||||
|
||||
```text
|
||||
Product
|
||||
ProductVariant
|
||||
Offer
|
||||
PriceObservation
|
||||
```
|
||||
|
||||
Diese dürfen nicht unnötig dupliziert werden.
|
||||
|
||||
Der Discovery-Slice kann zunächst mit SourceProductReference arbeiten und erst bei ausreichender Identität eine Verknüpfung zu canonical Product/ProductVariant herstellen.
|
||||
|
||||
## 24. API – minimaler Vertrag
|
||||
|
||||
### Taxonomie
|
||||
|
||||
```text
|
||||
POST /api/v1/sources/geizhals/categories/{category}/discover-taxonomy
|
||||
GET /api/v1/sources/geizhals/categories/{category}/filters
|
||||
```
|
||||
|
||||
### Mapping
|
||||
|
||||
```text
|
||||
GET /api/v1/criterion-mappings
|
||||
POST /api/v1/criterion-mappings
|
||||
PATCH /api/v1/criterion-mappings/{id}
|
||||
```
|
||||
|
||||
### DiscoverySearch
|
||||
|
||||
```text
|
||||
POST /api/v1/discovery-searches
|
||||
GET /api/v1/discovery-searches/{id}
|
||||
POST /api/v1/discovery-searches/{id}/generate-query
|
||||
POST /api/v1/discovery-searches/{id}/run
|
||||
```
|
||||
|
||||
### SearchRuns
|
||||
|
||||
```text
|
||||
GET /api/v1/discovery-searches/{id}/runs
|
||||
GET /api/v1/search-runs/{id}
|
||||
```
|
||||
|
||||
### Candidates
|
||||
|
||||
```text
|
||||
GET /api/v1/buying-projects/{id}/candidates
|
||||
GET /api/v1/buying-projects/{id}/shortlist
|
||||
```
|
||||
|
||||
Falls BuyingProject im aktuellen Code noch nicht existiert, muss die Integration mit der im Architecture Addendum vorgesehenen nächsten Domain-Erweiterung abgestimmt werden.
|
||||
|
||||
## 25. Tests
|
||||
|
||||
### Taxonomy
|
||||
|
||||
- Notebook-Kategorie kann on demand eingelesen werden
|
||||
- Filterdefinitionen werden reproduzierbar gespeichert
|
||||
- Optionen werden nicht dupliziert
|
||||
- Filteränderungen werden erkannt
|
||||
- BROKEN-Mapping wird nicht produktiv verwendet
|
||||
|
||||
### Query Generation
|
||||
|
||||
- identische Criteria + Mapping-Version erzeugen identische URL
|
||||
- RAM >= 24 erzeugt erwartetes Geizhals-Fragment
|
||||
- Preisbereich korrekt
|
||||
- mehrere Filter kombinierbar
|
||||
- INTERNAL_ONLY-Kriterien verändern die Source-URL nicht
|
||||
- veraltete Mapping-Version markiert Search als OUT_OF_SYNC
|
||||
|
||||
### Search Execution
|
||||
|
||||
- Pagination vollständig
|
||||
- Ergebniszahl korrekt
|
||||
- TOO_BROAD bei Überschreitung
|
||||
- kein stilles Top-N-Abschneiden
|
||||
- 429 → Backoff
|
||||
- 403/Captcha → BLOCKED
|
||||
|
||||
### Identity
|
||||
|
||||
- identische SKU wird wiedererkannt
|
||||
- identische EAN wird wiedererkannt
|
||||
- gleiche URL wird wiedererkannt
|
||||
- leicht geänderter Titel bei stabilen Specs wird erkannt
|
||||
- unsicheres Match erzeugt MATCH_UNCERTAIN
|
||||
- ähnliche Produktnamen werden nicht automatisch verschmolzen
|
||||
|
||||
### Diffing
|
||||
|
||||
- neuer Treffer → NEW
|
||||
- geänderter Preis → CHANGED
|
||||
- unverändert → UNCHANGED
|
||||
- einmal fehlend → MISSING
|
||||
- dreimal fehlend → DISAPPEARED
|
||||
- Wiederauftauchen reaktiviert bestehende Referenz
|
||||
|
||||
### Historie
|
||||
|
||||
- Preisänderungen bleiben historisiert
|
||||
- technische Änderungen bleiben historisiert
|
||||
- alte SearchRuns bleiben unverändert
|
||||
- Projektarchivierung löscht keine History
|
||||
|
||||
### Ranking
|
||||
|
||||
- nur interne Engine bestimmt Ranking
|
||||
- Source-Sortierung beeinflusst ProductScore nicht
|
||||
- Shortlistgröße Default 5
|
||||
- Shortlistgröße konfigurierbar
|
||||
|
||||
## 26. Implementierungsreihenfolge
|
||||
|
||||
1. Bestehende Implementierung gegen diese Spec und das Architecture Addendum prüfen.
|
||||
2. Notwendige BuyingProject-/ProjectCandidate-Minimalstruktur ergänzen, falls für Discovery erforderlich.
|
||||
3. Source Taxonomy + Mapping persistieren.
|
||||
4. Geizhals-Taxonomie-Pilot für Notebooks implementieren.
|
||||
5. reproduzierbare Query Generation implementieren.
|
||||
6. SearchRun + Pagination implementieren.
|
||||
7. SourceProductReference + ProductObservation implementieren.
|
||||
8. Identity Pilot implementieren.
|
||||
9. Diffing implementieren.
|
||||
10. interne Ranking-Engine anbinden.
|
||||
11. Shortlist bilden.
|
||||
12. Scheduling-Schnittstelle vorbereiten bzw. internen Scheduler minimal anbinden.
|
||||
13. Tests vollständig ausführen.
|
||||
14. Notebook-Pilot mit realer Suche validieren.
|
||||
|
||||
## 27. Definition of Done
|
||||
|
||||
- [ ] Geizhals-Notebook-Taxonomie wird on demand eingelesen und gespeichert.
|
||||
- [ ] Source-Filter und Optionen werden persistiert.
|
||||
- [ ] Filter-Mappings besitzen Lifecycle-Status.
|
||||
- [ ] BuyingProject-Kriterien erzeugen reproduzierbar eine Geizhals-URL.
|
||||
- [ ] Nicht abbildbare Kriterien bleiben INTERNAL_ONLY.
|
||||
- [ ] Aktive Suche kann vollständig ausgeführt werden.
|
||||
- [ ] Pagination wird vollständig verarbeitet.
|
||||
- [ ] TOO_BROAD wird erkannt.
|
||||
- [ ] SearchRuns werden gespeichert.
|
||||
- [ ] Produkte werden dauerhaft als SourceProductReference angelegt.
|
||||
- [ ] Preise und relevante Attribute werden historisiert.
|
||||
- [ ] NEW / CHANGED / UNCHANGED / MISSING / DISAPPEARED funktionieren.
|
||||
- [ ] DISAPPEARED erst nach drei aufeinanderfolgenden fehlenden Full Runs.
|
||||
- [ ] Unsichere Produktidentität wird nicht automatisch verschmolzen.
|
||||
- [ ] interne Engine bewertet Kandidaten.
|
||||
- [ ] Shortlist Default 5 funktioniert.
|
||||
- [ ] Suchfrequenz ist konfigurierbar, Default 180 Minuten.
|
||||
- [ ] 403/429/Captcha führen zu defensivem Stop/Backoff.
|
||||
- [ ] keine HTML-Archive.
|
||||
- [ ] keine Notebook-spezifische Businesslogik außerhalb des Geizhals-Pilotmappings.
|
||||
- [ ] Tests grün.
|
||||
- [ ] Ergebnisse des Notebook-Piloten dokumentiert.
|
||||
|
||||
# Coding-Agent-Arbeitsauftrag
|
||||
|
||||
Verwende als verbindliche Anforderungen:
|
||||
|
||||
1. `buying-agent-v0.1-spec-v1.1.md`
|
||||
2. `buying-agent-architecture-addendum-v1.md`
|
||||
3. `buying-agent-source-discovery-spec-v0.1.md`
|
||||
|
||||
Die neue Source-/Discovery-Spec ergänzt die bestehenden Dokumente. Bei Konflikten in ihrem Fachbereich hat sie Vorrang.
|
||||
|
||||
## Vorgehen
|
||||
|
||||
### 1. Zuerst analysieren
|
||||
|
||||
Prüfe den aktuellen Repository-Stand gegen alle drei Dokumente.
|
||||
|
||||
Berichte vor größeren Änderungen:
|
||||
|
||||
- welche Teile bereits kompatibel sind,
|
||||
- welche Domain-/DB-/API-Erweiterungen erforderlich sind,
|
||||
- ob bestehende Implementierung refaktoriert werden muss,
|
||||
- welche Änderungen rein additiv möglich sind.
|
||||
|
||||
### 2. Minimal integrieren
|
||||
|
||||
Implementiere nur den beschriebenen Source-/Discovery-Slice.
|
||||
|
||||
Nicht implementieren:
|
||||
|
||||
- Frontend
|
||||
- Keepa
|
||||
- Idealo
|
||||
- LLM-Automation
|
||||
- Auth/RBAC
|
||||
- Kaufautomatisierung
|
||||
- Warenkorb
|
||||
|
||||
### 3. Besonders beachten
|
||||
|
||||
- nicht notebook-spezifisch modellieren
|
||||
- Geizhals-spezifische Logik ausschließlich im Source Adapter / Mapping Layer
|
||||
- URL-Encoding nur aus verifizierten Source-Mappings
|
||||
- vollständige SearchRuns, kein stilles Top-N-Limit
|
||||
- konservatives Product Matching
|
||||
- keine automatische Zusammenführung bei Unsicherheit
|
||||
- alle Änderungen historisieren
|
||||
- Discovery, Ranking und Decision getrennt halten
|
||||
|
||||
### 4. Tests
|
||||
|
||||
Führe Unit-, Integration- und End-to-End-Tests aus.
|
||||
|
||||
Mindestens:
|
||||
|
||||
- Taxonomy Discovery
|
||||
- Query Generation
|
||||
- Pagination
|
||||
- Identity Matching
|
||||
- Diffing
|
||||
- History
|
||||
- Ranking
|
||||
- Shortlist
|
||||
- Backoff / BLOCKED
|
||||
|
||||
### 5. Abschlussbericht
|
||||
|
||||
Dokumentiere:
|
||||
|
||||
- geänderte Dateien
|
||||
- neue Migrationen
|
||||
- neue API-Endpunkte
|
||||
- Testergebnisse
|
||||
- real durchgeführten Notebook-Pilot
|
||||
- erkannte Geizhals-Eigenheiten
|
||||
- verbleibende Risiken
|
||||
- insbesondere Beobachtungen zur Stabilität der Produktidentität
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,671 @@
|
||||
# Buying Agent v0.1 – Implementierungsspezifikation
|
||||
|
||||
Status: implementierungsbereit · Spezifikationsversion: 1.1 · Stand: 2026-10-03
|
||||
|
||||
## Änderungen gegenüber Spezifikationsversion 1.0
|
||||
|
||||
- Preference-Scoring erhält ein explizites `strategy`-Feld; v0.1 implementiert ausschließlich `numeric_linear`, ohne vorsorgliche Plugin-Infrastruktur.
|
||||
- Produkt-Eignung und Angebots-Eignung werden getrennt als `product_eligibility` und `offer_eligibility` bewertet; `overall_eligibility` wird daraus abgeleitet.
|
||||
- Die Preisbewertung wird konzeptionell von der Identität eines Offers entkoppelt: v0.1 nutzt weiterhin nur die Historie des konkreten Offers, spätere Scoring-Versionen dürfen jedoch Variant- oder Market-Level-Referenzserien konsumieren.
|
||||
- `raw_value` wird bei JSONB als inhaltlich unverändert, nicht als bytegenau identisch beschrieben.
|
||||
- Ein verbindlicher `Future Extension Contract` dokumentiert, welche späteren Erweiterungen v0.1 nicht verbauen darf, ohne sie bereits zu implementieren.
|
||||
|
||||
## 1. Auftrag und Zielbild
|
||||
|
||||
Implementiere ein eigenständig lauffähiges Backend für einen generischen Buying-/Deal-Agenten. Das System bewertet, wie gut eine konkrete Produktvariante zu einem Bedarfsprofil passt und wie attraktiv ein bestimmtes Angebot preislich ist. Es liefert nachvollziehbare Entscheidungen, keine automatischen Käufe.
|
||||
|
||||
Notebooks sind der erste Anwendungsfall, nicht das Datenmodell. Weitere Kategorien wie GPU-Server, Monitore oder Kameras müssen durch neue Attribute und Profile ohne Änderungen an der Engine oder zusätzliche produktspezifische DB-Spalten möglich sein.
|
||||
|
||||
Diese Datei ist die vollständige Arbeitsgrundlage. Ein Zugriff auf den ursprünglichen Chat ist nicht erforderlich. Die dort beschlossene Architektur wird beibehalten. Die hier definierten Formeln, Fehlerfälle, API-Verträge und Zeitregeln konkretisieren bislang offene Punkte als verbindliche v0.1-Entscheidungen. Die Beispielwerte sind synthetische Testdaten und keine Markt- oder Herstellerangaben.
|
||||
|
||||
### Walking Skeleton
|
||||
|
||||
Der MVP muss diesen vollständigen Ablauf tatsächlich ausführen und dauerhaft speichern:
|
||||
|
||||
```text
|
||||
Profil → Produkt → Variante + Attribute → Angebot → Preisbeobachtungen
|
||||
→ Constraints → ProductScore → PriceScore → DealScore
|
||||
→ Klassifikation + Begründungen → gespeicherte Evaluation → API-Antwort
|
||||
```
|
||||
|
||||
Ein manuell erfasster Kandidat genügt für den ersten vertikalen Durchstich. Danach werden Grenzfälle und drei Beispielkandidaten ergänzt. Keine bloßen Interfaces ohne funktionsfähigen End-to-End-Pfad.
|
||||
|
||||
### Explizit nicht enthalten
|
||||
|
||||
- Keepa, Idealo, Geizhals oder andere externe Datenquellen und Scraper.
|
||||
- n8n, Scheduler, Benachrichtigungen und Hintergrundjobs.
|
||||
- LLM-Normalisierung oder LLM-Bewertung.
|
||||
- Web-UI; generierte OpenAPI-/Swagger-Dokumentation ist erlaubt.
|
||||
- Auto-Discovery, automatische Produktsuche oder automatische Zuordnung fremder Produkte.
|
||||
- Checkout, automatische Käufe, Nutzerkonten, Mehrmandantenfähigkeit und öffentliche Bereitstellung.
|
||||
- Wechselkursumrechnung, marktweite Preisprognosen, ML und Optimierung von Gewichten.
|
||||
|
||||
Diese Funktionen dürfen später über Adapter und Services ergänzt werden. Keine vorsorglichen Frameworks, leeren Integrationen oder Infrastruktur dafür bauen.
|
||||
|
||||
## 2. Stack und Architekturgrenzen
|
||||
|
||||
Verbindlich: Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2, Alembic, PostgreSQL 17, pytest und Docker Compose. Verwende einen synchronen SQLAlchemy-Zugriff mit psycopg 3; ein asynchroner Stack ist für v0.1 nicht erforderlich. Abhängigkeiten mit kompatiblen Versionen und einem eingecheckten Lockfile reproduzierbar festlegen.
|
||||
|
||||
Abhängigkeiten zeigen nach innen:
|
||||
|
||||
```text
|
||||
HTTP/API → Application Services → Domain / Constraint Engine / Scoring
|
||||
↓
|
||||
Repositories und Source-Adapter
|
||||
↓
|
||||
PostgreSQL
|
||||
```
|
||||
|
||||
- `domain` und `scoring` importieren weder FastAPI noch SQLAlchemy oder konkrete Quellen.
|
||||
- Die Engine ist eine reine Berechnung über explizite Eingaben. Kein Netzwerkzugriff, kein DB-Zugriff und kein implizites Lesen der Systemzeit.
|
||||
- Services laden Daten, erzeugen einen konsistenten Snapshot, rufen die Engine auf und speichern das Ergebnis atomar.
|
||||
- API-Schemas und ORM-Modelle sind getrennt von Domain-Modellen. Kleine Dataclasses oder Pydantic-Modelle sind für die Domain zulässig.
|
||||
- Regeln liegen als validierte Daten vor; keine Ausführung von Python, SQL, `eval` oder Templates aus Profilen.
|
||||
- n8n bleibt später ein externer Aufrufer. Die gesamte Buying Engine funktioniert ohne n8n.
|
||||
|
||||
## 3. Fachliche Entitäten und Identität
|
||||
|
||||
### Product
|
||||
|
||||
Produktfamilie bzw. Modell, beispielsweise „Example Mobile 13“. Felder: UUID `id`, `category`, `brand`, `name`, optional `model`, `created_at`. Keine RAM-, SSD- oder Preisfelder. Kategorie ist ein normalisierter String, kein Notebook-spezifisches Enum.
|
||||
|
||||
### ProductVariant
|
||||
|
||||
Eine eindeutig beschriebene technische Konfiguration eines Products. Felder: UUID `id`, `product_id`, `name`, optionale Hersteller-SKU `manufacturer_sku`, optionale Kennungen `identifiers` als JSON-Objekt, `created_at`.
|
||||
|
||||
RAM, Speicher, Farbe, Tastaturlayout und Ausstattung gehören zur Variante. Angebote unterschiedlicher Konfigurationen dürfen nicht zusammengeführt werden. In v0.1 verknüpft der Aufrufer Varianten explizit; keine heuristische Identitätsauflösung. Hersteller-SKUs sind nicht global eindeutig. Identische Namen sind kein Dublettenbeweis.
|
||||
|
||||
### Offer
|
||||
|
||||
Ein konkretes Händlerangebot für genau eine Variante. Felder: UUID `id`, `variant_id`, `source_id`, `external_id`, `seller`, optionale `url`, `market` (z. B. `DE`), `currency` (dreistelliger Großbuchstaben-Code), `condition` (`new`, `refurbished`, `used`), `created_at`.
|
||||
|
||||
Eindeutigkeit: `(source_id, external_id)`. Unterschiedliche Verkäufer, Zustände, Märkte oder Lieferumfänge sind unterschiedliche Angebote. Technisch abweichende Bundles erfordern eine passende Variante. Diese Identitätsfelder bleiben nach Erstellung unverändert. Ein Offer enthält keinen unabhängig veränderbaren „aktuellen Preis“.
|
||||
|
||||
### PriceObservation
|
||||
|
||||
Append-only-Beobachtung eines Offers zu einem Zeitpunkt. Felder: UUID `id`, `offer_id`, `source_id`, `external_observation_id`, `observed_at`, `ingested_at`, `item_price`, nullable `shipping_price`, `currency`, `availability` (`in_stock`, `out_of_stock`, `unknown`), `raw_payload` als JSON, `confidence`.
|
||||
|
||||
- Geld als `Decimal`/PostgreSQL `NUMERIC(12,2)`, nie binäre Floats. API-Geldwerte werden als Dezimalstrings übertragen.
|
||||
- `item_price > 0`; bekannte Versandkosten `>= 0`; null bedeutet unbekannt, nicht kostenlos.
|
||||
- Gesamtpreis = Bruttopreis inklusive bekannter obligatorischer Bestandteile + Versand. Nur direkt für jeden Käufer verfügbare Preise erfassen; keine Mitgliederrabatte, Finanzierung oder personalisierte Coupons modellieren.
|
||||
- Die Währung muss der Offer-Währung entsprechen. `source_id` muss der Offer-Quelle entsprechen.
|
||||
- Zeitzonen sind obligatorisch; Speicherung und Ausgabe in UTC. Zeitpunkte in der Zukunft relativ zur Ingestion werden abgewiesen.
|
||||
- Eindeutigkeit `(offer_id, external_observation_id)`: identischer Wiederholungsimport liefert den vorhandenen Datensatz, widersprüchlicher Inhalt HTTP 409. `ingested_at` ist serverseitig und zählt beim Inhaltsvergleich nicht mit.
|
||||
- Beobachtungen werden nicht überschrieben. Korrekturen werden als neue Beobachtung mit neuer externer ID erfasst; ein administrativer Korrekturworkflow ist nicht Bestandteil von v0.1.
|
||||
|
||||
## 4. Generische Attribute und Normalisierung
|
||||
|
||||
Attribute werden für v0.1 ausschließlich an Varianten erfasst. Es gibt keine implizite Vererbung von Product-Attributen. Eine spätere Erweiterung muss eine eigene, explizite Vorrangregel definieren.
|
||||
|
||||
### Attributdefinition
|
||||
|
||||
`attribute_definitions` enthält `key`, `value_type` (`number`, `string`, `boolean`, `string_list`), nullable `canonical_unit`, optionale `minimum`/`maximum` und `description`. Schlüssel sind stabile `snake_case`-Bezeichner. Neue Definitionen lassen sich per API ohne Migration anlegen. Bestehende Definitionen sind in v0.1 unveränderlich; geänderte Semantik erhält einen neuen Schlüssel.
|
||||
|
||||
Beispieldefinitionen:
|
||||
|
||||
| key | type | canonical_unit | Wertebereich |
|
||||
|---|---|---|---|
|
||||
| ram_gb | number | GB | >= 0 |
|
||||
| storage_gb | number | GB | >= 0 |
|
||||
| weight_kg | number | kg | > 0, im Eingabevalidator |
|
||||
| battery_score | number | null | 0–100 |
|
||||
| display_score | number | null | 0–100 |
|
||||
| cpu_score | number | null | 0–100 |
|
||||
| ports | string_list | null | beliebige Strings |
|
||||
|
||||
`*_score`-Attribute sind ausdrücklich manuelle Einschätzungen auf einer dokumentierten 0–100-Skala; die Engine erfindet keine Benchmarks oder Messwerte.
|
||||
|
||||
### Attributwert
|
||||
|
||||
Jede Zeile in `variant_attributes` ist eine unveränderliche Aussage einer Quelle:
|
||||
|
||||
```json
|
||||
{
|
||||
"attribute": "ram_gb",
|
||||
"raw_value": "24GB LPDDR5X RAM",
|
||||
"normalized_value": 24,
|
||||
"unit": "GB",
|
||||
"normalization_status": "normalized",
|
||||
"normalizer_version": "manual-v1",
|
||||
"confidence": 1.0,
|
||||
"source_id": "manual",
|
||||
"source_reference": "demo/notebook-a/ram",
|
||||
"observed_at": "2026-10-03T10:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
Zusätzlich serverseitig: UUID `id`, `variant_id`, `ingested_at`. `raw_value` ist verpflichtend und bleibt inhaltlich unverändert erhalten; es darf ein JSON-Skalar oder -Objekt sein. Bei Speicherung als JSONB ist keine bytegenaue Erhaltung der ursprünglichen JSON-Serialisierung garantiert. Falls eine spätere Quelle den exakten Response-Body benötigt, muss dieser separat als Rohtext gespeichert werden; das ist nicht Bestandteil von v0.1. `normalized_value` ist ein typisierter JSON-Wert. Bei `normalization_status=unknown` ist er null; ein optionales `normalization_error` erklärt den fehlenden Wert. Eine erfolglose Normalisierung ist kein Zahlenwert 0.
|
||||
|
||||
Manuelle Erfassung liefert Raw- und Normalized-Wert explizit. `ManualNormalizer` validiert Typ, Bereich und kanonische Einheit, analysiert aber keinen Freitext. Falsche Typen oder Einheiten werden mit 422 abgewiesen. Keine automatische Umwandlung von `"32"` zu 32 oder `true` zu 1. `confidence` liegt zwischen 0 und 1 und bezeichnet die Zuverlässigkeit der Aussage, nicht den Produktnutzen. Eine absichtlich unbekannte Aussage ist zulässig.
|
||||
|
||||
Bei mehreren Aussagen zu demselben Schlüssel verwendet der Snapshot die jüngste nach `(observed_at, ingested_at, id)` absteigend, jeweils nur bis zum Auswertungszeitpunkt bzw. Snapshot-Cutoff. Anschließend wird deren Confidence geprüft. Ein neuer unsicherer oder unbekannter Wert darf nicht stillschweigend durch einen älteren günstigen Wert ersetzt werden. Vorrang verschiedener Quellen ist damit zunächst bewusst einfach; spätere Auswahlstrategien benötigen eine neue Regelversion.
|
||||
|
||||
## 5. Product Profile und Versionierung
|
||||
|
||||
Ein Profil beschreibt Kategorie, Budget, harte Mindestanforderungen und gewichtete Präferenzen. Es besitzt eine stabile String-ID und eine positive Integer-Version. Jede `(id, version)` ist unveränderlich. Eine Änderung wird als neue Version gespeichert. Evaluationen müssen eine konkrete Version angeben; kein implizites „latest“.
|
||||
|
||||
Vollständiges Beispiel für `profiles/mobile-development-notebook.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "mobile-development-notebook",
|
||||
"version": 1,
|
||||
"name": "Mobiler Entwicklungsrechner",
|
||||
"category": "notebook",
|
||||
"currency": "EUR",
|
||||
"market": "DE",
|
||||
"allowed_conditions": ["new"],
|
||||
"budget": {"target": "1500.00", "maximum": "2000.00"},
|
||||
"scoring_version": "generic-v1",
|
||||
"constraints": [
|
||||
{"id": "min_ram", "attribute": "ram_gb", "operator": ">=", "value": 16, "mandatory": true},
|
||||
{"id": "min_storage", "attribute": "storage_gb", "operator": ">=", "value": 512, "mandatory": true},
|
||||
{"id": "max_weight", "attribute": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": true}
|
||||
],
|
||||
"preferences": [
|
||||
{"id": "ram", "attribute": "ram_gb", "strategy": "numeric_linear", "direction": "higher", "acceptable": 16, "ideal": 32, "weight": 0.30},
|
||||
{"id": "weight", "attribute": "weight_kg", "strategy": "numeric_linear", "direction": "lower", "acceptable": 1.6, "ideal": 1.0, "weight": 0.25},
|
||||
{"id": "battery", "attribute": "battery_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.20},
|
||||
{"id": "display", "attribute": "display_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.15},
|
||||
{"id": "cpu", "attribute": "cpu_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.10}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Jede Präferenz besitzt eine explizite `strategy`. v0.1 unterstützt ausschließlich `numeric_linear`. Das Profilformat ist jedoch so zu modellieren, dass spätere Scoring-Versionen zusätzliche klar benannte Strategien wie `boolean_preferred`, `categorical_match`, `set_contains` oder `categorical_rank` einführen können, ohne die Grundstruktur eines Profils zu ändern. Für v0.1 wird dafür weder ein Plugin-System noch dynamische Code-Ausführung gebaut; unbekannte Strategien werden bei der Profilvalidierung abgewiesen.
|
||||
|
||||
Validierung: `0 < target < maximum`, mindestens eine Präferenz, positive Gewichte, innerhalb der jeweiligen Liste eindeutige IDs, bekannte Attributschlüssel und eine vorhandene Scoring-Version. Die Summe der Gewichte muss nicht 1 sein, weil die Engine normalisiert. Für `numeric_linear` gilt: `higher` verlangt `ideal > acceptable`, `lower` verlangt `ideal < acceptable`, und das referenzierte Attribut muss numerisch sein. Kategorie, Währung, Markt und erlaubte Zustände sind verpflichtend.
|
||||
|
||||
Das Profil ist ein Beispiel aus der Architekturentscheidung, keine abschließend optimierte persönliche Kaufberatung. Die Auswahl eines Notebooks für mobile Entwicklung wird nicht mit Anforderungen an einen lokalen LLM-Server vermischt.
|
||||
|
||||
## 6. Constraint Engine und Eignungsstatus
|
||||
|
||||
Unterstütze ausschließlich folgende Operatoren; verwerfe ungültige Kombinationen bereits beim Anlegen des Profils:
|
||||
|
||||
| Operator | Attributtyp | Vergleichswert / Semantik |
|
||||
|---|---|---|
|
||||
| `>=`, `<=`, `>`, `<` | number | Zahl in kanonischer Einheit |
|
||||
| `==`, `!=` | number, string, boolean | gleicher skalarer Typ, exakter Vergleich |
|
||||
| `in` | number, string, boolean | nichtleere Liste gleichartiger skalarer Werte |
|
||||
| `contains` | string_list | einzelner String, exakte Mitgliedschaft |
|
||||
| `exists` | alle | kein `value`; verwertbarer normalisierter Wert vorhanden |
|
||||
|
||||
Keine impliziten Typkonvertierungen, String-Teilstrings oder case-insensitive Vergleiche. Leere String-Listen sind vorhandene Werte. Fehlend, null, nicht normalisiert oder Confidence unter der Regelgrenze ergibt `UNKNOWN`, auch bei `exists` und `!=`.
|
||||
|
||||
Jedes Ergebnis enthält Constraint-ID, Attribut, Operator, Sollwert, Istwert, Status `PASS|FAIL|UNKNOWN`, Mandatory-Flag, gewählte Attributzeilen-ID und Reason-Code.
|
||||
|
||||
### Product Eligibility
|
||||
|
||||
`product_eligibility` beschreibt ausschließlich, ob die konkrete Produktvariante fachlich zum Profil passt. Dazu gehören:
|
||||
|
||||
- Kategorie des Produkts/der Variante,
|
||||
- alle mandatory Produkt-Constraints,
|
||||
- technische bzw. produktbezogene Daten, die für diese Constraints benötigt werden.
|
||||
|
||||
Ableitung:
|
||||
|
||||
- mindestens ein bekannter produktbezogener harter `FAIL` → `product_eligibility=INELIGIBLE`;
|
||||
- sonst mindestens ein mandatory produktbezogener `UNKNOWN` → `product_eligibility=UNKNOWN`;
|
||||
- sonst → `product_eligibility=ELIGIBLE`.
|
||||
|
||||
Optionale Constraints ändern `product_eligibility` nicht. Optionales `FAIL` oder `UNKNOWN` reduziert ausschließlich den ProductScore um die im Scoring-Modell konfigurierte Punktzahl und bleibt in den Ergebnissen sichtbar.
|
||||
|
||||
Der ProductScore darf unabhängig von der aktuellen Angebotslage berechnet und als Diagnostik gespeichert werden, sofern das Profil und die vorhandenen Präferenzdaten dies zulassen. Ein technisch sehr gutes Produkt verliert seinen ProductScore nicht nur deshalb, weil das betrachtete Offer aktuell zu teuer oder nicht lieferbar ist.
|
||||
|
||||
### Offer Eligibility
|
||||
|
||||
`offer_eligibility` beschreibt ausschließlich die Eignung des konkret ausgewerteten Offers. Systemseitige Offer-Gates sind:
|
||||
|
||||
- Markt entspricht dem Profil,
|
||||
- Zustand ist in `allowed_conditions`,
|
||||
- Währung entspricht der Profilwährung,
|
||||
- aktuelle Preisbeobachtung ist vorhanden und frisch,
|
||||
- Preis-Confidence erfüllt die Mindestschwelle,
|
||||
- obligatorische Versandkosten sind bekannt,
|
||||
- Gesamtpreis <= Budgetmaximum,
|
||||
- aktuelle Verfügbarkeit ist `in_stock`.
|
||||
|
||||
Ein bekannter Verstoß ergibt `offer_eligibility=INELIGIBLE`. Fehlende/zu alte Preisbeobachtung, geringe Preis-Confidence, unbekannte Versandkosten oder unbekannte Verfügbarkeit ergeben `offer_eligibility=UNKNOWN`. `out_of_stock` ergibt `INELIGIBLE`. Ein Preis oberhalb des Budgets ist ein Angebotsproblem und macht nicht das Produkt selbst ungeeignet.
|
||||
|
||||
### Overall Eligibility
|
||||
|
||||
`overall_eligibility` wird ausschließlich aus Product und Offer Eligibility abgeleitet:
|
||||
|
||||
- wenn mindestens einer der beiden Status `INELIGIBLE` ist → `overall_eligibility=INELIGIBLE`;
|
||||
- sonst wenn mindestens einer `UNKNOWN` ist → `overall_eligibility=UNKNOWN`;
|
||||
- sonst → `overall_eligibility=ELIGIBLE`.
|
||||
|
||||
Alle Gates werden ausgewertet, damit die Antwort sämtliche Gründe enthält. Ein bekannter harter Fehler hat innerhalb der jeweiligen Ebene Vorrang vor unbekannten Daten. Das API-Feld `eligible` ist eine Convenience-Abbildung von `overall_eligibility`: `ELIGIBLE` → true, `INELIGIBLE` → false, `UNKNOWN` → null.
|
||||
|
||||
## 7. Scoring-Modell `generic-v1`
|
||||
|
||||
Alle Scores liegen zwischen 0 und 100. Rechne intern mit Decimal, runde nur endgültige Scores und angezeigte Komponenten auf zwei Dezimalstellen, `ROUND_HALF_UP`. Klassifikation anhand des ungerundeten DealScore. Der Snapshot enthält auch ungerundete Dezimalstrings für eine eindeutige Grenzfallprüfung.
|
||||
|
||||
Die unveränderliche Modelldefinition enthält mindestens:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "generic-v1",
|
||||
"algorithm_id": "linear-budget-history-v1",
|
||||
"supported_preference_strategies": ["numeric_linear"],
|
||||
"minimum_confidence": 0.8,
|
||||
"maximum_price_age_hours": 48,
|
||||
"optional_constraint_penalty_points": 5,
|
||||
"missing_preference_score": 0,
|
||||
"product_weight": 0.6,
|
||||
"price_weight": 0.4,
|
||||
"budget_weight": 0.4,
|
||||
"history_weight": 0.6,
|
||||
"history_window_days": 90,
|
||||
"minimum_history_days": 3,
|
||||
"history_discount_for_max_score": 0.2,
|
||||
"classification_upper_bounds": [
|
||||
{"upper": 60, "inclusive": false, "label": "IGNORE"},
|
||||
{"upper": 75, "inclusive": false, "label": "WATCH"},
|
||||
{"upper": 85, "inclusive": false, "label": "GOOD"},
|
||||
{"upper": 92, "inclusive": true, "label": "VERY_GOOD"},
|
||||
{"upper": 100, "inclusive": true, "label": "EXCEPTIONAL"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`clamp(x)=min(100,max(0,x))`. Modellvalidierung prüft positive Zeitfenster, gültige Confidence, nichtnegative Gewichte mit jeweiliger Summe 1, aufsteigende lückenlose Klassifikationsgrenzen, unterstützte Algorithmus-ID und eine nichtleere Liste unterstützter Preference-Strategien. Die Definition wird als kanonisches JSON inklusive SHA-256 gespeichert. Eine Version darf niemals mit geändertem Inhalt wiederverwendet werden. Neue Algorithmen oder Preference-Strategien werden als Code ergänzt und unter einer neuen Scoring-Version bzw. Algorithmus-ID registriert; JSON darf keinen ausführbaren Code enthalten.
|
||||
|
||||
### ProductScore
|
||||
|
||||
v0.1 implementiert ausschließlich die Preference-Strategie `numeric_linear`. Für eine solche numerische Präferenz mit Istwert `x`, akzeptablem Wert `a` und Idealwert `i`:
|
||||
|
||||
```text
|
||||
higher: component = clamp(50 + 50 × (x − a) / (i − a))
|
||||
lower: component = clamp(50 + 50 × (a − x) / (a − i))
|
||||
base_product_score = Σ(weight × component) / Σ(weight)
|
||||
product_score = clamp(base_product_score − 5 × Anzahl optionaler FAIL/UNKNOWN)
|
||||
```
|
||||
|
||||
Damit erhält ein akzeptabler Wert 50, ein idealer oder besserer Wert maximal 100. Ein fehlender oder unsicherer Präferenzwert erhält 0; sein Gewicht bleibt im Nenner. Fehlende Daten dürfen den Score nicht verbessern. `product_coverage` ist die Summe der Gewichte verwertbarer Präferenzen geteilt durch die Summe aller Gewichte. Confidence wird nicht nochmals als Score-Multiplikator verwendet.
|
||||
|
||||
Der ProductScore beschreibt Produkt-Fit und wird unabhängig von `offer_eligibility` berechnet. Auch bei `product_eligibility=INELIGIBLE` oder `UNKNOWN` darf er diagnostisch ausgegeben werden; Mandatory Gates bleiben jedoch autoritativ und können niemals durch einen hohen ProductScore überstimmt werden.
|
||||
|
||||
Die Strategy-Auswahl erfolgt explizit über das Preference-Feld `strategy`; es gibt keine dynamische Plugin-Suche. Unbekannte Strategien sind für `generic-v1` ein Validierungsfehler.
|
||||
|
||||
### Auswahl des aktuellen Preises und der Referenzhistorie
|
||||
|
||||
Der Evaluation-Request bezeichnet genau ein Offer. Kein implizites billigstes Angebot und kein Vermischen mehrerer Händler oder Varianten.
|
||||
|
||||
v0.1 verwendet als `reference_history` ausschließlich Preisbeobachtungen dieses konkreten Offers. Diese Bindung ist eine Entscheidung der Scoring-Version, nicht des generischen Preis-Scoring-Vertrags. Die reine Preis-Scoring-Funktion ist daher konzeptionell als Berechnung über `current_price`, `reference_history` und `budget` zu implementieren; sie darf fachlich nicht verlangen, dass die Referenzserie dauerhaft ein Offer-Objekt ist. Spätere Scoring-Versionen dürfen Variant-Level- oder Market-Level-Referenzserien bereitstellen, ohne Product, ProductVariant, Offer oder PriceObservation neu zu modellieren.
|
||||
|
||||
Für v0.1 gilt für die offer-bezogene Referenzserie:
|
||||
|
||||
1. Betrachte ausschließlich Beobachtungen dieses Offers mit `observed_at <= as_of` und `ingested_at <= snapshot_cutoff`.
|
||||
2. Aktuell ist die jüngste Zeile nach `(observed_at, ingested_at, id)` absteigend. Auch eine nicht verfügbare oder unsichere jüngste Zeile darf nicht durch eine ältere günstigere ersetzt werden.
|
||||
3. Frische: `as_of − observed_at <= 48 Stunden`, einschließlich Grenze.
|
||||
4. Historie endet strikt vor dem UTC-Kalendertag der aktuellen Beobachtung. Dadurch wird der aktuelle Angebotspreis nicht selbst zur historischen Referenz.
|
||||
5. Für jedes Fenster N ∈ {30,90,365} gilt `[current_day − N Tage, current_day)`. Wähle zunächst pro UTC-Tag die jüngste Zeile, danach filtere Tage mit `in_stock`, bekannter Gesamtsumme und Confidence >= 0.8. So verdrängt eine spätere unbekannte Beobachtung eine frühere bekannte desselben Tages.
|
||||
6. Berechne `median_30d`, `median_90d`, `lowest_365d` aus diesen Tageswerten. Bei gerader Anzahl ist der Median das arithmetische Mittel der beiden mittleren Werte. Gib die Zahl der vorhandenen Tage pro Fenster mit aus; keine Interpolation fehlender Tage.
|
||||
|
||||
Historienaggregate werden berechnet und im Evaluation-Snapshot gespeichert, nicht als vom Client beliebig vorgegebene Wahrheit übernommen. Die v0.1-Historie ist angebotsbezogen und keine Aussage über den Gesamtmarkt. `lowest_365d` darf als Minimum weniger beobachteter Tage angezeigt werden, muss dann ausdrücklich als Minimum der vorhandenen Daten bezeichnet sein. Der Snapshot speichert zusätzlich `price_reference_scope="offer"` und die IDs der verwendeten Beobachtungen, damit spätere Scoring-Versionen andere Scopes eindeutig unterscheiden können.
|
||||
|
||||
### PriceScore
|
||||
|
||||
Seien `p` aktueller Gesamtpreis, `t` Budgetziel, `m` Budgetmaximum, `r` 90-Tage-Median der von der Scoring-Version bereitgestellten Referenzhistorie:
|
||||
|
||||
```text
|
||||
budget_score = 100 falls p <= t
|
||||
budget_score = clamp(100 × (m − p) / (m − t)) sonst
|
||||
discount = (r − p) / r
|
||||
history_score = clamp(50 + 50 × discount / 0.20)
|
||||
price_score = 0.40 × budget_score + 0.60 × history_score
|
||||
```
|
||||
|
||||
Historien-Score: beim Median 50, bei 20 % günstiger 100, bei 20 % teurer 0. Nur bei mindestens drei verfügbaren Tagen im 90-Tage-Fenster ist `history_score` vorhanden. Andernfalls gilt `price_score=budget_score`, `price_basis=BUDGET_ONLY`, `history_score=null` und ein expliziter Hinweis `INSUFFICIENT_HISTORY`. Bei ausreichender Historie `price_basis=BUDGET_AND_HISTORY`. Kein Bonus für ein vermeintliches Jahrestief; 30-Tage-Median und Minimum dienen zunächst nur der Erklärung.
|
||||
|
||||
Ohne frischen, hinreichend sicheren und vollständig bekannten Gesamtpreis in Profilwährung ist der PriceScore null. Ein bekannter Preis oberhalb des Budgets kann diagnostisch bewertet werden, während `offer_eligibility` wegen des Budget-Gates `INELIGIBLE` bleibt. Bei falscher Währung wird ohne Wechselkursumrechnung kein PriceScore berechnet. Ausverkaufte Angebote dürfen ihren bekannten Preis diagnostisch behalten; `offer_eligibility` bleibt dennoch `INELIGIBLE`.
|
||||
|
||||
### DealScore und Klassifikation
|
||||
|
||||
Nur bei `overall_eligibility=ELIGIBLE` und vorhandenem PriceScore:
|
||||
|
||||
```text
|
||||
deal_score = 0.60 × product_score + 0.40 × price_score
|
||||
```
|
||||
|
||||
| Bereich des ungerundeten DealScore | Klassifikation |
|
||||
|---|---|
|
||||
| < 60 | IGNORE |
|
||||
| >= 60 und < 75 | WATCH |
|
||||
| >= 75 und < 85 | GOOD |
|
||||
| >= 85 und <= 92 | VERY_GOOD |
|
||||
| > 92 | EXCEPTIONAL |
|
||||
|
||||
Bei `overall_eligibility=INELIGIBLE`: DealScore null, Klassifikation `INELIGIBLE`. Bei `overall_eligibility=UNKNOWN`: DealScore null, Klassifikation `INSUFFICIENT_DATA`. ProductScore und vorhandener PriceScore dürfen als Diagnostik ausgegeben werden, ersetzen aber niemals die Gate-Ergebnisse. Die getrennten Product-/Offer-Status müssen in Snapshot und API sichtbar bleiben, damit ein gutes Produkt mit momentan schlechtem Angebot weiterhin beobachtbar bleibt.
|
||||
|
||||
### Nachvollziehbarkeit und Wiederholbarkeit
|
||||
|
||||
Jede Evaluation speichert Profil und Version, Modellversion und Hash, Engine-Build-ID, `as_of`, `snapshot_cutoff`, `product_eligibility`, `offer_eligibility`, `overall_eligibility`, vollständige tatsächlich verwendete Eingaben und IDs sowie alle Komponenten, Gates, Aggregate, Coverage, Price-Reference-Scope, Reason-Codes und verständliche Texte. Persistierte Evaluationen sind unveränderlich. Ein erneuter Request erzeugt eine neue Evaluation; ein GET berechnet nichts neu.
|
||||
|
||||
Historische Abfragen mit `as_of` rekonstruieren nur den Datenstand zum aktuellen Snapshot-Cutoff, einschließlich später eingegangener Rückdatierungen. Sie behaupten nicht, den damaligen Wissensstand wiederherzustellen. Exakte Wiederholung einer alten Bewertung erfolgt intern aus deren gespeichertem Snapshot und zugehöriger Algorithmusversion. Alte Algorithmen bleiben verfügbar; kein Replay-HTTP-Endpunkt für v0.1 erforderlich.
|
||||
|
||||
## 8. Source-Adapter
|
||||
|
||||
Implementiere einen kleinen `Protocol` und `ManualSource`. Quell-DTOs enthalten Rohdaten, Quellenkennung, externe IDs, Zeitpunkte und Confidence; sie enthalten keine ORM-Objekte und keine Scoring-Logik.
|
||||
|
||||
```python
|
||||
class ProductSource(Protocol):
|
||||
source_key: str
|
||||
def get_product(self, external_id: str) -> SourceProduct: ...
|
||||
def get_offers(self, external_product_id: str) -> list[SourceOffer]: ...
|
||||
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]: ...
|
||||
```
|
||||
|
||||
`ManualSource` liest einen validierten lokalen Demo-Datensatz, keine externe Website. Ein Import-Service nimmt den Adapter entgegen und legt über dieselben Application Services wie die API Produkte, Varianten, Attribute, Angebote und Beobachtungen an. Explizite Variantenzuordnung aus der Fixture; keine Matching-Engine. Wiederholter Demo-Import erzeugt keine Duplikate: stabile UUIDs für Produkte/Varianten/Attribute, stabile externe Offer-/Beobachtungs-IDs und identischer Inhalt; Widersprüche melden statt überschreiben.
|
||||
|
||||
Adapter normalisieren keine Preise oder technischen Eigenschaften im Scoring-Modul. Der Pfad lautet Quelle → validierte DTOs → Normalisierung → Persistenz → Evaluation. Für spätere Quellen genügen zunächst Adapter und Import-Mapping. Pagination, Rate Limits und Retry-Strategien erst zusammen mit der ersten realen Quelle einführen.
|
||||
|
||||
## 9. HTTP-API
|
||||
|
||||
Alle fachlichen Endpunkte unter `/api/v1`. UUIDs für Entitäten, String-IDs für Profile, Attribute und Scoring-Modelle. Pydantic-Schemas mit `extra=forbid`; Pflichtfelder und Enums im OpenAPI-Schema dokumentieren. JSON-Ausgabe nutzt `snake_case`.
|
||||
|
||||
| Methode / Pfad | Vertrag |
|
||||
|---|---|
|
||||
| GET `/health/live` | Prozess lebt, kein DB-Zugriff |
|
||||
| GET `/health/ready` | DB erreichbar und Migration auf Head; sonst 503 |
|
||||
| POST `/api/v1/attribute-definitions` | unveränderliche Definition anlegen |
|
||||
| GET `/api/v1/attribute-definitions` | Definitionen listen |
|
||||
| GET `/api/v1/sources` | registrierte Quellen, einschließlich `manual` |
|
||||
| GET `/api/v1/scoring-models/{version}` | vollständige Regeldefinition + Hash |
|
||||
| POST `/api/v1/profiles` | vollständiges Profil inklusive Version anlegen |
|
||||
| GET `/api/v1/profiles` | konkrete Profilversionen listen |
|
||||
| GET `/api/v1/profiles/{id}?version=1` | konkrete Version; Query verpflichtend |
|
||||
| POST `/api/v1/products` | Product anlegen |
|
||||
| GET `/api/v1/products/{id}` | Product lesen |
|
||||
| POST `/api/v1/products/{id}/variants` | Variante anlegen |
|
||||
| GET `/api/v1/variants/{id}` | Variante mit Attributaussagen lesen |
|
||||
| POST `/api/v1/variants/{id}/attributes` | einzelne unveränderliche Attributaussage hinzufügen |
|
||||
| POST `/api/v1/variants/{id}/offers` | Offer anlegen |
|
||||
| GET `/api/v1/offers/{id}` | Offer lesen |
|
||||
| POST `/api/v1/offers/{id}/prices` | einzelne Beobachtung hinzufügen bzw. identisch wiederholen |
|
||||
| GET `/api/v1/offers/{id}/prices` | gespeicherte Beobachtungen listen |
|
||||
| POST `/api/v1/evaluate/{variant_id}` | auswerten und Ergebnis atomar speichern |
|
||||
| GET `/api/v1/evaluations/{id}` | gespeichertes Ergebnis inklusive Snapshot lesen |
|
||||
|
||||
Listen verwenden `limit` (Default 50, maximal 200) und `offset` (>=0); Antwort `{items, limit, offset, total}`. Stabile Sortierung nach Primärschlüssel, bei Preisen `(observed_at, ingested_at, id)` aufsteigend. Keine Update-/Delete-Endpunkte in v0.1.
|
||||
|
||||
Evaluation-Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"profile_id": "mobile-development-notebook",
|
||||
"profile_version": 1,
|
||||
"offer_id": "33333333-3333-4333-8333-333333333333",
|
||||
"as_of": "2026-10-03T12:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
`as_of` ist optional; der Service setzt es einmal auf Serverzeit UTC. Zukunftswerte sind ungültig. Offer muss zur angegebenen Variante gehören (422). Das Profil legt die Scoring-Version fest; kein stiller Override. Fachlich unpassende Kategorie wird im Product-Gate gespeichert; Markt, Zustand, Währung, Preis und Verfügbarkeit werden im Offer-Gate gespeichert. Keiner dieser fachlichen Fälle ist ein HTTP-Fehler.
|
||||
|
||||
Evaluation-Response enthält mindestens:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "44444444-4444-4444-8444-444444444444",
|
||||
"profile_id": "mobile-development-notebook",
|
||||
"profile_version": 1,
|
||||
"scoring_version": "generic-v1",
|
||||
"product_eligibility": "ELIGIBLE",
|
||||
"offer_eligibility": "ELIGIBLE",
|
||||
"overall_eligibility": "ELIGIBLE",
|
||||
"eligible": true,
|
||||
"product_score": 90.21,
|
||||
"price_score": 98.24,
|
||||
"deal_score": 93.42,
|
||||
"classification": "EXCEPTIONAL",
|
||||
"price_basis": "BUDGET_AND_HISTORY",
|
||||
"product_coverage": 1.0,
|
||||
"failed_constraints": [],
|
||||
"unknown_constraints": [],
|
||||
"reasons": [
|
||||
{"code": "PRICE_BELOW_MEDIAN", "message": "Gesamtpreis 18,83 % unter dem Median der beobachteten 90-Tage-Historie."}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Zusätzlich verpflichtend: `variant_id`, `offer_id`, `as_of`, `snapshot_cutoff`, `scoring_hash`, `engine_build`, `product_eligibility`, `offer_eligibility`, `overall_eligibility`, `components`, sämtliche `constraint_results` und Offer-Gates, `price_statistics`, `price_reference_scope`, `input_snapshot` und `warnings`. Die Kurzantwort oben illustriert nur die Kernfelder. `failed_constraints` enthält alle `FAIL`-Ergebnisse; `unknown_constraints` alle `UNKNOWN`-Ergebnisse. Reason-Codes sind stabil, Texte dienen Menschen und dürfen nicht als Programmierschnittstelle ausgewertet werden.
|
||||
|
||||
POST liefert 201, identisch wiederholte Preisbeobachtung 200; GET liefert 200. Fehlende IDs: 404. Verletzte Eindeutigkeit/abweichender Wiederholungsinhalt: 409. Ungültige Payload/Referenzkombination: 422. Einheitliches Fehlerformat `{error: {code, message, details}}`, auch für Request-Validierung. Keine SQL-Details an Clients. Eine fachlich negative Evaluation ist eine erfolgreiche 201-Antwort.
|
||||
|
||||
## 10. Datenbank und Transaktionen
|
||||
|
||||
Alle Zeitstempel `TIMESTAMPTZ`; JSON-Daten `JSONB`; UUID-Primärschlüssel für Entitäten; Decimal-Geldfelder. `created_at`/`ingested_at` werden serverseitig gesetzt. Fremdschlüssel sind obligatorisch. Kein Cascade-Delete historischer Bewertungen oder Quelldaten.
|
||||
|
||||
| Tabelle | Wesentliche Spalten und Constraints |
|
||||
|---|---|
|
||||
| `attribute_definitions` | `key` PK, Typ, Einheit, Bereich, Beschreibung |
|
||||
| `sources` | `id` String-PK, `name`, `adapter_type`; Seed `manual` |
|
||||
| `scoring_models` | `version` PK, `algorithm_id`, `definition` JSONB, `definition_hash`, `created_at` |
|
||||
| `product_profiles` | zusammengesetzter PK `(id, version)`, `definition` JSONB, `category`, FK `scoring_version`, `definition_hash`, `created_at` |
|
||||
| `products` | `id`, Kategorie, Marke, Name, Modell, `created_at` |
|
||||
| `product_variants` | `id`, FK Product, Name, Hersteller-SKU, Kennungen JSONB, `created_at` |
|
||||
| `variant_attributes` | `id`, FK Variante/Definition/Quelle, Raw/Normalized JSONB, Einheit, Status, Normalizer-Version, Confidence, Source-Referenz, Beobachtungs-/Ingestion-Zeit, Fehler |
|
||||
| `offers` | `id`, FK Variante/Quelle, externe ID, Händler, URL, Markt, Währung, Zustand; UNIQUE Quelle/externe ID |
|
||||
| `price_observations` | `id`, FK Offer/Quelle, externe Beobachtungs-ID, Preise, Währung, Verfügbarkeit, Confidence, Raw JSONB, beide Zeiten; UNIQUE Offer/externe ID |
|
||||
| `evaluations` | `id`, FK Variante/Offer/Profilversion/Scoring-Version, Zeitpunkte, Hash, Build-ID, `product_eligibility`, `offer_eligibility`, `overall_eligibility`, drei nullable Scores, Klassifikation, `price_reference_scope`, vollständige Eingabe-/Ergebnis-Snapshots JSONB |
|
||||
| `evaluation_components` | `id`, FK Evaluation, `kind`, `component_key`, nullable Rohscore/Gewicht/Beitrag, `details` JSONB; UNIQUE Evaluation/kind/key |
|
||||
|
||||
`evaluation_components` enthält Präferenzen inklusive Strategy, optionale Abzüge, Budget-/History-Komponenten, Product-Gates und Offer-Gates. JSON-Snapshots sind die vollständige Replay-Grundlage; relationale Komponenten dienen Abfragen und dürfen bei Erstellung nicht davon abweichen.
|
||||
|
||||
DB-Checks mindestens für nichtnegative Preise, positiven Artikelpreis, Confidence 0–1, Scores 0–100, Profilversion >0 und gültige feste Enums. Typ-/Einheiten-/Profilvalidierung bleibt zusätzlich in der Application-Schicht. Redundante Indexspalten eines Profils müssen aus dessen validierter Definition erzeugt werden.
|
||||
|
||||
Indizes: Attribute `(variant_id, attribute_key, observed_at DESC, ingested_at DESC, id DESC)`, Preise `(offer_id, observed_at DESC, ingested_at DESC, id DESC)`, Offers `variant_id`, Evaluationen `(profile_id, profile_version, created_at DESC)` und `variant_id`.
|
||||
|
||||
Eine Evaluation liest und schreibt innerhalb einer PostgreSQL-Transaktion mit konsistentem Snapshot (`REPEATABLE READ`). Zu Beginn `snapshot_cutoff` festlegen; komplette Inputs, Ergebnis und Komponenten atomar committen. Bei Fehlern kein Teilergebnis. Insert-Konflikte über UNIQUE-Constraints behandeln, nicht allein per vorherigem SELECT.
|
||||
|
||||
Alembic ist die einzige produktive Schemaanlage; kein `metadata.create_all()` beim API-Start. Initialmigration enthält Tabellen, Constraints und Indizes. Bootstrap von Quellen und Regeldefinitionen separat, idempotent und inhaltlich geprüft. Ein Konflikt gleicher Version mit anderem Hash stoppt den Bootstrap sichtbar.
|
||||
|
||||
## 11. Repository und Lieferumfang
|
||||
|
||||
```text
|
||||
buying-agent/
|
||||
├── README.md
|
||||
├── Dockerfile
|
||||
├── docker-compose.yml
|
||||
├── .dockerignore
|
||||
├── .env.example
|
||||
├── .gitignore
|
||||
├── pyproject.toml
|
||||
├── uv.lock
|
||||
├── alembic.ini
|
||||
├── migrations/
|
||||
│ ├── env.py
|
||||
│ └── versions/
|
||||
├── app/
|
||||
│ ├── main.py
|
||||
│ ├── settings.py
|
||||
│ ├── api/
|
||||
│ │ ├── schemas.py
|
||||
│ │ ├── profiles.py
|
||||
│ │ ├── products.py
|
||||
│ │ ├── offers.py
|
||||
│ │ ├── evaluations.py
|
||||
│ │ └── health.py
|
||||
│ ├── domain/
|
||||
│ │ ├── models.py
|
||||
│ │ ├── attributes.py
|
||||
│ │ ├── constraints.py
|
||||
│ │ └── enums.py
|
||||
│ ├── scoring/
|
||||
│ │ ├── product.py
|
||||
│ │ ├── price.py
|
||||
│ │ ├── deal.py
|
||||
│ │ └── engine.py
|
||||
│ ├── normalization/
|
||||
│ │ ├── base.py
|
||||
│ │ └── manual.py
|
||||
│ ├── sources/
|
||||
│ │ ├── base.py
|
||||
│ │ └── manual.py
|
||||
│ ├── persistence/
|
||||
│ │ ├── db.py
|
||||
│ │ ├── models.py
|
||||
│ │ └── repositories.py
|
||||
│ └── services/
|
||||
│ ├── ingestion_service.py
|
||||
│ └── evaluation_service.py
|
||||
├── rules/generic-v1.json
|
||||
├── profiles/mobile-development-notebook.json
|
||||
├── fixtures/manual-demo.json
|
||||
├── scripts/
|
||||
│ ├── bootstrap.py
|
||||
│ └── demo.py
|
||||
└── tests/
|
||||
├── unit/
|
||||
├── integration/
|
||||
└── e2e/
|
||||
```
|
||||
|
||||
`__init__.py` nach Bedarf ergänzen. Kleine Dateien dürfen sinnvoll zusammengelegt werden; Architekturgrenzen sind verbindlich. Kein generisches Repository-Framework und kein Plugin-Discovery-System erforderlich. Das Lockfile-Beispiel verwendet uv; uv als Build-/Entwicklungswerkzeug mit dokumentierter Version verwenden.
|
||||
|
||||
README enthält Setup, Architektur, Datenmodell, Formeln, Fehlersemantik, alle Start-/Testkommandos, vollständige Beispielrequests sowie Grenzen der synthetischen Preisbewertung. Diese Spezifikation ebenfalls im Repository unter `docs/buying-agent-v0.1-spec-v1.1.md` ablegen.
|
||||
|
||||
## 12. Docker und lokale Ausführung
|
||||
|
||||
Compose enthält nur zwei dauerhaft laufende Services: `api` und `postgres`. PostgreSQL 17 mit benanntem Datenvolume und Healthcheck. API basiert auf Python 3.13, läuft als Nicht-Root und bindet den veröffentlichten Port an `127.0.0.1:8000`. Die Datenbank braucht keinen veröffentlichten Host-Port. Secrets und lokale DB-Zugänge über `.env`, nur Platzhalter in `.env.example`.
|
||||
|
||||
Für das lokale Einzelinstanz-Skeleton darf das API-Entrypoint nach erfolgreichem DB-Healthcheck sequenziell `alembic upgrade head`, Bootstrap und Uvicorn ausführen. Fehlgeschlagene Migrationen verhindern den API-Start. Diese Startmigration ist keine Strategie für spätere Mehrinstanz-Deployments.
|
||||
|
||||
Verbindlicher Startpfad:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
docker compose up --build -d
|
||||
docker compose exec api python -m scripts.demo
|
||||
```
|
||||
|
||||
Für PowerShell zusätzlich `Copy-Item .env.example .env` dokumentieren. `scripts.demo` legt Definitionen, Profil und synthetische Kandidaten über die API an, wiederholt sich ohne Duplikate und ruft Evaluationen mit explizitem `as_of` auf. Die Demo umfasst mindestens die Fälle A, B, C und D aus Abschnitt 13. Statische Fixtures nutzen den unten definierten historischen Zeitpunkt, damit sie nicht mit der Kalenderzeit altern. Der Adapterpfad wird separat im Integrationstest geprüft; die Demo muss den HTTP-Pfad abdecken.
|
||||
|
||||
Testabhängigkeiten in einer Docker-Build-Stage `test` bereitstellen. README dokumentiert einen ausführbaren Compose-Test-Override oder gleichwertigen Befehl mit dieser Stage und separater Testdatenbank. Tests dürfen niemals das normale Datenvolume leeren. Kein dauerhafter dritter Service nötig.
|
||||
|
||||
## 13. Deterministische Beispiele und Tests
|
||||
|
||||
### Referenzfall A
|
||||
|
||||
`as_of=2026-10-03T12:00:00Z`, aktuelle Beobachtung um 10:00 UTC; alle technischen Attribute vorher beobachtet. Kategorie Notebook, Markt DE, Zustand neu, EUR, lieferbar, Confidence jeweils 1.0, Versand 0.00.
|
||||
|
||||
| Attribut | Wert | Präferenzscore |
|
||||
|---|---:|---:|
|
||||
| ram_gb | 32 | 100 |
|
||||
| storage_gb | 1024 | nur Mindestconstraint |
|
||||
| weight_kg | 1.2 | 83.333333… |
|
||||
| battery_score | 90 | 87.5 |
|
||||
| display_score | 90 | 87.5 |
|
||||
| cpu_score | 90 | 87.5 |
|
||||
|
||||
Gesamtpreise der Historie: 1599.00 am 2026-09-01, 1599.00 am 2026-09-15, 1599.00 am 2026-10-01; jeweils 10:00 UTC. Aktuell 1298.00. Alle Zeilen betreffen dasselbe Offer und sind lieferbar. Fixtures werden vor Evaluation vollständig importiert.
|
||||
|
||||
Erwartungen:
|
||||
|
||||
```text
|
||||
product_eligibility = ELIGIBLE
|
||||
offer_eligibility = ELIGIBLE
|
||||
overall_eligibility = ELIGIBLE; eligible = true
|
||||
product_score = 90.21
|
||||
median_30d = 1599.00 (2 Tage)
|
||||
median_90d = 1599.00 (3 Tage)
|
||||
lowest_365d = 1599.00 (3 Tage)
|
||||
budget_score = 100
|
||||
history_score = 97.06
|
||||
price_score = 98.24
|
||||
deal_score = 93.42
|
||||
classification = EXCEPTIONAL
|
||||
product_coverage = 1.0
|
||||
```
|
||||
|
||||
Die Engine rechnet mit den ungerundeten Zwischenwerten. Die Response aus Abschnitt 9 bezieht sich auf diesen Fall. Das beobachtete Minimum von 1599.00 ist ausdrücklich kein belegtes Jahrestief des Marktes.
|
||||
|
||||
### Weitere Demokandidaten
|
||||
|
||||
- B: identisch zu A, aber 8 GB RAM → `product_eligibility=INELIGIBLE`, `offer_eligibility=ELIGIBLE`, `overall_eligibility=INELIGIBLE`, Mandatory RAM `FAIL`, ProductScore weiterhin diagnostisch vorhanden, kein DealScore.
|
||||
- C: identisch zu A, aber keine verwertbare Gewichtsangabe → `product_eligibility=UNKNOWN`, `offer_eligibility=ELIGIBLE`, `overall_eligibility=UNKNOWN`, Mandatory Gewicht `UNKNOWN`, ProductScore diagnostisch vorhanden, Klassifikation `INSUFFICIENT_DATA`, kein DealScore.
|
||||
- D: technisch identisch zu A, aber aktueller Gesamtpreis 2049.00 bei Budgetmaximum 2000.00 → `product_eligibility=ELIGIBLE`, `offer_eligibility=INELIGIBLE`, `overall_eligibility=INELIGIBLE`; ProductScore und diagnostischer PriceScore bleiben vorhanden, DealScore null. Dieser Fall beweist die Trennung von Produkt-Fit und Angebots-Fit.
|
||||
|
||||
### Unit-Tests
|
||||
|
||||
- Jeden Constraint-Operator mit PASS, FAIL, UNKNOWN; Gleichheitsgrenzen, strikte Typen, leere Listen und Confidence exakt 0.8 bzw. knapp darunter.
|
||||
- Preference-Strategie `numeric_linear`: akzeptabel 50, ideal 100, Clamping, lower/higher, nicht auf 1 summierte Gewichte, fehlende Werte und optionale Abzüge; unbekannte `strategy` wird für `generic-v1` abgewiesen.
|
||||
- Budget bei Ziel/Maximum/darüber; History-Score bei −20 %, 0 % und +20 % Rabatt; BUDGET_ONLY bei 0–2 Historientagen.
|
||||
- Tagesauswahl vor Qualitätsfilter, Fenstergrenzen, Ausschluss des aktuellen Tages, Median gerader/ungerader Anzahl, identische Zeitstempel und stabile Tie-Breaks.
|
||||
- Keine Vermischung von Offers, Varianten, Währungen oder Zuständen; aktuelle Zeile unbekannt/ausverkauft ohne Rückfall auf ältere Zeilen; ProductScore bleibt von Offer-Gates unabhängig.
|
||||
- Frische exakt 48 Stunden/knapp darüber; unbekannter Versand; fehlender Preis; Kombination Mandatory FAIL und UNKNOWN.
|
||||
- Klassifikationsgrenzen 60, 75, 85, 92 und unmittelbar darüber; Rundung darf keine Klasse verschieben.
|
||||
- Referenzfall A einschließlich exakter gerundeter Resultate; andere Kategorie mit eigenen Attributen ohne Engine-Änderung; Fall D zur Trennung von Product/Offer Eligibility.
|
||||
- Preis-Scoring-Funktion wird mit explizitem `current_price`, `reference_history` und `budget` getestet; Domain-Test darf dafür kein Offer-ORM-Objekt benötigen. `price_reference_scope=offer` ist für v0.1 verpflichtend.
|
||||
- Replay aus gespeichertem Input trotz später hinzugefügter Attribute/Preise liefert identische fachliche Ergebnisse; generierte IDs und Speicherzeit sind kein Bestandteil des Vergleichs.
|
||||
|
||||
### Integration und End-to-End
|
||||
|
||||
- Gegen echtes PostgreSQL 17, nicht SQLite als Ersatz für JSONB, Numeric, Constraints oder Isolation.
|
||||
- Migration auf leerer Test-DB; erneutes Upgrade ohne Änderung; initialer Downgrade/Upgrade nur in wegwerfbarer Test-DB.
|
||||
- Bootstrap identisch wiederholbar, veränderte Regel mit gleicher Version wird abgewiesen.
|
||||
- CRUD-Anlage-/Lesepfade, validierte Profilreferenzen, 404/409/422, Pagination, exakte Decimal- und UTC-Roundtrips.
|
||||
- Doppelte Preisimporte auch bei konkurrierenden Requests: eine Beobachtung, kein stiller Content-Overwrite.
|
||||
- Gescheiterte Evaluation-Persistenz hinterlässt weder Evaluation noch Komponenten.
|
||||
- HTTP-End-to-End: Profil → Produkt → Variante → Attribute → Offer → Historie → Evaluation → GET; danach API-Neustart und erneutes GET desselben unveränderten Ergebnisses.
|
||||
- ManualSource-Import nutzt dieselben Services und erzeugt bei Wiederholung keine Duplikate.
|
||||
- Einfügen neuer Preisbeobachtungen ändert alte Evaluationen nicht; neue Evaluation verwendet neue Inputs.
|
||||
|
||||
## 14. Implementierungsreihenfolge
|
||||
|
||||
1. Projekt, Settings, Docker, PostgreSQL und initiale Migration aufsetzen.
|
||||
2. Domain-Modelle, Definitions-/Profilvalidierung einschließlich `strategy`, getrennte Product-/Offer-Gates und reine Constraint-/Scoring-Funktionen samt Referenztests implementieren.
|
||||
3. Preis-Scoring als reine Berechnung über `current_price`, `reference_history` und `budget` implementieren; v0.1-Assembler liefert dafür ausschließlich offer-bezogene Historie und markiert `price_reference_scope=offer`.
|
||||
4. Repositories und transaktionalen Evaluation-Service integrieren.
|
||||
5. API, ManualSource und Bootstrap anschließen; ersten vollständigen Durchstich ausführen.
|
||||
6. Demo, Integrationstests, Fehlerfälle und nachvollziehbare Responses vervollständigen.
|
||||
7. README und sauberen Start auf leerem Volume prüfen; tatsächliche Testergebnisse dokumentieren.
|
||||
|
||||
Routineentscheidungen innerhalb dieser Spezifikation selbst treffen. Falls eine Vorgabe technisch widersprüchlich ist, den konkreten Konflikt dokumentieren und minimal auflösen; keine stillen fachlichen Formeländerungen oder Scope-Erweiterungen. Insbesondere keine Infrastruktur für spätere Preference-Strategien, Variant-/Market-Level-Preisreihen, n8n, LLMs oder Discovery vorwegnehmen.
|
||||
|
||||
## 15. Definition of Done
|
||||
|
||||
- [ ] Ein frischer Checkout startet mit den dokumentierten Docker-Kommandos; Migration und Readiness funktionieren.
|
||||
- [ ] PostgreSQL persistiert Daten über API-/Container-Neustarts hinweg.
|
||||
- [ ] Alle vier Entitäten Product, ProductVariant, Offer, PriceObservation sind getrennt und korrekt verknüpft.
|
||||
- [ ] Attribute sind generisch und bewahren Raw inhaltlich unverändert, Normalized, Quelle, Confidence und Normalizer-Version.
|
||||
- [ ] Beispielprofil und `generic-v1` liegen versioniert als Dateien vor und werden validiert gespeichert; jede Preference besitzt `strategy`, v0.1 akzeptiert ausschließlich `numeric_linear`.
|
||||
- [ ] Product-, Offer- und Overall-Eligibility werden getrennt berechnet, persistiert und über die API ausgegeben.
|
||||
- [ ] Ein technisch passendes Produkt behält ProductScore/Product Eligibility unabhängig davon, ob das konkrete Offer aktuell zu teuer, ausverkauft oder anderweitig ungeeignet ist.
|
||||
- [ ] Das Preis-Scoring ist fachlich über `current_price`, `reference_history` und `budget` entkoppelt; v0.1 verwendet ausschließlich `price_reference_scope=offer` und implementiert keine Variant-/Market-Level-Historie.
|
||||
- [ ] Demo liefert für A, B, C und D die festgelegten Ergebnisse über die API.
|
||||
- [ ] Scores, harte Gates, fehlende Daten, Historienbasis, Reference Scope und Begründungen sind sichtbar und reproduzierbar.
|
||||
- [ ] Evaluationen enthalten vollständige unveränderliche Snapshots; spätere Daten und Regelversionen verändern sie nicht.
|
||||
- [ ] Unit-, PostgreSQL-Integrations- und HTTP-End-to-End-Tests sind grün; Kommandos und Resultate sind dokumentiert.
|
||||
- [ ] Neue Kategorie und neue Attribute funktionieren ohne Schema- oder Engine-Änderung, solange sie von den in der gewählten Scoring-Version unterstützten Strategies/Operatoren abgedeckt werden.
|
||||
- [ ] Keine reale externe Quelle, kein n8n, keine LLM-Normalisierung, keine Web-UI, keine Auto-Discovery, keine Variant-/Market-Level-Preisaggregation und keine vorsorgliche Plugin-Infrastruktur wurden eingebaut.
|
||||
- [ ] README ermöglicht einem anderen Entwickler Setup, Demo, Tests und Verständnis der Grenzen ohne Chat-Kontext.
|
||||
|
||||
Abschlussbericht des Coding-Agenten: implementierter Umfang, relevante Entscheidungen, ausgeführte Prüfungen mit Ergebnissen und gegebenenfalls konkrete verbleibende Abweichungen. Nicht ausgeführte Tests ausdrücklich als solche benennen.
|
||||
|
||||
## 16. Future Extension Contract
|
||||
|
||||
v0.1 ist ein Walking Skeleton des Zielsystems. Die folgenden Fähigkeiten sind ausdrücklich **keine v0.1-Deliverables**, die Architektur darf ihre spätere Ergänzung jedoch nicht durch produktspezifische Schemata, fest verdrahtete Scoring-Annahmen oder n8n-interne Fachlogik unnötig blockieren:
|
||||
|
||||
- mehrere externe Produkt- und Preisquellen,
|
||||
- source-spezifische Search-/Discovery-Adapter,
|
||||
- Variant-Level- und Market-Level-Preisreferenzserien,
|
||||
- weitere Produktkategorien ohne produktspezifische DB-Spalten,
|
||||
- weitere explizit versionierte Preference-Strategien wie `boolean_preferred`, `categorical_match`, `set_contains` oder `categorical_rank`,
|
||||
- LLM-gestützte Normalisierung mit Raw-Wert, normalisiertem Wert, Confidence und Normalizer-Version,
|
||||
- n8n für Scheduling, Orchestrierung und Benachrichtigungen,
|
||||
- automatische Kandidatensuche und Kandidatenaufnahme,
|
||||
- mehrere parallele Buying-Profile und Watchlists.
|
||||
|
||||
Dieser Vertrag verlangt **keine vorsorgliche Infrastruktur** für diese Funktionen. Insbesondere keine Plugin-Registry, keinen Event-Bus, keine Microservices, keine zusätzlichen Preisaggregationstabellen, keine LLM-Abstraktionsschicht und keine n8n-Workflows bauen, solange sie nicht für den aktuellen Slice erforderlich sind.
|
||||
|
||||
Die stabilen Erweiterungspunkte von v0.1 sind:
|
||||
|
||||
1. Source-Adapter liefern validierte DTOs in den bestehenden Importpfad.
|
||||
2. Attribute und Constraints bleiben generisch und datengetrieben.
|
||||
3. Preference-Strategien werden explizit benannt und durch versionierte Scoring-Modelle unterstützt.
|
||||
4. PriceScore konsumiert eine von der Scoring-Version bereitgestellte Referenzserie statt dauerhaft an Offer-Identität gekoppelt zu sein.
|
||||
5. Product-, Offer- und Overall-Eligibility bleiben getrennt, damit Monitoring später gute Produkte unabhängig von momentan schlechten Angeboten verfolgen kann.
|
||||
6. n8n bleibt externer Orchestrator; die Buying Engine bleibt ohne n8n vollständig nutzbar und testbar.
|
||||
|
||||
@@ -1,7 +1,95 @@
|
||||
# BuyingAgent
|
||||
|
||||
Lokales Arbeitsverzeichnis, verbunden mit Gitea:
|
||||
Backend für einen generischen Buying-/Deal-Agenten. Das System bewertet, wie gut eine Produktvariante zu einem Profil passt und wie attraktiv ein konkretes Angebot preislich ist. Es kauft nicht.
|
||||
|
||||
`http://192.168.2.144:3000/Lars/BuyingAgent.git`
|
||||
v0.1 ist ein Walking Skeleton: Profil → Produkt → Variante → Angebot → Preisbeobachtungen → Evaluation. Notebooks sind der erste Anwendungsfall, nicht das Datenmodell.
|
||||
|
||||
Die eigentliche Spezifikation und Projektstruktur folgen.
|
||||
**Vorgaben:** `docs/buying-agent-v0.1-spec-v1.1.md` (fachlich) und `docs/buying-agent-architecture-addendum-v1.md` (Architektur, hat Vorrang bei Strukturfragen).
|
||||
|
||||
## Monorepo
|
||||
|
||||
```text
|
||||
backend/ System of Record: Domain, Scoring, Persistenz, /api/v1
|
||||
frontend/ Platzhalter — keine UI in v0.1
|
||||
docs/ Spezifikationen und Deployment
|
||||
docker-compose.yml
|
||||
```
|
||||
|
||||
Entwicklung auf `develop`. Push nach `develop` baut Dev auf dem Raspberry Pi. Merge `--no-ff` nach `main` deployt Production. Details: `docs/DEPLOYMENT.md`.
|
||||
|
||||
## Stack
|
||||
|
||||
Python 3.13, FastAPI, Pydantic 2, SQLAlchemy 2 (sync, psycopg 3), Alembic, PostgreSQL 17, pytest, Docker Compose. Build-Tool: uv 0.10.11.
|
||||
|
||||
Abhängigkeiten: HTTP → Services → Domain/Scoring → Repositories → PostgreSQL. `domain` und `scoring` importieren weder FastAPI noch SQLAlchemy.
|
||||
|
||||
## Lokal starten
|
||||
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
docker compose up --build -d
|
||||
docker compose exec api python -m scripts.demo
|
||||
```
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
docker compose up --build -d
|
||||
docker compose exec api python -m scripts.demo
|
||||
```
|
||||
|
||||
API: `http://127.0.0.1:8000`
|
||||
OpenAPI: `http://127.0.0.1:8000/docs`
|
||||
Live: `GET /health/live`
|
||||
Ready: `GET /health/ready`
|
||||
|
||||
Die Demo legt Definitionen, Profil und die Fälle A–D über HTTP an und wertet sie mit `as_of=2026-10-03T12:00:00Z` aus. Wiederholter Lauf erzeugt keine Duplikate.
|
||||
|
||||
## Tests
|
||||
|
||||
Lokal, isolierter Stack (tmpfs, kein Dev-/Prod-Volume):
|
||||
|
||||
```sh
|
||||
docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from api
|
||||
```
|
||||
|
||||
Nur Unit-Tests (ohne PostgreSQL), im Verzeichnis `backend/`:
|
||||
|
||||
```sh
|
||||
pytest tests/unit -ra --tb=short
|
||||
```
|
||||
|
||||
Auf dem Server nach Dev-Deploy (Gitea `test.yml`): pytest gegen `buyingagent_test` auf dem Dev-Postgres plus Smoke gegen `http://192.168.2.49:8095/health/ready`.
|
||||
|
||||
## Datenmodell (global)
|
||||
|
||||
Product, ProductVariant, Offer, PriceObservation, AttributeDefinition und Source sind **systemweit** und haben keine `user_id`/`workspace_id`. Workspace, BuyingProject und Benutzer kommen später und dürfen diese Daten nicht duplizieren.
|
||||
|
||||
Profile und Evaluationen können bereits parallel existieren; es gibt kein globales „aktuelles Kaufvorhaben“.
|
||||
|
||||
## Scoring `generic-v1`
|
||||
|
||||
- Preference-Strategie nur `numeric_linear`. Unbekannte Strategien werden beim Profil abgewiesen.
|
||||
- `product_eligibility`, `offer_eligibility` und `overall_eligibility` sind getrennt.
|
||||
- PriceScore ist eine reine Funktion über `current_price`, `reference_history` und `budget`. v0.1 setzt `price_reference_scope=offer`.
|
||||
- DealScore nur bei `overall_eligibility=ELIGIBLE` und vorhandenem PriceScore.
|
||||
- Klassifikation am **ungerundeten** DealScore; Anzeige auf zwei Dezimalen, `ROUND_HALF_UP`.
|
||||
- Geld intern `Decimal`/`NUMERIC(12,2)`, in der API als Dezimalstring.
|
||||
|
||||
Referenzfall A (synthetisch): ProductScore 90.21, PriceScore 98.24, DealScore 93.42, `EXCEPTIONAL`. Das ist keine Marktaussage.
|
||||
|
||||
## Fehler
|
||||
|
||||
Einheitliches Format `{error: {code, message, details}}`. 404 fehlende ID, 409 Konflikt/abweichender Wiederholungsimport, 422 ungültige Payload. Eine fachlich negative Evaluation ist HTTP 201.
|
||||
|
||||
## Grenzen von v0.1
|
||||
|
||||
Keine Web-UI, keine Authentifizierung, keine Workspaces, kein BuyingProject, kein n8n, keine externen Quellen, keine Auto-Discovery, keine Wechselkurse, keine Variant-/Market-Level-Preisaggregation, keine Plugin-Infrastruktur.
|
||||
|
||||
## Server
|
||||
|
||||
| Umgebung | Branch | Verzeichnis | API |
|
||||
|---|---|---|---|
|
||||
| Development | `develop` | `~/docker/buyingagent-dev` | `http://192.168.2.49:8095` |
|
||||
| Production | `main` | `~/docker/buyingagent` | `http://192.168.2.49:8006` |
|
||||
|
||||
Repo: `http://192.168.2.144:3000/Lars/BuyingAgent.git`
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,10 @@
|
||||
.git
|
||||
.venv
|
||||
.pytest_cache
|
||||
.mypy_cache
|
||||
.ruff_cache
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info
|
||||
.tests-tmp
|
||||
@@ -0,0 +1,42 @@
|
||||
FROM ghcr.io/astral-sh/uv:0.10.11-python3.13-trixie-slim AS base
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync --frozen --no-dev --no-install-project
|
||||
|
||||
FROM base AS runtime
|
||||
|
||||
COPY . .
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
||||
RUN uv sync --frozen --extra live --extra test --no-dev \
|
||||
&& playwright install --with-deps chromium \
|
||||
&& sed -i 's/\r$//' /app/scripts/startup.sh \
|
||||
&& chmod +x /app/scripts/startup.sh \
|
||||
&& useradd --create-home --uid 1000 app \
|
||||
&& chown -R app:app /app /ms-playwright
|
||||
|
||||
USER app
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["/app/scripts/startup.sh"]
|
||||
|
||||
FROM base AS test
|
||||
|
||||
COPY . .
|
||||
RUN uv sync --frozen --extra test \
|
||||
&& useradd --create-home --uid 1000 app \
|
||||
&& chown -R app:app /app
|
||||
|
||||
USER app
|
||||
CMD ["pytest", "tests", "-m", "not slow", "-ra", "--tb=short"]
|
||||
@@ -0,0 +1,39 @@
|
||||
[alembic]
|
||||
script_location = migrations
|
||||
prepend_sys_path = .
|
||||
path_separator = os
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,136 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import (
|
||||
CriterionDefinitionIn,
|
||||
CriterionMappingIn,
|
||||
CriterionMappingPatch,
|
||||
DiscoverySearchIn,
|
||||
DiscoverySearchPatch,
|
||||
SourceAttributeMappingIn,
|
||||
SourceAttributeMappingPatch,
|
||||
SourceCategoryIn,
|
||||
)
|
||||
from app.persistence.db import get_session
|
||||
from app.services.catalog_service import CatalogService
|
||||
from app.services.discovery_service import (
|
||||
CriterionDefinitionService,
|
||||
CriterionMappingService,
|
||||
DiscoverySearchService,
|
||||
SourceAttributeMappingService,
|
||||
TaxonomyService,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.get("/sources/{source_id}/categories")
|
||||
def list_source_categories(source_id: str, session: Session = Depends(get_session)):
|
||||
return {"items": CatalogService(session).source_categories(source_id)}
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/categories", status_code=201)
|
||||
def register_source_category(source_id: str, payload: SourceCategoryIn, session: Session = Depends(get_session)):
|
||||
return CatalogService(session).register_source_category(source_id, payload.model_dump())
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/categories/{category}/discover-taxonomy", status_code=201)
|
||||
def discover_taxonomy(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).discover(source_id, category)
|
||||
|
||||
|
||||
@router.get("/sources/{source_id}/categories/{category}/taxonomy-status")
|
||||
def taxonomy_status(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).taxonomy_status(source_id, category)
|
||||
|
||||
|
||||
@router.get("/sources/{source_id}/categories/{category}/filters")
|
||||
def list_filters(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).filters(source_id, category)
|
||||
|
||||
|
||||
@router.post("/criterion-definitions", status_code=201)
|
||||
def create_criterion_definition(payload: CriterionDefinitionIn, session: Session = Depends(get_session)):
|
||||
return CriterionDefinitionService(session).create(payload.model_dump())
|
||||
|
||||
|
||||
@router.get("/criterion-definitions")
|
||||
def list_criterion_definitions(category: str | None = None, session: Session = Depends(get_session)):
|
||||
return {"items": CriterionDefinitionService(session).list(category)}
|
||||
|
||||
|
||||
@router.get("/criterion-mappings")
|
||||
def list_mappings(source_category_id: UUID | None = None, session: Session = Depends(get_session)):
|
||||
return {"items": CriterionMappingService(session).list(source_category_id)}
|
||||
|
||||
|
||||
@router.post("/criterion-mappings", status_code=201)
|
||||
def create_mapping(payload: CriterionMappingIn, session: Session = Depends(get_session)):
|
||||
return CriterionMappingService(session).create(payload.model_dump())
|
||||
|
||||
|
||||
@router.patch("/criterion-mappings/{mapping_id}")
|
||||
def patch_mapping(mapping_id: UUID, payload: CriterionMappingPatch, session: Session = Depends(get_session)):
|
||||
return CriterionMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
||||
|
||||
|
||||
@router.get("/source-attribute-mappings")
|
||||
def list_attribute_mappings(session: Session = Depends(get_session)):
|
||||
return {"items": SourceAttributeMappingService(session).list()}
|
||||
|
||||
|
||||
@router.post("/source-attribute-mappings", status_code=201)
|
||||
def create_attribute_mapping(payload: SourceAttributeMappingIn, session: Session = Depends(get_session)):
|
||||
return SourceAttributeMappingService(session).create(payload.model_dump())
|
||||
|
||||
|
||||
@router.patch("/source-attribute-mappings/{mapping_id}")
|
||||
def patch_attribute_mapping(
|
||||
mapping_id: UUID,
|
||||
payload: SourceAttributeMappingPatch,
|
||||
session: Session = Depends(get_session),
|
||||
):
|
||||
return SourceAttributeMappingService(session).patch(mapping_id, payload.model_dump(exclude_unset=True))
|
||||
|
||||
|
||||
@router.post("/discovery-searches", status_code=201)
|
||||
def create_search(payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).create(payload.model_dump())
|
||||
|
||||
|
||||
@router.get("/discovery-searches")
|
||||
def list_searches(due: bool = Query(default=False), session: Session = Depends(get_session)):
|
||||
service = DiscoverySearchService(session)
|
||||
return {"items": service.list_due() if due else service.list_all()}
|
||||
|
||||
|
||||
@router.get("/discovery-searches/{search_id}")
|
||||
def get_search(search_id: UUID, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).get(search_id)
|
||||
|
||||
|
||||
@router.patch("/discovery-searches/{search_id}")
|
||||
def patch_search(search_id: UUID, payload: DiscoverySearchPatch, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).patch(search_id, payload.model_dump(exclude_unset=True))
|
||||
|
||||
|
||||
@router.post("/discovery-searches/{search_id}/generate-query")
|
||||
def generate_query(search_id: UUID, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).generate_query(search_id)
|
||||
|
||||
|
||||
@router.post("/discovery-searches/{search_id}/run", status_code=201)
|
||||
def run_search(search_id: UUID, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).run(search_id)
|
||||
|
||||
|
||||
@router.get("/discovery-searches/{search_id}/runs")
|
||||
def list_runs(search_id: UUID, session: Session = Depends(get_session)):
|
||||
return {"items": DiscoverySearchService(session).list_runs(search_id)}
|
||||
|
||||
|
||||
@router.get("/search-runs/{run_id}")
|
||||
def get_run(run_id: UUID, session: Session = Depends(get_session)):
|
||||
return DiscoverySearchService(session).get_run(run_id)
|
||||
@@ -0,0 +1,20 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import EvaluateIn
|
||||
from app.persistence.db import get_session
|
||||
from app.services.evaluation_service import evaluate_and_store, load_evaluation
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.post("/evaluate/{variant_id}", status_code=201)
|
||||
def evaluate(variant_id: UUID, payload: EvaluateIn):
|
||||
return evaluate_and_store(variant_id=variant_id, payload=payload.model_dump())
|
||||
|
||||
|
||||
@router.get("/evaluations/{evaluation_id}")
|
||||
def get_evaluation(evaluation_id: UUID, session: Session = Depends(get_session)):
|
||||
return load_evaluation(session, evaluation_id)
|
||||
@@ -0,0 +1,45 @@
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import text
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
from app.persistence.db import engine
|
||||
from app.settings import get_settings
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _alembic_at_head() -> bool:
|
||||
config = Config("alembic.ini")
|
||||
script = ScriptDirectory.from_config(config)
|
||||
with engine.connect() as connection:
|
||||
context = MigrationContext.configure(connection)
|
||||
return set(context.get_current_heads()) == set(script.get_heads())
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
def live() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/ready", response_model=None)
|
||||
def ready() -> JSONResponse | dict[str, str]:
|
||||
settings = get_settings()
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
if not _alembic_at_head():
|
||||
return JSONResponse(
|
||||
{"status": "not_ready", "reason": "migration_behind_head"},
|
||||
status_code=503,
|
||||
)
|
||||
except Exception:
|
||||
return JSONResponse({"status": "not_ready", "reason": "database"}, status_code=503)
|
||||
return {
|
||||
"status": "ready",
|
||||
"environment": settings.environment,
|
||||
"engine_build": settings.engine_build,
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import OfferIn, OfferOut, PriceIn, PriceOut, page, parse_page
|
||||
from app.errors import AppError
|
||||
from app.persistence.db import get_session
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
from app.services.ingestion_service import IngestionService
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.post("/variants/{variant_id}/offers", status_code=201)
|
||||
def create_offer(variant_id: UUID, payload: OfferIn, session: Session = Depends(get_session)) -> OfferOut:
|
||||
offer = IngestionService(session).create_offer(variant_id, payload.model_dump())
|
||||
return OfferOut.model_validate(offer.__dict__)
|
||||
|
||||
|
||||
@router.get("/offers/{offer_id}")
|
||||
def get_offer(offer_id: UUID, session: Session = Depends(get_session)) -> OfferOut:
|
||||
offer = CatalogRepository(session).get_offer(offer_id)
|
||||
if offer is None:
|
||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||
return OfferOut.model_validate(offer.__dict__)
|
||||
|
||||
|
||||
@router.post("/offers/{offer_id}/prices")
|
||||
def add_price(offer_id: UUID, payload: PriceIn, response: Response, session: Session = Depends(get_session)) -> PriceOut:
|
||||
observation, created = IngestionService(session).add_price(offer_id, payload.model_dump())
|
||||
response.status_code = 201 if created else 200
|
||||
return PriceOut.model_validate(observation.__dict__)
|
||||
|
||||
|
||||
@router.get("/offers/{offer_id}/prices")
|
||||
def list_prices(offer_id: UUID, limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||
if CatalogRepository(session).get_offer(offer_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||
limit, offset = parse_page(limit, offset)
|
||||
items, total = CatalogRepository(session).list_observations(offer_id, limit, offset)
|
||||
return page([PriceOut.model_validate(item.__dict__) for item in items], limit, offset, total)
|
||||
@@ -0,0 +1,63 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import AttributeIn, AttributeOut, ProductIn, ProductOut, VariantIn, VariantOut
|
||||
from app.errors import AppError
|
||||
from app.persistence.db import get_session
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
from app.services.ingestion_service import IngestionService
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.post("/products", status_code=201)
|
||||
def create_product(payload: ProductIn, session: Session = Depends(get_session)) -> ProductOut:
|
||||
product = IngestionService(session).create_product(payload.model_dump())
|
||||
return ProductOut.model_validate(product.__dict__)
|
||||
|
||||
|
||||
@router.get("/products/{product_id}")
|
||||
def get_product(product_id: UUID, session: Session = Depends(get_session)) -> ProductOut:
|
||||
product = CatalogRepository(session).get_product(product_id)
|
||||
if product is None:
|
||||
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
||||
return ProductOut.model_validate(product.__dict__)
|
||||
|
||||
|
||||
@router.post("/products/{product_id}/variants", status_code=201)
|
||||
def create_variant(product_id: UUID, payload: VariantIn, session: Session = Depends(get_session)):
|
||||
variant = IngestionService(session).create_variant(product_id, payload.model_dump())
|
||||
return {
|
||||
"id": variant.id,
|
||||
"product_id": variant.product_id,
|
||||
"name": variant.name,
|
||||
"manufacturer_sku": variant.manufacturer_sku,
|
||||
"identifiers": variant.identifiers,
|
||||
"created_at": variant.created_at,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/variants/{variant_id}")
|
||||
def get_variant(variant_id: UUID, session: Session = Depends(get_session)) -> VariantOut:
|
||||
repo = CatalogRepository(session)
|
||||
variant = repo.get_variant(variant_id)
|
||||
if variant is None:
|
||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||
attributes = [AttributeOut.model_validate(item.__dict__) for item in repo.list_attributes(variant_id)]
|
||||
return VariantOut(
|
||||
id=variant.id,
|
||||
product_id=variant.product_id,
|
||||
name=variant.name,
|
||||
manufacturer_sku=variant.manufacturer_sku,
|
||||
identifiers=variant.identifiers,
|
||||
created_at=variant.created_at,
|
||||
attributes=attributes,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/variants/{variant_id}/attributes", status_code=201)
|
||||
def add_attribute(variant_id: UUID, payload: AttributeIn, session: Session = Depends(get_session)) -> AttributeOut:
|
||||
statement = IngestionService(session).add_attribute(variant_id, payload.model_dump())
|
||||
return AttributeOut.model_validate(statement.__dict__)
|
||||
@@ -0,0 +1,100 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import AttributeDefinitionIn, page, parse_page
|
||||
from app.domain.serialize import jsonable
|
||||
from app.errors import AppError
|
||||
from app.persistence.db import get_session
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
from app.services.ingestion_service import IngestionService
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.post("/attribute-definitions", status_code=201)
|
||||
def create_attribute_definition(payload: AttributeDefinitionIn, session: Session = Depends(get_session)):
|
||||
definition = IngestionService(session).create_definition(payload.model_dump())
|
||||
return jsonable(definition.__dict__)
|
||||
|
||||
|
||||
@router.get("/attribute-definitions")
|
||||
def list_attribute_definitions(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||
limit, offset = parse_page(limit, offset)
|
||||
items, total = CatalogRepository(session).list_definitions(limit, offset)
|
||||
return page([jsonable(item.__dict__) for item in items], limit, offset, total)
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
def list_sources(session: Session = Depends(get_session)):
|
||||
items = CatalogRepository(session).list_sources()
|
||||
return {
|
||||
"items": [{"id": item.id, "name": item.name, "adapter_type": item.adapter_type} for item in items],
|
||||
"limit": len(items),
|
||||
"offset": 0,
|
||||
"total": len(items),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/scoring-models/{version}")
|
||||
def get_scoring_model(version: str, session: Session = Depends(get_session)):
|
||||
model = CatalogRepository(session).get_scoring_model(version)
|
||||
if model is None:
|
||||
raise AppError(404, "NOT_FOUND", "scoring model not found", {"version": version})
|
||||
return {
|
||||
"version": model.version,
|
||||
"algorithm_id": model.algorithm_id,
|
||||
"definition": model.definition,
|
||||
"definition_hash": model.definition_hash,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/profiles", status_code=201)
|
||||
def create_profile(payload: dict[str, Any] = Body(...), session: Session = Depends(get_session)):
|
||||
if not isinstance(payload, dict):
|
||||
raise AppError(422, "PROFILE_INVALID", "profile must be an object")
|
||||
row = IngestionService(session).create_profile(payload)
|
||||
return {
|
||||
"id": row.id,
|
||||
"version": row.version,
|
||||
"name": row.name,
|
||||
"category": row.category,
|
||||
"scoring_version": row.scoring_version,
|
||||
"definition": row.definition,
|
||||
"definition_hash": row.definition_hash,
|
||||
"created_at": row.created_at,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/profiles")
|
||||
def list_profiles(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||
limit, offset = parse_page(limit, offset)
|
||||
items, total = CatalogRepository(session).list_profiles(limit, offset)
|
||||
return page(
|
||||
[
|
||||
{
|
||||
"id": row.id,
|
||||
"version": row.version,
|
||||
"name": row.name,
|
||||
"category": row.category,
|
||||
"scoring_version": row.scoring_version,
|
||||
"definition_hash": row.definition_hash,
|
||||
"created_at": row.created_at,
|
||||
}
|
||||
for row in items
|
||||
],
|
||||
limit,
|
||||
offset,
|
||||
total,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/profiles/{profile_id}")
|
||||
def get_profile(profile_id: str, version: int | None = None, session: Session = Depends(get_session)):
|
||||
if version is None:
|
||||
raise AppError(422, "VERSION_REQUIRED", "query parameter version is required")
|
||||
profile = CatalogRepository(session).get_profile(profile_id, version)
|
||||
if profile is None:
|
||||
raise AppError(404, "NOT_FOUND", "profile version not found")
|
||||
return profile.definition
|
||||
@@ -0,0 +1,125 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import (
|
||||
BuyingProjectIn,
|
||||
BuyingProjectPatch,
|
||||
DiscoverySearchIn,
|
||||
ProjectCriteriaIn,
|
||||
page,
|
||||
parse_page,
|
||||
)
|
||||
from app.persistence.db import get_session
|
||||
from app.services.catalog_service import CatalogService
|
||||
from app.services.discovery_service import CandidateRankingService, DiscoverySearchService
|
||||
from app.services.project_service import ProjectService
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
|
||||
@router.get("/product-categories")
|
||||
def list_product_categories(session: Session = Depends(get_session)):
|
||||
return {"items": CatalogService(session).list_product_categories()}
|
||||
|
||||
|
||||
@router.get("/product-categories/{product_category_id}/sources")
|
||||
def list_category_sources(product_category_id: str, session: Session = Depends(get_session)):
|
||||
return {"items": CatalogService(session).sources_for_category(product_category_id)}
|
||||
|
||||
|
||||
@router.get("/product-categories/{product_category_id}/available-criteria")
|
||||
def available_criteria_for_category(
|
||||
product_category_id: str,
|
||||
source: str = Query(...),
|
||||
session: Session = Depends(get_session),
|
||||
):
|
||||
return CatalogService(session).available_criteria(product_category_id, source)
|
||||
|
||||
|
||||
@router.get("/buying-templates")
|
||||
def list_buying_templates(session: Session = Depends(get_session)):
|
||||
return {"items": CatalogService(session).list_templates()}
|
||||
|
||||
|
||||
@router.post("/buying-projects", status_code=201)
|
||||
def create_buying_project(payload: BuyingProjectIn, session: Session = Depends(get_session)):
|
||||
return ProjectService(session).create(payload.model_dump())
|
||||
|
||||
|
||||
@router.get("/buying-projects")
|
||||
def list_buying_projects(limit: int = 50, offset: int = 0, session: Session = Depends(get_session)):
|
||||
limit, offset = parse_page(limit, offset)
|
||||
items, total = ProjectService(session).list(limit, offset)
|
||||
return page(items, limit, offset, total)
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}")
|
||||
def get_buying_project(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return ProjectService(session).get(project_id)
|
||||
|
||||
|
||||
@router.patch("/buying-projects/{project_id}")
|
||||
def patch_buying_project(project_id: UUID, payload: BuyingProjectPatch, session: Session = Depends(get_session)):
|
||||
return ProjectService(session).patch(project_id, payload.model_dump(exclude_unset=True))
|
||||
|
||||
|
||||
@router.post("/buying-projects/{project_id}/criteria", status_code=201)
|
||||
def add_buying_project_criteria(
|
||||
project_id: UUID,
|
||||
payload: ProjectCriteriaIn,
|
||||
session: Session = Depends(get_session),
|
||||
):
|
||||
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
||||
|
||||
|
||||
@router.put("/buying-projects/{project_id}/criteria")
|
||||
def put_buying_project_criteria(
|
||||
project_id: UUID,
|
||||
payload: ProjectCriteriaIn,
|
||||
session: Session = Depends(get_session),
|
||||
):
|
||||
return ProjectService(session).replace_criteria(project_id, payload.model_dump())
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/criteria")
|
||||
def get_buying_project_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return ProjectService(session).get_criteria(project_id)
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/available-criteria")
|
||||
def available_criteria(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return ProjectService(session).available_criteria(project_id)
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/discovery-searches")
|
||||
def list_project_searches(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return {"items": DiscoverySearchService(session).list_for_project(project_id)}
|
||||
|
||||
|
||||
@router.post("/buying-projects/{project_id}/discovery-searches", status_code=201)
|
||||
def create_project_search(project_id: UUID, payload: DiscoverySearchIn, session: Session = Depends(get_session)):
|
||||
body = payload.model_dump()
|
||||
body["buying_project_id"] = project_id
|
||||
return DiscoverySearchService(session).create(body)
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/candidates")
|
||||
def list_candidates(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return {"items": CandidateRankingService(session).list_candidates(project_id)}
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}")
|
||||
def get_candidate(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
||||
return CandidateRankingService(session).get_candidate(project_id, candidate_id)
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/candidates/{candidate_id}/observations")
|
||||
def list_candidate_observations(project_id: UUID, candidate_id: UUID, session: Session = Depends(get_session)):
|
||||
return {"items": CandidateRankingService(session).list_observations(project_id, candidate_id)}
|
||||
|
||||
|
||||
@router.get("/buying-projects/{project_id}/shortlist")
|
||||
def list_shortlist(project_id: UUID, session: Session = Depends(get_session)):
|
||||
return {"items": CandidateRankingService(session).list_candidates(project_id, shortlisted_only=True)}
|
||||
@@ -0,0 +1,281 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AttributeDefinitionIn(StrictModel):
|
||||
key: str
|
||||
value_type: str
|
||||
canonical_unit: str | None = None
|
||||
minimum: Decimal | None = None
|
||||
maximum: Decimal | None = None
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class AttributeDefinitionOut(StrictModel):
|
||||
key: str
|
||||
value_type: str
|
||||
canonical_unit: str | None
|
||||
minimum: Decimal | None
|
||||
maximum: Decimal | None
|
||||
description: str | None
|
||||
|
||||
|
||||
class ProductIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
category: str
|
||||
brand: str
|
||||
name: str
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class ProductOut(StrictModel):
|
||||
id: UUID
|
||||
category: str
|
||||
brand: str
|
||||
name: str
|
||||
model: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class VariantIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
name: str
|
||||
manufacturer_sku: str | None = None
|
||||
identifiers: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AttributeIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
attribute: str
|
||||
raw_value: Any
|
||||
normalized_value: Any | None = None
|
||||
unit: str | None = None
|
||||
normalization_status: str
|
||||
normalizer_version: str | None = None
|
||||
confidence: Decimal
|
||||
source_id: str
|
||||
source_reference: str
|
||||
observed_at: datetime
|
||||
normalization_error: str | None = None
|
||||
|
||||
|
||||
class AttributeOut(StrictModel):
|
||||
id: UUID
|
||||
variant_id: UUID
|
||||
attribute: str
|
||||
raw_value: Any
|
||||
normalized_value: Any | None
|
||||
unit: str | None
|
||||
normalization_status: str
|
||||
normalizer_version: str
|
||||
confidence: Decimal
|
||||
source_id: str
|
||||
source_reference: str
|
||||
observed_at: datetime
|
||||
ingested_at: datetime
|
||||
normalization_error: str | None
|
||||
|
||||
|
||||
class VariantOut(StrictModel):
|
||||
id: UUID
|
||||
product_id: UUID
|
||||
name: str
|
||||
manufacturer_sku: str | None
|
||||
identifiers: dict[str, Any]
|
||||
created_at: datetime
|
||||
attributes: list[AttributeOut]
|
||||
|
||||
|
||||
class OfferIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
source_id: str
|
||||
external_id: str
|
||||
seller: str
|
||||
url: str | None = None
|
||||
market: str
|
||||
currency: str
|
||||
condition: str
|
||||
|
||||
|
||||
class OfferOut(StrictModel):
|
||||
id: UUID
|
||||
variant_id: UUID
|
||||
source_id: str
|
||||
external_id: str
|
||||
seller: str
|
||||
url: str | None
|
||||
market: str
|
||||
currency: str
|
||||
condition: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class PriceIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
source_id: str
|
||||
external_observation_id: str
|
||||
observed_at: datetime
|
||||
item_price: Decimal
|
||||
shipping_price: Decimal | None = None
|
||||
currency: str
|
||||
availability: str
|
||||
raw_payload: Any
|
||||
confidence: Decimal
|
||||
|
||||
@field_serializer("item_price", "shipping_price")
|
||||
def serialize_money(self, value: Decimal | None) -> str | None:
|
||||
return None if value is None else format(value, "f")
|
||||
|
||||
|
||||
class PriceOut(StrictModel):
|
||||
id: UUID
|
||||
offer_id: UUID
|
||||
source_id: str
|
||||
external_observation_id: str
|
||||
observed_at: datetime
|
||||
ingested_at: datetime
|
||||
item_price: Decimal
|
||||
shipping_price: Decimal | None
|
||||
currency: str
|
||||
availability: str
|
||||
raw_payload: Any
|
||||
confidence: Decimal
|
||||
|
||||
@field_serializer("item_price", "shipping_price")
|
||||
def serialize_money(self, value: Decimal | None) -> str | None:
|
||||
return None if value is None else format(Decimal(value).quantize(Decimal("0.01")), "f")
|
||||
|
||||
|
||||
class EvaluateIn(StrictModel):
|
||||
profile_id: str
|
||||
profile_version: int
|
||||
offer_id: UUID
|
||||
as_of: datetime | None = None
|
||||
|
||||
|
||||
class BuyingProjectIn(StrictModel):
|
||||
id: UUID | None = None
|
||||
name: str
|
||||
product_category_id: str | None = None
|
||||
market: str | None = None
|
||||
currency: str | None = None
|
||||
target_budget: Decimal | None = None
|
||||
template_id: str | None = None
|
||||
template_version: int | None = None
|
||||
profile_id: str | None = None
|
||||
profile_version: int | None = None
|
||||
workspace_id: str | None = None
|
||||
shortlist_size: int = 5
|
||||
schedule_interval_minutes: int | None = None
|
||||
criteria: dict[str, Any]
|
||||
|
||||
|
||||
class BuyingProjectPatch(StrictModel):
|
||||
name: str | None = None
|
||||
target_budget: Decimal | None = None
|
||||
shortlist_size: int | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class ProjectCriteriaIn(StrictModel):
|
||||
items: list[dict[str, Any]]
|
||||
|
||||
|
||||
class SourceCategoryIn(StrictModel):
|
||||
source_url: str
|
||||
name: str | None = None
|
||||
product_category_id: str | None = None
|
||||
product_category_name: str | None = None
|
||||
|
||||
|
||||
class CriterionDefinitionIn(StrictModel):
|
||||
key: str
|
||||
value_type: str
|
||||
unit: str | None = None
|
||||
semantic: str | None = None
|
||||
category: str | None = None
|
||||
description: str | None = None
|
||||
origin: str | None = None
|
||||
source: str | None = None
|
||||
product_category_id: str | None = None
|
||||
label: str | None = None
|
||||
scoring_attribute_key: str | None = None
|
||||
|
||||
|
||||
class DiscoverySearchIn(StrictModel):
|
||||
buying_project_id: UUID | None = None
|
||||
source_id: str
|
||||
source_category_id: UUID
|
||||
schedule_interval_minutes: int = 180
|
||||
broad_result_threshold: int = 300
|
||||
|
||||
|
||||
class DiscoverySearchPatch(StrictModel):
|
||||
schedule_interval_minutes: int | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class SourceAttributeMappingIn(StrictModel):
|
||||
source_id: str
|
||||
source_category_id: UUID | None = None
|
||||
raw_name: str
|
||||
criterion_definition_id: UUID
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class SourceAttributeMappingPatch(StrictModel):
|
||||
raw_name: str | None = None
|
||||
criterion_definition_id: UUID | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class CriterionMappingIn(StrictModel):
|
||||
criterion_definition_id: UUID | None = None
|
||||
criterion_key: str | None = None
|
||||
source_id: str
|
||||
source_category_id: UUID
|
||||
external_filter_key: str
|
||||
source_value: str | None = None
|
||||
encoding: dict[str, Any] = Field(default_factory=lambda: {"url_param": "xf"})
|
||||
status: str | None = None
|
||||
value_type: str | None = None
|
||||
unit: str | None = None
|
||||
semantic: str | None = None
|
||||
description: str | None = None
|
||||
operator: str | None = None
|
||||
probe_value: Any = None
|
||||
|
||||
|
||||
class CriterionMappingPatch(StrictModel):
|
||||
source_value: str | None = None
|
||||
encoding: dict[str, Any] | None = None
|
||||
status: str | None = None
|
||||
probe_value: Any = None
|
||||
operator: str | None = None
|
||||
|
||||
|
||||
class Page(StrictModel):
|
||||
items: list[Any]
|
||||
limit: int
|
||||
offset: int
|
||||
total: int
|
||||
|
||||
|
||||
def page(items: list[Any], limit: int, offset: int, total: int) -> dict[str, Any]:
|
||||
return {"items": items, "limit": limit, "offset": offset, "total": total}
|
||||
|
||||
|
||||
def parse_page(limit: int = 50, offset: int = 0) -> tuple[int, int]:
|
||||
from app.errors import AppError
|
||||
|
||||
if limit < 1 or limit > 200 or offset < 0:
|
||||
raise AppError(422, "PAGINATION", "limit must be 1-200 and offset >= 0")
|
||||
return limit, offset
|
||||
@@ -0,0 +1,12 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.settings import get_settings
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
override = get_settings().clock_now
|
||||
if override is not None:
|
||||
if override.tzinfo is None:
|
||||
return override.replace(tzinfo=timezone.utc)
|
||||
return override.astimezone(timezone.utc)
|
||||
return datetime.now(timezone.utc)
|
||||
@@ -0,0 +1,38 @@
|
||||
from app.discovery.criteria import parse_project_criteria
|
||||
from app.discovery.diff import diff_run, next_lifecycle, observations_changed
|
||||
from app.discovery.enums import (
|
||||
CandidateStatus,
|
||||
DiffStatus,
|
||||
LifecycleStatus,
|
||||
MappingStatus,
|
||||
MatchStatus,
|
||||
ProjectStatus,
|
||||
RunStatus,
|
||||
SearchStatus,
|
||||
)
|
||||
from app.discovery.identity import IdentityDecision, IdentitySignals, decide_identity, spec_fingerprint
|
||||
from app.discovery.mapping import verify_roundtrip
|
||||
from app.discovery.ranking import rank_candidates
|
||||
from app.discovery.urlgen import generate_search_url
|
||||
|
||||
__all__ = [
|
||||
"CandidateStatus",
|
||||
"DiffStatus",
|
||||
"IdentityDecision",
|
||||
"IdentitySignals",
|
||||
"LifecycleStatus",
|
||||
"MappingStatus",
|
||||
"MatchStatus",
|
||||
"ProjectStatus",
|
||||
"RunStatus",
|
||||
"SearchStatus",
|
||||
"decide_identity",
|
||||
"diff_run",
|
||||
"generate_search_url",
|
||||
"next_lifecycle",
|
||||
"observations_changed",
|
||||
"parse_project_criteria",
|
||||
"rank_candidates",
|
||||
"spec_fingerprint",
|
||||
"verify_roundtrip",
|
||||
]
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
SPEC_ALIASES = {
|
||||
"ram_gb": ("ram", "arbeitsspeicher"),
|
||||
"storage_gb": ("speicher", "ssd", "festplatte"),
|
||||
"weight_kg": ("gewicht",),
|
||||
}
|
||||
|
||||
|
||||
def merge_observation_attributes(
|
||||
raw_attributes: dict[str, Any],
|
||||
normalized_attributes: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
used_norm: set[str] = set()
|
||||
for name, raw_value in raw_attributes.items():
|
||||
if raw_value is None or str(raw_value).strip() == "":
|
||||
continue
|
||||
fold = str(name).casefold().rstrip(":")
|
||||
norm_key = next((key for key, aliases in SPEC_ALIASES.items() if fold in aliases), None)
|
||||
norm_value = normalized_attributes.get(norm_key) if norm_key else None
|
||||
if norm_key:
|
||||
used_norm.add(norm_key)
|
||||
items.append(
|
||||
{
|
||||
"attribute_key": norm_key or fold.replace(" ", "_"),
|
||||
"raw_name": str(name).rstrip(":"),
|
||||
"raw_value": raw_value,
|
||||
"normalized_value": norm_value,
|
||||
}
|
||||
)
|
||||
for key, value in normalized_attributes.items():
|
||||
if key in used_norm or value is None:
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"attribute_key": key,
|
||||
"raw_name": key,
|
||||
"raw_value": None,
|
||||
"normalized_value": value,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def is_derived_duplicate(name: str, present: set[str]) -> bool:
|
||||
fold = name.casefold()
|
||||
aliases = SPEC_ALIASES.get(fold)
|
||||
return bool(aliases and present.intersection(aliases))
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.validation import DomainValidationError, canonical_json, sha256_text
|
||||
|
||||
|
||||
def parse_project_criteria(definition: dict[str, Any]) -> dict[str, Any]:
|
||||
items = definition.get("items")
|
||||
if not isinstance(items, list) or not items:
|
||||
raise DomainValidationError("criteria.items must be a non-empty list")
|
||||
seen: set[str] = set()
|
||||
canonical_items: list[dict[str, Any]] = []
|
||||
for raw in items:
|
||||
criterion_id = raw.get("criterion_definition_id")
|
||||
key = raw.get("criterion")
|
||||
item_id = str(raw.get("id") or criterion_id or key or "")
|
||||
if not item_id:
|
||||
raise DomainValidationError("each criterion needs id, criterion_definition_id or criterion")
|
||||
if item_id in seen:
|
||||
raise DomainValidationError("duplicate criterion id", {"id": item_id})
|
||||
seen.add(item_id)
|
||||
if not criterion_id and not key:
|
||||
raise DomainValidationError("criterion_definition_id or criterion is required", {"id": item_id})
|
||||
if not raw.get("operator"):
|
||||
raise DomainValidationError("operator is required", {"id": item_id})
|
||||
enabled = raw.get("enabled_for_discovery")
|
||||
if enabled is None:
|
||||
enabled = raw.get("mandatory", True)
|
||||
canonical_items.append(
|
||||
{
|
||||
"id": item_id,
|
||||
"criterion_definition_id": None if criterion_id is None else UUID(str(criterion_id)),
|
||||
"criterion": None if key is None else str(key),
|
||||
"operator": str(raw["operator"]),
|
||||
"value": raw.get("value"),
|
||||
"enabled_for_discovery": bool(enabled),
|
||||
"mandatory": bool(enabled),
|
||||
"value_type": raw.get("value_type"),
|
||||
"unit": raw.get("unit"),
|
||||
"semantic": raw.get("semantic"),
|
||||
"category": raw.get("category"),
|
||||
}
|
||||
)
|
||||
canonical = {
|
||||
"items": [
|
||||
{
|
||||
"id": item["id"],
|
||||
"criterion_definition_id": None
|
||||
if item["criterion_definition_id"] is None
|
||||
else str(item["criterion_definition_id"]),
|
||||
"criterion": item["criterion"],
|
||||
"operator": item["operator"],
|
||||
"value": item["value"],
|
||||
"enabled_for_discovery": item["enabled_for_discovery"],
|
||||
}
|
||||
for item in canonical_items
|
||||
]
|
||||
}
|
||||
return {
|
||||
"items": canonical_items,
|
||||
"definition": canonical,
|
||||
"definition_hash": sha256_text(canonical_json(canonical)),
|
||||
}
|
||||
|
||||
|
||||
def criterion_payload(row) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.item_id,
|
||||
"criterion_definition_id": row.criterion_definition_id,
|
||||
"criterion": row.criterion_key,
|
||||
"operator": row.operator,
|
||||
"value": row.value,
|
||||
"enabled_for_discovery": row.enabled_for_discovery,
|
||||
"mandatory": row.enabled_for_discovery,
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.persistence.discovery_models import CriterionDefinitionRecord
|
||||
|
||||
|
||||
def get_definition_by_key(session: Session, key: str) -> CriterionDefinitionRecord | None:
|
||||
return session.scalars(select(CriterionDefinitionRecord).where(CriterionDefinitionRecord.key == key)).first()
|
||||
|
||||
|
||||
def get_definition(session: Session, definition_id: UUID) -> CriterionDefinitionRecord | None:
|
||||
return session.get(CriterionDefinitionRecord, definition_id)
|
||||
|
||||
|
||||
def upsert_definition(session: Session, payload: dict[str, Any]) -> CriterionDefinitionRecord:
|
||||
key = str(payload["key"])
|
||||
existing = get_definition_by_key(session, key)
|
||||
if existing is not None:
|
||||
if payload.get("label"):
|
||||
existing.label = str(payload["label"])
|
||||
if payload.get("scoring_attribute_key"):
|
||||
existing.scoring_attribute_key = str(payload["scoring_attribute_key"])
|
||||
if payload.get("product_category_id"):
|
||||
existing.product_category_id = str(payload["product_category_id"])
|
||||
if payload.get("origin"):
|
||||
existing.origin = str(payload["origin"])
|
||||
if payload.get("source"):
|
||||
existing.source = str(payload["source"])
|
||||
return existing
|
||||
row = CriterionDefinitionRecord(
|
||||
id=payload.get("id") or uuid4(),
|
||||
key=key,
|
||||
origin=str(payload.get("origin") or "SOURCE"),
|
||||
source=payload.get("source"),
|
||||
product_category_id=payload.get("product_category_id"),
|
||||
label=str(payload.get("label") or key),
|
||||
value_type=str(payload["value_type"]),
|
||||
unit=payload.get("unit"),
|
||||
semantic=payload.get("semantic"),
|
||||
category=payload.get("category") or payload.get("product_category_id"),
|
||||
description=payload.get("description"),
|
||||
scoring_attribute_key=payload.get("scoring_attribute_key"),
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.discovery.enums import DiffStatus, LifecycleStatus, RunStatus
|
||||
|
||||
|
||||
RELEVANT_ATTR_IGNORE = {"title", "html", "presentation", "raw_title"}
|
||||
|
||||
|
||||
def observations_changed(previous: dict[str, Any] | None, current: dict[str, Any] | None) -> bool:
|
||||
if previous is None or current is None:
|
||||
return previous != current
|
||||
if _money(previous.get("best_price")) != _money(current.get("best_price")):
|
||||
return True
|
||||
if previous.get("availability") != current.get("availability"):
|
||||
return True
|
||||
if previous.get("seller_count") != current.get("seller_count"):
|
||||
return True
|
||||
prev_attrs = _technical(previous.get("normalized_attributes") or {})
|
||||
curr_attrs = _technical(current.get("normalized_attributes") or {})
|
||||
return prev_attrs != curr_attrs
|
||||
|
||||
|
||||
def _money(value: Any) -> Decimal | None:
|
||||
if value is None:
|
||||
return None
|
||||
return Decimal(str(value))
|
||||
|
||||
|
||||
def _technical(attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
return {key: attrs[key] for key in sorted(attrs) if key not in RELEVANT_ATTR_IGNORE}
|
||||
|
||||
|
||||
def diff_run(
|
||||
*,
|
||||
previous_ids: set[Any],
|
||||
current_ids: set[Any],
|
||||
previous_obs: dict[Any, dict[str, Any]],
|
||||
current_obs: dict[Any, dict[str, Any]],
|
||||
run_status: str,
|
||||
) -> dict[Any, DiffStatus]:
|
||||
result: dict[Any, DiffStatus] = {}
|
||||
for ref_id in current_ids:
|
||||
if ref_id not in previous_ids:
|
||||
result[ref_id] = DiffStatus.NEW
|
||||
elif observations_changed(previous_obs.get(ref_id), current_obs.get(ref_id)):
|
||||
result[ref_id] = DiffStatus.CHANGED
|
||||
else:
|
||||
result[ref_id] = DiffStatus.UNCHANGED
|
||||
if run_status == RunStatus.SUCCESS.value:
|
||||
for ref_id in previous_ids - current_ids:
|
||||
result[ref_id] = DiffStatus.MISSING
|
||||
return result
|
||||
|
||||
|
||||
def next_lifecycle(
|
||||
*,
|
||||
current: str,
|
||||
missing_in_success_run: bool,
|
||||
consecutive_missing: int,
|
||||
reappeared: bool,
|
||||
) -> tuple[str, int]:
|
||||
if current == LifecycleStatus.ARCHIVED.value:
|
||||
return current, consecutive_missing
|
||||
if reappeared:
|
||||
return LifecycleStatus.ACTIVE.value, 0
|
||||
if missing_in_success_run:
|
||||
streak = consecutive_missing + 1
|
||||
if streak >= 3:
|
||||
return LifecycleStatus.DISAPPEARED.value, streak
|
||||
return LifecycleStatus.ACTIVE.value, streak
|
||||
return current, consecutive_missing
|
||||
@@ -0,0 +1,87 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ProjectStatus(StrEnum):
|
||||
ACTIVE = "ACTIVE"
|
||||
PURCHASED = "PURCHASED"
|
||||
ARCHIVED = "ARCHIVED"
|
||||
|
||||
|
||||
class TaxonomyStatus(StrEnum):
|
||||
NOT_LEARNED = "NOT_LEARNED"
|
||||
LEARNING = "LEARNING"
|
||||
READY = "READY"
|
||||
DEGRADED = "DEGRADED"
|
||||
BROKEN = "BROKEN"
|
||||
BLOCKED = "BLOCKED"
|
||||
|
||||
|
||||
class MappingStatus(StrEnum):
|
||||
DISCOVERED = "DISCOVERED"
|
||||
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
|
||||
VERIFIED = "VERIFIED"
|
||||
BROKEN = "BROKEN"
|
||||
DISABLED = "DISABLED"
|
||||
|
||||
|
||||
class FilterType(StrEnum):
|
||||
BOOLEAN = "BOOLEAN"
|
||||
ENUM = "ENUM"
|
||||
MULTI_ENUM = "MULTI_ENUM"
|
||||
NUMERIC_RANGE = "NUMERIC_RANGE"
|
||||
NUMERIC_THRESHOLD = "NUMERIC_THRESHOLD"
|
||||
TEXT = "TEXT"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class CriterionOrigin(StrEnum):
|
||||
SOURCE = "SOURCE"
|
||||
INTERNAL = "INTERNAL"
|
||||
|
||||
|
||||
class SearchStatus(StrEnum):
|
||||
ACTIVE = "ACTIVE"
|
||||
OUT_OF_SYNC = "OUT_OF_SYNC"
|
||||
BLOCKED = "BLOCKED"
|
||||
DISABLED = "DISABLED"
|
||||
TOO_BROAD = "TOO_BROAD"
|
||||
PAUSED = "PAUSED"
|
||||
|
||||
|
||||
class RunStatus(StrEnum):
|
||||
RUNNING = "RUNNING"
|
||||
SUCCESS = "SUCCESS"
|
||||
PARTIAL = "PARTIAL"
|
||||
FAILED = "FAILED"
|
||||
BLOCKED = "BLOCKED"
|
||||
TOO_BROAD = "TOO_BROAD"
|
||||
|
||||
|
||||
class RankingStatus(StrEnum):
|
||||
NOT_CONFIGURED = "NOT_CONFIGURED"
|
||||
CONFIGURED = "CONFIGURED"
|
||||
|
||||
|
||||
class DiffStatus(StrEnum):
|
||||
NEW = "NEW"
|
||||
CHANGED = "CHANGED"
|
||||
UNCHANGED = "UNCHANGED"
|
||||
MISSING = "MISSING"
|
||||
|
||||
|
||||
class LifecycleStatus(StrEnum):
|
||||
ACTIVE = "ACTIVE"
|
||||
DISAPPEARED = "DISAPPEARED"
|
||||
ARCHIVED = "ARCHIVED"
|
||||
|
||||
|
||||
class MatchStatus(StrEnum):
|
||||
MATCHED = "MATCHED"
|
||||
MATCH_UNCERTAIN = "MATCH_UNCERTAIN"
|
||||
UNCERTAIN = "MATCH_UNCERTAIN"
|
||||
NEW_REFERENCE = "NEW_REFERENCE"
|
||||
|
||||
|
||||
class CandidateStatus(StrEnum):
|
||||
CANDIDATE = "CANDIDATE"
|
||||
SHORTLISTED = "SHORTLISTED"
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.discovery.enums import MatchStatus
|
||||
|
||||
PRESENTATION_KEYS = {"title", "html", "presentation", "raw_title"}
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"\s+", " ", title.casefold().strip())
|
||||
|
||||
|
||||
def spec_fingerprint(normalized_attributes: dict[str, Any], keys: tuple[str, ...] | None = None) -> str:
|
||||
"""Fingerprint from whatever technical keys the observation actually has."""
|
||||
use = keys if keys is not None else tuple(sorted(key for key in normalized_attributes if key not in PRESENTATION_KEYS))
|
||||
parts = []
|
||||
for key in use:
|
||||
value = normalized_attributes.get(key)
|
||||
parts.append(f"{key}={'' if value is None else str(value).casefold().strip()}")
|
||||
return "|".join(parts)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IdentitySignals:
|
||||
source_external_id: str | None
|
||||
gtin_ean: str | None
|
||||
manufacturer_sku: str | None
|
||||
source_url: str | None
|
||||
title: str
|
||||
manufacturer: str | None
|
||||
model: str | None
|
||||
fingerprint: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IdentityDecision:
|
||||
match_status: MatchStatus
|
||||
existing_id: Any | None
|
||||
reason: str
|
||||
|
||||
|
||||
def decide_identity(incoming: IdentitySignals, known: list[tuple[Any, IdentitySignals]]) -> IdentityDecision:
|
||||
"""Conservative matching. Similar titles never merge by themselves."""
|
||||
for ref_id, existing in known:
|
||||
if incoming.source_external_id and incoming.source_external_id == existing.source_external_id:
|
||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_external_id")
|
||||
if incoming.gtin_ean and incoming.gtin_ean == existing.gtin_ean:
|
||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "gtin_ean")
|
||||
if incoming.manufacturer_sku and incoming.manufacturer_sku == existing.manufacturer_sku:
|
||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "manufacturer_sku")
|
||||
if incoming.source_url and incoming.source_url == existing.source_url:
|
||||
return IdentityDecision(MatchStatus.MATCHED, ref_id, "source_url")
|
||||
|
||||
title_hits = [
|
||||
(ref_id, existing)
|
||||
for ref_id, existing in known
|
||||
if normalize_title(incoming.title) == normalize_title(existing.title)
|
||||
]
|
||||
if len(title_hits) == 1 and incoming.fingerprint and incoming.fingerprint == title_hits[0][1].fingerprint:
|
||||
return IdentityDecision(MatchStatus.MATCHED, title_hits[0][0], "title+fingerprint")
|
||||
|
||||
manufacturer_model = [
|
||||
(ref_id, existing)
|
||||
for ref_id, existing in known
|
||||
if incoming.manufacturer
|
||||
and incoming.model
|
||||
and incoming.manufacturer.casefold() == (existing.manufacturer or "").casefold()
|
||||
and incoming.model.casefold() == (existing.model or "").casefold()
|
||||
]
|
||||
if len(manufacturer_model) == 1:
|
||||
existing = manufacturer_model[0][1]
|
||||
if incoming.fingerprint and existing.fingerprint and incoming.fingerprint != existing.fingerprint:
|
||||
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "manufacturer+model conflict with specs")
|
||||
return IdentityDecision(MatchStatus.MATCHED, manufacturer_model[0][0], "manufacturer+model")
|
||||
|
||||
if title_hits:
|
||||
return IdentityDecision(MatchStatus.MATCH_UNCERTAIN, None, "title only")
|
||||
return IdentityDecision(MatchStatus.NEW_REFERENCE, None, "no match")
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
from app.discovery.enums import MappingStatus
|
||||
from app.discovery.urlgen import decode_xf_roundtrip, generate_search_url
|
||||
|
||||
|
||||
def verify_roundtrip(
|
||||
*,
|
||||
category_url: str,
|
||||
criterion: dict[str, Any],
|
||||
mapping: dict[str, Any],
|
||||
) -> bool:
|
||||
"""True when encoding a single mapping round-trips to the same source parameter."""
|
||||
probe = {**mapping, "status": MappingStatus.VERIFIED.value}
|
||||
url, internal = generate_search_url(
|
||||
category_url=category_url,
|
||||
criteria_items=[criterion],
|
||||
mappings=[probe],
|
||||
)
|
||||
if internal:
|
||||
return False
|
||||
encoding = mapping.get("encoding") or {}
|
||||
param = str(encoding.get("url_param") or "xf")
|
||||
if param == "xf":
|
||||
expected_key = str(mapping["external_filter_key"])
|
||||
expected_value = str(mapping.get("source_value") or "")
|
||||
decoded = decode_xf_roundtrip(url)
|
||||
return any(key == expected_key and value == expected_value.replace("+", " ") for key, value in decoded)
|
||||
expected = str(criterion.get("value"))
|
||||
query = parse_qsl(urlsplit(url).query, keep_blank_values=True)
|
||||
return any(name == param and value == expected for name, value in query)
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app.clock import now_utc
|
||||
from app.domain.attributes import select_attribute_snapshot
|
||||
from app.domain.constraints import evaluate_constraint
|
||||
from app.domain.enums import Eligibility, NormalizationStatus
|
||||
from app.domain.models import AttributeStatement, Product
|
||||
from app.scoring.engine import _product_eligibility
|
||||
from app.scoring.mathutil import round_score
|
||||
from app.scoring.product import score_product
|
||||
|
||||
|
||||
def rank_candidates(
|
||||
*,
|
||||
profile,
|
||||
scoring_model,
|
||||
definitions,
|
||||
candidates: list[dict[str, Any]],
|
||||
shortlist_size: int,
|
||||
as_of: datetime | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
scored: list[dict[str, Any]] = []
|
||||
when = as_of or now_utc()
|
||||
for candidate in candidates:
|
||||
attrs = _statements(candidate, when)
|
||||
snapshot = select_attribute_snapshot(
|
||||
attrs,
|
||||
as_of=when,
|
||||
snapshot_cutoff=when,
|
||||
)
|
||||
constraint_results = tuple(
|
||||
evaluate_constraint(
|
||||
constraint,
|
||||
snapshot.get(constraint.attribute),
|
||||
definitions[constraint.attribute],
|
||||
scoring_model.minimum_confidence,
|
||||
)
|
||||
for constraint in profile.constraints
|
||||
if constraint.attribute in definitions
|
||||
)
|
||||
product = Product(
|
||||
id=uuid4(),
|
||||
category=profile.category,
|
||||
brand=str(candidate.get("manufacturer") or "unknown"),
|
||||
name=str(candidate.get("source_title") or candidate.get("name") or "candidate"),
|
||||
model=candidate.get("model"),
|
||||
created_at=when,
|
||||
)
|
||||
_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
||||
product_score, coverage, _components = score_product(
|
||||
profile.preferences,
|
||||
snapshot,
|
||||
constraint_results,
|
||||
scoring_model,
|
||||
)
|
||||
scored.append(
|
||||
{
|
||||
**candidate,
|
||||
"product_eligibility": product_eligibility.value,
|
||||
"product_score": round_score(product_score),
|
||||
"product_coverage": coverage,
|
||||
"eligible": product_eligibility == Eligibility.ELIGIBLE,
|
||||
}
|
||||
)
|
||||
scored.sort(key=lambda item: (not item["eligible"], -(item["product_score"] or Decimal("0"))))
|
||||
for index, item in enumerate(scored):
|
||||
item["candidate_status"] = "SHORTLISTED" if index < shortlist_size and item["eligible"] else "CANDIDATE"
|
||||
item["rank"] = index + 1
|
||||
return scored
|
||||
|
||||
|
||||
def _statements(candidate: dict[str, Any], when: datetime) -> tuple[AttributeStatement, ...]:
|
||||
raw = candidate.get("normalized_attributes") or {}
|
||||
now = when
|
||||
items = []
|
||||
variant_id = candidate.get("variant_id") or uuid4()
|
||||
for key, value in raw.items():
|
||||
items.append(
|
||||
AttributeStatement(
|
||||
id=uuid4(),
|
||||
variant_id=variant_id if isinstance(variant_id, UUID) else uuid4(),
|
||||
attribute=key,
|
||||
raw_value=value,
|
||||
normalized_value=value,
|
||||
unit=None,
|
||||
normalization_status=NormalizationStatus.NORMALIZED,
|
||||
normalizer_version="discovery-v1",
|
||||
confidence=Decimal("1"),
|
||||
source_id="geizhals",
|
||||
source_reference="discovery",
|
||||
observed_at=now,
|
||||
ingested_at=now,
|
||||
)
|
||||
)
|
||||
return tuple(items)
|
||||
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlsplit, urlunsplit
|
||||
|
||||
from app.discovery.enums import MappingStatus
|
||||
|
||||
|
||||
def generate_search_url(
|
||||
*,
|
||||
category_url: str,
|
||||
criteria_items: list[dict[str, Any]],
|
||||
mappings: list[dict[str, Any]],
|
||||
) -> tuple[str, list[str]]:
|
||||
"""Build a source URL from VERIFIED mappings only. INTERNAL_ONLY keys are skipped."""
|
||||
by_criterion = {item["criterion_key"]: item for item in mappings if item.get("status") == MappingStatus.VERIFIED.value}
|
||||
fragments: list[tuple[str, str]] = []
|
||||
internal_only: list[str] = []
|
||||
for criterion in criteria_items:
|
||||
key = str(criterion["criterion"])
|
||||
mapping = by_criterion.get(key)
|
||||
if mapping is None:
|
||||
internal_only.append(key)
|
||||
continue
|
||||
fragments.extend(_encode_mapping(mapping, criterion))
|
||||
fragments.sort()
|
||||
base = urlsplit(category_url)
|
||||
query = dict(parse_qsl(base.query, keep_blank_values=True))
|
||||
extra: list[tuple[str, str]] = []
|
||||
xf_values: list[str] = []
|
||||
for name, value in fragments:
|
||||
if name == "xf":
|
||||
xf_values.append(value)
|
||||
continue
|
||||
if name in query:
|
||||
query[name] = value
|
||||
else:
|
||||
extra.append((name, value))
|
||||
if xf_values:
|
||||
extra.append(("xf", "~".join(xf_values)))
|
||||
encoded = urlencode(list(query.items()) + extra, doseq=True, safe="+", quote_via=quote)
|
||||
url = urlunsplit((base.scheme, base.netloc, base.path, encoded, base.fragment))
|
||||
return url, internal_only
|
||||
|
||||
|
||||
def _encode_mapping(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
encoding = mapping.get("encoding") or {}
|
||||
param = str(encoding.get("url_param") or "xf")
|
||||
if encoding.get("encoding_type") == "RANGE_PARAMETERS" or param in {"bpmin", "bpmax"}:
|
||||
return _encode_range(encoding, criterion)
|
||||
if encoding.get("encoding_type") == "RANGE_TOKEN":
|
||||
return _encode_range_token(mapping, criterion)
|
||||
if param == "xf":
|
||||
return _encode_xf(mapping, criterion)
|
||||
if param in {"minp", "maxp", "prmin", "prmax", "asuch", "fs"}:
|
||||
return [(param, str(criterion.get("value")))]
|
||||
template = encoding.get("value_template")
|
||||
if template:
|
||||
return [(param, str(template).format(value=criterion.get("value"), **mapping))]
|
||||
raise ValueError(f"unsupported encoding for mapping {mapping.get('criterion_key')}")
|
||||
|
||||
|
||||
def _encode_range(encoding: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
operator = str(criterion.get("operator") or "<=")
|
||||
value = criterion.get("value")
|
||||
min_param = str(encoding.get("min_parameter") or encoding.get("url_param") or "bpmin")
|
||||
max_param = str(encoding.get("max_parameter") or "bpmax")
|
||||
if operator == "between":
|
||||
if isinstance(value, dict):
|
||||
items: list[tuple[str, str]] = []
|
||||
if value.get("min") is not None:
|
||||
items.append((min_param, str(value["min"])))
|
||||
if value.get("max") is not None:
|
||||
items.append((max_param, str(value["max"])))
|
||||
return items
|
||||
if isinstance(value, list) and len(value) == 2:
|
||||
return [(min_param, str(value[0])), (max_param, str(value[1]))]
|
||||
if operator in {">=", ">"}:
|
||||
return [(min_param, str(value))]
|
||||
return [(max_param, str(value))]
|
||||
|
||||
|
||||
def _encode_range_token(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
||||
if looked_up is not None:
|
||||
return [("xf", f"{filter_key}_{_plus(looked_up)}")]
|
||||
operator = str(criterion.get("operator") or "<=")
|
||||
value = criterion.get("value")
|
||||
if operator == "between":
|
||||
if isinstance(value, dict):
|
||||
raw = f"{value.get('min') or ''}*{value.get('max') or ''}"
|
||||
elif isinstance(value, list) and len(value) == 2:
|
||||
raw = f"{value[0]}*{value[1]}"
|
||||
else:
|
||||
raw = f"*{value}"
|
||||
elif operator in {">=", ">"}:
|
||||
raw = f"{value}*"
|
||||
else:
|
||||
raw = f"*{value}"
|
||||
return [("xf", f"{filter_key}_{raw}")]
|
||||
|
||||
|
||||
def _lookup_option(mapping: dict[str, Any], value: Any) -> str | None:
|
||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
||||
if value in options:
|
||||
return str(options[value])
|
||||
if str(value) in options:
|
||||
return str(options[str(value)])
|
||||
return None
|
||||
|
||||
|
||||
def _encode_xf(mapping: dict[str, Any], criterion: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
filter_key = str(mapping["external_filter_key"])
|
||||
raw = criterion.get("value")
|
||||
values = raw if isinstance(raw, list) else [raw]
|
||||
fragments: list[tuple[str, str]] = []
|
||||
for value in values:
|
||||
if value is None or value == "":
|
||||
continue
|
||||
source_value = _xf_value(mapping, {**criterion, "value": value})
|
||||
fragments.append(("xf", f"{filter_key}_{_plus(source_value)}"))
|
||||
if fragments:
|
||||
return fragments
|
||||
source_value = _xf_value(mapping, criterion)
|
||||
return [("xf", f"{filter_key}_{_plus(source_value)}")]
|
||||
|
||||
|
||||
def _xf_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||
looked_up = _lookup_option(mapping, criterion.get("value"))
|
||||
if looked_up is not None:
|
||||
return looked_up
|
||||
options = (mapping.get("encoding") or {}).get("options") or {}
|
||||
value = criterion.get("value")
|
||||
if options and value is not None and value != "":
|
||||
return str(value)
|
||||
return str(mapping.get("source_value") or _threshold_value(mapping, criterion))
|
||||
|
||||
|
||||
def _threshold_value(mapping: dict[str, Any], criterion: dict[str, Any]) -> str:
|
||||
option = mapping.get("source_value")
|
||||
if option:
|
||||
return str(option)
|
||||
value = criterion.get("value")
|
||||
unit = mapping.get("unit") or ""
|
||||
operator = criterion.get("operator")
|
||||
if operator in {">=", ">"} and unit:
|
||||
return f"ab {value}{unit}"
|
||||
return str(value)
|
||||
|
||||
|
||||
def _plus(value: str) -> str:
|
||||
return value.replace(" ", "+")
|
||||
|
||||
|
||||
def decode_xf_roundtrip(url: str) -> list[tuple[str, str]]:
|
||||
parts = urlsplit(url)
|
||||
decoded: list[tuple[str, str]] = []
|
||||
for name, value in parse_qsl(parts.query, keep_blank_values=True):
|
||||
if name != "xf":
|
||||
continue
|
||||
for part in value.replace("|", "~").split("~"):
|
||||
if "_" not in part:
|
||||
continue
|
||||
key, raw = part.split("_", 1)
|
||||
decoded.append((key, raw.replace("+", " ")))
|
||||
return decoded
|
||||
|
||||
|
||||
def same_origin_url(base: str, maybe_relative: str) -> str:
|
||||
return urljoin(base, maybe_relative)
|
||||
Whitespace-only changes.
@@ -0,0 +1,45 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Iterable
|
||||
|
||||
from app.domain.enums import NormalizationStatus
|
||||
from app.domain.models import AttributeStatement
|
||||
|
||||
|
||||
def statement_is_usable(statement: AttributeStatement | None, minimum_confidence: Decimal) -> bool:
|
||||
if statement is None:
|
||||
return False
|
||||
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
||||
return False
|
||||
if statement.normalized_value is None:
|
||||
return False
|
||||
return statement.confidence >= minimum_confidence
|
||||
|
||||
|
||||
def select_attribute_snapshot(
|
||||
statements: Iterable[AttributeStatement],
|
||||
*,
|
||||
as_of: datetime,
|
||||
snapshot_cutoff: datetime,
|
||||
) -> dict[str, AttributeStatement]:
|
||||
"""Pick the newest statement per key at or before the evaluation cutoff."""
|
||||
chosen: dict[str, AttributeStatement] = {}
|
||||
for statement in statements:
|
||||
if statement.observed_at > as_of or statement.ingested_at > snapshot_cutoff:
|
||||
continue
|
||||
current = chosen.get(statement.attribute)
|
||||
if current is None or _is_newer(statement, current):
|
||||
chosen[statement.attribute] = statement
|
||||
return chosen
|
||||
|
||||
|
||||
def _is_newer(candidate: AttributeStatement, current: AttributeStatement) -> bool:
|
||||
return (
|
||||
candidate.observed_at,
|
||||
candidate.ingested_at,
|
||||
candidate.id,
|
||||
) > (
|
||||
current.observed_at,
|
||||
current.ingested_at,
|
||||
current.id,
|
||||
)
|
||||
@@ -0,0 +1,148 @@
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.attributes import statement_is_usable
|
||||
from app.domain.enums import ConstraintOperator, ConstraintStatus, ValueType
|
||||
from app.domain.models import AttributeDefinition, AttributeStatement, Constraint, ConstraintResult
|
||||
|
||||
_NUMBER_OPS = {
|
||||
ConstraintOperator.GE,
|
||||
ConstraintOperator.LE,
|
||||
ConstraintOperator.GT,
|
||||
ConstraintOperator.LT,
|
||||
}
|
||||
_EQUALITY_OPS = {ConstraintOperator.EQ, ConstraintOperator.NE}
|
||||
_SCALAR_TYPES = {ValueType.NUMBER, ValueType.STRING, ValueType.BOOLEAN}
|
||||
|
||||
|
||||
def operator_allowed_for(value_type: ValueType, operator: ConstraintOperator) -> bool:
|
||||
if operator == ConstraintOperator.EXISTS:
|
||||
return True
|
||||
if operator in _NUMBER_OPS:
|
||||
return value_type == ValueType.NUMBER
|
||||
if operator in _EQUALITY_OPS:
|
||||
return value_type in _SCALAR_TYPES
|
||||
if operator == ConstraintOperator.IN:
|
||||
return value_type in _SCALAR_TYPES
|
||||
if operator == ConstraintOperator.CONTAINS:
|
||||
return value_type == ValueType.STRING_LIST
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_constraint(
|
||||
constraint: Constraint,
|
||||
statement: AttributeStatement | None,
|
||||
definition: AttributeDefinition,
|
||||
minimum_confidence: Decimal,
|
||||
) -> ConstraintResult:
|
||||
usable = statement_is_usable(statement, minimum_confidence)
|
||||
actual = statement.normalized_value if usable else None
|
||||
attribute_row_id = statement.id if statement is not None else None
|
||||
|
||||
if not usable:
|
||||
return ConstraintResult(
|
||||
constraint_id=constraint.id,
|
||||
attribute=constraint.attribute,
|
||||
operator=constraint.operator.value,
|
||||
expected=constraint.value,
|
||||
actual=None,
|
||||
status=ConstraintStatus.UNKNOWN.value,
|
||||
mandatory=constraint.mandatory,
|
||||
attribute_row_id=attribute_row_id,
|
||||
reason_code="ATTRIBUTE_UNUSABLE",
|
||||
)
|
||||
|
||||
passed = _compare(constraint.operator, actual, constraint.value, definition.value_type)
|
||||
if passed is None:
|
||||
status = ConstraintStatus.UNKNOWN
|
||||
reason = "TYPE_MISMATCH"
|
||||
elif passed:
|
||||
status = ConstraintStatus.PASS
|
||||
reason = "CONSTRAINT_PASS"
|
||||
else:
|
||||
status = ConstraintStatus.FAIL
|
||||
reason = "CONSTRAINT_FAIL"
|
||||
return ConstraintResult(
|
||||
constraint_id=constraint.id,
|
||||
attribute=constraint.attribute,
|
||||
operator=constraint.operator.value,
|
||||
expected=constraint.value,
|
||||
actual=actual,
|
||||
status=status.value,
|
||||
mandatory=constraint.mandatory,
|
||||
attribute_row_id=attribute_row_id,
|
||||
reason_code=reason,
|
||||
)
|
||||
|
||||
|
||||
def _as_number(value: Any) -> Decimal | None:
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
if isinstance(value, int):
|
||||
return Decimal(value)
|
||||
if isinstance(value, float):
|
||||
return Decimal(str(value))
|
||||
if isinstance(value, str):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _same_scalar(left: Any, right: Any, value_type: ValueType) -> bool | None:
|
||||
if value_type == ValueType.NUMBER:
|
||||
left_n = _as_number(left)
|
||||
right_n = _as_number(right)
|
||||
if left_n is None or right_n is None:
|
||||
return None
|
||||
return left_n == right_n
|
||||
if value_type == ValueType.STRING:
|
||||
if not isinstance(left, str) or not isinstance(right, str):
|
||||
return None
|
||||
return left == right
|
||||
if value_type == ValueType.BOOLEAN:
|
||||
if not isinstance(left, bool) or not isinstance(right, bool):
|
||||
return None
|
||||
return left == right
|
||||
return None
|
||||
|
||||
|
||||
def _compare(
|
||||
operator: ConstraintOperator,
|
||||
actual: Any,
|
||||
expected: Any,
|
||||
value_type: ValueType,
|
||||
) -> bool | None:
|
||||
if operator == ConstraintOperator.EXISTS:
|
||||
return True
|
||||
if operator in _NUMBER_OPS:
|
||||
actual_n = _as_number(actual)
|
||||
expected_n = _as_number(expected)
|
||||
if actual_n is None or expected_n is None:
|
||||
return None
|
||||
if operator == ConstraintOperator.GE:
|
||||
return actual_n >= expected_n
|
||||
if operator == ConstraintOperator.LE:
|
||||
return actual_n <= expected_n
|
||||
if operator == ConstraintOperator.GT:
|
||||
return actual_n > expected_n
|
||||
return actual_n < expected_n
|
||||
if operator == ConstraintOperator.EQ:
|
||||
return _same_scalar(actual, expected, value_type)
|
||||
if operator == ConstraintOperator.NE:
|
||||
equal = _same_scalar(actual, expected, value_type)
|
||||
return None if equal is None else (not equal)
|
||||
if operator == ConstraintOperator.IN:
|
||||
if not isinstance(expected, list) or not expected:
|
||||
return None
|
||||
matches = [_same_scalar(actual, item, value_type) for item in expected]
|
||||
if any(item is None for item in matches):
|
||||
return None
|
||||
return any(matches)
|
||||
if operator == ConstraintOperator.CONTAINS:
|
||||
if not isinstance(actual, list) or not isinstance(expected, str):
|
||||
return None
|
||||
if any(not isinstance(item, str) for item in actual):
|
||||
return None
|
||||
return expected in actual
|
||||
return None
|
||||
@@ -0,0 +1,81 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ValueType(StrEnum):
|
||||
NUMBER = "number"
|
||||
STRING = "string"
|
||||
BOOLEAN = "boolean"
|
||||
STRING_LIST = "string_list"
|
||||
|
||||
|
||||
class OfferCondition(StrEnum):
|
||||
NEW = "new"
|
||||
REFURBISHED = "refurbished"
|
||||
USED = "used"
|
||||
|
||||
|
||||
class Availability(StrEnum):
|
||||
IN_STOCK = "in_stock"
|
||||
OUT_OF_STOCK = "out_of_stock"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class NormalizationStatus(StrEnum):
|
||||
NORMALIZED = "normalized"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ConstraintStatus(StrEnum):
|
||||
PASS = "PASS"
|
||||
FAIL = "FAIL"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class Eligibility(StrEnum):
|
||||
ELIGIBLE = "ELIGIBLE"
|
||||
INELIGIBLE = "INELIGIBLE"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class PreferenceStrategy(StrEnum):
|
||||
NUMERIC_LINEAR = "numeric_linear"
|
||||
|
||||
|
||||
class PreferenceDirection(StrEnum):
|
||||
HIGHER = "higher"
|
||||
LOWER = "lower"
|
||||
|
||||
|
||||
class ConstraintOperator(StrEnum):
|
||||
GE = ">="
|
||||
LE = "<="
|
||||
GT = ">"
|
||||
LT = "<"
|
||||
EQ = "=="
|
||||
NE = "!="
|
||||
IN = "in"
|
||||
CONTAINS = "contains"
|
||||
EXISTS = "exists"
|
||||
|
||||
|
||||
class Classification(StrEnum):
|
||||
IGNORE = "IGNORE"
|
||||
WATCH = "WATCH"
|
||||
GOOD = "GOOD"
|
||||
VERY_GOOD = "VERY_GOOD"
|
||||
EXCEPTIONAL = "EXCEPTIONAL"
|
||||
INELIGIBLE = "INELIGIBLE"
|
||||
INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
|
||||
|
||||
|
||||
class PriceBasis(StrEnum):
|
||||
BUDGET_ONLY = "BUDGET_ONLY"
|
||||
BUDGET_AND_HISTORY = "BUDGET_AND_HISTORY"
|
||||
|
||||
|
||||
class PriceReferenceScope(StrEnum):
|
||||
OFFER = "offer"
|
||||
|
||||
|
||||
class AdapterType(StrEnum):
|
||||
MANUAL = "manual"
|
||||
@@ -0,0 +1,275 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.enums import (
|
||||
Availability,
|
||||
ConstraintOperator,
|
||||
Eligibility,
|
||||
NormalizationStatus,
|
||||
OfferCondition,
|
||||
PreferenceDirection,
|
||||
PreferenceStrategy,
|
||||
PriceBasis,
|
||||
PriceReferenceScope,
|
||||
ValueType,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Budget:
|
||||
target: Decimal
|
||||
maximum: Decimal
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Constraint:
|
||||
id: str
|
||||
attribute: str
|
||||
operator: ConstraintOperator
|
||||
value: Any = None
|
||||
mandatory: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Preference:
|
||||
id: str
|
||||
attribute: str
|
||||
strategy: PreferenceStrategy
|
||||
direction: PreferenceDirection
|
||||
acceptable: Decimal
|
||||
ideal: Decimal
|
||||
weight: Decimal
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClassificationBound:
|
||||
upper: Decimal
|
||||
inclusive: bool
|
||||
label: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoringModel:
|
||||
version: str
|
||||
algorithm_id: str
|
||||
supported_preference_strategies: tuple[str, ...]
|
||||
minimum_confidence: Decimal
|
||||
maximum_price_age_hours: int
|
||||
optional_constraint_penalty_points: Decimal
|
||||
missing_preference_score: Decimal
|
||||
product_weight: Decimal
|
||||
price_weight: Decimal
|
||||
budget_weight: Decimal
|
||||
history_weight: Decimal
|
||||
history_window_days: int
|
||||
minimum_history_days: int
|
||||
history_discount_for_max_score: Decimal
|
||||
classification_upper_bounds: tuple[ClassificationBound, ...]
|
||||
definition: dict[str, Any]
|
||||
definition_hash: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Profile:
|
||||
id: str
|
||||
version: int
|
||||
name: str
|
||||
category: str
|
||||
currency: str
|
||||
market: str
|
||||
allowed_conditions: tuple[OfferCondition, ...]
|
||||
budget: Budget
|
||||
scoring_version: str
|
||||
constraints: tuple[Constraint, ...]
|
||||
preferences: tuple[Preference, ...]
|
||||
definition: dict[str, Any]
|
||||
definition_hash: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AttributeDefinition:
|
||||
key: str
|
||||
value_type: ValueType
|
||||
canonical_unit: str | None
|
||||
minimum: Decimal | None
|
||||
maximum: Decimal | None
|
||||
description: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Product:
|
||||
id: UUID
|
||||
category: str
|
||||
brand: str
|
||||
name: str
|
||||
model: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProductVariant:
|
||||
id: UUID
|
||||
product_id: UUID
|
||||
name: str
|
||||
manufacturer_sku: str | None
|
||||
identifiers: dict[str, Any]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AttributeStatement:
|
||||
id: UUID
|
||||
variant_id: UUID
|
||||
attribute: str
|
||||
raw_value: Any
|
||||
normalized_value: Any
|
||||
unit: str | None
|
||||
normalization_status: NormalizationStatus
|
||||
normalizer_version: str
|
||||
confidence: Decimal
|
||||
source_id: str
|
||||
source_reference: str
|
||||
observed_at: datetime
|
||||
ingested_at: datetime
|
||||
normalization_error: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Offer:
|
||||
id: UUID
|
||||
variant_id: UUID
|
||||
source_id: str
|
||||
external_id: str
|
||||
seller: str
|
||||
url: str | None
|
||||
market: str
|
||||
currency: str
|
||||
condition: OfferCondition
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PriceObservation:
|
||||
id: UUID
|
||||
offer_id: UUID
|
||||
source_id: str
|
||||
external_observation_id: str
|
||||
observed_at: datetime
|
||||
ingested_at: datetime
|
||||
item_price: Decimal
|
||||
shipping_price: Decimal | None
|
||||
currency: str
|
||||
availability: Availability
|
||||
raw_payload: Any
|
||||
confidence: Decimal
|
||||
|
||||
@property
|
||||
def total_price(self) -> Decimal | None:
|
||||
if self.shipping_price is None:
|
||||
return None
|
||||
return self.item_price + self.shipping_price
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConstraintResult:
|
||||
constraint_id: str
|
||||
attribute: str
|
||||
operator: str
|
||||
expected: Any
|
||||
actual: Any
|
||||
status: str
|
||||
mandatory: bool
|
||||
attribute_row_id: UUID | None
|
||||
reason_code: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GateResult:
|
||||
key: str
|
||||
status: str
|
||||
reason_code: str
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoreComponent:
|
||||
kind: str
|
||||
component_key: str
|
||||
raw_score: Decimal | None
|
||||
weight: Decimal | None
|
||||
contribution: Decimal | None
|
||||
details: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HistoryWindow:
|
||||
days: int
|
||||
values: tuple[Decimal, ...]
|
||||
observation_ids: tuple[UUID, ...]
|
||||
|
||||
@property
|
||||
def day_count(self) -> int:
|
||||
return len(self.values)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PriceStatistics:
|
||||
current_total: Decimal | None
|
||||
median_30d: Decimal | None
|
||||
median_30d_days: int
|
||||
median_90d: Decimal | None
|
||||
median_90d_days: int
|
||||
lowest_365d: Decimal | None
|
||||
lowest_365d_days: int
|
||||
lowest_365d_is_full_year: bool
|
||||
used_observation_ids: tuple[UUID, ...]
|
||||
current_observation_id: UUID | None
|
||||
price_reference_scope: PriceReferenceScope = PriceReferenceScope.OFFER
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Reason:
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvaluationResult:
|
||||
profile_id: str
|
||||
profile_version: int
|
||||
scoring_version: str
|
||||
scoring_hash: str
|
||||
engine_build: str
|
||||
variant_id: UUID
|
||||
offer_id: UUID
|
||||
as_of: datetime
|
||||
snapshot_cutoff: datetime
|
||||
product_eligibility: Eligibility
|
||||
offer_eligibility: Eligibility
|
||||
overall_eligibility: Eligibility
|
||||
eligible: bool | None
|
||||
product_score: Decimal | None
|
||||
product_score_raw: Decimal | None
|
||||
price_score: Decimal | None
|
||||
price_score_raw: Decimal | None
|
||||
deal_score: Decimal | None
|
||||
deal_score_raw: Decimal | None
|
||||
classification: str
|
||||
price_basis: PriceBasis | None
|
||||
product_coverage: Decimal
|
||||
constraint_results: tuple[ConstraintResult, ...]
|
||||
product_gates: tuple[GateResult, ...]
|
||||
offer_gates: tuple[GateResult, ...]
|
||||
components: tuple[ScoreComponent, ...]
|
||||
price_statistics: PriceStatistics
|
||||
price_reference_scope: PriceReferenceScope
|
||||
reasons: tuple[Reason, ...]
|
||||
warnings: tuple[str, ...]
|
||||
input_snapshot: dict[str, Any]
|
||||
failed_constraints: tuple[ConstraintResult, ...]
|
||||
unknown_constraints: tuple[ConstraintResult, ...]
|
||||
@@ -0,0 +1,26 @@
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.timeutil import to_iso_z
|
||||
|
||||
|
||||
def jsonable(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (str, int, bool)):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
return format(value, "f")
|
||||
if isinstance(value, UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return to_iso_z(value)
|
||||
if isinstance(value, date):
|
||||
return value.isoformat()
|
||||
if isinstance(value, dict):
|
||||
return {str(key): jsonable(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [jsonable(item) for item in value]
|
||||
if hasattr(value, "value"):
|
||||
return jsonable(value.value)
|
||||
return value
|
||||
@@ -0,0 +1,11 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def require_utc(value: datetime, field_name: str) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError(f"{field_name} must include a timezone")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def to_iso_z(value: datetime) -> str:
|
||||
return require_utc(value, "datetime").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,307 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.constraints import operator_allowed_for
|
||||
from app.domain.enums import (
|
||||
ConstraintOperator,
|
||||
OfferCondition,
|
||||
PreferenceDirection,
|
||||
PreferenceStrategy,
|
||||
ValueType,
|
||||
)
|
||||
from app.domain.models import (
|
||||
AttributeDefinition,
|
||||
Budget,
|
||||
ClassificationBound,
|
||||
Constraint,
|
||||
Preference,
|
||||
Profile,
|
||||
ScoringModel,
|
||||
)
|
||||
|
||||
SNAKE_CASE = r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$"
|
||||
CURRENCY = r"^[A-Z]{3}$"
|
||||
SUPPORTED_ALGORITHM = "linear-budget-history-v1"
|
||||
|
||||
|
||||
class DomainValidationError(ValueError):
|
||||
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
def canonical_json(data: Any) -> str:
|
||||
return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _decimal(value: Any, field: str) -> Decimal:
|
||||
if isinstance(value, bool) or value is None:
|
||||
raise DomainValidationError(f"{field} must be a decimal", {"field": field})
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except Exception as exc:
|
||||
raise DomainValidationError(f"{field} must be a decimal", {"field": field}) from exc
|
||||
|
||||
|
||||
def _require(mapping: dict[str, Any], key: str) -> Any:
|
||||
if key not in mapping:
|
||||
raise DomainValidationError(f"missing field {key}", {"field": key})
|
||||
return mapping[key]
|
||||
|
||||
|
||||
def validate_attribute_key(key: str) -> None:
|
||||
import re
|
||||
|
||||
if not re.fullmatch(SNAKE_CASE, key):
|
||||
raise DomainValidationError("attribute key must be snake_case", {"key": key})
|
||||
|
||||
|
||||
def validate_currency(value: str) -> str:
|
||||
import re
|
||||
|
||||
if not re.fullmatch(CURRENCY, value):
|
||||
raise DomainValidationError("currency must be a 3-letter uppercase code", {"currency": value})
|
||||
return value
|
||||
|
||||
|
||||
def parse_scoring_model(definition: dict[str, Any]) -> ScoringModel:
|
||||
version = str(_require(definition, "version"))
|
||||
algorithm_id = str(_require(definition, "algorithm_id"))
|
||||
if algorithm_id != SUPPORTED_ALGORITHM:
|
||||
raise DomainValidationError("unsupported algorithm_id", {"algorithm_id": algorithm_id})
|
||||
strategies = tuple(str(item) for item in _require(definition, "supported_preference_strategies"))
|
||||
if not strategies:
|
||||
raise DomainValidationError("supported_preference_strategies must not be empty")
|
||||
if PreferenceStrategy.NUMERIC_LINEAR.value not in strategies:
|
||||
raise DomainValidationError("generic-v1 must support numeric_linear")
|
||||
|
||||
min_conf = _decimal(_require(definition, "minimum_confidence"), "minimum_confidence")
|
||||
if not Decimal("0") <= min_conf <= Decimal("1"):
|
||||
raise DomainValidationError("minimum_confidence must be between 0 and 1")
|
||||
|
||||
max_age = int(_require(definition, "maximum_price_age_hours"))
|
||||
history_window = int(_require(definition, "history_window_days"))
|
||||
min_history = int(_require(definition, "minimum_history_days"))
|
||||
if max_age <= 0 or history_window <= 0 or min_history <= 0:
|
||||
raise DomainValidationError("time windows must be positive")
|
||||
|
||||
product_weight = _decimal(_require(definition, "product_weight"), "product_weight")
|
||||
price_weight = _decimal(_require(definition, "price_weight"), "price_weight")
|
||||
budget_weight = _decimal(_require(definition, "budget_weight"), "budget_weight")
|
||||
history_weight = _decimal(_require(definition, "history_weight"), "history_weight")
|
||||
for name, weight in (
|
||||
("product_weight", product_weight),
|
||||
("price_weight", price_weight),
|
||||
("budget_weight", budget_weight),
|
||||
("history_weight", history_weight),
|
||||
):
|
||||
if weight < 0:
|
||||
raise DomainValidationError(f"{name} must be non-negative")
|
||||
if product_weight + price_weight != Decimal("1"):
|
||||
raise DomainValidationError("product_weight + price_weight must equal 1")
|
||||
if budget_weight + history_weight != Decimal("1"):
|
||||
raise DomainValidationError("budget_weight + history_weight must equal 1")
|
||||
|
||||
bounds_raw = _require(definition, "classification_upper_bounds")
|
||||
bounds: list[ClassificationBound] = []
|
||||
previous = Decimal("0")
|
||||
for index, raw in enumerate(bounds_raw):
|
||||
upper = _decimal(raw["upper"], f"classification_upper_bounds[{index}].upper")
|
||||
inclusive = bool(raw["inclusive"])
|
||||
label = str(raw["label"])
|
||||
if upper < previous or (index and upper == previous):
|
||||
raise DomainValidationError("classification bounds must be strictly ascending")
|
||||
if index == 0 and upper <= 0:
|
||||
raise DomainValidationError("classification bounds must start above 0")
|
||||
bounds.append(ClassificationBound(upper=upper, inclusive=inclusive, label=label))
|
||||
previous = upper
|
||||
if not bounds or bounds[-1].upper != Decimal("100"):
|
||||
raise DomainValidationError("classification bounds must end at 100")
|
||||
|
||||
canonical = json.loads(canonical_json(definition))
|
||||
return ScoringModel(
|
||||
version=version,
|
||||
algorithm_id=algorithm_id,
|
||||
supported_preference_strategies=strategies,
|
||||
minimum_confidence=min_conf,
|
||||
maximum_price_age_hours=max_age,
|
||||
optional_constraint_penalty_points=_decimal(
|
||||
_require(definition, "optional_constraint_penalty_points"),
|
||||
"optional_constraint_penalty_points",
|
||||
),
|
||||
missing_preference_score=_decimal(
|
||||
_require(definition, "missing_preference_score"),
|
||||
"missing_preference_score",
|
||||
),
|
||||
product_weight=product_weight,
|
||||
price_weight=price_weight,
|
||||
budget_weight=budget_weight,
|
||||
history_weight=history_weight,
|
||||
history_window_days=history_window,
|
||||
minimum_history_days=min_history,
|
||||
history_discount_for_max_score=_decimal(
|
||||
_require(definition, "history_discount_for_max_score"),
|
||||
"history_discount_for_max_score",
|
||||
),
|
||||
classification_upper_bounds=tuple(bounds),
|
||||
definition=canonical,
|
||||
definition_hash=sha256_text(canonical_json(canonical)),
|
||||
)
|
||||
|
||||
|
||||
def parse_profile(
|
||||
definition: dict[str, Any],
|
||||
*,
|
||||
attribute_definitions: dict[str, AttributeDefinition],
|
||||
scoring_versions: set[str],
|
||||
supported_strategies: set[str],
|
||||
) -> Profile:
|
||||
profile_id = str(_require(definition, "id"))
|
||||
version = int(_require(definition, "version"))
|
||||
if version <= 0:
|
||||
raise DomainValidationError("profile version must be > 0")
|
||||
name = str(_require(definition, "name"))
|
||||
category = str(_require(definition, "category"))
|
||||
if not category:
|
||||
raise DomainValidationError("category is required")
|
||||
currency = validate_currency(str(_require(definition, "currency")))
|
||||
market = str(_require(definition, "market"))
|
||||
if not market:
|
||||
raise DomainValidationError("market is required")
|
||||
raw_conditions = _require(definition, "allowed_conditions")
|
||||
if not raw_conditions:
|
||||
raise DomainValidationError("allowed_conditions is required")
|
||||
conditions = tuple(OfferCondition(item) for item in raw_conditions)
|
||||
scoring_version = str(_require(definition, "scoring_version"))
|
||||
if scoring_version not in scoring_versions:
|
||||
raise DomainValidationError("unknown scoring_version", {"scoring_version": scoring_version})
|
||||
|
||||
budget_raw = _require(definition, "budget")
|
||||
target = _decimal(_require(budget_raw, "target"), "budget.target")
|
||||
maximum = _decimal(_require(budget_raw, "maximum"), "budget.maximum")
|
||||
if not (Decimal("0") < target < maximum):
|
||||
raise DomainValidationError("budget must satisfy 0 < target < maximum")
|
||||
|
||||
constraints = _parse_constraints(_require(definition, "constraints"), attribute_definitions)
|
||||
preferences = _parse_preferences(
|
||||
_require(definition, "preferences"),
|
||||
attribute_definitions,
|
||||
supported_strategies,
|
||||
)
|
||||
canonical = json.loads(canonical_json(definition))
|
||||
return Profile(
|
||||
id=profile_id,
|
||||
version=version,
|
||||
name=name,
|
||||
category=category,
|
||||
currency=currency,
|
||||
market=market,
|
||||
allowed_conditions=conditions,
|
||||
budget=Budget(target=target, maximum=maximum),
|
||||
scoring_version=scoring_version,
|
||||
constraints=constraints,
|
||||
preferences=preferences,
|
||||
definition=canonical,
|
||||
definition_hash=sha256_text(canonical_json(canonical)),
|
||||
)
|
||||
|
||||
|
||||
def _parse_constraints(
|
||||
raw_items: list[dict[str, Any]],
|
||||
attribute_definitions: dict[str, AttributeDefinition],
|
||||
) -> tuple[Constraint, ...]:
|
||||
seen: set[str] = set()
|
||||
items: list[Constraint] = []
|
||||
for raw in raw_items:
|
||||
constraint_id = str(_require(raw, "id"))
|
||||
if constraint_id in seen:
|
||||
raise DomainValidationError("duplicate constraint id", {"id": constraint_id})
|
||||
seen.add(constraint_id)
|
||||
attribute = str(_require(raw, "attribute"))
|
||||
definition = attribute_definitions.get(attribute)
|
||||
if definition is None:
|
||||
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
||||
operator = ConstraintOperator(str(_require(raw, "operator")))
|
||||
if not operator_allowed_for(definition.value_type, operator):
|
||||
raise DomainValidationError(
|
||||
"operator is not valid for attribute type",
|
||||
{"attribute": attribute, "operator": operator.value, "value_type": definition.value_type.value},
|
||||
)
|
||||
value = raw.get("value")
|
||||
if operator == ConstraintOperator.EXISTS:
|
||||
if "value" in raw and raw["value"] is not None:
|
||||
raise DomainValidationError("exists must not have a value", {"id": constraint_id})
|
||||
value = None
|
||||
elif operator == ConstraintOperator.IN:
|
||||
if not isinstance(value, list) or not value:
|
||||
raise DomainValidationError("in requires a non-empty list", {"id": constraint_id})
|
||||
elif operator == ConstraintOperator.CONTAINS:
|
||||
if not isinstance(value, str):
|
||||
raise DomainValidationError("contains requires a string", {"id": constraint_id})
|
||||
items.append(
|
||||
Constraint(
|
||||
id=constraint_id,
|
||||
attribute=attribute,
|
||||
operator=operator,
|
||||
value=value,
|
||||
mandatory=bool(raw.get("mandatory", True)),
|
||||
)
|
||||
)
|
||||
return tuple(items)
|
||||
|
||||
|
||||
def _parse_preferences(
|
||||
raw_items: list[dict[str, Any]],
|
||||
attribute_definitions: dict[str, AttributeDefinition],
|
||||
supported_strategies: set[str],
|
||||
) -> tuple[Preference, ...]:
|
||||
if not raw_items:
|
||||
raise DomainValidationError("at least one preference is required")
|
||||
seen: set[str] = set()
|
||||
items: list[Preference] = []
|
||||
for raw in raw_items:
|
||||
preference_id = str(_require(raw, "id"))
|
||||
if preference_id in seen:
|
||||
raise DomainValidationError("duplicate preference id", {"id": preference_id})
|
||||
seen.add(preference_id)
|
||||
attribute = str(_require(raw, "attribute"))
|
||||
definition = attribute_definitions.get(attribute)
|
||||
if definition is None:
|
||||
raise DomainValidationError("unknown attribute key", {"attribute": attribute})
|
||||
strategy = str(_require(raw, "strategy"))
|
||||
if strategy not in supported_strategies:
|
||||
raise DomainValidationError("unsupported preference strategy", {"strategy": strategy})
|
||||
if strategy != PreferenceStrategy.NUMERIC_LINEAR.value:
|
||||
raise DomainValidationError("v0.1 only supports numeric_linear", {"strategy": strategy})
|
||||
if definition.value_type != ValueType.NUMBER:
|
||||
raise DomainValidationError("numeric_linear requires a number attribute", {"attribute": attribute})
|
||||
direction = PreferenceDirection(str(_require(raw, "direction")))
|
||||
acceptable = _decimal(_require(raw, "acceptable"), "acceptable")
|
||||
ideal = _decimal(_require(raw, "ideal"), "ideal")
|
||||
weight = _decimal(_require(raw, "weight"), "weight")
|
||||
if weight <= 0:
|
||||
raise DomainValidationError("preference weight must be positive", {"id": preference_id})
|
||||
if direction == PreferenceDirection.HIGHER and not ideal > acceptable:
|
||||
raise DomainValidationError("higher requires ideal > acceptable", {"id": preference_id})
|
||||
if direction == PreferenceDirection.LOWER and not ideal < acceptable:
|
||||
raise DomainValidationError("lower requires ideal < acceptable", {"id": preference_id})
|
||||
items.append(
|
||||
Preference(
|
||||
id=preference_id,
|
||||
attribute=attribute,
|
||||
strategy=PreferenceStrategy.NUMERIC_LINEAR,
|
||||
direction=direction,
|
||||
acceptable=acceptable,
|
||||
ideal=ideal,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
return tuple(items)
|
||||
@@ -0,0 +1,20 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
def error_body(code: str, message: str, details: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return {"error": {"code": code, "message": message, "details": details or {}}}
|
||||
@@ -0,0 +1,48 @@
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.discovery import router as discovery_router
|
||||
from app.api.evaluations import router as evaluations_router
|
||||
from app.api.health import router as health_router
|
||||
from app.api.offers import router as offers_router
|
||||
from app.api.products import router as products_router
|
||||
from app.api.profiles import router as profiles_router
|
||||
from app.api.projects import router as projects_router
|
||||
from app.errors import AppError, error_body
|
||||
from app.settings import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
app = FastAPI(
|
||||
title="Buying Agent",
|
||||
version="0.1.0",
|
||||
description="Generic buying and deal evaluation API. The backend is the system of record; no frontend or n8n is required.",
|
||||
)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
app.include_router(health_router)
|
||||
app.include_router(profiles_router)
|
||||
app.include_router(products_router)
|
||||
app.include_router(offers_router)
|
||||
app.include_router(evaluations_router)
|
||||
app.include_router(projects_router)
|
||||
app.include_router(discovery_router)
|
||||
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
def handle_app_error(_request: Request, exc: AppError) -> JSONResponse:
|
||||
return JSONResponse(error_body(exc.code, exc.message, exc.details), status_code=exc.status_code)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
def handle_validation_error(_request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
error_body("VALIDATION_ERROR", "request validation failed", {"errors": exc.errors()}),
|
||||
status_code=422,
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,11 @@
|
||||
from typing import Protocol
|
||||
|
||||
from app.domain.models import AttributeDefinition, AttributeStatement
|
||||
|
||||
|
||||
class Normalizer(Protocol):
|
||||
version: str
|
||||
|
||||
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
||||
"""Validate an already normalized statement. Must not parse free text."""
|
||||
...
|
||||
@@ -0,0 +1,61 @@
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.enums import NormalizationStatus, ValueType
|
||||
from app.domain.models import AttributeDefinition, AttributeStatement
|
||||
from app.errors import AppError
|
||||
|
||||
|
||||
class ManualNormalizer:
|
||||
version = "manual-v1"
|
||||
|
||||
def validate(self, definition: AttributeDefinition, statement: AttributeStatement) -> AttributeStatement:
|
||||
if statement.normalization_status == NormalizationStatus.UNKNOWN:
|
||||
if statement.normalized_value is not None:
|
||||
raise AppError(
|
||||
422,
|
||||
"NORMALIZATION_INCONSISTENT",
|
||||
"unknown statements must not have a normalized_value",
|
||||
)
|
||||
return statement
|
||||
if statement.normalization_status != NormalizationStatus.NORMALIZED:
|
||||
raise AppError(422, "NORMALIZATION_STATUS", "unsupported normalization_status")
|
||||
if statement.normalized_value is None:
|
||||
raise AppError(422, "NORMALIZATION_MISSING", "normalized_value is required")
|
||||
_assert_type(definition, statement.normalized_value)
|
||||
if definition.value_type == ValueType.NUMBER:
|
||||
number = Decimal(str(statement.normalized_value))
|
||||
if definition.key == "weight_kg" and number <= 0:
|
||||
raise AppError(422, "VALUE_RANGE", "weight_kg must be > 0")
|
||||
if definition.minimum is not None and number < definition.minimum:
|
||||
raise AppError(422, "VALUE_RANGE", f"{definition.key} is below the minimum")
|
||||
if definition.maximum is not None and number > definition.maximum:
|
||||
raise AppError(422, "VALUE_RANGE", f"{definition.key} is above the maximum")
|
||||
if definition.canonical_unit and statement.unit != definition.canonical_unit:
|
||||
raise AppError(
|
||||
422,
|
||||
"UNIT_MISMATCH",
|
||||
"unit must match the canonical unit",
|
||||
{"expected": definition.canonical_unit, "actual": statement.unit},
|
||||
)
|
||||
if definition.canonical_unit is None and statement.unit not in {None, ""}:
|
||||
raise AppError(422, "UNIT_MISMATCH", "this attribute has no canonical unit")
|
||||
return statement
|
||||
|
||||
|
||||
def _assert_type(definition: AttributeDefinition, value: Any) -> None:
|
||||
if definition.value_type == ValueType.NUMBER:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, Decimal)):
|
||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a number")
|
||||
return
|
||||
if definition.value_type == ValueType.STRING:
|
||||
if not isinstance(value, str):
|
||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a string")
|
||||
return
|
||||
if definition.value_type == ValueType.BOOLEAN:
|
||||
if not isinstance(value, bool):
|
||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a boolean")
|
||||
return
|
||||
if definition.value_type == ValueType.STRING_LIST:
|
||||
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
||||
raise AppError(422, "TYPE_MISMATCH", "normalized_value must be a list of strings")
|
||||
Whitespace-only changes.
@@ -0,0 +1,22 @@
|
||||
from collections.abc import Generator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.settings import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
engine = create_engine(settings.database_url, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def get_session() -> Generator[Session, None, None]:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,324 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, ForeignKeyConstraint, Integer, Numeric, String, Text, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.persistence.models import Base
|
||||
|
||||
|
||||
class ProductCategoryRecord(Base):
|
||||
__tablename__ = "product_categories"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="ACTIVE")
|
||||
|
||||
|
||||
class BuyingProjectRecord(Base):
|
||||
__tablename__ = "buying_projects"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["template_id", "template_version"],
|
||||
["product_profiles.id", "product_profiles.version"],
|
||||
name="fk_buying_projects_template",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
workspace_id: Mapped[str | None] = mapped_column(String(64))
|
||||
product_category_id: Mapped[str] = mapped_column(ForeignKey("product_categories.id"), nullable=False)
|
||||
market: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||
target_budget: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
template_id: Mapped[str | None] = mapped_column(String(128))
|
||||
template_version: Mapped[int | None] = mapped_column(Integer)
|
||||
shortlist_size: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
archived_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
@property
|
||||
def profile_id(self) -> str | None:
|
||||
return self.template_id
|
||||
|
||||
@property
|
||||
def profile_version(self) -> int | None:
|
||||
return self.template_version
|
||||
|
||||
|
||||
class BuyingProjectCriteriaRecord(Base):
|
||||
__tablename__ = "buying_project_criteria"
|
||||
|
||||
project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), primary_key=True)
|
||||
version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ProjectCriterionRecord(Base):
|
||||
__tablename__ = "project_criteria"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["project_id", "version"],
|
||||
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
||||
name="fk_project_criteria_version",
|
||||
),
|
||||
UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
project_id: Mapped[UUID] = mapped_column(nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
item_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||
criterion_key: Mapped[str | None] = mapped_column(String(128))
|
||||
operator: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
value: Mapped[Any] = mapped_column(JSONB)
|
||||
enabled_for_discovery: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
@property
|
||||
def mandatory(self) -> bool:
|
||||
return self.enabled_for_discovery
|
||||
|
||||
@property
|
||||
def criteria_version(self) -> int:
|
||||
return self.version
|
||||
|
||||
|
||||
class SourceCategoryRecord(Base):
|
||||
__tablename__ = "source_categories"
|
||||
__table_args__ = (UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
external_category_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
source_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
||||
canonical_category: Mapped[str | None] = mapped_column(String(128))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
taxonomy_status: Mapped[str] = mapped_column(String(32), nullable=False, default="NOT_LEARNED")
|
||||
last_taxonomy_scan_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class SourceFilterDefinitionRecord(Base):
|
||||
__tablename__ = "source_filter_definitions"
|
||||
__table_args__ = (UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
group_label: Mapped[str | None] = mapped_column(String(128))
|
||||
filter_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
|
||||
|
||||
class SourceFilterOptionRecord(Base):
|
||||
__tablename__ = "source_filter_options"
|
||||
__table_args__ = (UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), nullable=False)
|
||||
source_value: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||
operator: Mapped[str | None] = mapped_column(String(16))
|
||||
unit: Mapped[str | None] = mapped_column(String(32))
|
||||
|
||||
|
||||
class SourceFilterRangeRecord(Base):
|
||||
__tablename__ = "source_filter_ranges"
|
||||
|
||||
filter_id: Mapped[UUID] = mapped_column(ForeignKey("source_filter_definitions.id"), primary_key=True)
|
||||
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
step: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
unit: Mapped[str | None] = mapped_column(String(32))
|
||||
|
||||
|
||||
class CriterionDefinitionRecord(Base):
|
||||
__tablename__ = "criterion_definitions"
|
||||
__table_args__ = (UniqueConstraint("key", name="uq_criterion_definition_key"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="SOURCE")
|
||||
source: Mapped[str | None] = mapped_column(String(64))
|
||||
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
||||
label: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
unit: Mapped[str | None] = mapped_column(String(32))
|
||||
semantic: Mapped[str | None] = mapped_column(String(128))
|
||||
category: Mapped[str | None] = mapped_column(String(128))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
scoring_attribute_key: Mapped[str | None] = mapped_column(String(128))
|
||||
|
||||
|
||||
class SourceAttributeMappingRecord(Base):
|
||||
__tablename__ = "source_attribute_mappings"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
source_category_id: Mapped[UUID | None] = mapped_column(ForeignKey("source_categories.id"))
|
||||
raw_name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class CriterionMappingRecord(Base):
|
||||
__tablename__ = "criterion_mappings"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
criterion_definition_id: Mapped[UUID] = mapped_column(ForeignKey("criterion_definitions.id"), nullable=False)
|
||||
criterion_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||
external_filter_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
source_value: Mapped[str | None] = mapped_column(String(256))
|
||||
encoding: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class DiscoverySearchRecord(Base):
|
||||
__tablename__ = "discovery_searches"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
schedule_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, default=180)
|
||||
broad_result_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=300)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_successful_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SearchRunRecord(Base):
|
||||
__tablename__ = "search_runs"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
discovery_search_id: Mapped[UUID] = mapped_column(ForeignKey("discovery_searches.id"), nullable=False)
|
||||
criteria_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
mapping_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
generated_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
new_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
changed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
unchanged_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
missing_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
http_status_summary: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class SourceProductReferenceRecord(Base):
|
||||
__tablename__ = "source_product_references"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
source_category_id: Mapped[UUID] = mapped_column(ForeignKey("source_categories.id"), nullable=False)
|
||||
source_external_id: Mapped[str | None] = mapped_column(String(256))
|
||||
source_url: Mapped[str | None] = mapped_column(Text)
|
||||
source_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
manufacturer: Mapped[str | None] = mapped_column(String(256))
|
||||
model: Mapped[str | None] = mapped_column(String(256))
|
||||
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
||||
gtin_ean: Mapped[str | None] = mapped_column(String(32))
|
||||
normalized_spec_fingerprint: Mapped[str | None] = mapped_column(Text)
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
lifecycle_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
consecutive_missing_success_runs: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
||||
|
||||
|
||||
class ProductObservationRecord(Base):
|
||||
__tablename__ = "product_observations"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
raw_title: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
best_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
currency: Mapped[str | None] = mapped_column(String(3))
|
||||
availability: Mapped[str | None] = mapped_column(String(32))
|
||||
seller_count: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class ProductObservationAttributeRecord(Base):
|
||||
__tablename__ = "product_observation_attributes"
|
||||
__table_args__ = (UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
observation_id: Mapped[UUID] = mapped_column(ForeignKey("product_observations.id"), nullable=False)
|
||||
criterion_definition_id: Mapped[UUID | None] = mapped_column(ForeignKey("criterion_definitions.id"))
|
||||
attribute_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
raw_name: Mapped[str | None] = mapped_column(String(256))
|
||||
raw_value: Mapped[Any] = mapped_column(JSONB)
|
||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||
unit: Mapped[str | None] = mapped_column(String(32))
|
||||
|
||||
|
||||
class SearchRunItemRecord(Base):
|
||||
__tablename__ = "search_run_items"
|
||||
__table_args__ = (UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
search_run_id: Mapped[UUID] = mapped_column(ForeignKey("search_runs.id"), nullable=False)
|
||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||
observation_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_observations.id"))
|
||||
diff_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
|
||||
|
||||
class ProjectCandidateRecord(Base):
|
||||
__tablename__ = "project_candidates"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
buying_project_id: Mapped[UUID] = mapped_column(ForeignKey("buying_projects.id"), nullable=False)
|
||||
source_product_reference_id: Mapped[UUID] = mapped_column(ForeignKey("source_product_references.id"), nullable=False)
|
||||
product_variant_id: Mapped[UUID | None] = mapped_column(ForeignKey("product_variants.id"))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
match_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
product_eligibility: Mapped[str | None] = mapped_column(String(16))
|
||||
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class CandidateStatusHistoryRecord(Base):
|
||||
__tablename__ = "candidate_status_history"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
candidate_id: Mapped[UUID] = mapped_column(ForeignKey("project_candidates.id"), nullable=False)
|
||||
from_status: Mapped[str | None] = mapped_column(String(32))
|
||||
to_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
reason: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,105 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from app.domain.enums import (
|
||||
Availability,
|
||||
NormalizationStatus,
|
||||
OfferCondition,
|
||||
ValueType,
|
||||
)
|
||||
from app.domain.models import (
|
||||
AttributeDefinition,
|
||||
AttributeStatement,
|
||||
Offer,
|
||||
PriceObservation,
|
||||
Product,
|
||||
ProductVariant,
|
||||
)
|
||||
from app.persistence import models as orm
|
||||
|
||||
|
||||
def decimal_or_none(value) -> Decimal | None:
|
||||
return None if value is None else Decimal(str(value))
|
||||
|
||||
|
||||
def to_definition(row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
||||
return AttributeDefinition(
|
||||
key=row.key,
|
||||
value_type=ValueType(row.value_type),
|
||||
canonical_unit=row.canonical_unit,
|
||||
minimum=decimal_or_none(row.minimum),
|
||||
maximum=decimal_or_none(row.maximum),
|
||||
description=row.description,
|
||||
)
|
||||
|
||||
|
||||
def to_product(row: orm.ProductRecord) -> Product:
|
||||
return Product(
|
||||
id=row.id,
|
||||
category=row.category,
|
||||
brand=row.brand,
|
||||
name=row.name,
|
||||
model=row.model,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def to_variant(row: orm.ProductVariantRecord) -> ProductVariant:
|
||||
return ProductVariant(
|
||||
id=row.id,
|
||||
product_id=row.product_id,
|
||||
name=row.name,
|
||||
manufacturer_sku=row.manufacturer_sku,
|
||||
identifiers=row.identifiers or {},
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def to_attribute(row: orm.VariantAttributeRecord) -> AttributeStatement:
|
||||
return AttributeStatement(
|
||||
id=row.id,
|
||||
variant_id=row.variant_id,
|
||||
attribute=row.attribute_key,
|
||||
raw_value=row.raw_value,
|
||||
normalized_value=row.normalized_value,
|
||||
unit=row.unit,
|
||||
normalization_status=NormalizationStatus(row.normalization_status),
|
||||
normalizer_version=row.normalizer_version,
|
||||
confidence=Decimal(str(row.confidence)),
|
||||
source_id=row.source_id,
|
||||
source_reference=row.source_reference,
|
||||
observed_at=row.observed_at,
|
||||
ingested_at=row.ingested_at,
|
||||
normalization_error=row.normalization_error,
|
||||
)
|
||||
|
||||
|
||||
def to_offer(row: orm.OfferRecord) -> Offer:
|
||||
return Offer(
|
||||
id=row.id,
|
||||
variant_id=row.variant_id,
|
||||
source_id=row.source_id,
|
||||
external_id=row.external_id,
|
||||
seller=row.seller,
|
||||
url=row.url,
|
||||
market=row.market,
|
||||
currency=row.currency,
|
||||
condition=OfferCondition(row.condition),
|
||||
created_at=row.created_at,
|
||||
)
|
||||
|
||||
|
||||
def to_observation(row: orm.PriceObservationRecord) -> PriceObservation:
|
||||
return PriceObservation(
|
||||
id=row.id,
|
||||
offer_id=row.offer_id,
|
||||
source_id=row.source_id,
|
||||
external_observation_id=row.external_observation_id,
|
||||
observed_at=row.observed_at,
|
||||
ingested_at=row.ingested_at,
|
||||
item_price=Decimal(str(row.item_price)),
|
||||
shipping_price=decimal_or_none(row.shipping_price),
|
||||
currency=row.currency,
|
||||
availability=Availability(row.availability),
|
||||
raw_payload=row.raw_payload,
|
||||
confidence=Decimal(str(row.confidence)),
|
||||
)
|
||||
@@ -0,0 +1,249 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
ForeignKeyConstraint,
|
||||
Numeric,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class AttributeDefinitionRecord(Base):
|
||||
__tablename__ = "attribute_definitions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
||||
name="ck_attribute_definitions_value_type",
|
||||
),
|
||||
)
|
||||
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
value_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
canonical_unit: Mapped[str | None] = mapped_column(String(32))
|
||||
minimum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
||||
maximum: Mapped[Decimal | None] = mapped_column(Numeric(12, 4))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class SourceRecord(Base):
|
||||
__tablename__ = "sources"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
adapter_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
|
||||
|
||||
class ScoringModelRecord(Base):
|
||||
__tablename__ = "scoring_models"
|
||||
|
||||
version: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
algorithm_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ProductProfileRecord(Base):
|
||||
__tablename__ = "product_profiles"
|
||||
__table_args__ = (
|
||||
CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
version: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
||||
definition: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ProductRecord(Base):
|
||||
__tablename__ = "products"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
category: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
brand: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
model: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
variants: Mapped[list["ProductVariantRecord"]] = relationship(back_populates="product")
|
||||
|
||||
|
||||
class ProductVariantRecord(Base):
|
||||
__tablename__ = "product_variants"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
product_id: Mapped[UUID] = mapped_column(ForeignKey("products.id"), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
manufacturer_sku: Mapped[str | None] = mapped_column(String(128))
|
||||
identifiers: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
product: Mapped[ProductRecord] = relationship(back_populates="variants")
|
||||
|
||||
|
||||
class VariantAttributeRecord(Base):
|
||||
__tablename__ = "variant_attributes"
|
||||
__table_args__ = (
|
||||
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
||||
CheckConstraint(
|
||||
"normalization_status IN ('normalized', 'unknown')",
|
||||
name="ck_variant_attributes_status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False)
|
||||
attribute_key: Mapped[str] = mapped_column(ForeignKey("attribute_definitions.key"), nullable=False)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
raw_value: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
||||
normalized_value: Mapped[Any] = mapped_column(JSONB)
|
||||
unit: Mapped[str | None] = mapped_column(String(32))
|
||||
normalization_status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
normalizer_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
||||
source_reference: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
normalization_error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class OfferRecord(Base):
|
||||
__tablename__ = "offers"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
||||
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
||||
CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
external_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
seller: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
url: Mapped[str | None] = mapped_column(Text)
|
||||
market: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||
condition: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class PriceObservationRecord(Base):
|
||||
__tablename__ = "price_observations"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
||||
CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
||||
CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
||||
CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
||||
CheckConstraint(
|
||||
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
||||
name="ck_price_obs_availability",
|
||||
),
|
||||
CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("sources.id"), nullable=False)
|
||||
external_observation_id: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
item_price: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
||||
shipping_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
|
||||
currency: Mapped[str] = mapped_column(String(3), nullable=False)
|
||||
availability: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
raw_payload: Mapped[Any] = mapped_column(JSONB, nullable=False)
|
||||
confidence: Mapped[Decimal] = mapped_column(Numeric(4, 3), nullable=False)
|
||||
|
||||
|
||||
class EvaluationRecord(Base):
|
||||
__tablename__ = "evaluations"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["profile_id", "profile_version"],
|
||||
["product_profiles.id", "product_profiles.version"],
|
||||
name="fk_evaluations_profile",
|
||||
),
|
||||
CheckConstraint(
|
||||
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_product_eligibility",
|
||||
),
|
||||
CheckConstraint(
|
||||
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_offer_eligibility",
|
||||
),
|
||||
CheckConstraint(
|
||||
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_overall_eligibility",
|
||||
),
|
||||
CheckConstraint(
|
||||
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
||||
name="ck_eval_product_score",
|
||||
),
|
||||
CheckConstraint(
|
||||
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
||||
name="ck_eval_price_score",
|
||||
),
|
||||
CheckConstraint(
|
||||
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
||||
name="ck_eval_deal_score",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
variant_id: Mapped[UUID] = mapped_column(ForeignKey("product_variants.id"), nullable=False, index=True)
|
||||
offer_id: Mapped[UUID] = mapped_column(ForeignKey("offers.id"), nullable=False)
|
||||
profile_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
profile_version: Mapped[int] = mapped_column(nullable=False)
|
||||
scoring_version: Mapped[str] = mapped_column(ForeignKey("scoring_models.version"), nullable=False)
|
||||
as_of: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
snapshot_cutoff: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
scoring_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
engine_build: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
product_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
offer_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
overall_eligibility: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
product_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||
price_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||
deal_score: Mapped[Decimal | None] = mapped_column(Numeric(6, 2))
|
||||
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
price_reference_scope: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
input_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
result_snapshot: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
components: Mapped[list["EvaluationComponentRecord"]] = relationship(back_populates="evaluation")
|
||||
|
||||
|
||||
class EvaluationComponentRecord(Base):
|
||||
__tablename__ = "evaluation_components"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True)
|
||||
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"), nullable=False)
|
||||
kind: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
component_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
raw_score: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||
weight: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||
contribution: Mapped[Decimal | None] = mapped_column(Numeric(12, 6))
|
||||
details: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
|
||||
evaluation: Mapped[EvaluationRecord] = relationship(back_populates="components")
|
||||
@@ -0,0 +1,183 @@
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
||||
from app.domain.validation import parse_profile, parse_scoring_model
|
||||
from app.persistence import models as orm
|
||||
from app.persistence.mapping import to_attribute, to_definition, to_observation, to_offer, to_product, to_variant
|
||||
|
||||
|
||||
class CatalogRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def get_definition(self, key: str) -> AttributeDefinition | None:
|
||||
row = self.session.get(orm.AttributeDefinitionRecord, key)
|
||||
return None if row is None else to_definition(row)
|
||||
|
||||
def list_definitions(self, limit: int, offset: int) -> tuple[list[AttributeDefinition], int]:
|
||||
total = self.session.scalar(select(func.count()).select_from(orm.AttributeDefinitionRecord)) or 0
|
||||
rows = self.session.scalars(
|
||||
select(orm.AttributeDefinitionRecord).order_by(orm.AttributeDefinitionRecord.key).limit(limit).offset(offset)
|
||||
).all()
|
||||
return [to_definition(row) for row in rows], total
|
||||
|
||||
def add_definition(self, row: orm.AttributeDefinitionRecord) -> AttributeDefinition:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_definition(row)
|
||||
|
||||
def list_sources(self) -> list[orm.SourceRecord]:
|
||||
return list(self.session.scalars(select(orm.SourceRecord).order_by(orm.SourceRecord.id)).all())
|
||||
|
||||
def get_source(self, source_id: str) -> orm.SourceRecord | None:
|
||||
return self.session.get(orm.SourceRecord, source_id)
|
||||
|
||||
def get_scoring_model(self, version: str):
|
||||
row = self.session.get(orm.ScoringModelRecord, version)
|
||||
return None if row is None else parse_scoring_model(row.definition)
|
||||
|
||||
def scoring_versions(self) -> set[str]:
|
||||
return set(self.session.scalars(select(orm.ScoringModelRecord.version)).all())
|
||||
|
||||
def all_definitions(self) -> dict[str, AttributeDefinition]:
|
||||
rows = self.session.scalars(select(orm.AttributeDefinitionRecord)).all()
|
||||
return {row.key: to_definition(row) for row in rows}
|
||||
|
||||
def get_profile(self, profile_id: str, version: int):
|
||||
row = self.session.get(orm.ProductProfileRecord, (profile_id, version))
|
||||
if row is None:
|
||||
return None
|
||||
model = self.get_scoring_model(row.scoring_version)
|
||||
if model is None:
|
||||
return None
|
||||
return parse_profile(
|
||||
row.definition,
|
||||
attribute_definitions=self.all_definitions(),
|
||||
scoring_versions=self.scoring_versions(),
|
||||
supported_strategies=set(model.supported_preference_strategies),
|
||||
)
|
||||
|
||||
def add_profile(self, row: orm.ProductProfileRecord):
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
def list_profiles(self, limit: int, offset: int) -> tuple[list[orm.ProductProfileRecord], int]:
|
||||
total = self.session.scalar(select(func.count()).select_from(orm.ProductProfileRecord)) or 0
|
||||
rows = self.session.scalars(
|
||||
select(orm.ProductProfileRecord)
|
||||
.order_by(orm.ProductProfileRecord.id, orm.ProductProfileRecord.version)
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
).all()
|
||||
return list(rows), total
|
||||
|
||||
def get_product(self, product_id: UUID) -> Product | None:
|
||||
row = self.session.get(orm.ProductRecord, product_id)
|
||||
return None if row is None else to_product(row)
|
||||
|
||||
def add_product(self, row: orm.ProductRecord) -> Product:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_product(row)
|
||||
|
||||
def get_variant(self, variant_id: UUID) -> ProductVariant | None:
|
||||
row = self.session.get(orm.ProductVariantRecord, variant_id)
|
||||
return None if row is None else to_variant(row)
|
||||
|
||||
def add_variant(self, row: orm.ProductVariantRecord) -> ProductVariant:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_variant(row)
|
||||
|
||||
def list_attributes(self, variant_id: UUID) -> list[AttributeStatement]:
|
||||
rows = self.session.scalars(
|
||||
select(orm.VariantAttributeRecord)
|
||||
.where(orm.VariantAttributeRecord.variant_id == variant_id)
|
||||
.order_by(
|
||||
orm.VariantAttributeRecord.observed_at,
|
||||
orm.VariantAttributeRecord.ingested_at,
|
||||
orm.VariantAttributeRecord.id,
|
||||
)
|
||||
).all()
|
||||
return [to_attribute(row) for row in rows]
|
||||
|
||||
def get_attribute(self, attribute_id: UUID) -> AttributeStatement | None:
|
||||
row = self.session.get(orm.VariantAttributeRecord, attribute_id)
|
||||
return None if row is None else to_attribute(row)
|
||||
|
||||
def add_attribute(self, row: orm.VariantAttributeRecord) -> AttributeStatement:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_attribute(row)
|
||||
|
||||
def get_offer(self, offer_id: UUID) -> Offer | None:
|
||||
row = self.session.get(orm.OfferRecord, offer_id)
|
||||
return None if row is None else to_offer(row)
|
||||
|
||||
def get_offer_by_external(self, source_id: str, external_id: str) -> Offer | None:
|
||||
row = self.session.scalar(
|
||||
select(orm.OfferRecord).where(
|
||||
orm.OfferRecord.source_id == source_id,
|
||||
orm.OfferRecord.external_id == external_id,
|
||||
)
|
||||
)
|
||||
return None if row is None else to_offer(row)
|
||||
|
||||
def add_offer(self, row: orm.OfferRecord) -> Offer:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_offer(row)
|
||||
|
||||
def list_observations(self, offer_id: UUID, limit: int | None = None, offset: int = 0) -> tuple[list[PriceObservation], int]:
|
||||
total = (
|
||||
self.session.scalar(
|
||||
select(func.count())
|
||||
.select_from(orm.PriceObservationRecord)
|
||||
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
query = (
|
||||
select(orm.PriceObservationRecord)
|
||||
.where(orm.PriceObservationRecord.offer_id == offer_id)
|
||||
.order_by(
|
||||
orm.PriceObservationRecord.observed_at,
|
||||
orm.PriceObservationRecord.ingested_at,
|
||||
orm.PriceObservationRecord.id,
|
||||
)
|
||||
.offset(offset)
|
||||
)
|
||||
if limit is not None:
|
||||
query = query.limit(limit)
|
||||
rows = self.session.scalars(query).all()
|
||||
return [to_observation(row) for row in rows], total
|
||||
|
||||
def get_observation_by_external(self, offer_id: UUID, external_id: str) -> orm.PriceObservationRecord | None:
|
||||
return self.session.scalar(
|
||||
select(orm.PriceObservationRecord).where(
|
||||
orm.PriceObservationRecord.offer_id == offer_id,
|
||||
orm.PriceObservationRecord.external_observation_id == external_id,
|
||||
)
|
||||
)
|
||||
|
||||
def add_observation(self, row: orm.PriceObservationRecord) -> PriceObservation:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return to_observation(row)
|
||||
|
||||
def get_evaluation(self, evaluation_id: UUID) -> orm.EvaluationRecord | None:
|
||||
return self.session.get(orm.EvaluationRecord, evaluation_id)
|
||||
|
||||
def add_evaluation(self, row: orm.EvaluationRecord) -> orm.EvaluationRecord:
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
def add_component(self, row: orm.EvaluationComponentRecord) -> None:
|
||||
self.session.add(row)
|
||||
Whitespace-only changes.
@@ -0,0 +1,42 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from app.domain.enums import Classification, Eligibility
|
||||
from app.domain.models import ScoringModel
|
||||
from app.scoring.mathutil import clamp
|
||||
|
||||
|
||||
def classify_deal_score(score: Decimal, model: ScoringModel) -> str:
|
||||
for bound in model.classification_upper_bounds:
|
||||
if bound.inclusive and score <= bound.upper:
|
||||
return bound.label
|
||||
if not bound.inclusive and score < bound.upper:
|
||||
return bound.label
|
||||
return Classification.EXCEPTIONAL.value
|
||||
|
||||
|
||||
def score_deal(product_score: Decimal, price_score: Decimal, model: ScoringModel) -> Decimal:
|
||||
return clamp(model.product_weight * product_score + model.price_weight * price_score)
|
||||
|
||||
|
||||
def eligibility_from_parts(product: Eligibility, offer: Eligibility) -> Eligibility:
|
||||
if Eligibility.INELIGIBLE in {product, offer}:
|
||||
return Eligibility.INELIGIBLE
|
||||
if Eligibility.UNKNOWN in {product, offer}:
|
||||
return Eligibility.UNKNOWN
|
||||
return Eligibility.ELIGIBLE
|
||||
|
||||
|
||||
def derive_eligible(overall: Eligibility) -> bool | None:
|
||||
if overall == Eligibility.ELIGIBLE:
|
||||
return True
|
||||
if overall == Eligibility.INELIGIBLE:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def classification_for(overall: Eligibility, deal_score: Decimal | None, model: ScoringModel) -> str:
|
||||
if overall == Eligibility.INELIGIBLE:
|
||||
return Classification.INELIGIBLE.value
|
||||
if overall == Eligibility.UNKNOWN or deal_score is None:
|
||||
return Classification.INSUFFICIENT_DATA.value
|
||||
return classify_deal_score(deal_score, model)
|
||||
@@ -0,0 +1,545 @@
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.attributes import select_attribute_snapshot
|
||||
from app.domain.constraints import evaluate_constraint
|
||||
from app.domain.enums import (
|
||||
Availability,
|
||||
ConstraintStatus,
|
||||
Eligibility,
|
||||
PriceReferenceScope,
|
||||
)
|
||||
from app.domain.models import (
|
||||
AttributeDefinition,
|
||||
AttributeStatement,
|
||||
ConstraintResult,
|
||||
EvaluationResult,
|
||||
GateResult,
|
||||
Offer,
|
||||
PriceObservation,
|
||||
Product,
|
||||
ProductVariant,
|
||||
Profile,
|
||||
Reason,
|
||||
ScoreComponent,
|
||||
ScoringModel,
|
||||
)
|
||||
from app.domain.serialize import jsonable
|
||||
from app.scoring.deal import classification_for, derive_eligible, eligibility_from_parts, score_deal
|
||||
from app.scoring.mathutil import round_score
|
||||
from app.scoring.price import (
|
||||
build_offer_reference_history,
|
||||
price_statistics_from_windows,
|
||||
score_price,
|
||||
select_current_observation,
|
||||
)
|
||||
from app.scoring.product import score_product
|
||||
|
||||
|
||||
def evaluate_deal(
|
||||
*,
|
||||
profile: Profile,
|
||||
scoring_model: ScoringModel,
|
||||
product: Product,
|
||||
variant: ProductVariant,
|
||||
offer: Offer,
|
||||
attributes: tuple[AttributeStatement, ...],
|
||||
observations: tuple[PriceObservation, ...],
|
||||
definitions: dict[str, AttributeDefinition],
|
||||
as_of: datetime,
|
||||
snapshot_cutoff: datetime,
|
||||
engine_build: str,
|
||||
) -> EvaluationResult:
|
||||
observations = tuple(item for item in observations if item.offer_id == offer.id)
|
||||
snapshot = select_attribute_snapshot(attributes, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
||||
constraint_results = tuple(
|
||||
evaluate_constraint(
|
||||
constraint,
|
||||
snapshot.get(constraint.attribute),
|
||||
definitions[constraint.attribute],
|
||||
scoring_model.minimum_confidence,
|
||||
)
|
||||
for constraint in profile.constraints
|
||||
)
|
||||
product_gates, product_eligibility = _product_eligibility(profile, product, constraint_results)
|
||||
product_score_raw, coverage, preference_components = score_product(
|
||||
profile.preferences,
|
||||
snapshot,
|
||||
constraint_results,
|
||||
scoring_model,
|
||||
)
|
||||
|
||||
current = select_current_observation(observations, as_of=as_of, snapshot_cutoff=snapshot_cutoff)
|
||||
offer_gates, offer_eligibility = _offer_eligibility(profile, offer, current, scoring_model, as_of)
|
||||
overall = eligibility_from_parts(product_eligibility, offer_eligibility)
|
||||
|
||||
windows: dict[int, Any] = {}
|
||||
price_score_raw: Decimal | None = None
|
||||
budget_score: Decimal | None = None
|
||||
history_score: Decimal | None = None
|
||||
price_basis = None
|
||||
history_warning: str | None = None
|
||||
if _can_score_price(current, profile, scoring_model, as_of):
|
||||
windows = build_offer_reference_history(observations, current, scoring_model)
|
||||
price_score_raw, budget_score, history_score, price_basis, history_warning = score_price(
|
||||
current_price=current.total_price,
|
||||
reference_history=windows[90].values,
|
||||
budget=profile.budget,
|
||||
model=scoring_model,
|
||||
)
|
||||
|
||||
stats = price_statistics_from_windows(current, windows)
|
||||
deal_score_raw = None
|
||||
if overall == Eligibility.ELIGIBLE and price_score_raw is not None:
|
||||
deal_score_raw = score_deal(product_score_raw, price_score_raw, scoring_model)
|
||||
|
||||
classification = classification_for(overall, deal_score_raw, scoring_model)
|
||||
reasons, warnings = _reasons(
|
||||
constraint_results,
|
||||
product_gates,
|
||||
offer_gates,
|
||||
stats,
|
||||
current,
|
||||
history_warning,
|
||||
price_score_raw,
|
||||
)
|
||||
components = _all_components(
|
||||
preference_components,
|
||||
product_gates,
|
||||
offer_gates,
|
||||
budget_score,
|
||||
history_score,
|
||||
scoring_model,
|
||||
)
|
||||
input_snapshot = {
|
||||
"profile": profile.definition,
|
||||
"scoring_model": scoring_model.definition,
|
||||
"product": jsonable(_product_dict(product)),
|
||||
"variant": jsonable(_variant_dict(variant)),
|
||||
"offer": jsonable(_offer_dict(offer)),
|
||||
"attribute_snapshot": {
|
||||
key: jsonable(_attribute_dict(value))
|
||||
for key, value in snapshot.items()
|
||||
},
|
||||
"current_observation": None if current is None else jsonable(_observation_dict(current)),
|
||||
"history_observation_ids": [str(item) for item in stats.used_observation_ids],
|
||||
"as_of": jsonable(as_of),
|
||||
"snapshot_cutoff": jsonable(snapshot_cutoff),
|
||||
"price_reference_scope": PriceReferenceScope.OFFER.value,
|
||||
"unrounded": {
|
||||
"product_score": jsonable(product_score_raw),
|
||||
"price_score": jsonable(price_score_raw),
|
||||
"deal_score": jsonable(deal_score_raw),
|
||||
"budget_score": jsonable(budget_score),
|
||||
"history_score": jsonable(history_score),
|
||||
},
|
||||
}
|
||||
return EvaluationResult(
|
||||
profile_id=profile.id,
|
||||
profile_version=profile.version,
|
||||
scoring_version=scoring_model.version,
|
||||
scoring_hash=scoring_model.definition_hash,
|
||||
engine_build=engine_build,
|
||||
variant_id=variant.id,
|
||||
offer_id=offer.id,
|
||||
as_of=as_of,
|
||||
snapshot_cutoff=snapshot_cutoff,
|
||||
product_eligibility=product_eligibility,
|
||||
offer_eligibility=offer_eligibility,
|
||||
overall_eligibility=overall,
|
||||
eligible=derive_eligible(overall),
|
||||
product_score=round_score(product_score_raw),
|
||||
product_score_raw=product_score_raw,
|
||||
price_score=None if price_score_raw is None else round_score(price_score_raw),
|
||||
price_score_raw=price_score_raw,
|
||||
deal_score=None if deal_score_raw is None else round_score(deal_score_raw),
|
||||
deal_score_raw=deal_score_raw,
|
||||
classification=classification,
|
||||
price_basis=price_basis,
|
||||
product_coverage=coverage,
|
||||
constraint_results=constraint_results,
|
||||
product_gates=product_gates,
|
||||
offer_gates=offer_gates,
|
||||
components=components,
|
||||
price_statistics=stats,
|
||||
price_reference_scope=PriceReferenceScope.OFFER,
|
||||
reasons=reasons,
|
||||
warnings=warnings,
|
||||
input_snapshot=input_snapshot,
|
||||
failed_constraints=tuple(
|
||||
item for item in constraint_results if item.status == ConstraintStatus.FAIL.value
|
||||
),
|
||||
unknown_constraints=tuple(
|
||||
item for item in constraint_results if item.status == ConstraintStatus.UNKNOWN.value
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def statement_price_confidence(observation: PriceObservation, model: ScoringModel) -> bool:
|
||||
return observation.confidence >= model.minimum_confidence
|
||||
|
||||
|
||||
def _fresh_enough(observation: PriceObservation, as_of: datetime, model: ScoringModel) -> bool:
|
||||
return as_of - observation.observed_at <= timedelta(hours=model.maximum_price_age_hours)
|
||||
|
||||
|
||||
def _product_eligibility(
|
||||
profile: Profile,
|
||||
product: Product,
|
||||
constraint_results: tuple[ConstraintResult, ...],
|
||||
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
||||
category_ok = product.category == profile.category
|
||||
category_gate = GateResult(
|
||||
key="category",
|
||||
status="PASS" if category_ok else "FAIL",
|
||||
reason_code="CATEGORY_MATCH" if category_ok else "CATEGORY_MISMATCH",
|
||||
details={"expected": profile.category, "actual": product.category},
|
||||
)
|
||||
mandatory = [item for item in constraint_results if item.mandatory]
|
||||
if not category_ok or any(item.status == ConstraintStatus.FAIL.value for item in mandatory):
|
||||
eligibility = Eligibility.INELIGIBLE
|
||||
elif any(item.status == ConstraintStatus.UNKNOWN.value for item in mandatory):
|
||||
eligibility = Eligibility.UNKNOWN
|
||||
else:
|
||||
eligibility = Eligibility.ELIGIBLE
|
||||
return (category_gate,), eligibility
|
||||
|
||||
|
||||
def _offer_eligibility(
|
||||
profile: Profile,
|
||||
offer: Offer,
|
||||
current: PriceObservation | None,
|
||||
model: ScoringModel,
|
||||
as_of: datetime,
|
||||
) -> tuple[tuple[GateResult, ...], Eligibility]:
|
||||
gates: list[GateResult] = []
|
||||
|
||||
market_ok = offer.market == profile.market
|
||||
gates.append(_gate("market", market_ok, "MARKET_MATCH", "MARKET_MISMATCH", offer.market, profile.market))
|
||||
|
||||
condition_ok = offer.condition in profile.allowed_conditions
|
||||
gates.append(
|
||||
_gate(
|
||||
"condition",
|
||||
condition_ok,
|
||||
"CONDITION_ALLOWED",
|
||||
"CONDITION_NOT_ALLOWED",
|
||||
offer.condition.value,
|
||||
[item.value for item in profile.allowed_conditions],
|
||||
)
|
||||
)
|
||||
|
||||
currency_ok = offer.currency == profile.currency
|
||||
gates.append(_gate("currency", currency_ok, "CURRENCY_MATCH", "CURRENCY_MISMATCH", offer.currency, profile.currency))
|
||||
|
||||
if current is None:
|
||||
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_MISSING", {}))
|
||||
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_MISSING", {}))
|
||||
gates.append(GateResult("price_confidence", "UNKNOWN", "PRICE_MISSING", {}))
|
||||
gates.append(GateResult("shipping", "UNKNOWN", "SHIPPING_UNKNOWN", {}))
|
||||
gates.append(GateResult("budget", "UNKNOWN", "PRICE_MISSING", {}))
|
||||
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
||||
return tuple(gates), _combine_offer_gates(gates)
|
||||
|
||||
fresh = _fresh_enough(current, as_of, model)
|
||||
if not fresh:
|
||||
gates.append(GateResult("price_present", "UNKNOWN", "PRICE_STALE", {"observed_at": current.observed_at.isoformat()}))
|
||||
gates.append(GateResult("price_fresh", "UNKNOWN", "PRICE_STALE", {}))
|
||||
else:
|
||||
gates.append(GateResult("price_present", "PASS", "PRICE_PRESENT", {"observation_id": str(current.id)}))
|
||||
gates.append(GateResult("price_fresh", "PASS", "PRICE_FRESH", {}))
|
||||
|
||||
confident = statement_price_confidence(current, model)
|
||||
gates.append(
|
||||
GateResult(
|
||||
"price_confidence",
|
||||
"PASS" if confident else "UNKNOWN",
|
||||
"PRICE_CONFIDENCE_OK" if confident else "PRICE_LOW_CONFIDENCE",
|
||||
{"confidence": str(current.confidence)},
|
||||
)
|
||||
)
|
||||
|
||||
shipping_known = current.shipping_price is not None
|
||||
gates.append(
|
||||
GateResult(
|
||||
"shipping",
|
||||
"PASS" if shipping_known else "UNKNOWN",
|
||||
"SHIPPING_KNOWN" if shipping_known else "SHIPPING_UNKNOWN",
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
total = current.total_price
|
||||
if total is None or not currency_ok:
|
||||
gates.append(GateResult("budget", "UNKNOWN", "PRICE_INCOMPLETE", {}))
|
||||
elif total <= profile.budget.maximum:
|
||||
gates.append(GateResult("budget", "PASS", "WITHIN_BUDGET", {"total": str(total)}))
|
||||
else:
|
||||
gates.append(GateResult("budget", "FAIL", "OVER_BUDGET", {"total": str(total)}))
|
||||
|
||||
if current.availability == Availability.IN_STOCK:
|
||||
gates.append(GateResult("availability", "PASS", "IN_STOCK", {}))
|
||||
elif current.availability == Availability.OUT_OF_STOCK:
|
||||
gates.append(GateResult("availability", "FAIL", "OUT_OF_STOCK", {}))
|
||||
else:
|
||||
gates.append(GateResult("availability", "UNKNOWN", "AVAILABILITY_UNKNOWN", {}))
|
||||
|
||||
if current.currency != offer.currency:
|
||||
gates.append(GateResult("observation_currency", "FAIL", "CURRENCY_MISMATCH", {}))
|
||||
|
||||
if current.source_id != offer.source_id:
|
||||
gates.append(GateResult("observation_source", "FAIL", "SOURCE_MISMATCH", {}))
|
||||
|
||||
return tuple(gates), _combine_offer_gates(gates)
|
||||
|
||||
|
||||
def _gate(key: str, ok: bool, pass_code: str, fail_code: str, actual: Any, expected: Any) -> GateResult:
|
||||
return GateResult(
|
||||
key=key,
|
||||
status="PASS" if ok else "FAIL",
|
||||
reason_code=pass_code if ok else fail_code,
|
||||
details={"actual": actual, "expected": expected},
|
||||
)
|
||||
|
||||
|
||||
def _can_score_price(
|
||||
current: PriceObservation | None,
|
||||
profile: Profile,
|
||||
model: ScoringModel,
|
||||
as_of: datetime,
|
||||
) -> bool:
|
||||
if current is None:
|
||||
return False
|
||||
if current.currency != profile.currency:
|
||||
return False
|
||||
if current.total_price is None:
|
||||
return False
|
||||
if not _fresh_enough(current, as_of, model):
|
||||
return False
|
||||
return statement_price_confidence(current, model)
|
||||
|
||||
|
||||
def _combine_offer_gates(gates: Sequence[GateResult]) -> Eligibility:
|
||||
statuses = [gate.status for gate in gates]
|
||||
if "FAIL" in statuses:
|
||||
return Eligibility.INELIGIBLE
|
||||
if "UNKNOWN" in statuses:
|
||||
return Eligibility.UNKNOWN
|
||||
return Eligibility.ELIGIBLE
|
||||
|
||||
|
||||
def _reasons(
|
||||
constraints: tuple[ConstraintResult, ...],
|
||||
product_gates: tuple[GateResult, ...],
|
||||
offer_gates: tuple[GateResult, ...],
|
||||
stats,
|
||||
current: PriceObservation | None,
|
||||
history_warning: str | None,
|
||||
price_score_raw: Decimal | None,
|
||||
) -> tuple[tuple[Reason, ...], tuple[str, ...]]:
|
||||
reasons: list[Reason] = []
|
||||
warnings: list[str] = []
|
||||
for gate in (*product_gates, *offer_gates):
|
||||
if gate.status != "PASS":
|
||||
reasons.append(Reason(gate.reason_code, _gate_message(gate)))
|
||||
for item in constraints:
|
||||
if item.status != ConstraintStatus.PASS.value:
|
||||
reasons.append(Reason(item.reason_code, _constraint_message(item)))
|
||||
if history_warning:
|
||||
warnings.append(history_warning)
|
||||
reasons.append(Reason(history_warning, "Zu wenige Historientage für den 90-Tage-Median."))
|
||||
if stats.lowest_365d is not None and not stats.lowest_365d_is_full_year:
|
||||
warnings.append("LOWEST_365D_PARTIAL")
|
||||
reasons.append(
|
||||
Reason(
|
||||
"LOWEST_365D_PARTIAL",
|
||||
f"lowest_365d {stats.lowest_365d} ist das Minimum der vorhandenen {stats.lowest_365d_days} Tage, kein belegtes Jahrestief.",
|
||||
)
|
||||
)
|
||||
if (
|
||||
price_score_raw is not None
|
||||
and current is not None
|
||||
and current.total_price is not None
|
||||
and stats.median_90d is not None
|
||||
and stats.median_90d != 0
|
||||
):
|
||||
delta = (stats.median_90d - current.total_price) / stats.median_90d * Decimal("100")
|
||||
percent = round_score(delta)
|
||||
if current.total_price < stats.median_90d:
|
||||
reasons.append(
|
||||
Reason(
|
||||
"PRICE_BELOW_MEDIAN",
|
||||
f"Gesamtpreis {_de_decimal(percent)} % unter dem Median der beobachteten 90-Tage-Historie.",
|
||||
)
|
||||
)
|
||||
elif current.total_price > stats.median_90d:
|
||||
reasons.append(
|
||||
Reason(
|
||||
"PRICE_ABOVE_MEDIAN",
|
||||
f"Gesamtpreis {_de_decimal(percent.copy_abs())} % über dem Median der beobachteten 90-Tage-Historie.",
|
||||
)
|
||||
)
|
||||
else:
|
||||
reasons.append(Reason("PRICE_AT_MEDIAN", "Gesamtpreis entspricht dem Median der 90-Tage-Historie."))
|
||||
return tuple(reasons), tuple(warnings)
|
||||
|
||||
|
||||
def _de_decimal(value: Decimal) -> str:
|
||||
return format(value, "f").replace(".", ",")
|
||||
|
||||
|
||||
def _gate_message(gate: GateResult) -> str:
|
||||
messages = {
|
||||
"CATEGORY_MISMATCH": "Die Produktkategorie entspricht nicht dem Profil.",
|
||||
"MARKET_MISMATCH": "Der Angebotsmarkt entspricht nicht dem Profil.",
|
||||
"CONDITION_NOT_ALLOWED": "Der Angebotszustand ist im Profil nicht erlaubt.",
|
||||
"CURRENCY_MISMATCH": "Die Währung entspricht nicht dem Profil.",
|
||||
"PRICE_MISSING": "Es liegt keine aktuelle Preisbeobachtung vor.",
|
||||
"PRICE_STALE": "Die aktuelle Preisbeobachtung ist nicht frisch genug.",
|
||||
"PRICE_LOW_CONFIDENCE": "Die Preis-Confidence liegt unter der Mindestschwelle.",
|
||||
"SHIPPING_UNKNOWN": "Obligatorische Versandkosten sind unbekannt.",
|
||||
"OVER_BUDGET": "Der Gesamtpreis liegt über dem Budgetmaximum.",
|
||||
"OUT_OF_STOCK": "Das Angebot ist nicht lieferbar.",
|
||||
"AVAILABILITY_UNKNOWN": "Die Verfügbarkeit ist unbekannt.",
|
||||
"SOURCE_MISMATCH": "Die Beobachtungsquelle weicht vom Angebot ab.",
|
||||
}
|
||||
return messages.get(gate.reason_code, gate.reason_code)
|
||||
|
||||
|
||||
def _constraint_message(item: ConstraintResult) -> str:
|
||||
if item.status == ConstraintStatus.FAIL.value:
|
||||
return f"Constraint {item.constraint_id} ist nicht erfüllt."
|
||||
return f"Constraint {item.constraint_id} kann nicht bewertet werden."
|
||||
|
||||
|
||||
def _all_components(
|
||||
preference_components: tuple[ScoreComponent, ...],
|
||||
product_gates: tuple[GateResult, ...],
|
||||
offer_gates: tuple[GateResult, ...],
|
||||
budget_score: Decimal | None,
|
||||
history_score: Decimal | None,
|
||||
model: ScoringModel,
|
||||
) -> tuple[ScoreComponent, ...]:
|
||||
extra: list[ScoreComponent] = list(preference_components)
|
||||
if budget_score is not None:
|
||||
extra.append(
|
||||
ScoreComponent(
|
||||
kind="budget",
|
||||
component_key="budget",
|
||||
raw_score=budget_score,
|
||||
weight=model.budget_weight,
|
||||
contribution=model.budget_weight * budget_score,
|
||||
details={},
|
||||
)
|
||||
)
|
||||
if history_score is not None:
|
||||
extra.append(
|
||||
ScoreComponent(
|
||||
kind="history",
|
||||
component_key="history",
|
||||
raw_score=history_score,
|
||||
weight=model.history_weight,
|
||||
contribution=model.history_weight * history_score,
|
||||
details={},
|
||||
)
|
||||
)
|
||||
for gate in product_gates:
|
||||
extra.append(
|
||||
ScoreComponent(
|
||||
kind="product_gate",
|
||||
component_key=gate.key,
|
||||
raw_score=None,
|
||||
weight=None,
|
||||
contribution=None,
|
||||
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
||||
)
|
||||
)
|
||||
for gate in offer_gates:
|
||||
extra.append(
|
||||
ScoreComponent(
|
||||
kind="offer_gate",
|
||||
component_key=gate.key,
|
||||
raw_score=None,
|
||||
weight=None,
|
||||
contribution=None,
|
||||
details={"status": gate.status, "reason_code": gate.reason_code, **gate.details},
|
||||
)
|
||||
)
|
||||
return tuple(extra)
|
||||
|
||||
|
||||
def _product_dict(product: Product) -> dict[str, Any]:
|
||||
return {
|
||||
"id": product.id,
|
||||
"category": product.category,
|
||||
"brand": product.brand,
|
||||
"name": product.name,
|
||||
"model": product.model,
|
||||
"created_at": product.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _variant_dict(variant: ProductVariant) -> dict[str, Any]:
|
||||
return {
|
||||
"id": variant.id,
|
||||
"product_id": variant.product_id,
|
||||
"name": variant.name,
|
||||
"manufacturer_sku": variant.manufacturer_sku,
|
||||
"identifiers": variant.identifiers,
|
||||
"created_at": variant.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _offer_dict(offer: Offer) -> dict[str, Any]:
|
||||
return {
|
||||
"id": offer.id,
|
||||
"variant_id": offer.variant_id,
|
||||
"source_id": offer.source_id,
|
||||
"external_id": offer.external_id,
|
||||
"seller": offer.seller,
|
||||
"url": offer.url,
|
||||
"market": offer.market,
|
||||
"currency": offer.currency,
|
||||
"condition": offer.condition,
|
||||
"created_at": offer.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _attribute_dict(statement: AttributeStatement) -> dict[str, Any]:
|
||||
return {
|
||||
"id": statement.id,
|
||||
"variant_id": statement.variant_id,
|
||||
"attribute": statement.attribute,
|
||||
"raw_value": statement.raw_value,
|
||||
"normalized_value": statement.normalized_value,
|
||||
"unit": statement.unit,
|
||||
"normalization_status": statement.normalization_status,
|
||||
"normalizer_version": statement.normalizer_version,
|
||||
"confidence": statement.confidence,
|
||||
"source_id": statement.source_id,
|
||||
"source_reference": statement.source_reference,
|
||||
"observed_at": statement.observed_at,
|
||||
"ingested_at": statement.ingested_at,
|
||||
"normalization_error": statement.normalization_error,
|
||||
}
|
||||
|
||||
|
||||
def _observation_dict(observation: PriceObservation) -> dict[str, Any]:
|
||||
return {
|
||||
"id": observation.id,
|
||||
"offer_id": observation.offer_id,
|
||||
"source_id": observation.source_id,
|
||||
"external_observation_id": observation.external_observation_id,
|
||||
"observed_at": observation.observed_at,
|
||||
"ingested_at": observation.ingested_at,
|
||||
"item_price": observation.item_price,
|
||||
"shipping_price": observation.shipping_price,
|
||||
"currency": observation.currency,
|
||||
"availability": observation.availability,
|
||||
"raw_payload": observation.raw_payload,
|
||||
"confidence": observation.confidence,
|
||||
"total_price": observation.total_price,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from decimal import ROUND_HALF_UP, Decimal
|
||||
|
||||
HUNDRED = Decimal("100")
|
||||
ZERO = Decimal("0")
|
||||
TWO_PLACES = Decimal("0.01")
|
||||
|
||||
|
||||
def clamp(value: Decimal) -> Decimal:
|
||||
return min(HUNDRED, max(ZERO, value))
|
||||
|
||||
|
||||
def round_score(value: Decimal) -> Decimal:
|
||||
return value.quantize(TWO_PLACES, rounding=ROUND_HALF_UP)
|
||||
|
||||
|
||||
def as_decimal(value: object) -> Decimal:
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
if isinstance(value, bool) or value is None:
|
||||
raise TypeError("value is not numeric")
|
||||
return Decimal(str(value))
|
||||
@@ -0,0 +1,148 @@
|
||||
from collections.abc import Sequence
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.enums import Availability, PriceBasis
|
||||
from app.domain.models import Budget, HistoryWindow, PriceObservation, PriceStatistics, ScoringModel
|
||||
from app.scoring.mathutil import clamp
|
||||
|
||||
|
||||
def total_or_none(observation: PriceObservation) -> Decimal | None:
|
||||
return observation.total_price
|
||||
|
||||
|
||||
def select_current_observation(
|
||||
observations: Sequence[PriceObservation],
|
||||
*,
|
||||
as_of: datetime,
|
||||
snapshot_cutoff: datetime,
|
||||
) -> PriceObservation | None:
|
||||
eligible = [
|
||||
item
|
||||
for item in observations
|
||||
if item.observed_at <= as_of and item.ingested_at <= snapshot_cutoff
|
||||
]
|
||||
if not eligible:
|
||||
return None
|
||||
return max(eligible, key=lambda item: (item.observed_at, item.ingested_at, item.id))
|
||||
|
||||
|
||||
def median(values: Sequence[Decimal]) -> Decimal | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
mid = len(ordered) // 2
|
||||
if len(ordered) % 2:
|
||||
return ordered[mid]
|
||||
return (ordered[mid - 1] + ordered[mid]) / Decimal("2")
|
||||
|
||||
|
||||
def _quality_ok(observation: PriceObservation, minimum_confidence: Decimal) -> bool:
|
||||
return (
|
||||
observation.availability == Availability.IN_STOCK
|
||||
and observation.total_price is not None
|
||||
and observation.confidence >= minimum_confidence
|
||||
)
|
||||
|
||||
|
||||
def daily_series(
|
||||
observations: Sequence[PriceObservation],
|
||||
*,
|
||||
window_start: date,
|
||||
window_end_exclusive: date,
|
||||
minimum_confidence: Decimal,
|
||||
) -> tuple[tuple[Decimal, ...], tuple[UUID, ...]]:
|
||||
by_day: dict[date, PriceObservation] = {}
|
||||
for item in observations:
|
||||
day = item.observed_at.date()
|
||||
if day < window_start or day >= window_end_exclusive:
|
||||
continue
|
||||
current = by_day.get(day)
|
||||
if current is None or (item.observed_at, item.ingested_at, item.id) > (
|
||||
current.observed_at,
|
||||
current.ingested_at,
|
||||
current.id,
|
||||
):
|
||||
by_day[day] = item
|
||||
values: list[Decimal] = []
|
||||
ids: list[UUID] = []
|
||||
for day in sorted(by_day):
|
||||
observation = by_day[day]
|
||||
if _quality_ok(observation, minimum_confidence):
|
||||
values.append(observation.total_price or Decimal("0"))
|
||||
ids.append(observation.id)
|
||||
return tuple(values), tuple(ids)
|
||||
|
||||
|
||||
def build_offer_reference_history(
|
||||
observations: Sequence[PriceObservation],
|
||||
current: PriceObservation,
|
||||
model: ScoringModel,
|
||||
) -> dict[int, HistoryWindow]:
|
||||
current_day = current.observed_at.date()
|
||||
windows: dict[int, HistoryWindow] = {}
|
||||
for days in (30, 90, 365):
|
||||
values, ids = daily_series(
|
||||
observations,
|
||||
window_start=current_day - timedelta(days=days),
|
||||
window_end_exclusive=current_day,
|
||||
minimum_confidence=model.minimum_confidence,
|
||||
)
|
||||
windows[days] = HistoryWindow(days=days, values=values, observation_ids=ids)
|
||||
return windows
|
||||
|
||||
|
||||
def score_price(
|
||||
*,
|
||||
current_price: Decimal,
|
||||
reference_history: Sequence[Decimal],
|
||||
budget: Budget,
|
||||
model: ScoringModel,
|
||||
) -> tuple[Decimal, Decimal, Decimal | None, PriceBasis, str | None]:
|
||||
"""Pure price score over current_price, reference_history and budget."""
|
||||
if current_price <= budget.target:
|
||||
budget_score = Decimal("100")
|
||||
else:
|
||||
budget_score = clamp(
|
||||
Decimal("100") * (budget.maximum - current_price) / (budget.maximum - budget.target)
|
||||
)
|
||||
|
||||
if len(reference_history) < model.minimum_history_days:
|
||||
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
||||
|
||||
reference = median(reference_history)
|
||||
if reference is None or reference == 0:
|
||||
return budget_score, budget_score, None, PriceBasis.BUDGET_ONLY, "INSUFFICIENT_HISTORY"
|
||||
|
||||
discount = (reference - current_price) / reference
|
||||
history_score = clamp(
|
||||
Decimal("50") + Decimal("50") * discount / model.history_discount_for_max_score
|
||||
)
|
||||
price_score = model.budget_weight * budget_score + model.history_weight * history_score
|
||||
return price_score, budget_score, history_score, PriceBasis.BUDGET_AND_HISTORY, None
|
||||
|
||||
|
||||
def price_statistics_from_windows(
|
||||
current: PriceObservation | None,
|
||||
windows: dict[int, HistoryWindow],
|
||||
) -> PriceStatistics:
|
||||
window_30 = windows.get(30, HistoryWindow(30, (), ()))
|
||||
window_90 = windows.get(90, HistoryWindow(90, (), ()))
|
||||
window_365 = windows.get(365, HistoryWindow(365, (), ()))
|
||||
used = window_30.observation_ids + window_90.observation_ids + window_365.observation_ids
|
||||
if current is not None:
|
||||
used = (current.id,) + used
|
||||
lowest = min(window_365.values) if window_365.values else None
|
||||
return PriceStatistics(
|
||||
current_total=None if current is None else current.total_price,
|
||||
median_30d=median(window_30.values),
|
||||
median_30d_days=window_30.day_count,
|
||||
median_90d=median(window_90.values),
|
||||
median_90d_days=window_90.day_count,
|
||||
lowest_365d=lowest,
|
||||
lowest_365d_days=window_365.day_count,
|
||||
lowest_365d_is_full_year=window_365.day_count >= 365,
|
||||
used_observation_ids=tuple(dict.fromkeys(used)),
|
||||
current_observation_id=None if current is None else current.id,
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from app.domain.attributes import statement_is_usable
|
||||
from app.domain.enums import ConstraintStatus, PreferenceDirection
|
||||
from app.domain.models import (
|
||||
AttributeStatement,
|
||||
ConstraintResult,
|
||||
Preference,
|
||||
ScoreComponent,
|
||||
ScoringModel,
|
||||
)
|
||||
from app.scoring.mathutil import as_decimal, clamp
|
||||
|
||||
|
||||
def score_numeric_linear(value: Decimal, acceptable: Decimal, ideal: Decimal, direction: PreferenceDirection) -> Decimal:
|
||||
if direction == PreferenceDirection.HIGHER:
|
||||
return clamp(Decimal("50") + Decimal("50") * (value - acceptable) / (ideal - acceptable))
|
||||
return clamp(Decimal("50") + Decimal("50") * (acceptable - value) / (acceptable - ideal))
|
||||
|
||||
|
||||
def score_product(
|
||||
preferences: tuple[Preference, ...],
|
||||
snapshot: dict[str, AttributeStatement],
|
||||
constraint_results: tuple[ConstraintResult, ...],
|
||||
model: ScoringModel,
|
||||
) -> tuple[Decimal, Decimal, tuple[ScoreComponent, ...]]:
|
||||
components: list[ScoreComponent] = []
|
||||
weighted = Decimal("0")
|
||||
total_weight = sum((pref.weight for pref in preferences), Decimal("0"))
|
||||
usable_weight = Decimal("0")
|
||||
|
||||
for preference in preferences:
|
||||
statement = snapshot.get(preference.attribute)
|
||||
usable = statement_is_usable(statement, model.minimum_confidence)
|
||||
if usable:
|
||||
raw = score_numeric_linear(
|
||||
as_decimal(statement.normalized_value),
|
||||
preference.acceptable,
|
||||
preference.ideal,
|
||||
preference.direction,
|
||||
)
|
||||
usable_weight += preference.weight
|
||||
else:
|
||||
raw = model.missing_preference_score
|
||||
contribution = preference.weight * raw
|
||||
weighted += contribution
|
||||
components.append(
|
||||
ScoreComponent(
|
||||
kind="preference",
|
||||
component_key=preference.id,
|
||||
raw_score=raw,
|
||||
weight=preference.weight,
|
||||
contribution=contribution,
|
||||
details={
|
||||
"attribute": preference.attribute,
|
||||
"strategy": preference.strategy.value,
|
||||
"direction": preference.direction.value,
|
||||
"acceptable": preference.acceptable,
|
||||
"ideal": preference.ideal,
|
||||
"usable": usable,
|
||||
"attribute_row_id": None if statement is None else str(statement.id),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
base = weighted / total_weight if total_weight else Decimal("0")
|
||||
optional_issues = [
|
||||
result
|
||||
for result in constraint_results
|
||||
if not result.mandatory and result.status in {ConstraintStatus.FAIL.value, ConstraintStatus.UNKNOWN.value}
|
||||
]
|
||||
penalty = model.optional_constraint_penalty_points * Decimal(len(optional_issues))
|
||||
product_score = clamp(base - penalty)
|
||||
if optional_issues:
|
||||
components.append(
|
||||
ScoreComponent(
|
||||
kind="optional_penalty",
|
||||
component_key="optional_constraints",
|
||||
raw_score=penalty,
|
||||
weight=None,
|
||||
contribution=-penalty,
|
||||
details={"count": len(optional_issues), "constraint_ids": [item.constraint_id for item in optional_issues]},
|
||||
)
|
||||
)
|
||||
coverage = (usable_weight / total_weight) if total_weight else Decimal("0")
|
||||
return product_score, coverage, tuple(components)
|
||||
Whitespace-only changes.
@@ -0,0 +1,334 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discovery.enums import FilterType, MappingStatus, TaxonomyStatus
|
||||
from app.domain.serialize import jsonable
|
||||
from app.errors import AppError
|
||||
from app.persistence.discovery_models import (
|
||||
CriterionDefinitionRecord,
|
||||
CriterionMappingRecord,
|
||||
ProductCategoryRecord,
|
||||
SourceCategoryRecord,
|
||||
SourceFilterDefinitionRecord,
|
||||
SourceFilterOptionRecord,
|
||||
SourceFilterRangeRecord,
|
||||
)
|
||||
from app.discovery.definitions import upsert_definition
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
|
||||
|
||||
FILTER_OPERATORS = {
|
||||
FilterType.BOOLEAN.value: ["=", "!="],
|
||||
FilterType.ENUM.value: ["=", "!="],
|
||||
FilterType.MULTI_ENUM.value: ["in", "not_in"],
|
||||
FilterType.NUMERIC_RANGE.value: [">=", "<=", "between"],
|
||||
FilterType.NUMERIC_THRESHOLD.value: [">=", ">", "<=", "<"],
|
||||
FilterType.TEXT.value: ["=", "contains"],
|
||||
}
|
||||
|
||||
|
||||
class CatalogService:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
self.catalog = CatalogRepository(session)
|
||||
|
||||
def list_product_categories(self) -> list[dict[str, Any]]:
|
||||
rows = self.session.scalars(select(ProductCategoryRecord).order_by(ProductCategoryRecord.name)).all()
|
||||
return [{"id": row.id, "name": row.name, "status": row.status} for row in rows]
|
||||
|
||||
def get_product_category(self, category_id: str) -> ProductCategoryRecord:
|
||||
row = self.session.get(ProductCategoryRecord, category_id)
|
||||
if row is None:
|
||||
raise AppError(404, "NOT_FOUND", "product category not found", {"id": category_id})
|
||||
return row
|
||||
|
||||
def sources_for_category(self, product_category_id: str) -> list[dict[str, Any]]:
|
||||
self.get_product_category(product_category_id)
|
||||
rows = self.session.scalars(
|
||||
select(SourceCategoryRecord)
|
||||
.where(SourceCategoryRecord.product_category_id == product_category_id)
|
||||
.order_by(SourceCategoryRecord.source_id)
|
||||
).all()
|
||||
sources = {item.id: item for item in self.catalog.list_sources()}
|
||||
return [
|
||||
{
|
||||
"source": row.source_id,
|
||||
"source_category_id": row.id,
|
||||
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
|
||||
"external_category_key": row.external_category_key,
|
||||
"status": row.status,
|
||||
"taxonomy_status": row.taxonomy_status,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def register_source_category(self, source_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if self.catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
from app.sources.geizhals.category_url import parse_category_key, slugify
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
source_url = str(payload["source_url"]).strip()
|
||||
category_key = parse_category_key(source_url)
|
||||
name = str(payload.get("name") or category_key)
|
||||
product_category_id = str(payload.get("product_category_id") or slugify(name))
|
||||
ensure_product_category(self.session, product_category_id, str(payload.get("product_category_name") or name))
|
||||
category = ensure_source_category(
|
||||
self.session,
|
||||
source_id=source_id,
|
||||
external_category_key=category_key,
|
||||
product_category_id=product_category_id,
|
||||
name=name,
|
||||
source_url=source_url,
|
||||
)
|
||||
category.source_url = source_url
|
||||
category.name = name
|
||||
category.product_category_id = product_category_id
|
||||
self.session.flush()
|
||||
TaxonomyService(self.session).discover(source_id, category_key)
|
||||
items = self.source_categories(source_id)
|
||||
return next(item for item in items if item["external_category_key"] == category_key)
|
||||
|
||||
def source_categories(self, source_id: str) -> list[dict[str, Any]]:
|
||||
if self.catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
rows = self.session.scalars(
|
||||
select(SourceCategoryRecord)
|
||||
.where(SourceCategoryRecord.source_id == source_id)
|
||||
.order_by(SourceCategoryRecord.external_category_key)
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"id": row.id,
|
||||
"source": row.source_id,
|
||||
"source_category_id": row.id,
|
||||
"external_category_key": row.external_category_key,
|
||||
"name": row.name,
|
||||
"product_category_id": row.product_category_id,
|
||||
"status": row.status,
|
||||
"taxonomy_status": row.taxonomy_status,
|
||||
"source_url": row.source_url,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def available_criteria(self, product_category_id: str, source_id: str) -> dict[str, Any]:
|
||||
self.get_product_category(product_category_id)
|
||||
if self.catalog.get_source(source_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "source not found", {"source_id": source_id})
|
||||
source_category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
if source_category is not None and self._should_learn(source_category, source_id):
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
|
||||
source_category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
criteria: list[dict[str, Any]] = []
|
||||
if source_category is not None:
|
||||
filters = self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord)
|
||||
.where(SourceFilterDefinitionRecord.source_category_id == source_category.id)
|
||||
.order_by(SourceFilterDefinitionRecord.group_label, SourceFilterDefinitionRecord.label)
|
||||
).all()
|
||||
mappings = {
|
||||
row.external_filter_key: row
|
||||
for row in self.session.scalars(
|
||||
select(CriterionMappingRecord).where(
|
||||
CriterionMappingRecord.source_category_id == source_category.id
|
||||
)
|
||||
)
|
||||
}
|
||||
for filt in filters:
|
||||
mapping = mappings.get(filt.external_filter_key)
|
||||
definition = None if mapping is None else self.session.get(CriterionDefinitionRecord, mapping.criterion_definition_id)
|
||||
if definition is None:
|
||||
definition = upsert_definition(
|
||||
self.session,
|
||||
{
|
||||
"key": f"{source_id}:{source_category.external_category_key}:{filt.external_filter_key}",
|
||||
"origin": "SOURCE",
|
||||
"source": source_id,
|
||||
"product_category_id": product_category_id,
|
||||
"label": filt.label,
|
||||
"value_type": "number"
|
||||
if filt.filter_type in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}
|
||||
else "string",
|
||||
"unit": filt.canonical_unit,
|
||||
"category": product_category_id,
|
||||
},
|
||||
)
|
||||
self.session.flush()
|
||||
options = self.session.scalars(
|
||||
select(SourceFilterOptionRecord)
|
||||
.where(SourceFilterOptionRecord.filter_id == filt.id)
|
||||
.order_by(SourceFilterOptionRecord.source_value)
|
||||
).all()
|
||||
rng = self.session.get(SourceFilterRangeRecord, filt.id)
|
||||
criteria.append(
|
||||
self._criterion_out(
|
||||
definition,
|
||||
filt,
|
||||
mapping,
|
||||
options,
|
||||
rng,
|
||||
)
|
||||
)
|
||||
internals = self.session.scalars(
|
||||
select(CriterionDefinitionRecord)
|
||||
.where(
|
||||
CriterionDefinitionRecord.origin == "INTERNAL",
|
||||
(CriterionDefinitionRecord.product_category_id == product_category_id)
|
||||
| (CriterionDefinitionRecord.category == product_category_id),
|
||||
)
|
||||
.order_by(CriterionDefinitionRecord.label)
|
||||
).all()
|
||||
seen = {item["criterion_definition_id"] for item in criteria}
|
||||
for definition in internals:
|
||||
if definition.id in seen:
|
||||
continue
|
||||
criteria.append(self._criterion_out(definition, None, None, [], None))
|
||||
return jsonable(
|
||||
{
|
||||
"product_category_id": product_category_id,
|
||||
"source": source_id,
|
||||
"taxonomy_status": None if source_category is None else source_category.taxonomy_status,
|
||||
"criteria": criteria,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_learn(self, source_category: SourceCategoryRecord, source_id: str) -> bool:
|
||||
if source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
return True
|
||||
if source_category.taxonomy_status != TaxonomyStatus.READY.value:
|
||||
return False
|
||||
from app.settings import get_settings
|
||||
|
||||
if get_settings().geizhals_use_fixtures or source_id != "geizhals":
|
||||
return False
|
||||
keys = {
|
||||
row.external_filter_key
|
||||
for row in self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord).where(
|
||||
SourceFilterDefinitionRecord.source_category_id == source_category.id
|
||||
)
|
||||
)
|
||||
}
|
||||
return len(keys) < 40
|
||||
|
||||
def list_templates(self) -> list[dict[str, Any]]:
|
||||
rows, _total = self.catalog.list_profiles(200, 0)
|
||||
return [
|
||||
{
|
||||
"id": row.id,
|
||||
"version": row.version,
|
||||
"name": row.definition.get("name") or row.id,
|
||||
"product_category_id": row.definition.get("category"),
|
||||
"market": row.definition.get("market"),
|
||||
"currency": row.definition.get("currency"),
|
||||
"description": row.definition.get("description"),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def _criterion_out(
|
||||
self,
|
||||
definition: CriterionDefinitionRecord,
|
||||
filt: SourceFilterDefinitionRecord | None,
|
||||
mapping: CriterionMappingRecord | None,
|
||||
options: list[SourceFilterOptionRecord],
|
||||
rng: SourceFilterRangeRecord | None,
|
||||
) -> dict[str, Any]:
|
||||
filter_type = None if filt is None else filt.filter_type
|
||||
return {
|
||||
"criterion_definition_id": definition.id,
|
||||
"label": definition.label,
|
||||
"group_label": None if filt is None else (filt.group_label or "Allgemein"),
|
||||
"description": definition.description,
|
||||
"origin": definition.origin,
|
||||
"source": definition.source,
|
||||
"product_category_id": definition.product_category_id,
|
||||
"filter_type": filter_type,
|
||||
"unit": definition.unit or (None if filt is None else filt.canonical_unit),
|
||||
"operators": FILTER_OPERATORS.get(filter_type or "", ["=", ">=", "<="]),
|
||||
"options": [
|
||||
{
|
||||
"source_value": option.source_value,
|
||||
"normalized_value": option.normalized_value,
|
||||
"operator": option.operator,
|
||||
"unit": option.unit,
|
||||
}
|
||||
for option in options
|
||||
],
|
||||
"range": None
|
||||
if rng is None
|
||||
else {"minimum": rng.minimum, "maximum": rng.maximum, "step": rng.step, "unit": rng.unit},
|
||||
"mapping_status": None if mapping is None else mapping.status,
|
||||
"supports_discovery": mapping is not None
|
||||
and mapping.status == MappingStatus.VERIFIED.value
|
||||
and filt is not None,
|
||||
"scoring_attribute_key": definition.scoring_attribute_key,
|
||||
}
|
||||
|
||||
|
||||
def ensure_product_category(session: Session, category_id: str, name: str | None = None) -> ProductCategoryRecord:
|
||||
row = session.get(ProductCategoryRecord, category_id)
|
||||
if row is not None:
|
||||
return row
|
||||
row = ProductCategoryRecord(id=category_id, name=name or category_id.title(), status="ACTIVE")
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def ensure_source_category(
|
||||
session: Session,
|
||||
*,
|
||||
source_id: str,
|
||||
external_category_key: str,
|
||||
product_category_id: str,
|
||||
name: str,
|
||||
source_url: str,
|
||||
) -> SourceCategoryRecord:
|
||||
row = session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.external_category_key == external_category_key,
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
row = SourceCategoryRecord(
|
||||
id=uuid4(),
|
||||
source_id=source_id,
|
||||
external_category_key=external_category_key,
|
||||
name=name,
|
||||
source_url=source_url,
|
||||
product_category_id=product_category_id,
|
||||
canonical_category=product_category_id,
|
||||
status="ACTIVE",
|
||||
taxonomy_status=TaxonomyStatus.NOT_LEARNED.value,
|
||||
last_taxonomy_scan_at=None,
|
||||
last_verified_at=None,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
if row.product_category_id is None:
|
||||
row.product_category_id = product_category_id
|
||||
if not row.source_url:
|
||||
row.source_url = source_url
|
||||
session.flush()
|
||||
return row
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,210 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.clock import now_utc
|
||||
from app.domain.serialize import jsonable
|
||||
from app.domain.timeutil import require_utc, to_iso_z
|
||||
from app.errors import AppError
|
||||
from app.persistence import models as orm
|
||||
from app.persistence.db import engine
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
from app.scoring.engine import evaluate_deal
|
||||
from app.scoring.mathutil import round_score
|
||||
from app.settings import get_settings
|
||||
from sqlalchemy.orm import Session as SASession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
|
||||
def evaluate_and_store(
|
||||
*,
|
||||
variant_id: UUID,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc()
|
||||
as_of = payload.get("as_of")
|
||||
if as_of is None:
|
||||
as_of = now
|
||||
else:
|
||||
as_of = require_utc(as_of, "as_of")
|
||||
if as_of > now:
|
||||
raise AppError(422, "AS_OF_IN_FUTURE", "as_of must not be in the future")
|
||||
snapshot_cutoff = now
|
||||
|
||||
maker = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
with engine.connect().execution_options(isolation_level="REPEATABLE READ") as connection:
|
||||
with SASession(bind=connection) as session:
|
||||
with session.begin():
|
||||
repo = CatalogRepository(session)
|
||||
variant = repo.get_variant(variant_id)
|
||||
if variant is None:
|
||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||
offer = repo.get_offer(payload["offer_id"])
|
||||
if offer is None:
|
||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(payload["offer_id"])})
|
||||
if offer.variant_id != variant_id:
|
||||
raise AppError(422, "OFFER_VARIANT_MISMATCH", "offer does not belong to the variant")
|
||||
product = repo.get_product(variant.product_id)
|
||||
if product is None:
|
||||
raise AppError(404, "NOT_FOUND", "product not found")
|
||||
profile = repo.get_profile(payload["profile_id"], int(payload["profile_version"]))
|
||||
if profile is None:
|
||||
raise AppError(404, "NOT_FOUND", "profile version not found")
|
||||
model = repo.get_scoring_model(profile.scoring_version)
|
||||
if model is None:
|
||||
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version is missing")
|
||||
attributes = tuple(repo.list_attributes(variant_id))
|
||||
observations, _ = repo.list_observations(offer.id, limit=None)
|
||||
result = evaluate_deal(
|
||||
profile=profile,
|
||||
scoring_model=model,
|
||||
product=product,
|
||||
variant=variant,
|
||||
offer=offer,
|
||||
attributes=attributes,
|
||||
observations=tuple(observations),
|
||||
definitions=repo.all_definitions(),
|
||||
as_of=as_of,
|
||||
snapshot_cutoff=snapshot_cutoff,
|
||||
engine_build=get_settings().engine_build,
|
||||
)
|
||||
response = evaluation_to_response(result, evaluation_id=uuid4(), created_at=now)
|
||||
record = orm.EvaluationRecord(
|
||||
id=response["id"],
|
||||
variant_id=variant.id,
|
||||
offer_id=offer.id,
|
||||
profile_id=profile.id,
|
||||
profile_version=profile.version,
|
||||
scoring_version=model.version,
|
||||
as_of=as_of,
|
||||
snapshot_cutoff=snapshot_cutoff,
|
||||
scoring_hash=model.definition_hash,
|
||||
engine_build=result.engine_build,
|
||||
product_eligibility=result.product_eligibility.value,
|
||||
offer_eligibility=result.offer_eligibility.value,
|
||||
overall_eligibility=result.overall_eligibility.value,
|
||||
product_score=result.product_score,
|
||||
price_score=result.price_score,
|
||||
deal_score=result.deal_score,
|
||||
classification=result.classification,
|
||||
price_reference_scope=result.price_reference_scope.value,
|
||||
input_snapshot=result.input_snapshot,
|
||||
result_snapshot=response,
|
||||
created_at=now,
|
||||
)
|
||||
repo.add_evaluation(record)
|
||||
for component in result.components:
|
||||
repo.add_component(
|
||||
orm.EvaluationComponentRecord(
|
||||
id=uuid4(),
|
||||
evaluation_id=record.id,
|
||||
kind=component.kind,
|
||||
component_key=component.component_key,
|
||||
raw_score=component.raw_score,
|
||||
weight=component.weight,
|
||||
contribution=component.contribution,
|
||||
details=jsonable(component.details),
|
||||
)
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def load_evaluation(session: Session, evaluation_id: UUID) -> dict[str, Any]:
|
||||
record = CatalogRepository(session).get_evaluation(evaluation_id)
|
||||
if record is None:
|
||||
raise AppError(404, "NOT_FOUND", "evaluation not found", {"id": str(evaluation_id)})
|
||||
return record.result_snapshot
|
||||
|
||||
|
||||
def evaluation_to_response(result, *, evaluation_id: UUID, created_at: datetime) -> dict[str, Any]:
|
||||
stats = result.price_statistics
|
||||
return {
|
||||
"id": str(evaluation_id),
|
||||
"profile_id": result.profile_id,
|
||||
"profile_version": result.profile_version,
|
||||
"scoring_version": result.scoring_version,
|
||||
"variant_id": str(result.variant_id),
|
||||
"offer_id": str(result.offer_id),
|
||||
"as_of": to_iso_z(result.as_of),
|
||||
"snapshot_cutoff": to_iso_z(result.snapshot_cutoff),
|
||||
"scoring_hash": result.scoring_hash,
|
||||
"engine_build": result.engine_build,
|
||||
"product_eligibility": result.product_eligibility.value,
|
||||
"offer_eligibility": result.offer_eligibility.value,
|
||||
"overall_eligibility": result.overall_eligibility.value,
|
||||
"eligible": result.eligible,
|
||||
"product_score": _num(result.product_score),
|
||||
"price_score": _num(result.price_score),
|
||||
"deal_score": _num(result.deal_score),
|
||||
"classification": result.classification,
|
||||
"price_basis": None if result.price_basis is None else result.price_basis.value,
|
||||
"product_coverage": _num(result.product_coverage),
|
||||
"failed_constraints": [_constraint(item) for item in result.failed_constraints],
|
||||
"unknown_constraints": [_constraint(item) for item in result.unknown_constraints],
|
||||
"constraint_results": [_constraint(item) for item in result.constraint_results],
|
||||
"product_gates": [_gate(item) for item in result.product_gates],
|
||||
"offer_gates": [_gate(item) for item in result.offer_gates],
|
||||
"components": [_component(item) for item in result.components],
|
||||
"price_statistics": {
|
||||
"current_total": _money(stats.current_total),
|
||||
"median_30d": _money(stats.median_30d),
|
||||
"median_30d_days": stats.median_30d_days,
|
||||
"median_90d": _money(stats.median_90d),
|
||||
"median_90d_days": stats.median_90d_days,
|
||||
"lowest_365d": _money(stats.lowest_365d),
|
||||
"lowest_365d_days": stats.lowest_365d_days,
|
||||
"lowest_365d_is_full_year": stats.lowest_365d_is_full_year,
|
||||
"used_observation_ids": [str(item) for item in stats.used_observation_ids],
|
||||
"current_observation_id": None if stats.current_observation_id is None else str(stats.current_observation_id),
|
||||
},
|
||||
"price_reference_scope": result.price_reference_scope.value,
|
||||
"reasons": [{"code": item.code, "message": item.message} for item in result.reasons],
|
||||
"warnings": list(result.warnings),
|
||||
"input_snapshot": result.input_snapshot,
|
||||
"created_at": to_iso_z(created_at),
|
||||
}
|
||||
|
||||
|
||||
def _num(value: Decimal | None) -> float | None:
|
||||
return None if value is None else float(round_score(value))
|
||||
|
||||
|
||||
def _money(value: Decimal | None) -> str | None:
|
||||
return None if value is None else format(value.quantize(Decimal("0.01")), "f")
|
||||
|
||||
|
||||
def _constraint(item) -> dict[str, Any]:
|
||||
return {
|
||||
"constraint_id": item.constraint_id,
|
||||
"attribute": item.attribute,
|
||||
"operator": item.operator,
|
||||
"expected": jsonable(item.expected),
|
||||
"actual": jsonable(item.actual),
|
||||
"status": item.status,
|
||||
"mandatory": item.mandatory,
|
||||
"attribute_row_id": None if item.attribute_row_id is None else str(item.attribute_row_id),
|
||||
"reason_code": item.reason_code,
|
||||
}
|
||||
|
||||
|
||||
def _gate(item) -> dict[str, Any]:
|
||||
return {
|
||||
"key": item.key,
|
||||
"status": item.status,
|
||||
"reason_code": item.reason_code,
|
||||
"details": jsonable(item.details),
|
||||
}
|
||||
|
||||
|
||||
def _component(item) -> dict[str, Any]:
|
||||
return {
|
||||
"kind": item.kind,
|
||||
"component_key": item.component_key,
|
||||
"raw_score": None if item.raw_score is None else float(round_score(item.raw_score)),
|
||||
"weight": None if item.weight is None else float(item.weight),
|
||||
"contribution": None if item.contribution is None else float(round_score(item.contribution)),
|
||||
"details": jsonable(item.details),
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
from app.errors import AppError
|
||||
from app.services.ingestion_service import IngestionService
|
||||
from app.sources.base import ProductSource
|
||||
|
||||
|
||||
class ImportService:
|
||||
def __init__(self, ingestion: IngestionService) -> None:
|
||||
self.ingestion = ingestion
|
||||
|
||||
def import_product(self, source: ProductSource, external_id: str) -> None:
|
||||
product = source.get_product(external_id)
|
||||
try:
|
||||
self.ingestion.create_product(
|
||||
{
|
||||
"id": product.id,
|
||||
"category": product.category,
|
||||
"brand": product.brand,
|
||||
"name": product.name,
|
||||
"model": product.model,
|
||||
}
|
||||
)
|
||||
except AppError as exc:
|
||||
if exc.status_code != 409:
|
||||
raise
|
||||
for variant in product.variants:
|
||||
try:
|
||||
self.ingestion.create_variant(
|
||||
product.id,
|
||||
{
|
||||
"id": variant.id,
|
||||
"name": variant.name,
|
||||
"manufacturer_sku": variant.manufacturer_sku,
|
||||
"identifiers": variant.identifiers,
|
||||
},
|
||||
)
|
||||
except AppError as exc:
|
||||
if exc.status_code != 409:
|
||||
raise
|
||||
for attribute in variant.attributes:
|
||||
try:
|
||||
self.ingestion.add_attribute(
|
||||
variant.id,
|
||||
{
|
||||
"id": attribute.id,
|
||||
"attribute": attribute.attribute,
|
||||
"raw_value": attribute.raw_value,
|
||||
"normalized_value": attribute.normalized_value,
|
||||
"unit": attribute.unit,
|
||||
"normalization_status": attribute.normalization_status,
|
||||
"normalizer_version": attribute.normalizer_version,
|
||||
"confidence": attribute.confidence,
|
||||
"source_id": source.source_key,
|
||||
"source_reference": attribute.source_reference,
|
||||
"observed_at": attribute.observed_at,
|
||||
"normalization_error": attribute.normalization_error,
|
||||
},
|
||||
)
|
||||
except AppError as exc:
|
||||
if exc.status_code != 409:
|
||||
raise
|
||||
for offer in source.get_offers(external_id):
|
||||
try:
|
||||
self.ingestion.create_offer(
|
||||
offer.variant_id,
|
||||
{
|
||||
"id": offer.id,
|
||||
"source_id": source.source_key,
|
||||
"external_id": offer.external_id,
|
||||
"seller": offer.seller,
|
||||
"url": offer.url,
|
||||
"market": offer.market,
|
||||
"currency": offer.currency,
|
||||
"condition": offer.condition,
|
||||
},
|
||||
)
|
||||
except AppError as exc:
|
||||
if exc.status_code != 409:
|
||||
raise
|
||||
for price in source.get_price_history(offer.external_id):
|
||||
self.ingestion.add_price(
|
||||
offer.id,
|
||||
{
|
||||
"id": price.id,
|
||||
"source_id": source.source_key,
|
||||
"external_observation_id": price.external_observation_id,
|
||||
"observed_at": price.observed_at,
|
||||
"item_price": price.item_price,
|
||||
"shipping_price": price.shipping_price,
|
||||
"currency": price.currency,
|
||||
"availability": price.availability,
|
||||
"raw_payload": price.raw_payload,
|
||||
"confidence": price.confidence,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,263 @@
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.domain.enums import Availability, NormalizationStatus, OfferCondition, ValueType
|
||||
from app.domain.models import AttributeDefinition, AttributeStatement, Offer, PriceObservation, Product, ProductVariant
|
||||
from app.clock import now_utc
|
||||
from app.domain.timeutil import require_utc
|
||||
from app.domain.validation import DomainValidationError, parse_profile, validate_attribute_key, validate_currency
|
||||
from app.errors import AppError
|
||||
from app.normalization.manual import ManualNormalizer
|
||||
from app.persistence import models as orm
|
||||
from app.persistence.mapping import to_observation
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return now_utc()
|
||||
|
||||
|
||||
def _uuid(value: UUID | None) -> UUID:
|
||||
return value or uuid4()
|
||||
|
||||
|
||||
class IngestionService:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
self.repo = CatalogRepository(session)
|
||||
self.normalizer = ManualNormalizer()
|
||||
|
||||
def create_definition(self, payload: dict[str, Any]) -> AttributeDefinition:
|
||||
key = str(payload["key"])
|
||||
validate_attribute_key(key)
|
||||
row = orm.AttributeDefinitionRecord(
|
||||
key=key,
|
||||
value_type=ValueType(payload["value_type"]).value,
|
||||
canonical_unit=payload.get("canonical_unit"),
|
||||
minimum=payload.get("minimum"),
|
||||
maximum=payload.get("maximum"),
|
||||
description=payload.get("description"),
|
||||
)
|
||||
if self.repo.get_definition(key) is not None:
|
||||
raise AppError(409, "CONFLICT", "attribute definition already exists", {"key": key})
|
||||
return self.repo.add_definition(row)
|
||||
|
||||
def create_profile(self, payload: dict[str, Any]) -> orm.ProductProfileRecord:
|
||||
try:
|
||||
model_versions = self.repo.scoring_versions()
|
||||
scoring_version = payload.get("scoring_version")
|
||||
model = self.repo.get_scoring_model(scoring_version) if scoring_version else None
|
||||
if model is None:
|
||||
raise AppError(422, "UNKNOWN_SCORING_VERSION", "scoring version does not exist")
|
||||
profile = parse_profile(
|
||||
payload,
|
||||
attribute_definitions=self.repo.all_definitions(),
|
||||
scoring_versions=model_versions,
|
||||
supported_strategies=set(model.supported_preference_strategies),
|
||||
)
|
||||
except DomainValidationError as exc:
|
||||
raise AppError(422, "PROFILE_INVALID", str(exc), exc.details) from exc
|
||||
except AppError:
|
||||
raise
|
||||
row = orm.ProductProfileRecord(
|
||||
id=profile.id,
|
||||
version=profile.version,
|
||||
name=profile.name,
|
||||
category=profile.category,
|
||||
scoring_version=profile.scoring_version,
|
||||
definition=profile.definition,
|
||||
definition_hash=profile.definition_hash,
|
||||
created_at=_now(),
|
||||
)
|
||||
if self.repo.get_profile(profile.id, profile.version) is not None:
|
||||
raise AppError(409, "CONFLICT", "profile version already exists", {"id": profile.id, "version": profile.version})
|
||||
return self.repo.add_profile(row)
|
||||
|
||||
def create_product(self, payload: dict[str, Any]) -> Product:
|
||||
product_id = _uuid(payload.get("id"))
|
||||
if self.repo.get_product(product_id) is not None:
|
||||
raise AppError(409, "CONFLICT", "product already exists", {"id": str(product_id)})
|
||||
row = orm.ProductRecord(
|
||||
id=product_id,
|
||||
category=str(payload["category"]),
|
||||
brand=str(payload["brand"]),
|
||||
name=str(payload["name"]),
|
||||
model=payload.get("model"),
|
||||
created_at=_now(),
|
||||
)
|
||||
return self.repo.add_product(row)
|
||||
|
||||
def create_variant(self, product_id: UUID, payload: dict[str, Any]) -> ProductVariant:
|
||||
if self.repo.get_product(product_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "product not found", {"id": str(product_id)})
|
||||
variant_id = _uuid(payload.get("id"))
|
||||
if self.repo.get_variant(variant_id) is not None:
|
||||
raise AppError(409, "CONFLICT", "variant already exists", {"id": str(variant_id)})
|
||||
row = orm.ProductVariantRecord(
|
||||
id=variant_id,
|
||||
product_id=product_id,
|
||||
name=str(payload["name"]),
|
||||
manufacturer_sku=payload.get("manufacturer_sku"),
|
||||
identifiers=payload.get("identifiers") or {},
|
||||
created_at=_now(),
|
||||
)
|
||||
return self.repo.add_variant(row)
|
||||
|
||||
def add_attribute(self, variant_id: UUID, payload: dict[str, Any]) -> AttributeStatement:
|
||||
variant = self.repo.get_variant(variant_id)
|
||||
if variant is None:
|
||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||
definition = self.repo.get_definition(payload["attribute"])
|
||||
if definition is None:
|
||||
raise AppError(422, "UNKNOWN_ATTRIBUTE", "attribute definition does not exist")
|
||||
if self.repo.get_source(payload["source_id"]) is None:
|
||||
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
||||
observed_at = require_utc(payload["observed_at"], "observed_at")
|
||||
ingested_at = _now()
|
||||
if observed_at > ingested_at:
|
||||
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
||||
confidence = Decimal(str(payload["confidence"]))
|
||||
if not Decimal("0") <= confidence <= Decimal("1"):
|
||||
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
||||
statement = AttributeStatement(
|
||||
id=_uuid(payload.get("id")),
|
||||
variant_id=variant_id,
|
||||
attribute=definition.key,
|
||||
raw_value=payload["raw_value"],
|
||||
normalized_value=payload.get("normalized_value"),
|
||||
unit=payload.get("unit"),
|
||||
normalization_status=NormalizationStatus(payload["normalization_status"]),
|
||||
normalizer_version=payload.get("normalizer_version") or self.normalizer.version,
|
||||
confidence=confidence,
|
||||
source_id=payload["source_id"],
|
||||
source_reference=payload["source_reference"],
|
||||
observed_at=observed_at,
|
||||
ingested_at=ingested_at,
|
||||
normalization_error=payload.get("normalization_error"),
|
||||
)
|
||||
statement = self.normalizer.validate(definition, statement)
|
||||
if self.repo.get_attribute(statement.id) is not None:
|
||||
raise AppError(409, "CONFLICT", "attribute statement already exists", {"id": str(statement.id)})
|
||||
row = orm.VariantAttributeRecord(
|
||||
id=statement.id,
|
||||
variant_id=variant_id,
|
||||
attribute_key=statement.attribute,
|
||||
source_id=statement.source_id,
|
||||
raw_value=statement.raw_value,
|
||||
normalized_value=statement.normalized_value,
|
||||
unit=statement.unit,
|
||||
normalization_status=statement.normalization_status.value,
|
||||
normalizer_version=statement.normalizer_version,
|
||||
confidence=statement.confidence,
|
||||
source_reference=statement.source_reference,
|
||||
observed_at=statement.observed_at,
|
||||
ingested_at=statement.ingested_at,
|
||||
normalization_error=statement.normalization_error,
|
||||
)
|
||||
return self.repo.add_attribute(row)
|
||||
|
||||
def create_offer(self, variant_id: UUID, payload: dict[str, Any]) -> Offer:
|
||||
if self.repo.get_variant(variant_id) is None:
|
||||
raise AppError(404, "NOT_FOUND", "variant not found", {"id": str(variant_id)})
|
||||
if self.repo.get_source(payload["source_id"]) is None:
|
||||
raise AppError(422, "UNKNOWN_SOURCE", "source does not exist")
|
||||
offer_id = _uuid(payload.get("id"))
|
||||
if self.repo.get_offer(offer_id) is not None:
|
||||
raise AppError(409, "CONFLICT", "offer already exists", {"id": str(offer_id)})
|
||||
if self.repo.get_offer_by_external(payload["source_id"], str(payload["external_id"])) is not None:
|
||||
raise AppError(409, "CONFLICT", "offer source/external_id already exists")
|
||||
row = orm.OfferRecord(
|
||||
id=offer_id,
|
||||
variant_id=variant_id,
|
||||
source_id=payload["source_id"],
|
||||
external_id=str(payload["external_id"]),
|
||||
seller=str(payload["seller"]),
|
||||
url=payload.get("url"),
|
||||
market=str(payload["market"]),
|
||||
currency=validate_currency(str(payload["currency"])),
|
||||
condition=OfferCondition(payload["condition"]).value,
|
||||
created_at=_now(),
|
||||
)
|
||||
return self.repo.add_offer(row)
|
||||
|
||||
def add_price(self, offer_id: UUID, payload: dict[str, Any]) -> tuple[PriceObservation, bool]:
|
||||
offer = self.repo.get_offer(offer_id)
|
||||
if offer is None:
|
||||
raise AppError(404, "NOT_FOUND", "offer not found", {"id": str(offer_id)})
|
||||
if payload["source_id"] != offer.source_id:
|
||||
raise AppError(422, "SOURCE_MISMATCH", "source_id must match the offer")
|
||||
currency = validate_currency(str(payload["currency"]))
|
||||
if currency != offer.currency:
|
||||
raise AppError(422, "CURRENCY_MISMATCH", "currency must match the offer")
|
||||
observed_at = require_utc(payload["observed_at"], "observed_at")
|
||||
ingested_at = _now()
|
||||
if observed_at > ingested_at:
|
||||
raise AppError(422, "FUTURE_OBSERVATION", "observed_at must not be after ingestion")
|
||||
item_price = Decimal(str(payload["item_price"]))
|
||||
if item_price <= 0:
|
||||
raise AppError(422, "ITEM_PRICE", "item_price must be > 0")
|
||||
shipping = payload.get("shipping_price")
|
||||
shipping_price = None if shipping is None else Decimal(str(shipping))
|
||||
if shipping_price is not None and shipping_price < 0:
|
||||
raise AppError(422, "SHIPPING_PRICE", "shipping_price must be >= 0")
|
||||
confidence = Decimal(str(payload["confidence"]))
|
||||
if not Decimal("0") <= confidence <= Decimal("1"):
|
||||
raise AppError(422, "CONFIDENCE_RANGE", "confidence must be between 0 and 1")
|
||||
external_id = str(payload["external_observation_id"])
|
||||
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
||||
incoming = {
|
||||
"source_id": payload["source_id"],
|
||||
"observed_at": observed_at,
|
||||
"item_price": item_price,
|
||||
"shipping_price": shipping_price,
|
||||
"currency": currency,
|
||||
"availability": Availability(payload["availability"]).value,
|
||||
"raw_payload": payload["raw_payload"],
|
||||
"confidence": confidence,
|
||||
}
|
||||
if existing is not None:
|
||||
if _observation_content(existing) != incoming:
|
||||
raise AppError(409, "CONFLICT", "price observation content differs")
|
||||
return to_observation(existing), False
|
||||
row = orm.PriceObservationRecord(
|
||||
id=_uuid(payload.get("id")),
|
||||
offer_id=offer_id,
|
||||
source_id=payload["source_id"],
|
||||
external_observation_id=external_id,
|
||||
observed_at=observed_at,
|
||||
ingested_at=ingested_at,
|
||||
item_price=item_price,
|
||||
shipping_price=shipping_price,
|
||||
currency=currency,
|
||||
availability=incoming["availability"],
|
||||
raw_payload=payload["raw_payload"],
|
||||
confidence=confidence,
|
||||
)
|
||||
try:
|
||||
return self.repo.add_observation(row), True
|
||||
except IntegrityError:
|
||||
self.session.rollback()
|
||||
existing = self.repo.get_observation_by_external(offer_id, external_id)
|
||||
if existing is None:
|
||||
raise
|
||||
if _observation_content(existing) != incoming:
|
||||
raise AppError(409, "CONFLICT", "price observation content differs")
|
||||
return to_observation(existing), False
|
||||
|
||||
|
||||
def _observation_content(row: orm.PriceObservationRecord) -> dict[str, Any]:
|
||||
return {
|
||||
"source_id": row.source_id,
|
||||
"observed_at": row.observed_at,
|
||||
"item_price": Decimal(str(row.item_price)),
|
||||
"shipping_price": None if row.shipping_price is None else Decimal(str(row.shipping_price)),
|
||||
"currency": row.currency,
|
||||
"availability": row.availability,
|
||||
"raw_payload": row.raw_payload,
|
||||
"confidence": Decimal(str(row.confidence)),
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.clock import now_utc
|
||||
from app.discovery.criteria import criterion_payload, parse_project_criteria
|
||||
from app.discovery.definitions import get_definition, get_definition_by_key, upsert_definition
|
||||
from app.discovery.enums import CandidateStatus, DiffStatus, ProjectStatus, RankingStatus, SearchStatus
|
||||
from app.domain.serialize import jsonable
|
||||
from app.domain.validation import DomainValidationError
|
||||
from app.errors import AppError
|
||||
from app.persistence.discovery_models import (
|
||||
BuyingProjectCriteriaRecord,
|
||||
BuyingProjectRecord,
|
||||
DiscoverySearchRecord,
|
||||
ProductCategoryRecord,
|
||||
ProjectCandidateRecord,
|
||||
ProjectCriterionRecord,
|
||||
SearchRunItemRecord,
|
||||
SearchRunRecord,
|
||||
)
|
||||
from app.persistence.repositories import CatalogRepository
|
||||
from app.services.catalog_service import CatalogService, ensure_product_category
|
||||
|
||||
|
||||
INACTIVE_SEARCH = {SearchStatus.BLOCKED.value, SearchStatus.DISABLED.value, SearchStatus.PAUSED.value}
|
||||
|
||||
|
||||
def _uuid(value: UUID | None) -> UUID:
|
||||
return value or uuid4()
|
||||
|
||||
|
||||
class ProjectService:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
self.catalog = CatalogRepository(session)
|
||||
self.categories = CatalogService(session)
|
||||
|
||||
def create(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
template_id = payload.get("template_id") or payload.get("profile_id")
|
||||
template_version = payload.get("template_version") or payload.get("profile_version")
|
||||
profile = None
|
||||
if template_id is not None:
|
||||
if template_version is None:
|
||||
raise AppError(422, "VALIDATION_ERROR", "template_version is required when template_id is set")
|
||||
profile = self.catalog.get_profile(str(template_id), int(template_version))
|
||||
if profile is None:
|
||||
raise AppError(404, "NOT_FOUND", "template version not found")
|
||||
product_category_id = payload.get("product_category_id") or (None if profile is None else profile.category)
|
||||
if not product_category_id:
|
||||
raise AppError(422, "VALIDATION_ERROR", "product_category_id is required")
|
||||
if self.session.get(ProductCategoryRecord, product_category_id) is None:
|
||||
ensure_product_category(self.session, str(product_category_id))
|
||||
market = payload.get("market") or (None if profile is None else profile.market)
|
||||
currency = payload.get("currency") or (None if profile is None else profile.currency)
|
||||
if not market or not currency:
|
||||
raise AppError(422, "VALIDATION_ERROR", "market and currency are required")
|
||||
target_budget = payload.get("target_budget")
|
||||
if target_budget is None and profile is not None and getattr(profile, "budget", None) is not None:
|
||||
target_budget = getattr(profile.budget, "target", None)
|
||||
criteria_payload = payload.get("criteria")
|
||||
if not criteria_payload:
|
||||
raise AppError(422, "CRITERIA_REQUIRED", "criteria are required")
|
||||
try:
|
||||
parsed = parse_project_criteria(criteria_payload)
|
||||
except DomainValidationError as exc:
|
||||
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
||||
shortlist_size = int(payload.get("shortlist_size") or 5)
|
||||
if shortlist_size < 1:
|
||||
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
||||
now = now_utc()
|
||||
project = BuyingProjectRecord(
|
||||
id=_uuid(payload.get("id")),
|
||||
name=str(payload["name"]),
|
||||
status=ProjectStatus.ACTIVE.value,
|
||||
workspace_id=payload.get("workspace_id"),
|
||||
product_category_id=str(product_category_id),
|
||||
market=str(market),
|
||||
currency=str(currency),
|
||||
target_budget=None if target_budget is None else Decimal(str(target_budget)),
|
||||
template_id=None if profile is None else profile.id,
|
||||
template_version=None if profile is None else profile.version,
|
||||
shortlist_size=shortlist_size,
|
||||
criteria_version=1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
archived_at=None,
|
||||
)
|
||||
self.session.add(project)
|
||||
self.session.flush()
|
||||
self._store_criteria(project.id, 1, parsed, str(product_category_id), now)
|
||||
return self.get(project.id)
|
||||
|
||||
def replace_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
project = self._require_project(project_id)
|
||||
try:
|
||||
parsed = parse_project_criteria(payload)
|
||||
except DomainValidationError as exc:
|
||||
raise AppError(422, "CRITERIA_INVALID", str(exc), exc.details) from exc
|
||||
current = self.latest_criteria(project_id)
|
||||
version = (current.version if current is not None else 0) + 1
|
||||
now = now_utc()
|
||||
self._store_criteria(project.id, version, parsed, project.product_category_id, now)
|
||||
project.criteria_version = version
|
||||
project.updated_at = now
|
||||
for search in self.session.scalars(
|
||||
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
||||
):
|
||||
if search.status not in INACTIVE_SEARCH:
|
||||
search.status = SearchStatus.OUT_OF_SYNC.value
|
||||
search.updated_at = now
|
||||
self.session.flush()
|
||||
return self.get(project.id)
|
||||
|
||||
def add_criteria(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self.replace_criteria(project_id, payload)
|
||||
|
||||
def patch(self, project_id: UUID, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
project = self._require_project(project_id)
|
||||
allowed = {"name", "target_budget", "shortlist_size", "status"}
|
||||
unknown = set(payload) - allowed
|
||||
if unknown:
|
||||
raise AppError(422, "VALIDATION_ERROR", "field cannot be patched", {"fields": sorted(unknown)})
|
||||
if "name" in payload:
|
||||
project.name = str(payload["name"])
|
||||
if "target_budget" in payload:
|
||||
project.target_budget = None if payload["target_budget"] is None else Decimal(str(payload["target_budget"]))
|
||||
if "shortlist_size" in payload:
|
||||
size = int(payload["shortlist_size"])
|
||||
if size < 1:
|
||||
raise AppError(422, "VALIDATION_ERROR", "shortlist_size must be > 0")
|
||||
project.shortlist_size = size
|
||||
if "status" in payload:
|
||||
status = str(payload["status"])
|
||||
if status not in {item.value for item in ProjectStatus}:
|
||||
raise AppError(422, "VALIDATION_ERROR", "invalid project status")
|
||||
project.status = status
|
||||
if status == ProjectStatus.ARCHIVED.value:
|
||||
project.archived_at = now_utc()
|
||||
project.updated_at = now_utc()
|
||||
self.session.flush()
|
||||
return self.get(project.id)
|
||||
|
||||
def get(self, project_id: UUID) -> dict[str, Any]:
|
||||
project = self._require_project(project_id)
|
||||
criteria = self.latest_criteria(project_id)
|
||||
return self._out(project, criteria)
|
||||
|
||||
def get_criteria(self, project_id: UUID) -> dict[str, Any]:
|
||||
project = self._require_project(project_id)
|
||||
criteria = self.latest_criteria(project_id)
|
||||
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
||||
return jsonable(
|
||||
{
|
||||
"project_id": project.id,
|
||||
"criteria_version": None if criteria is None else criteria.version,
|
||||
"items": items,
|
||||
}
|
||||
)
|
||||
|
||||
def list(self, limit: int, offset: int) -> tuple[list[dict[str, Any]], int]:
|
||||
total = self.session.scalar(select(func.count()).select_from(BuyingProjectRecord)) or 0
|
||||
rows = self.session.scalars(
|
||||
select(BuyingProjectRecord).order_by(BuyingProjectRecord.created_at).limit(limit).offset(offset)
|
||||
).all()
|
||||
return [self._summary(row) for row in rows], total
|
||||
|
||||
def latest_criteria(self, project_id: UUID) -> BuyingProjectCriteriaRecord | None:
|
||||
return self.session.scalars(
|
||||
select(BuyingProjectCriteriaRecord)
|
||||
.where(BuyingProjectCriteriaRecord.project_id == project_id)
|
||||
.order_by(BuyingProjectCriteriaRecord.version.desc())
|
||||
).first()
|
||||
|
||||
def criteria_version(self, project_id: UUID, version: int) -> BuyingProjectCriteriaRecord | None:
|
||||
return self.session.get(BuyingProjectCriteriaRecord, (project_id, version))
|
||||
|
||||
def criteria_items(self, project_id: UUID, version: int) -> list[dict[str, Any]]:
|
||||
rows = self.session.scalars(
|
||||
select(ProjectCriterionRecord)
|
||||
.where(ProjectCriterionRecord.project_id == project_id, ProjectCriterionRecord.version == version)
|
||||
.order_by(ProjectCriterionRecord.item_id)
|
||||
).all()
|
||||
return [criterion_payload(row) for row in rows]
|
||||
|
||||
def available_criteria(self, project_id: UUID) -> dict[str, Any]:
|
||||
project = self._require_project(project_id)
|
||||
sources = self.categories.sources_for_category(project.product_category_id)
|
||||
source = sources[0]["source"] if sources else "geizhals"
|
||||
return self.categories.available_criteria(project.product_category_id, source)
|
||||
|
||||
def _store_criteria(
|
||||
self,
|
||||
project_id: UUID,
|
||||
version: int,
|
||||
parsed: dict[str, Any],
|
||||
category: str | None,
|
||||
created_at,
|
||||
) -> None:
|
||||
self.session.add(
|
||||
BuyingProjectCriteriaRecord(
|
||||
project_id=project_id,
|
||||
version=version,
|
||||
definition_hash=parsed["definition_hash"],
|
||||
created_at=created_at,
|
||||
)
|
||||
)
|
||||
self.session.flush()
|
||||
for item in parsed["items"]:
|
||||
definition = self._resolve_definition(item, category)
|
||||
self.session.add(
|
||||
ProjectCriterionRecord(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
version=version,
|
||||
item_id=item["id"],
|
||||
criterion_definition_id=definition.id,
|
||||
criterion_key=definition.key,
|
||||
operator=item["operator"],
|
||||
value=item["value"],
|
||||
enabled_for_discovery=item["enabled_for_discovery"],
|
||||
created_at=created_at,
|
||||
)
|
||||
)
|
||||
self.session.flush()
|
||||
|
||||
def _resolve_definition(self, item: dict[str, Any], category: str | None):
|
||||
if item.get("criterion_definition_id") is not None:
|
||||
definition = get_definition(self.session, item["criterion_definition_id"])
|
||||
if definition is None:
|
||||
raise AppError(422, "UNKNOWN_CRITERION", "criterion definition not found")
|
||||
return definition
|
||||
existing = get_definition_by_key(self.session, str(item["criterion"]))
|
||||
if existing is not None:
|
||||
return existing
|
||||
if not item.get("value_type"):
|
||||
raise AppError(
|
||||
422,
|
||||
"UNKNOWN_CRITERION",
|
||||
"criterion definition missing; provide value_type or create it first",
|
||||
{"criterion": item["criterion"]},
|
||||
)
|
||||
return upsert_definition(
|
||||
self.session,
|
||||
{
|
||||
"key": item["criterion"],
|
||||
"origin": "INTERNAL",
|
||||
"product_category_id": item.get("category") or category,
|
||||
"label": item["criterion"],
|
||||
"value_type": item["value_type"],
|
||||
"unit": item.get("unit"),
|
||||
"semantic": item.get("semantic"),
|
||||
"category": item.get("category") or category,
|
||||
"scoring_attribute_key": item["criterion"],
|
||||
},
|
||||
)
|
||||
|
||||
def _require_project(self, project_id: UUID) -> BuyingProjectRecord:
|
||||
project = self.session.get(BuyingProjectRecord, project_id)
|
||||
if project is None:
|
||||
raise AppError(404, "NOT_FOUND", "buying project not found")
|
||||
return project
|
||||
|
||||
def _out(self, project: BuyingProjectRecord, criteria: BuyingProjectCriteriaRecord | None) -> dict[str, Any]:
|
||||
items = [] if criteria is None else self.criteria_items(project.id, criteria.version)
|
||||
return jsonable(
|
||||
{
|
||||
"id": project.id,
|
||||
"name": project.name,
|
||||
"status": project.status,
|
||||
"workspace_id": project.workspace_id,
|
||||
"product_category_id": project.product_category_id,
|
||||
"market": project.market,
|
||||
"currency": project.currency,
|
||||
"target_budget": project.target_budget,
|
||||
"template_id": project.template_id,
|
||||
"template_version": project.template_version,
|
||||
"profile_id": project.template_id,
|
||||
"profile_version": project.template_version,
|
||||
"shortlist_size": project.shortlist_size,
|
||||
"created_at": project.created_at,
|
||||
"updated_at": project.updated_at,
|
||||
"archived_at": project.archived_at,
|
||||
"criteria_version": project.criteria_version if criteria is None else criteria.version,
|
||||
"criteria": None if criteria is None else {"items": items},
|
||||
"criteria_hash": None if criteria is None else criteria.definition_hash,
|
||||
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
||||
if project.template_id is None
|
||||
else RankingStatus.CONFIGURED.value,
|
||||
}
|
||||
)
|
||||
|
||||
def _summary(self, project: BuyingProjectRecord) -> dict[str, Any]:
|
||||
category = self.session.get(ProductCategoryRecord, project.product_category_id)
|
||||
searches = list(
|
||||
self.session.scalars(
|
||||
select(DiscoverySearchRecord).where(DiscoverySearchRecord.buying_project_id == project.id)
|
||||
)
|
||||
)
|
||||
active = next((row for row in searches if row.status not in INACTIVE_SEARCH), searches[0] if searches else None)
|
||||
last_run = None
|
||||
if active is not None:
|
||||
last_run = self.session.scalars(
|
||||
select(SearchRunRecord)
|
||||
.where(SearchRunRecord.discovery_search_id == active.id)
|
||||
.order_by(SearchRunRecord.started_at.desc())
|
||||
).first()
|
||||
candidates = list(
|
||||
self.session.scalars(
|
||||
select(ProjectCandidateRecord).where(ProjectCandidateRecord.buying_project_id == project.id)
|
||||
)
|
||||
)
|
||||
counts = {"NEW": 0, "CHANGED": 0, "MISSING": 0}
|
||||
if last_run is not None:
|
||||
for item in self.session.scalars(
|
||||
select(SearchRunItemRecord).where(SearchRunItemRecord.search_run_id == last_run.id)
|
||||
):
|
||||
if item.diff_status in counts:
|
||||
counts[item.diff_status] += 1
|
||||
return jsonable(
|
||||
{
|
||||
"id": project.id,
|
||||
"name": project.name,
|
||||
"product_category": None
|
||||
if category is None
|
||||
else {"id": category.id, "name": category.name, "status": category.status},
|
||||
"status": project.status,
|
||||
"last_successful_run_at": None if active is None else active.last_successful_run_at,
|
||||
"next_run_at": None if active is None else active.next_run_at,
|
||||
"discovery_status": None if active is None else active.status,
|
||||
"candidate_count": len(candidates),
|
||||
"shortlist_count": sum(1 for row in candidates if row.status == CandidateStatus.SHORTLISTED.value),
|
||||
"new_count": last_run.new_count if last_run is not None else counts[DiffStatus.NEW.value],
|
||||
"changed_count": last_run.changed_count if last_run is not None else counts[DiffStatus.CHANGED.value],
|
||||
"missing_count": last_run.missing_count if last_run is not None else counts[DiffStatus.MISSING.value],
|
||||
"current_search_result_count": None if last_run is None else last_run.result_count,
|
||||
"ranking_status": RankingStatus.NOT_CONFIGURED.value
|
||||
if project.template_id is None
|
||||
else RankingStatus.CONFIGURED.value,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
db_host: str = "postgres"
|
||||
db_port: int = 5432
|
||||
db_name: str = "buyingagent"
|
||||
db_user: str = "buyingagent"
|
||||
db_password: str = "change_me"
|
||||
environment: str = "development"
|
||||
engine_build: str = "dev"
|
||||
clock_now: datetime | None = None
|
||||
cors_allowed_origins: str = "http://localhost:5173,http://127.0.0.1:5173"
|
||||
geizhals_use_fixtures: bool = False
|
||||
geizhals_playwright: bool = True
|
||||
|
||||
@property
|
||||
def cors_origins(self) -> list[str]:
|
||||
return [item.strip() for item in self.cors_allowed_origins.split(",") if item.strip()]
|
||||
|
||||
@property
|
||||
def database_url(self) -> str:
|
||||
return (
|
||||
f"postgresql+psycopg://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
Whitespace-only changes.
@@ -0,0 +1,85 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceProduct:
|
||||
source_key: str
|
||||
external_id: str
|
||||
id: UUID
|
||||
category: str
|
||||
brand: str
|
||||
name: str
|
||||
model: str | None
|
||||
variants: tuple["SourceVariant", ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceVariant:
|
||||
id: UUID
|
||||
name: str
|
||||
manufacturer_sku: str | None
|
||||
identifiers: dict[str, Any]
|
||||
attributes: tuple["SourceAttribute", ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceAttribute:
|
||||
id: UUID
|
||||
attribute: str
|
||||
raw_value: Any
|
||||
normalized_value: Any
|
||||
unit: str | None
|
||||
normalization_status: str
|
||||
normalizer_version: str
|
||||
confidence: Decimal
|
||||
source_reference: str
|
||||
observed_at: datetime
|
||||
normalization_error: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceOffer:
|
||||
id: UUID
|
||||
variant_id: UUID
|
||||
external_id: str
|
||||
seller: str
|
||||
url: str | None
|
||||
market: str
|
||||
currency: str
|
||||
condition: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourcePriceObservation:
|
||||
id: UUID | None
|
||||
external_offer_id: str
|
||||
external_observation_id: str
|
||||
observed_at: datetime
|
||||
item_price: Decimal
|
||||
shipping_price: Decimal | None
|
||||
currency: str
|
||||
availability: str
|
||||
raw_payload: Any
|
||||
confidence: Decimal
|
||||
|
||||
|
||||
class ProductSource(Protocol):
|
||||
source_key: str
|
||||
|
||||
def get_product(self, external_id: str) -> SourceProduct: ...
|
||||
def get_offers(self, external_product_id: str) -> list[SourceOffer]: ...
|
||||
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]: ...
|
||||
|
||||
|
||||
class DiscoverySourceAdapter(Protocol):
|
||||
source_key: str
|
||||
|
||||
def discover_taxonomy(self, category_ref: str) -> dict[str, Any]: ...
|
||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]: ...
|
||||
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]: ...
|
||||
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]: ...
|
||||
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]: ...
|
||||
@@ -0,0 +1,3 @@
|
||||
from app.sources.geizhals.adapter import GeizhalsAdapter
|
||||
|
||||
__all__ = ["GeizhalsAdapter"]
|
||||
@@ -0,0 +1,130 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.discovery.enums import FilterType
|
||||
from app.errors import AppError
|
||||
from app.settings import get_settings
|
||||
from app.sources.geizhals.category_url import parse_category_key, with_extended_filters
|
||||
from app.sources.geizhals.live_page import load_category_page, load_result_page
|
||||
from app.sources.geizhals.result_parser import parse_result_html, with_page
|
||||
from app.sources.geizhals.taxonomy_api import fetch_taxonomy_api
|
||||
from app.sources.geizhals.taxonomy_parser import parse_taxonomy_html
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
||||
|
||||
|
||||
class GeizhalsAdapter:
|
||||
source_key = "geizhals"
|
||||
|
||||
def __init__(self, fixture_dir: Path | None = None, live: bool | None = None) -> None:
|
||||
self.fixture_dir = fixture_dir or FIXTURE_DIR
|
||||
settings = get_settings()
|
||||
self.live = (not settings.geizhals_use_fixtures) if live is None else live
|
||||
|
||||
def discover_taxonomy(self, category_ref: str, source_url: str | None = None) -> dict[str, Any]:
|
||||
if not self.live:
|
||||
return self._load(f"{category_ref}-taxonomy.json")
|
||||
url = with_extended_filters(source_url or f"https://geizhals.de/?cat={category_ref}")
|
||||
category_ref = parse_category_key(url)
|
||||
api_payload = fetch_taxonomy_api(source_url or url, category_ref=category_ref)
|
||||
if api_payload and api_payload.get("filters"):
|
||||
return api_payload
|
||||
html, mode, status = load_category_page(
|
||||
url,
|
||||
category_ref=category_ref,
|
||||
playwright=get_settings().geizhals_playwright,
|
||||
)
|
||||
payload = parse_taxonomy_html(html, category_ref=category_ref, source_url=source_url or url)
|
||||
if payload.get("blocked") or not payload.get("filters"):
|
||||
raise AppError(
|
||||
422,
|
||||
"TAXONOMY_EMPTY",
|
||||
"Geizhals-Taxonomie konnte nicht aus der Kategorieseite gelesen werden",
|
||||
{"url": url, "http_status": status, "mode": mode},
|
||||
)
|
||||
payload["discovery_mode"] = mode
|
||||
payload["http_status"] = status
|
||||
payload["taxonomy_status"] = "DEGRADED" if mode == "recorded_html" else "READY"
|
||||
return payload
|
||||
|
||||
def build_query(self, criteria: list[dict[str, Any]], mappings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
from app.discovery.urlgen import generate_search_url
|
||||
|
||||
category_url = next((str(item["category_url"]) for item in mappings if item.get("category_url")), None)
|
||||
category_url = category_url or f"https://geizhals.de/?cat={self._category(mappings, criteria)}"
|
||||
url, internal_only = generate_search_url(
|
||||
category_url=category_url,
|
||||
criteria_items=criteria,
|
||||
mappings=mappings,
|
||||
)
|
||||
return {"url": url, "internal_only": internal_only}
|
||||
|
||||
def fetch_results(self, query: dict[str, Any], pagination: dict[str, Any]) -> dict[str, Any]:
|
||||
page = int(pagination.get("page") or 1)
|
||||
if not self.live:
|
||||
category = parse_qs(urlparse(query["url"]).query).get("cat", ["nb"])[0]
|
||||
return self._load(f"{category}-search-page{page}.json")
|
||||
url = with_page(str(query["url"]), page)
|
||||
html, mode, status, blocked = load_result_page(url, playwright=get_settings().geizhals_playwright)
|
||||
if blocked:
|
||||
return {
|
||||
"page": page,
|
||||
"page_count": page,
|
||||
"result_count": 0,
|
||||
"http_status": status or 403,
|
||||
"blocked": True,
|
||||
"challenge": "captcha" if status in {401, 403} else None,
|
||||
"items": [],
|
||||
"discovery_mode": mode,
|
||||
}
|
||||
payload = parse_result_html(html, page_url=url)
|
||||
payload["page"] = page
|
||||
payload["http_status"] = status
|
||||
payload["discovery_mode"] = mode
|
||||
return payload
|
||||
|
||||
def parse_results(self, response: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if response.get("html") and not response.get("items"):
|
||||
return list(parse_result_html(str(response["html"]), page_url=str(response.get("url") or "")).get("items") or [])
|
||||
return list(response.get("items") or [])
|
||||
|
||||
def normalize_result(self, result: dict[str, Any]) -> dict[str, Any]:
|
||||
attrs = result.get("normalized_attributes") or {}
|
||||
return {
|
||||
"source_external_id": result.get("source_external_id"),
|
||||
"source_url": result.get("source_url"),
|
||||
"source_title": result.get("source_title"),
|
||||
"manufacturer": result.get("manufacturer"),
|
||||
"model": result.get("model"),
|
||||
"manufacturer_sku": result.get("manufacturer_sku"),
|
||||
"gtin_ean": result.get("gtin_ean"),
|
||||
"best_price": result.get("best_price"),
|
||||
"currency": result.get("currency") or "EUR",
|
||||
"availability": result.get("availability") or "unknown",
|
||||
"seller_count": result.get("seller_count"),
|
||||
"raw_attributes": result.get("raw_attributes") or {},
|
||||
"normalized_attributes": attrs,
|
||||
}
|
||||
|
||||
def _load(self, name: str) -> dict[str, Any]:
|
||||
path = self.fixture_dir / name
|
||||
if not path.exists():
|
||||
raise AppError(404, "FIXTURE_NOT_FOUND", f"geizhals fixture missing: {name}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
def _category(self, mappings: list[dict[str, Any]], criteria: list[dict[str, Any]]) -> str:
|
||||
for mapping in mappings:
|
||||
if mapping.get("category_key"):
|
||||
return str(mapping["category_key"])
|
||||
return "nb"
|
||||
|
||||
|
||||
def filter_type(raw: str) -> str:
|
||||
try:
|
||||
return FilterType(raw).value
|
||||
except ValueError:
|
||||
return FilterType.UNKNOWN.value
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from app.errors import AppError
|
||||
|
||||
CATEGORY_PATH_RE = re.compile(r"/cat/([^/?#]+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_category_key(source_url: str) -> str:
|
||||
parts = urlsplit(source_url.strip())
|
||||
query = parse_qs(parts.query)
|
||||
if query.get("cat"):
|
||||
return str(query["cat"][0])
|
||||
match = CATEGORY_PATH_RE.search(parts.path)
|
||||
if match:
|
||||
return match.group(1)
|
||||
raise AppError(422, "SOURCE_URL_INVALID", "Quellpfad muss eine Geizhals-Kategorie enthalten (cat=…)")
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
text = value.strip().casefold().translate(str.maketrans("äöüß", "aous"))
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
||||
return text or "category"
|
||||
|
||||
|
||||
def with_extended_filters(source_url: str) -> str:
|
||||
parts = urlsplit(source_url.strip())
|
||||
query = [(name, value) for name, value in _query_items(parts.query) if name != "promode"]
|
||||
query.append(("promode", "true"))
|
||||
return urlunsplit((parts.scheme or "https", parts.netloc, parts.path or "/", urlencode(query), parts.fragment))
|
||||
|
||||
|
||||
def _query_items(query: str) -> list[tuple[str, str]]:
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
return parse_qsl(query, keep_blank_values=True)
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.errors import AppError
|
||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "geizhals"
|
||||
BROWSER_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||
}
|
||||
EXPAND_LABELS = (
|
||||
"Erweiterte Filter",
|
||||
"Alle Filter ausklappen",
|
||||
"Immer alle Filter anzeigen",
|
||||
"weitere Filter anzeigen",
|
||||
)
|
||||
MORE_OPTION_SELECTOR = 'button:has-text("weitere"), a:has-text("weitere")'
|
||||
|
||||
|
||||
def load_result_page(url: str, *, playwright: bool = True) -> tuple[str, str, int, bool]:
|
||||
status, html = 0, ""
|
||||
try:
|
||||
status, html = _fetch_http(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_http", status, False
|
||||
except Exception:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status, False
|
||||
except Exception:
|
||||
pass
|
||||
if html and not is_challenge_html(html) and status and status < 400:
|
||||
return html, "live_http", status, False
|
||||
return html, "blocked", status or 403, True
|
||||
|
||||
|
||||
def load_category_page(url: str, *, category_ref: str, playwright: bool = True) -> tuple[str, str, int]:
|
||||
status, html = 0, ""
|
||||
if playwright:
|
||||
try:
|
||||
status, html = _fetch_playwright(url, expand_filters=True)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_playwright", status
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
status, html = _fetch_http(url)
|
||||
if not _blocked(status, html):
|
||||
return html, "live_http", status
|
||||
except Exception:
|
||||
status, html = status or 0, html or ""
|
||||
recorded = FIXTURE_DIR / f"{category_ref}-category.html"
|
||||
if recorded.exists():
|
||||
return recorded.read_text(encoding="utf-8"), "recorded_html", 0
|
||||
raise AppError(
|
||||
423,
|
||||
"SOURCE_BLOCKED",
|
||||
"Geizhals blockiert den Taxonomie-Abruf (403/Challenge)",
|
||||
{"url": url, "http_status": status},
|
||||
)
|
||||
|
||||
|
||||
def _fetch_http(url: str) -> tuple[int, str]:
|
||||
import httpx
|
||||
|
||||
response = httpx.get(url, headers=BROWSER_HEADERS, follow_redirects=True, timeout=25.0)
|
||||
return response.status_code, response.text
|
||||
|
||||
|
||||
def _fetch_playwright(url: str, *, expand_filters: bool = False) -> tuple[int, str]:
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError as exc:
|
||||
raise AppError(
|
||||
423,
|
||||
"SOURCE_BLOCKED",
|
||||
"Geizhals verlangt einen Browser-Abruf, Playwright ist nicht installiert",
|
||||
) from exc
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=True)
|
||||
page = browser.new_page(locale="de-DE", extra_http_headers=BROWSER_HEADERS)
|
||||
response = page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
page.wait_for_timeout(4000)
|
||||
if expand_filters:
|
||||
_expand_filter_sections(page)
|
||||
for label in EXPAND_LABELS:
|
||||
button = page.get_by_text(label, exact=False)
|
||||
if button.count():
|
||||
try:
|
||||
button.first.click(timeout=4000)
|
||||
page.wait_for_timeout(800)
|
||||
except Exception:
|
||||
continue
|
||||
if expand_filters:
|
||||
_expand_filter_sections(page)
|
||||
_expand_more_options(page)
|
||||
html = page.content()
|
||||
status = response.status if response is not None else 200
|
||||
browser.close()
|
||||
return status, html
|
||||
|
||||
|
||||
def _expand_filter_sections(page: object) -> None:
|
||||
locator = page.locator('button[aria-label*="aufklappen"]')
|
||||
for _ in range(40):
|
||||
if locator.count() == 0:
|
||||
return
|
||||
try:
|
||||
locator.first.click(timeout=3000)
|
||||
page.wait_for_timeout(700)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _expand_more_options(page: object) -> None:
|
||||
locator = page.locator(MORE_OPTION_SELECTOR)
|
||||
for _ in range(80):
|
||||
if locator.count() == 0:
|
||||
return
|
||||
try:
|
||||
locator.first.click(timeout=2000)
|
||||
page.wait_for_timeout(350)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _blocked(status: int, html: str) -> bool:
|
||||
if status in {401, 403, 429}:
|
||||
return True
|
||||
return is_challenge_html(html)
|
||||
@@ -0,0 +1,287 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit
|
||||
|
||||
from app.sources.geizhals.taxonomy_parser import is_challenge_html, node_text, parse_html_tree
|
||||
|
||||
PRODUCT_ID_RE = re.compile(r"-a(\d+)\.html", re.IGNORECASE)
|
||||
PRICE_RE = re.compile(r"(\d{1,3}(?:\.\d{3})+|\d+),(\d{2})")
|
||||
OFFER_RE = re.compile(r"(\d+)\s*Angebote?", re.IGNORECASE)
|
||||
PRODUCT_COUNT_RE = re.compile(r"(\d[\d.]*)\s*Produkte", re.IGNORECASE)
|
||||
CAPACITY_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(TB|GB)", re.IGNORECASE)
|
||||
WEIGHT_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*kg", re.IGNORECASE)
|
||||
AVAILABILITY_BY_CODE = {"0": "out_of_stock", "1": "limited", "2": "in_stock"}
|
||||
AVAILABILITY_BY_LABEL = {
|
||||
"lagernd": "in_stock",
|
||||
"kurzfristig lieferbar": "limited",
|
||||
"kurzfristig": "limited",
|
||||
"nicht verfügbar": "out_of_stock",
|
||||
}
|
||||
|
||||
|
||||
def parse_result_html(html: str, *, page_url: str) -> dict[str, Any]:
|
||||
if is_challenge_html(html):
|
||||
return {"items": [], "page_count": 1, "result_count": 0, "blocked": True}
|
||||
root = parse_html_tree(html)
|
||||
cards = _find_cards(root)
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for card in cards:
|
||||
item = _parse_card(card, page_url)
|
||||
if item is None or item["source_external_id"] in seen:
|
||||
continue
|
||||
seen.add(item["source_external_id"])
|
||||
items.append(item)
|
||||
page_count = _page_count(root)
|
||||
result_count = _result_count(root) or len(items)
|
||||
return {
|
||||
"items": items,
|
||||
"page_count": page_count,
|
||||
"result_count": result_count,
|
||||
"blocked": False,
|
||||
}
|
||||
|
||||
|
||||
def with_page(url: str, page: int) -> str:
|
||||
parts = urlsplit(url)
|
||||
query = [(name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "pg"]
|
||||
if page > 1:
|
||||
query.append(("pg", str(page)))
|
||||
return parts._replace(query=urlencode(query, doseq=True)).geturl()
|
||||
|
||||
|
||||
def _find_cards(root: Any) -> list[Any]:
|
||||
cards: list[Any] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
classes = node.attrs.get("class", "")
|
||||
if "galleryview__item" in classes or "listview__item" in classes:
|
||||
cards.append(node)
|
||||
return
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
return cards
|
||||
|
||||
|
||||
def _parse_card(card: Any, page_url: str) -> dict[str, Any] | None:
|
||||
link = _first_link(card, "name-link") or _first_product_link(card)
|
||||
if link is None:
|
||||
return None
|
||||
href = link.attrs.get("href") or ""
|
||||
match = PRODUCT_ID_RE.search(href)
|
||||
if match is None:
|
||||
return None
|
||||
title = (link.attrs.get("title") or node_text(link)).strip()
|
||||
if not title:
|
||||
return None
|
||||
source_url = urljoin(page_url, href.split("#")[0])
|
||||
raw_attributes = _definition_list(card)
|
||||
sku = _first_text_by_class(card, "galleryview__mpn") or _first_text_by_class(card, "listview__mpn")
|
||||
price_text = _first_text_by_class(card, "price")
|
||||
offers_text = _first_text_by_class(card, "offercount") or ""
|
||||
availability_node = _first_by_attr(card, "data-availability")
|
||||
manufacturer, model = _split_title(title)
|
||||
return {
|
||||
"source_external_id": match.group(1),
|
||||
"source_url": source_url,
|
||||
"source_title": re.sub(r"\s+", " ", title),
|
||||
"manufacturer": manufacturer,
|
||||
"model": model,
|
||||
"manufacturer_sku": sku,
|
||||
"gtin_ean": None,
|
||||
"best_price": _parse_price(price_text),
|
||||
"currency": "EUR" if price_text and "€" in price_text else None,
|
||||
"availability": _availability(availability_node),
|
||||
"seller_count": _seller_count(offers_text),
|
||||
"raw_attributes": raw_attributes,
|
||||
"normalized_attributes": _normalize_attributes(raw_attributes),
|
||||
}
|
||||
|
||||
|
||||
def _first_link(node: Any, fragment: str) -> Any | None:
|
||||
classes = node.attrs.get("class", "")
|
||||
if node.tag == "a" and fragment in classes:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_link(child, fragment)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_product_link(node: Any) -> Any | None:
|
||||
if node.tag == "a" and PRODUCT_ID_RE.search(node.attrs.get("href") or ""):
|
||||
href = node.attrs.get("href") or ""
|
||||
if "#ratings" not in href and "#offerlist" not in href:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_product_link(child)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_text_by_class(node: Any, fragment: str) -> str | None:
|
||||
found = _first_by_class(node, fragment)
|
||||
if found is None:
|
||||
return None
|
||||
text = node_text(found).strip()
|
||||
return re.sub(r"\s+", " ", text) or None
|
||||
|
||||
|
||||
def _first_by_class(node: Any, fragment: str) -> Any | None:
|
||||
if fragment in node.attrs.get("class", ""):
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_by_class(child, fragment)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _first_by_attr(node: Any, name: str) -> Any | None:
|
||||
if name in node.attrs:
|
||||
return node
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
found = _first_by_attr(child, name)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _definition_list(node: Any) -> dict[str, str]:
|
||||
pairs: dict[str, str] = {}
|
||||
|
||||
def walk(current: Any) -> None:
|
||||
if current.tag == "dl":
|
||||
pending: str | None = None
|
||||
for child in current.children:
|
||||
if isinstance(child, str):
|
||||
continue
|
||||
if child.tag == "dt":
|
||||
pending = re.sub(r"\s+", " ", node_text(child)).strip().rstrip(":")
|
||||
elif child.tag == "dd" and pending:
|
||||
value = re.sub(r"\s+", " ", node_text(child)).strip()
|
||||
if value:
|
||||
pairs[pending] = value
|
||||
pending = None
|
||||
return
|
||||
for child in current.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
return pairs
|
||||
|
||||
|
||||
def _split_title(title: str) -> tuple[str | None, str | None]:
|
||||
parts = title.split(None, 1)
|
||||
if not parts:
|
||||
return None, None
|
||||
if len(parts) == 1:
|
||||
return parts[0], None
|
||||
return parts[0], parts[1]
|
||||
|
||||
|
||||
def _parse_price(text: str | None) -> str | None:
|
||||
if not text:
|
||||
return None
|
||||
match = PRICE_RE.search(text.replace("\xa0", " "))
|
||||
if match is None:
|
||||
return None
|
||||
whole = match.group(1).replace(".", "")
|
||||
return f"{whole}.{match.group(2)}"
|
||||
|
||||
|
||||
def _seller_count(text: str) -> int | None:
|
||||
match = OFFER_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def _availability(node: Any | None) -> str:
|
||||
if node is None:
|
||||
return "unknown"
|
||||
code = AVAILABILITY_BY_CODE.get(node.attrs.get("data-availability", ""))
|
||||
if code:
|
||||
return code
|
||||
label = (node.attrs.get("aria-label") or node_text(node)).strip().casefold()
|
||||
return AVAILABILITY_BY_LABEL.get(label, "unknown")
|
||||
|
||||
|
||||
def _normalize_attributes(raw: dict[str, str]) -> dict[str, Any]:
|
||||
normalized: dict[str, Any] = {}
|
||||
for label, value in raw.items():
|
||||
key = label.casefold()
|
||||
capacity = _capacity_gb(value)
|
||||
if capacity is not None and key in {"ram", "arbeitsspeicher"}:
|
||||
normalized["ram_gb"] = capacity
|
||||
elif capacity is not None and key in {"speicher", "ssd", "festplatte"}:
|
||||
normalized["storage_gb"] = capacity
|
||||
weight = _weight_kg(value)
|
||||
if weight is not None and "gewicht" in key:
|
||||
normalized["weight_kg"] = weight
|
||||
return normalized
|
||||
|
||||
|
||||
def _capacity_gb(text: str) -> float | int | None:
|
||||
match = CAPACITY_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
value = float(match.group(1).replace(",", "."))
|
||||
if match.group(2).upper() == "TB":
|
||||
value *= 1024
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def _weight_kg(text: str) -> float | None:
|
||||
match = WEIGHT_RE.search(text)
|
||||
if match is None:
|
||||
return None
|
||||
return float(match.group(1).replace(",", "."))
|
||||
|
||||
|
||||
def _page_count(root: Any) -> int:
|
||||
pages = {1}
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
href = node.attrs.get("href") or ""
|
||||
if "pg=" in href and ("pagination" in node.attrs.get("class", "") or _in_pagination(node)):
|
||||
for name, value in parse_qsl(urlsplit(href).query):
|
||||
if name == "pg" and value.isdigit():
|
||||
pages.add(int(value))
|
||||
for child in node.children:
|
||||
if not isinstance(child, str):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
return max(pages)
|
||||
|
||||
|
||||
def _in_pagination(node: Any) -> bool:
|
||||
current = node
|
||||
while current is not None:
|
||||
classes = current.attrs.get("class", "")
|
||||
ident = current.attrs.get("id", "")
|
||||
if "pagination" in classes or ident == "pagination":
|
||||
return True
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
|
||||
def _result_count(root: Any) -> int | None:
|
||||
counts = [
|
||||
int(match.replace(".", ""))
|
||||
for match in PRODUCT_COUNT_RE.findall(node_text(root))
|
||||
]
|
||||
return max(counts) if counts else None
|
||||
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from app.discovery.enums import FilterType
|
||||
from app.sources.geizhals.live_page import BROWSER_HEADERS
|
||||
from app.sources.geizhals.taxonomy_parser import _group_sort, _option_from_link
|
||||
|
||||
TYPE_MAP = {
|
||||
"enum": FilterType.ENUM.value,
|
||||
"substring": FilterType.ENUM.value,
|
||||
"range_from": FilterType.NUMERIC_THRESHOLD.value,
|
||||
"range_to": FilterType.NUMERIC_THRESHOLD.value,
|
||||
}
|
||||
GROUP_RULES = (
|
||||
("Anschlüsse", ("anschluss", "usb", "thunderbolt", "docking")),
|
||||
("Display", ("display", "pixeldichte", "screen-to-body", "bildwiederhol", "farbraum", "farbräume")),
|
||||
("CPU", ("cpu", "ai-rechen")),
|
||||
("RAM erweitert", ("ram",)),
|
||||
("Speicher", ("speicher", "ssd", "hdd", "flash", "m.2", "cardreader", "laufwerk")),
|
||||
("Grafik", ("grafik", "gpu", "igpu", "directx")),
|
||||
("Wireless", ("wlan", "wifi", "bluetooth", "wireless")),
|
||||
("Mobilfunk", ("mobilfunk", "navigation")),
|
||||
("Akku", ("akku", "netzteil")),
|
||||
("Abmessungen", ("breite", "tiefe", "höhe", "volumen", "gewicht", "verhältnis")),
|
||||
("Eingabe", ("eingabe", "tastatur", "touchpad", "webcam", "kamera")),
|
||||
("Betriebssystem", ("betriebssystem", "sprachversion", "chrome auto")),
|
||||
)
|
||||
|
||||
|
||||
def categorylist_api_url(source_url: str) -> str:
|
||||
parts = urlsplit(source_url)
|
||||
query = parse_qs(parts.query)
|
||||
category = (query.get("cat") or [""])[0] or "nb"
|
||||
hloc = (query.get("hloc") or ["de"])[0]
|
||||
return (
|
||||
"https://geizhals.de/api/gh0/categorylist/"
|
||||
f"?cat={category}&xf=&sort=t&hloc={hloc}&v=e&filters_only=1"
|
||||
)
|
||||
|
||||
|
||||
def fetch_taxonomy_api(source_url: str, *, category_ref: str) -> dict[str, Any] | None:
|
||||
url = categorylist_api_url(source_url)
|
||||
try:
|
||||
response = httpx.get(
|
||||
url,
|
||||
headers={**BROWSER_HEADERS, "Accept": "application/json"},
|
||||
follow_redirects=True,
|
||||
timeout=25.0,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
return None
|
||||
return parse_taxonomy_api(
|
||||
response.json(),
|
||||
category_ref=category_ref,
|
||||
source_url=source_url,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def parse_taxonomy_api(payload: dict[str, Any], *, category_ref: str, source_url: str) -> dict[str, Any]:
|
||||
raw_filters = ((payload.get("response") or {}).get("filters") or [])
|
||||
filters = [_filter_from_api(item) for item in raw_filters if isinstance(item, dict)]
|
||||
filters = [item for item in filters if item["options"] or item.get("range")]
|
||||
if not any(item["external_filter_key"] == "price" for item in filters):
|
||||
filters.append(_price_filter())
|
||||
filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower()))
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
"name": "Notebooks" if category_ref == "nb" else category_ref,
|
||||
"canonical_category": "notebook" if category_ref == "nb" else None,
|
||||
"source_url": source_url,
|
||||
"filters": filters,
|
||||
"blocked": False,
|
||||
"discovery_mode": "live_api",
|
||||
"taxonomy_status": "READY",
|
||||
}
|
||||
|
||||
|
||||
def _filter_from_api(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
key = str(raw.get("id") or "")
|
||||
label = str(raw.get("title") or key)
|
||||
source_type = str(raw.get("type") or "enum")
|
||||
filter_type = TYPE_MAP.get(source_type, FilterType.ENUM.value)
|
||||
options: list[dict[str, Any]] = []
|
||||
encoding_options: dict[str, str] = {}
|
||||
for value in raw.get("values") or []:
|
||||
query = str(value.get("id") or "")
|
||||
if "_" in query:
|
||||
query = query.split("_", 1)[1].replace("+", " ")
|
||||
option = _option_from_link(str(value.get("name") or query), query)
|
||||
if source_type == "range_from":
|
||||
option["operator"] = ">="
|
||||
elif source_type == "range_to":
|
||||
option["operator"] = "<="
|
||||
options.append(option)
|
||||
encoding_options[option["source_value"]] = query
|
||||
if option.get("normalized_value") is not None:
|
||||
encoding_options[str(option["normalized_value"])] = query
|
||||
group_label = _group_label(label)
|
||||
return {
|
||||
"external_filter_key": key,
|
||||
"label": label,
|
||||
"filter_type": filter_type,
|
||||
"canonical_unit": next((item.get("unit") for item in options if item.get("unit")), None),
|
||||
"group_label": group_label,
|
||||
"group_sort": _group_sort(group_label),
|
||||
"options": options,
|
||||
"encoding": {
|
||||
"url_param": "xf",
|
||||
"encoding_type": "OPTION_TOKEN",
|
||||
"options": encoding_options,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _price_filter() -> dict[str, Any]:
|
||||
return {
|
||||
"external_filter_key": "price",
|
||||
"label": "Preis",
|
||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
||||
"canonical_unit": "EUR",
|
||||
"group_label": "Allgemein",
|
||||
"group_sort": _group_sort("Allgemein"),
|
||||
"options": [],
|
||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
||||
"encoding": {
|
||||
"url_param": "bpmin",
|
||||
"min_parameter": "bpmin",
|
||||
"max_parameter": "bpmax",
|
||||
"encoding_type": "RANGE_PARAMETERS",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _group_label(title: str) -> str:
|
||||
text = title.casefold()
|
||||
for group, needles in GROUP_RULES:
|
||||
if any(needle in text for needle in needles):
|
||||
return group
|
||||
return "Allgemein"
|
||||
@@ -0,0 +1,581 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, unquote, urlparse
|
||||
|
||||
from app.discovery.enums import FilterType
|
||||
|
||||
SKIP_OPTION_LABELS = {
|
||||
"alle",
|
||||
"entfernen",
|
||||
"überspringen",
|
||||
"aktualisieren",
|
||||
"abonnieren",
|
||||
"filter zurücksetzen",
|
||||
"nur beliebteste filter anzeigen",
|
||||
"immer alle filter anzeigen",
|
||||
"filter überspringen",
|
||||
"einfache filter",
|
||||
"erweiterte filter",
|
||||
"weitere",
|
||||
"weniger",
|
||||
"alle anzeigen",
|
||||
"weniger anzeigen",
|
||||
}
|
||||
PRODUCT_COUNT_RE = re.compile(r"\(\d[\d.]*\s*(produkte)?\)\s*$", re.IGNORECASE)
|
||||
OPTION_TAGS = {"a", "button", "label", "input", "span"}
|
||||
GROUP_ORDER = (
|
||||
"Allgemein",
|
||||
"Display",
|
||||
"CPU",
|
||||
"RAM",
|
||||
"RAM erweitert",
|
||||
"Speicher",
|
||||
"Grafik",
|
||||
"Anschlüsse",
|
||||
"Wireless",
|
||||
"Mobilfunk",
|
||||
"Eingabe",
|
||||
"Betriebssystem",
|
||||
"Akku",
|
||||
"Abmessungen",
|
||||
"Sonstiges",
|
||||
)
|
||||
RANGE_MIN_NAMES = {"bpmin", "minp", "prmin"}
|
||||
RANGE_MAX_NAMES = {"bpmax", "maxp", "prmax"}
|
||||
TEXT_NAMES = {"asuch", "fs"}
|
||||
HEADING_TAGS = {"th", "h2", "h3", "h4", "h5", "legend", "dt", "caption", "label", "strong"}
|
||||
NUMBER_RE = re.compile(r"(-?\d+(?:[.,]\d+)?)")
|
||||
THRESHOLD_RE = re.compile(r"^(ab|bis)\s+", re.IGNORECASE)
|
||||
UNIT_RE = re.compile(r"(GB|TB|GHz|MHz|Hz|cd/m²|ppi|kg|\"|″|mm|%|€|EUR)$", re.IGNORECASE)
|
||||
TYPE_MAP = {
|
||||
"text": FilterType.ENUM.value,
|
||||
"color": FilterType.ENUM.value,
|
||||
"text_range": FilterType.NUMERIC_THRESHOLD.value,
|
||||
"numeric_range": FilterType.NUMERIC_RANGE.value,
|
||||
}
|
||||
|
||||
|
||||
class _Node:
|
||||
def __init__(self, tag: str, attrs: dict[str, str]) -> None:
|
||||
self.tag = tag
|
||||
self.attrs = attrs
|
||||
self.children: list[_Node | str] = []
|
||||
self.parent: _Node | None = None
|
||||
|
||||
|
||||
class _TreeBuilder(HTMLParser):
|
||||
VOID = {
|
||||
"area",
|
||||
"base",
|
||||
"br",
|
||||
"col",
|
||||
"embed",
|
||||
"hr",
|
||||
"img",
|
||||
"input",
|
||||
"link",
|
||||
"meta",
|
||||
"param",
|
||||
"source",
|
||||
"track",
|
||||
"wbr",
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = _Node("document", {})
|
||||
self._stack = [self.root]
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
node = _Node(tag, {key: value or "" for key, value in attrs})
|
||||
parent = self._stack[-1]
|
||||
node.parent = parent
|
||||
parent.children.append(node)
|
||||
if tag not in self.VOID:
|
||||
self._stack.append(node)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
for index in range(len(self._stack) - 1, 0, -1):
|
||||
if self._stack[index].tag == tag:
|
||||
del self._stack[index:]
|
||||
break
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
text = data.strip()
|
||||
if text:
|
||||
self._stack[-1].children.append(text)
|
||||
|
||||
|
||||
def parse_html_tree(html: str) -> _Node:
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(html)
|
||||
return builder.root
|
||||
|
||||
|
||||
def node_text(node: _Node) -> str:
|
||||
return _node_text(node)
|
||||
|
||||
|
||||
def parse_taxonomy_html(html: str, *, category_ref: str, source_url: str) -> dict[str, Any]:
|
||||
if is_challenge_html(html):
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
"name": category_ref,
|
||||
"source_url": source_url,
|
||||
"filters": [],
|
||||
"blocked": True,
|
||||
}
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(html)
|
||||
groups: dict[str, dict[str, Any]] = {}
|
||||
_collect_declared_filters(builder.root, groups)
|
||||
_collect_xf_links(builder.root, groups)
|
||||
_collect_named_inputs(builder.root, groups)
|
||||
filters = [
|
||||
group
|
||||
for group in groups.values()
|
||||
if _usable_group(group)
|
||||
]
|
||||
for group in filters:
|
||||
section = group.get("group_label") or "Allgemein"
|
||||
group["group_label"] = section
|
||||
group["group_sort"] = _group_sort(section)
|
||||
filters.sort(key=lambda item: (item["group_sort"], item["group_label"].lower(), item["label"].lower(), item["external_filter_key"]))
|
||||
return {
|
||||
"source": "geizhals",
|
||||
"external_category_key": category_ref,
|
||||
"name": _category_name(builder.root, category_ref),
|
||||
"canonical_category": "notebook" if category_ref == "nb" else None,
|
||||
"source_url": source_url,
|
||||
"filters": filters,
|
||||
"blocked": False,
|
||||
}
|
||||
|
||||
|
||||
def is_challenge_html(html: str) -> bool:
|
||||
sample = html.lower()
|
||||
markers = (
|
||||
"cf-mitigated",
|
||||
"cf-challenge",
|
||||
"just a moment",
|
||||
"enable javascript and cookies",
|
||||
"checking your browser",
|
||||
"verification successful. waiting for geizhals",
|
||||
)
|
||||
return any(marker in sample for marker in markers)
|
||||
|
||||
|
||||
def _collect_declared_filters(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
||||
filter_id = node.attrs.get("data-filter-id")
|
||||
if filter_id:
|
||||
source_type = node.attrs.get("data-filter-type") or ""
|
||||
label = _declared_label(node) or f"Filter {filter_id}"
|
||||
group = groups.setdefault(filter_id, _empty_group(filter_id, label, source_type))
|
||||
section = _section_label(node)
|
||||
if section:
|
||||
group["group_label"] = section
|
||||
if label and group["label"].startswith("Filter "):
|
||||
group["label"] = label
|
||||
if source_type:
|
||||
group["filter_type"] = TYPE_MAP.get(source_type, group["filter_type"])
|
||||
if source_type == "numeric_range":
|
||||
group["encoding"]["encoding_type"] = "RANGE_TOKEN"
|
||||
group["encoding"]["value_template"] = "{min}*{max}"
|
||||
group.setdefault("range", {"minimum": None, "maximum": None, "step": None, "unit": group.get("canonical_unit")})
|
||||
_collect_xf_links(node, groups, required_key=filter_id, allow_histogram=source_type == "numeric_range")
|
||||
return
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_collect_declared_filters(child, groups)
|
||||
|
||||
|
||||
def _collect_xf_links(
|
||||
node: _Node,
|
||||
groups: dict[str, dict[str, Any]],
|
||||
required_key: str | None = None,
|
||||
allow_histogram: bool = False,
|
||||
) -> None:
|
||||
if node.tag in OPTION_TAGS:
|
||||
label = node.attrs.get("aria-label") or node.attrs.get("title") or _node_text(node)
|
||||
if _usable_option(node, label, allow_histogram=allow_histogram):
|
||||
for key, query_value in _node_xf_tokens(node):
|
||||
if required_key is not None and key != required_key:
|
||||
continue
|
||||
if required_key is None and not _has_structural_label(node):
|
||||
continue
|
||||
filter_label = _declared_label(_filter_root(node)) or _nearest_group_label(node)
|
||||
group = groups.setdefault(key, _empty_group(key, filter_label, None))
|
||||
section = _section_label(node)
|
||||
if section:
|
||||
group["group_label"] = section
|
||||
if filter_label and (group["label"].startswith("Filter ") or group["label"] == key):
|
||||
group["label"] = filter_label
|
||||
option = _option_from_link(label, query_value)
|
||||
existing = {item["source_value"] for item in group["options"]}
|
||||
if option["source_value"] not in existing:
|
||||
group["options"].append(option)
|
||||
group["encoding"].setdefault("options", {})
|
||||
group["encoding"]["options"][option["source_value"]] = query_value
|
||||
if option.get("normalized_value") is not None:
|
||||
group["encoding"]["options"][str(option["normalized_value"])] = query_value
|
||||
if group["filter_type"] not in {FilterType.NUMERIC_RANGE.value, FilterType.NUMERIC_THRESHOLD.value}:
|
||||
group["filter_type"] = _classify_options(group["options"])
|
||||
if group["canonical_unit"] is None:
|
||||
group["canonical_unit"] = option.get("unit")
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_collect_xf_links(child, groups, required_key=required_key, allow_histogram=allow_histogram)
|
||||
|
||||
|
||||
def _collect_named_inputs(node: _Node, groups: dict[str, dict[str, Any]]) -> None:
|
||||
inputs: list[_Node] = []
|
||||
_find_inputs(node, inputs)
|
||||
names = {item.attrs.get("name", ""): item for item in inputs if item.attrs.get("name")}
|
||||
if names.keys() & RANGE_MIN_NAMES or names.keys() & RANGE_MAX_NAMES:
|
||||
min_name = next((name for name in RANGE_MIN_NAMES if name in names), "bpmin")
|
||||
max_name = next((name for name in RANGE_MAX_NAMES if name in names), "bpmax")
|
||||
label = _nearest_group_label(names.get(min_name) or names.get(max_name) or node) or "Preis"
|
||||
section = _section_label(names.get(min_name) or names.get(max_name) or node) or "Allgemein"
|
||||
groups["price"] = {
|
||||
"external_filter_key": "price",
|
||||
"label": "Preis" if "preis" not in label.lower() else label,
|
||||
"filter_type": FilterType.NUMERIC_RANGE.value,
|
||||
"canonical_unit": "EUR",
|
||||
"group_label": section,
|
||||
"group_sort": _group_sort(section),
|
||||
"options": [],
|
||||
"range": {"minimum": 0, "maximum": None, "step": 1, "unit": "EUR"},
|
||||
"encoding": {
|
||||
"url_param": min_name,
|
||||
"min_parameter": min_name,
|
||||
"max_parameter": max_name,
|
||||
"encoding_type": "RANGE_PARAMETERS",
|
||||
},
|
||||
}
|
||||
for name in TEXT_NAMES:
|
||||
if name not in names:
|
||||
continue
|
||||
label = _nearest_group_label(names[name]) or "Suchbegriffe"
|
||||
section = _section_label(names[name]) or "Allgemein"
|
||||
groups[name] = {
|
||||
"external_filter_key": name,
|
||||
"label": label,
|
||||
"filter_type": FilterType.TEXT.value,
|
||||
"canonical_unit": None,
|
||||
"group_label": section,
|
||||
"group_sort": _group_sort(section),
|
||||
"options": [],
|
||||
"encoding": {"url_param": name, "encoding_type": "TEXT_PARAMETER", "value_template": "{value}"},
|
||||
}
|
||||
|
||||
|
||||
def _find_inputs(node: _Node, found: list[_Node]) -> None:
|
||||
if node.tag == "input":
|
||||
found.append(node)
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
_find_inputs(child, found)
|
||||
|
||||
|
||||
def _node_xf_tokens(node: _Node) -> list[tuple[str, str]]:
|
||||
tokens = _xf_tokens(node.attrs.get("href") or "")
|
||||
if tokens:
|
||||
return tokens
|
||||
for attr in ("data-xf", "data-filter-value", "data-value", "value"):
|
||||
raw = node.attrs.get(attr) or ""
|
||||
if attr == "value" and node.attrs.get("name") not in {"xf", ""} and node.tag != "input":
|
||||
continue
|
||||
parsed = _xf_raw_token(raw)
|
||||
if parsed:
|
||||
return [parsed]
|
||||
return []
|
||||
|
||||
|
||||
def _xf_raw_token(raw: str) -> tuple[str, str] | None:
|
||||
value = unquote(raw).replace("|", "~").split("~")[0]
|
||||
if "_" not in value:
|
||||
return None
|
||||
key, token = value.split("_", 1)
|
||||
if key.isdigit() and token:
|
||||
return key, token.replace("+", " ")
|
||||
return None
|
||||
|
||||
|
||||
def _xf_tokens(href: str) -> list[tuple[str, str]]:
|
||||
if not href or href.startswith("#") or href.startswith("javascript:"):
|
||||
return []
|
||||
query = urlparse(href).query
|
||||
tokens: list[tuple[str, str]] = []
|
||||
for name, value in parse_qsl(query, keep_blank_values=True):
|
||||
if name != "xf" or not value:
|
||||
continue
|
||||
for part in unquote(value).replace("|", "~").split("~"):
|
||||
if "_" not in part:
|
||||
continue
|
||||
key, raw = part.split("_", 1)
|
||||
if key.isdigit() and raw:
|
||||
tokens.append((key, raw.replace("+", " ")))
|
||||
return tokens
|
||||
|
||||
|
||||
def _option_from_link(label: str, query_value: str) -> dict[str, Any]:
|
||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
||||
operator = None
|
||||
if cleaned.lower().startswith("ab "):
|
||||
operator = ">="
|
||||
elif cleaned.lower().startswith("bis "):
|
||||
operator = "<="
|
||||
number = _first_number(cleaned)
|
||||
unit = _unit(cleaned)
|
||||
return {
|
||||
"source_value": cleaned or query_value,
|
||||
"query_value": query_value,
|
||||
"normalized_value": number,
|
||||
"operator": operator,
|
||||
"unit": unit,
|
||||
}
|
||||
|
||||
|
||||
def _classify_options(options: list[dict[str, Any]]) -> str:
|
||||
if not options:
|
||||
return FilterType.UNKNOWN.value
|
||||
labels = [str(item["source_value"]).lower() for item in options]
|
||||
if all(item in {"ja", "nein", "egal", "true", "false"} for item in labels):
|
||||
return FilterType.BOOLEAN.value
|
||||
if all(THRESHOLD_RE.match(str(item["source_value"])) for item in options):
|
||||
return FilterType.NUMERIC_THRESHOLD.value
|
||||
return FilterType.ENUM.value
|
||||
|
||||
|
||||
def _empty_group(key: str, label: str | None, source_type: str | None) -> dict[str, Any]:
|
||||
encoding_type = "OPTION_TOKEN"
|
||||
if source_type == "numeric_range":
|
||||
encoding_type = "RANGE_TOKEN"
|
||||
return {
|
||||
"external_filter_key": key,
|
||||
"label": label or f"Filter {key}",
|
||||
"filter_type": TYPE_MAP.get(source_type or "", FilterType.ENUM.value),
|
||||
"canonical_unit": None,
|
||||
"group_label": None,
|
||||
"group_sort": _group_sort("Allgemein"),
|
||||
"options": [],
|
||||
"encoding": {
|
||||
"url_param": "xf",
|
||||
"encoding_type": encoding_type,
|
||||
"options": {},
|
||||
**({"value_template": "{min}*{max}"} if encoding_type == "RANGE_TOKEN" else {}),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _declared_label(node: _Node | None) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
attr_id = node.attrs.get("id", "")
|
||||
if attr_id.startswith("filter-element__title-label"):
|
||||
return _clean_label(_node_text(node)) or None
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
found = _declared_label(child)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _filter_root(node: _Node) -> _Node | None:
|
||||
current: _Node | None = node
|
||||
while current is not None:
|
||||
if current.attrs.get("data-filter-id"):
|
||||
return current
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _nearest_group_label(node: _Node | None) -> str | None:
|
||||
current = node
|
||||
while current is not None:
|
||||
declared = _declared_label(current)
|
||||
if declared:
|
||||
return declared
|
||||
if current.tag in HEADING_TAGS:
|
||||
text = _clean_label(_node_text(current))
|
||||
if text:
|
||||
return text
|
||||
previous = _previous_element(current)
|
||||
while previous is not None:
|
||||
if previous.tag in HEADING_TAGS:
|
||||
text = _clean_label(_node_text(previous))
|
||||
if text:
|
||||
return text
|
||||
previous = _previous_element(previous)
|
||||
labeled = current.attrs.get("aria-label") or current.attrs.get("data-filter-name") or current.attrs.get("title")
|
||||
if labeled:
|
||||
text = _clean_label(labeled)
|
||||
if text and len(text.split()) <= 6:
|
||||
return text
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _has_structural_label(node: _Node) -> bool:
|
||||
if _filter_root(node) is not None:
|
||||
return True
|
||||
current: _Node | None = node
|
||||
while current is not None:
|
||||
if current.tag in {"th", "td", "tr", "table", "dt", "dd"}:
|
||||
return _nearest_group_label(node) is not None
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
|
||||
def _usable_option(node: _Node, label: str, *, allow_histogram: bool) -> bool:
|
||||
classes = f"{node.attrs.get('class', '')} {node.attrs.get('id', '')}".lower()
|
||||
if "skip-link" in classes or "add-to-columns" in classes:
|
||||
return False
|
||||
if "bucket" in classes and not allow_histogram:
|
||||
return False
|
||||
cleaned = PRODUCT_COUNT_RE.sub("", re.sub(r"\s+", " ", label)).strip()
|
||||
if not cleaned or _skip_option(cleaned):
|
||||
return False
|
||||
if "überspringen" in cleaned.lower() or "aufklappen" in cleaned.lower() or "zuklappen" in cleaned.lower():
|
||||
return False
|
||||
if cleaned.lower().startswith("weitere"):
|
||||
return False
|
||||
if not allow_histogram and re.fullmatch(r"\d[\d.]*\s*produkte", cleaned, re.IGNORECASE):
|
||||
return False
|
||||
if len(cleaned.split()) > 12:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _section_label(node: _Node | None) -> str | None:
|
||||
current = node
|
||||
while current is not None:
|
||||
classes = current.attrs.get("class", "").lower()
|
||||
if "filter-row" in classes or current.attrs.get("data-filter-row-id"):
|
||||
text = _strip_section_suffix(
|
||||
current.attrs.get("data-filter-row-name")
|
||||
or current.attrs.get("aria-label")
|
||||
or _row_heading(current)
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
labeled = current.attrs.get("aria-label") or ""
|
||||
if re.search(r"(auf|zu)klappen", labeled, re.IGNORECASE):
|
||||
text = _strip_section_suffix(labeled)
|
||||
if text:
|
||||
return text
|
||||
current = current.parent
|
||||
return None
|
||||
|
||||
|
||||
def _row_heading(node: _Node) -> str:
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node) and child.tag in {"button", "h2", "h3", "h4", "legend", "strong"}:
|
||||
text = _clean_label(_node_text(child))
|
||||
if text:
|
||||
return text
|
||||
return _clean_label(node.attrs.get("aria-label") or "")
|
||||
|
||||
|
||||
def _strip_section_suffix(text: str | None) -> str:
|
||||
cleaned = _clean_label(text or "")
|
||||
return re.sub(r"\s*(aufklappen|zuklappen|einklappen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
||||
|
||||
|
||||
def _group_sort(label: str) -> int:
|
||||
try:
|
||||
return GROUP_ORDER.index(label)
|
||||
except ValueError:
|
||||
return len(GROUP_ORDER)
|
||||
|
||||
|
||||
def _usable_group(group: dict[str, Any]) -> bool:
|
||||
if group["filter_type"] == FilterType.TEXT.value:
|
||||
return True
|
||||
if group.get("range") and group["filter_type"] == FilterType.NUMERIC_RANGE.value:
|
||||
return True
|
||||
if not group.get("options") and not group.get("range"):
|
||||
return False
|
||||
label = str(group.get("label") or "")
|
||||
if label.startswith("Filter ") and len(group.get("options") or []) < 2:
|
||||
return False
|
||||
if len(label.split()) > 8:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _previous_element(node: _Node) -> _Node | None:
|
||||
if node.parent is None:
|
||||
return None
|
||||
siblings = [child for child in node.parent.children if isinstance(child, _Node)]
|
||||
try:
|
||||
index = siblings.index(node)
|
||||
except ValueError:
|
||||
return None
|
||||
return siblings[index - 1] if index > 0 else None
|
||||
|
||||
|
||||
def _node_text(node: _Node) -> str:
|
||||
parts: list[str] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, str):
|
||||
parts.append(child)
|
||||
else:
|
||||
parts.append(_node_text(child))
|
||||
return " ".join(part for part in parts if part).strip()
|
||||
|
||||
|
||||
def _clean_label(text: str) -> str:
|
||||
cleaned = re.sub(r"\s+", " ", text).strip().rstrip(":")
|
||||
cleaned = re.sub(r"\s*(überspringen|entfernen)$", "", cleaned, flags=re.IGNORECASE).strip()
|
||||
if not cleaned or _skip_option(cleaned):
|
||||
return ""
|
||||
return cleaned
|
||||
|
||||
|
||||
def _skip_option(label: str) -> bool:
|
||||
return label.strip().lower() in SKIP_OPTION_LABELS
|
||||
|
||||
|
||||
def _first_number(text: str) -> float | int | None:
|
||||
match = NUMBER_RE.search(text.replace(",", "."))
|
||||
if match is None:
|
||||
return None
|
||||
raw = match.group(1)
|
||||
value = float(raw)
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def _unit(text: str) -> str | None:
|
||||
match = UNIT_RE.search(text.replace(" ", ""))
|
||||
if match is None:
|
||||
if "kg" in text.lower():
|
||||
return "kg"
|
||||
return None
|
||||
unit = match.group(1)
|
||||
if unit in {'"', "″"}:
|
||||
return "inch"
|
||||
return unit
|
||||
|
||||
|
||||
def _category_name(root: _Node, fallback: str) -> str:
|
||||
titles: list[str] = []
|
||||
|
||||
def walk(node: _Node) -> None:
|
||||
if node.tag in {"h1", "title"}:
|
||||
text = _node_text(node)
|
||||
if text:
|
||||
titles.append(text)
|
||||
for child in node.children:
|
||||
if isinstance(child, _Node):
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
if titles:
|
||||
return titles[0].split("»")[0].strip() or fallback
|
||||
return "Notebooks" if fallback == "nb" else fallback
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from app.errors import AppError
|
||||
from app.sources.base import (
|
||||
SourceAttribute,
|
||||
SourceOffer,
|
||||
SourcePriceObservation,
|
||||
SourceProduct,
|
||||
SourceVariant,
|
||||
)
|
||||
|
||||
|
||||
def _parse_dt(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
class ManualSource:
|
||||
source_key = "manual"
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._data = json.loads(path.read_text(encoding="utf-8"))
|
||||
self._products = {item["external_id"]: item for item in self._data["products"]}
|
||||
self._offers = {item["external_id"]: item for item in self._data.get("offers", [])}
|
||||
self._prices = self._data.get("prices", [])
|
||||
|
||||
def list_external_product_ids(self) -> list[str]:
|
||||
return list(self._products)
|
||||
|
||||
def get_product(self, external_id: str) -> SourceProduct:
|
||||
raw = self._products.get(external_id)
|
||||
if raw is None:
|
||||
raise AppError(404, "NOT_FOUND", "manual product not found", {"external_id": external_id})
|
||||
variants = []
|
||||
for variant in raw["variants"]:
|
||||
attributes = tuple(
|
||||
SourceAttribute(
|
||||
id=UUID(item["id"]),
|
||||
attribute=item["attribute"],
|
||||
raw_value=item["raw_value"],
|
||||
normalized_value=item.get("normalized_value"),
|
||||
unit=item.get("unit"),
|
||||
normalization_status=item["normalization_status"],
|
||||
normalizer_version=item.get("normalizer_version", "manual-v1"),
|
||||
confidence=Decimal(str(item["confidence"])),
|
||||
source_reference=item["source_reference"],
|
||||
observed_at=_parse_dt(item["observed_at"]),
|
||||
normalization_error=item.get("normalization_error"),
|
||||
)
|
||||
for item in variant["attributes"]
|
||||
)
|
||||
variants.append(
|
||||
SourceVariant(
|
||||
id=UUID(variant["id"]),
|
||||
name=variant["name"],
|
||||
manufacturer_sku=variant.get("manufacturer_sku"),
|
||||
identifiers=variant.get("identifiers") or {},
|
||||
attributes=attributes,
|
||||
)
|
||||
)
|
||||
return SourceProduct(
|
||||
source_key=self.source_key,
|
||||
external_id=external_id,
|
||||
id=UUID(raw["id"]),
|
||||
category=raw["category"],
|
||||
brand=raw["brand"],
|
||||
name=raw["name"],
|
||||
model=raw.get("model"),
|
||||
variants=tuple(variants),
|
||||
)
|
||||
|
||||
def get_offers(self, external_product_id: str) -> list[SourceOffer]:
|
||||
product = self.get_product(external_product_id)
|
||||
variant_ids = {variant.id for variant in product.variants}
|
||||
return [
|
||||
SourceOffer(
|
||||
id=UUID(item["id"]),
|
||||
variant_id=UUID(item["variant_id"]),
|
||||
external_id=item["external_id"],
|
||||
seller=item["seller"],
|
||||
url=item.get("url"),
|
||||
market=item["market"],
|
||||
currency=item["currency"],
|
||||
condition=item["condition"],
|
||||
)
|
||||
for item in self._offers.values()
|
||||
if UUID(item["variant_id"]) in variant_ids
|
||||
]
|
||||
|
||||
def get_price_history(self, external_offer_id: str) -> list[SourcePriceObservation]:
|
||||
return [
|
||||
SourcePriceObservation(
|
||||
id=UUID(item["id"]) if item.get("id") else None,
|
||||
external_offer_id=external_offer_id,
|
||||
external_observation_id=item["external_observation_id"],
|
||||
observed_at=_parse_dt(item["observed_at"]),
|
||||
item_price=Decimal(str(item["item_price"])),
|
||||
shipping_price=None if item.get("shipping_price") is None else Decimal(str(item["shipping_price"])),
|
||||
currency=item["currency"],
|
||||
availability=item["availability"],
|
||||
raw_payload=item["raw_payload"],
|
||||
confidence=Decimal(str(item["confidence"])),
|
||||
)
|
||||
for item in self._prices
|
||||
if item["external_offer_id"] == external_offer_id
|
||||
]
|
||||
@@ -0,0 +1,137 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<title>Notebooks Preisvergleich Geizhals Deutschland</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Notebooks</h1>
|
||||
<div class="filter-row" data-filter-row-id="allgemein">
|
||||
<button aria-label="Allgemein aufklappen">Allgemein</button>
|
||||
<div data-filter-type="text_range" data-filter-id="17023">
|
||||
<span id="filter-element__title-label--17023">RAM</span>
|
||||
<a href="?cat=nb&xf=17023_ab+16GB&promode=true">ab 16GB</a>
|
||||
<a href="?cat=nb&xf=17023_ab+32GB&promode=true">ab 32GB</a>
|
||||
</div>
|
||||
<div data-filter-type="text" data-filter-id="1">
|
||||
<span id="filter-element__title-label--1">Hersteller</span>
|
||||
<a href="?cat=nb&xf=1_Apple" aria-label="Apple (234 Produkte)">Apple</a>
|
||||
<a href="?cat=nb&xf=1_ASUS">ASUS</a>
|
||||
<button type="button" data-xf="1_Lenovo">Lenovo</button>
|
||||
<button type="button">weitere 12</button>
|
||||
</div>
|
||||
<form>
|
||||
<label>Suchbegriffe</label>
|
||||
<input name="asuch" type="text">
|
||||
<label>Preisbereich (€)</label>
|
||||
<input name="bpmin" type="number" placeholder="Preis ab">
|
||||
<input name="bpmax" type="number" placeholder="Preis bis">
|
||||
</form>
|
||||
</div>
|
||||
<div class="filter-row" data-filter-row-id="display">
|
||||
<button aria-label="Display aufklappen">Display</button>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Display-Größe:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12128_13">13.x"</a>
|
||||
<a href="?cat=nb&xf=12128_14">14.x"</a>
|
||||
<a href="?cat=nb&xf=12128_15">15.x"</a>
|
||||
<a href="?cat=nb&xf=12128_16">16.x"</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Display-Auflösung:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=9_1920x1080">1920x1080 (Full HD)</a>
|
||||
<a href="?cat=nb&xf=9_2560x1600">2560x1600 (WQXGA)</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Kerne:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12822_8">8 (Octa-Core)</a>
|
||||
<a href="?cat=nb&xf=12822_10">10</a>
|
||||
<a href="?cat=nb&xf=12822_14">14-Core</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Kerne ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12822_8">ab 8</a>
|
||||
<a href="?cat=nb&xf=12822_10">ab 10</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Threads ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12823_12">ab 12</a>
|
||||
<a href="?cat=nb&xf=12823_16">ab 16</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>CPU-Turbotakt:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=9682_4500">ab 4.5GHz</a>
|
||||
<a href="?cat=nb&xf=9682_5000">ab 5GHz</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>RAM ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=12_16384">ab 16GB</a>
|
||||
<a href="?cat=nb&xf=12_32768">ab 32GB</a>
|
||||
<a href="?cat=nb&xf=12_65536">ab 64GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>RAM-Typ:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=7520_DDR5">DDR5</a>
|
||||
<a href="?cat=nb&xf=7520_LPDDR5">LPDDR5</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Speichertyp:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=82_SSD">SSD</a>
|
||||
<a href="?cat=nb&xf=82_M.2">M.2</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>SSD-Kapazität:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=2991_1000">ab 1TB</a>
|
||||
<a href="?cat=nb&xf=2991_2000">ab 2TB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Grafik-Typ:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=8149_zzx">dedizierte Grafik (dGPU)</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>GPU - Speichergröße ab:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=7436_2048">ab 2GB</a>
|
||||
<a href="?cat=nb&xf=7436_4096">ab 4GB</a>
|
||||
<a href="?cat=nb&xf=7436_8192">ab 8GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>GPU - Speichergröße:</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=16692_8192">8GB</a>
|
||||
<a href="?cat=nb&xf=16692_12288">12GB</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>USB-C mit DisplayPort gesamt (inkl. Thunderbolt):</th>
|
||||
<td>
|
||||
<a href="?cat=nb&xf=13731_1">ab 1x</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,52 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<body>
|
||||
<a href="#productlist">6431 Produkte</a>
|
||||
<nav class="pagination">
|
||||
<a class="pagination__page" href="?cat=nb&pg=2">Seite 2</a>
|
||||
<a class="pagination__page last-page" href="?cat=nb&pg=4">Seite 4</a>
|
||||
</nav>
|
||||
<div class="galleryview__item card">
|
||||
<h3 class="galleryview__name">
|
||||
<a class="galleryview__name-link" href="acme-notebook-14-a1000001.html" title="Acme Notebook 14, 32GB RAM, 1TB SSD">
|
||||
Acme Notebook 14, 32GB RAM, 1TB SSD
|
||||
</a>
|
||||
</h3>
|
||||
<span class="galleryview__mpn">ACM-14-32</span>
|
||||
<dl>
|
||||
<dt>Betriebssystem</dt><dd>Linux</dd>
|
||||
<dt>Display</dt><dd>14"</dd>
|
||||
<dt>CPU</dt><dd>Acme 8-Core</dd>
|
||||
<dt>RAM</dt><dd>32GB</dd>
|
||||
<dt>Speicher</dt><dd>1TB SSD</dd>
|
||||
<dt>Grafik</dt><dd>Acme GPU</dd>
|
||||
</dl>
|
||||
<div class="galleryview__price">
|
||||
<a class="galleryview__price-link" href="acme-notebook-14-a1000001.html#offerlist">
|
||||
<span class="price">€ 1298,00</span>
|
||||
</a>
|
||||
<span class="availability" data-availability="2" aria-label="lagernd"></span>
|
||||
</div>
|
||||
<a class="galleryview__offercount-link">8 Angebote</a>
|
||||
</div>
|
||||
<div class="galleryview__item card">
|
||||
<h3 class="galleryview__name">
|
||||
<a class="galleryview__name-link" href="acme-notebook-13-a1000002.html" title="Acme Notebook 13, 8GB RAM">
|
||||
Acme Notebook 13, 8GB RAM
|
||||
</a>
|
||||
</h3>
|
||||
<span class="galleryview__mpn">ACM-13-8</span>
|
||||
<dl>
|
||||
<dt>RAM</dt><dd>8GB</dd>
|
||||
<dt>Speicher</dt><dd>512GB SSD</dd>
|
||||
</dl>
|
||||
<div class="galleryview__price">
|
||||
<a class="galleryview__price-link" href="acme-notebook-13-a1000002.html#offerlist">
|
||||
<span class="price">€ 899,00</span>
|
||||
</a>
|
||||
<span class="availability" data-availability="1" aria-label="kurzfristig lieferbar"></span>
|
||||
</div>
|
||||
<a class="galleryview__offercount-link">3 Angebote</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"page": 1,
|
||||
"page_count": 2,
|
||||
"result_count": 3,
|
||||
"http_status": 200,
|
||||
"items": [
|
||||
{
|
||||
"source_external_id": "gh-nb-a",
|
||||
"source_url": "https://geizhals.de/example-mobile-13-a.html",
|
||||
"source_title": "Example Mobile 13 32GB",
|
||||
"manufacturer": "Example",
|
||||
"model": "Mobile 13",
|
||||
"manufacturer_sku": "EM13-32-1T",
|
||||
"gtin_ean": "4000000000001",
|
||||
"best_price": "1298.00",
|
||||
"currency": "EUR",
|
||||
"availability": "in_stock",
|
||||
"seller_count": 8,
|
||||
"raw_attributes": {"RAM": "32GB"},
|
||||
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||
},
|
||||
{
|
||||
"source_external_id": "gh-nb-b",
|
||||
"source_url": "https://geizhals.de/example-mobile-13-b.html",
|
||||
"source_title": "Example Mobile 13 8GB",
|
||||
"manufacturer": "Example",
|
||||
"model": "Mobile 13",
|
||||
"manufacturer_sku": "EM13-8-1T",
|
||||
"gtin_ean": "4000000000002",
|
||||
"best_price": "1298.00",
|
||||
"currency": "EUR",
|
||||
"availability": "in_stock",
|
||||
"seller_count": 5,
|
||||
"raw_attributes": {"RAM": "8GB"},
|
||||
"normalized_attributes": {"ram_gb": 8, "storage_gb": 1024, "weight_kg": 1.2, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"page": 2,
|
||||
"page_count": 2,
|
||||
"result_count": 3,
|
||||
"http_status": 200,
|
||||
"items": [
|
||||
{
|
||||
"source_external_id": "gh-nb-c",
|
||||
"source_url": "https://geizhals.de/example-mobile-13-c.html",
|
||||
"source_title": "Example Mobile 13 unknown weight",
|
||||
"manufacturer": "Example",
|
||||
"model": "Mobile 13 UW",
|
||||
"manufacturer_sku": "EM13-32-UW",
|
||||
"gtin_ean": "4000000000003",
|
||||
"best_price": "1298.00",
|
||||
"currency": "EUR",
|
||||
"availability": "in_stock",
|
||||
"seller_count": 3,
|
||||
"raw_attributes": {"RAM": "32GB"},
|
||||
"normalized_attributes": {"ram_gb": 32, "storage_gb": 1024, "battery_score": 90, "display_score": 90, "cpu_score": 90}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"source": "geizhals",
|
||||
"external_category_key": "nb",
|
||||
"name": "Notebooks",
|
||||
"canonical_category": "notebook",
|
||||
"source_url": "https://geizhals.de/?cat=nb",
|
||||
"filters": [
|
||||
{
|
||||
"external_filter_key": "17023",
|
||||
"label": "Arbeitsspeicher",
|
||||
"filter_type": "NUMERIC_THRESHOLD",
|
||||
"canonical_unit": "GB",
|
||||
"options": [
|
||||
{"source_value": "ab 16GB", "normalized_value": 16, "operator": ">=", "unit": "GB"},
|
||||
{"source_value": "ab 24GB", "normalized_value": 24, "operator": ">=", "unit": "GB"},
|
||||
{"source_value": "ab 32GB", "normalized_value": 32, "operator": ">=", "unit": "GB"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"external_filter_key": "price",
|
||||
"label": "Preis",
|
||||
"filter_type": "NUMERIC_RANGE",
|
||||
"canonical_unit": "EUR",
|
||||
"range": {"minimum": 0, "maximum": null, "step": 1, "unit": "EUR"}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"products": [
|
||||
{
|
||||
"id": "11111111-1111-4111-8111-111111111111",
|
||||
"external_id": "example-mobile-13",
|
||||
"category": "notebook",
|
||||
"brand": "Example",
|
||||
"name": "Example Mobile 13",
|
||||
"model": "EM13",
|
||||
"variants": [
|
||||
{
|
||||
"id": "22222222-2222-4222-8222-222222222222",
|
||||
"name": "32 GB / 1 TB",
|
||||
"manufacturer_sku": "EM13-32-1T",
|
||||
"identifiers": {"sku": "EM13-32-1T"},
|
||||
"attributes": [
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "aaaaaaa1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-a/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "22222222-2222-4222-8222-22222222222b",
|
||||
"name": "8 GB / 1 TB",
|
||||
"manufacturer_sku": "EM13-8-1T",
|
||||
"identifiers": {"sku": "EM13-8-1T"},
|
||||
"attributes": [
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "8GB RAM", "normalized_value": 8, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "1.2 kg", "normalized_value": 1.2, "unit": "kg", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/weight", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "bbbbbbb1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-b/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "22222222-2222-4222-8222-22222222222c",
|
||||
"name": "32 GB / unknown weight",
|
||||
"manufacturer_sku": "EM13-32-UW",
|
||||
"identifiers": {},
|
||||
"attributes": [
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000001", "attribute": "ram_gb", "raw_value": "32GB LPDDR5X RAM", "normalized_value": 32, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/ram", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000002", "attribute": "storage_gb", "raw_value": "1024GB SSD", "normalized_value": 1024, "unit": "GB", "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/storage", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000003", "attribute": "weight_kg", "raw_value": "weight unknown", "normalized_value": null, "unit": "kg", "normalization_status": "unknown", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/weight", "observed_at": "2026-10-03T09:00:00Z", "normalization_error": "manual unknown"},
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000004", "attribute": "battery_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/battery", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000005", "attribute": "display_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/display", "observed_at": "2026-10-03T09:00:00Z"},
|
||||
{"id": "ccccccc1-0000-4000-8000-000000000006", "attribute": "cpu_score", "raw_value": 90, "normalized_value": 90, "normalization_status": "normalized", "normalizer_version": "manual-v1", "confidence": 1.0, "source_reference": "demo/notebook-c/cpu", "observed_at": "2026-10-03T09:00:00Z"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"offers": [
|
||||
{"id": "33333333-3333-4333-8333-333333333333", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-a", "seller": "DemoShop", "url": "https://example.test/a", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||
{"id": "33333333-3333-4333-8333-33333333333b", "variant_id": "22222222-2222-4222-8222-22222222222b", "external_id": "demo-offer-b", "seller": "DemoShop", "url": "https://example.test/b", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||
{"id": "33333333-3333-4333-8333-33333333333c", "variant_id": "22222222-2222-4222-8222-22222222222c", "external_id": "demo-offer-c", "seller": "DemoShop", "url": "https://example.test/c", "market": "DE", "currency": "EUR", "condition": "new"},
|
||||
{"id": "33333333-3333-4333-8333-33333333333d", "variant_id": "22222222-2222-4222-8222-222222222222", "external_id": "demo-offer-d", "seller": "OtherShop", "url": "https://example.test/d", "market": "DE", "currency": "EUR", "condition": "new"}
|
||||
],
|
||||
"prices": [
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000a1", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000a2", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000a3", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000a4", "external_offer_id": "demo-offer-a", "external_observation_id": "a-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000b1", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000b2", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000b3", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000b4", "external_offer_id": "demo-offer-b", "external_observation_id": "b-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000c1", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000c2", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000c3", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000c4", "external_offer_id": "demo-offer-c", "external_observation_id": "c-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "1298.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1298.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000d1", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-01", "observed_at": "2026-09-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000d2", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-09-15", "observed_at": "2026-09-15T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000d3", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-01", "observed_at": "2026-10-01T10:00:00Z", "item_price": "1599.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "1599.00"}},
|
||||
{"id": "ddddddd1-0000-4000-8000-0000000000d4", "external_offer_id": "demo-offer-d", "external_observation_id": "d-2026-10-03", "observed_at": "2026-10-03T10:00:00Z", "item_price": "2049.00", "shipping_price": "0.00", "currency": "EUR", "availability": "in_stock", "confidence": 1.0, "raw_payload": {"total": "2049.00"}}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from app.settings import get_settings
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
config.set_main_option("sqlalchemy.url", get_settings().database_url)
|
||||
target_metadata = None
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: str | None = ${repr(down_revision)}
|
||||
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
|
||||
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Deploy skeleton — domain schema follows in 0002.
|
||||
|
||||
Revision ID: 0001_deploy_skeleton
|
||||
Revises:
|
||||
Create Date: 2026-10-03
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
revision: str = "0001_deploy_skeleton"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Global product/price schema for v0.1. No user or workspace columns.
|
||||
|
||||
Revision ID: 0002_domain_schema
|
||||
Revises: 0001_deploy_skeleton
|
||||
Create Date: 2026-10-03
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0002_domain_schema"
|
||||
down_revision: str | None = "0001_deploy_skeleton"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"attribute_definitions",
|
||||
sa.Column("key", sa.String(128), primary_key=True),
|
||||
sa.Column("value_type", sa.String(32), nullable=False),
|
||||
sa.Column("canonical_unit", sa.String(32)),
|
||||
sa.Column("minimum", sa.Numeric(12, 4)),
|
||||
sa.Column("maximum", sa.Numeric(12, 4)),
|
||||
sa.Column("description", sa.Text()),
|
||||
sa.CheckConstraint(
|
||||
"value_type IN ('number', 'string', 'boolean', 'string_list')",
|
||||
name="ck_attribute_definitions_value_type",
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"sources",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(128), nullable=False),
|
||||
sa.Column("adapter_type", sa.String(64), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"scoring_models",
|
||||
sa.Column("version", sa.String(64), primary_key=True),
|
||||
sa.Column("algorithm_id", sa.String(128), nullable=False),
|
||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"product_profiles",
|
||||
sa.Column("id", sa.String(128), primary_key=True),
|
||||
sa.Column("version", sa.Integer(), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("category", sa.String(128), nullable=False),
|
||||
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("version > 0", name="ck_product_profiles_version"),
|
||||
)
|
||||
op.create_table(
|
||||
"products",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("category", sa.String(128), nullable=False),
|
||||
sa.Column("brand", sa.String(128), nullable=False),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("model", sa.String(128)),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"product_variants",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("product_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("products.id"), nullable=False),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("manufacturer_sku", sa.String(128)),
|
||||
sa.Column("identifiers", postgresql.JSONB, nullable=False, server_default=sa.text("'{}'::jsonb")),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_product_variants_product_id", "product_variants", ["product_id"])
|
||||
op.create_table(
|
||||
"variant_attributes",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||
sa.Column("attribute_key", sa.String(128), sa.ForeignKey("attribute_definitions.key"), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("raw_value", postgresql.JSONB, nullable=False),
|
||||
sa.Column("normalized_value", postgresql.JSONB),
|
||||
sa.Column("unit", sa.String(32)),
|
||||
sa.Column("normalization_status", sa.String(32), nullable=False),
|
||||
sa.Column("normalizer_version", sa.String(64), nullable=False),
|
||||
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
||||
sa.Column("source_reference", sa.String(256), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("normalization_error", sa.Text()),
|
||||
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_variant_attributes_confidence"),
|
||||
sa.CheckConstraint(
|
||||
"normalization_status IN ('normalized', 'unknown')",
|
||||
name="ck_variant_attributes_status",
|
||||
),
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX ix_variant_attributes_lookup ON variant_attributes "
|
||||
"(variant_id, attribute_key, observed_at DESC, ingested_at DESC, id DESC)"
|
||||
)
|
||||
op.create_table(
|
||||
"offers",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("external_id", sa.String(256), nullable=False),
|
||||
sa.Column("seller", sa.String(256), nullable=False),
|
||||
sa.Column("url", sa.Text()),
|
||||
sa.Column("market", sa.String(16), nullable=False),
|
||||
sa.Column("currency", sa.String(3), nullable=False),
|
||||
sa.Column("condition", sa.String(32), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("source_id", "external_id", name="uq_offers_source_external"),
|
||||
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_offers_currency"),
|
||||
sa.CheckConstraint("condition IN ('new', 'refurbished', 'used')", name="ck_offers_condition"),
|
||||
)
|
||||
op.create_index("ix_offers_variant_id", "offers", ["variant_id"])
|
||||
op.create_table(
|
||||
"price_observations",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("external_observation_id", sa.String(256), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("item_price", sa.Numeric(12, 2), nullable=False),
|
||||
sa.Column("shipping_price", sa.Numeric(12, 2)),
|
||||
sa.Column("currency", sa.String(3), nullable=False),
|
||||
sa.Column("availability", sa.String(32), nullable=False),
|
||||
sa.Column("raw_payload", postgresql.JSONB, nullable=False),
|
||||
sa.Column("confidence", sa.Numeric(4, 3), nullable=False),
|
||||
sa.UniqueConstraint("offer_id", "external_observation_id", name="uq_price_obs_offer_external"),
|
||||
sa.CheckConstraint("item_price > 0", name="ck_price_obs_item_price"),
|
||||
sa.CheckConstraint("shipping_price IS NULL OR shipping_price >= 0", name="ck_price_obs_shipping"),
|
||||
sa.CheckConstraint("confidence >= 0 AND confidence <= 1", name="ck_price_obs_confidence"),
|
||||
sa.CheckConstraint(
|
||||
"availability IN ('in_stock', 'out_of_stock', 'unknown')",
|
||||
name="ck_price_obs_availability",
|
||||
),
|
||||
sa.CheckConstraint("currency ~ '^[A-Z]{3}$'", name="ck_price_obs_currency"),
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX ix_price_observations_lookup ON price_observations "
|
||||
"(offer_id, observed_at DESC, ingested_at DESC, id DESC)"
|
||||
)
|
||||
op.create_table(
|
||||
"evaluations",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id"), nullable=False),
|
||||
sa.Column("offer_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("offers.id"), nullable=False),
|
||||
sa.Column("profile_id", sa.String(128), nullable=False),
|
||||
sa.Column("profile_version", sa.Integer(), nullable=False),
|
||||
sa.Column("scoring_version", sa.String(64), sa.ForeignKey("scoring_models.version"), nullable=False),
|
||||
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("snapshot_cutoff", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("scoring_hash", sa.String(64), nullable=False),
|
||||
sa.Column("engine_build", sa.String(64), nullable=False),
|
||||
sa.Column("product_eligibility", sa.String(16), nullable=False),
|
||||
sa.Column("offer_eligibility", sa.String(16), nullable=False),
|
||||
sa.Column("overall_eligibility", sa.String(16), nullable=False),
|
||||
sa.Column("product_score", sa.Numeric(6, 2)),
|
||||
sa.Column("price_score", sa.Numeric(6, 2)),
|
||||
sa.Column("deal_score", sa.Numeric(6, 2)),
|
||||
sa.Column("classification", sa.String(32), nullable=False),
|
||||
sa.Column("price_reference_scope", sa.String(32), nullable=False),
|
||||
sa.Column("input_snapshot", postgresql.JSONB, nullable=False),
|
||||
sa.Column("result_snapshot", postgresql.JSONB, nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["profile_id", "profile_version"],
|
||||
["product_profiles.id", "product_profiles.version"],
|
||||
name="fk_evaluations_profile",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"product_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_product_eligibility",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"offer_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_offer_eligibility",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"overall_eligibility IN ('ELIGIBLE', 'INELIGIBLE', 'UNKNOWN')",
|
||||
name="ck_eval_overall_eligibility",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"product_score IS NULL OR (product_score >= 0 AND product_score <= 100)",
|
||||
name="ck_eval_product_score",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"price_score IS NULL OR (price_score >= 0 AND price_score <= 100)",
|
||||
name="ck_eval_price_score",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"deal_score IS NULL OR (deal_score >= 0 AND deal_score <= 100)",
|
||||
name="ck_eval_deal_score",
|
||||
),
|
||||
)
|
||||
op.create_index("ix_evaluations_variant_id", "evaluations", ["variant_id"])
|
||||
op.execute(
|
||||
"CREATE INDEX ix_evaluations_profile_created ON evaluations "
|
||||
"(profile_id, profile_version, created_at DESC)"
|
||||
)
|
||||
op.create_table(
|
||||
"evaluation_components",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("evaluation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("evaluations.id"), nullable=False),
|
||||
sa.Column("kind", sa.String(64), nullable=False),
|
||||
sa.Column("component_key", sa.String(128), nullable=False),
|
||||
sa.Column("raw_score", sa.Numeric(12, 6)),
|
||||
sa.Column("weight", sa.Numeric(12, 6)),
|
||||
sa.Column("contribution", sa.Numeric(12, 6)),
|
||||
sa.Column("details", postgresql.JSONB, nullable=False),
|
||||
sa.UniqueConstraint("evaluation_id", "kind", "component_key", name="uq_eval_component"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("evaluation_components")
|
||||
op.drop_table("evaluations")
|
||||
op.drop_table("price_observations")
|
||||
op.drop_table("offers")
|
||||
op.drop_table("variant_attributes")
|
||||
op.drop_table("product_variants")
|
||||
op.drop_table("products")
|
||||
op.drop_table("product_profiles")
|
||||
op.drop_table("scoring_models")
|
||||
op.drop_table("sources")
|
||||
op.drop_table("attribute_definitions")
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Discovery, taxonomy, search runs, and project candidates.
|
||||
|
||||
Revision ID: 0003_discovery_schema
|
||||
Revises: 0002_domain_schema
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0003_discovery_schema"
|
||||
down_revision: str | None = "0002_domain_schema"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"buying_projects",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("workspace_id", sa.String(64)),
|
||||
sa.Column("profile_id", sa.String(128), nullable=False),
|
||||
sa.Column("profile_version", sa.Integer(), nullable=False),
|
||||
sa.Column("shortlist_size", sa.Integer(), nullable=False, server_default="5"),
|
||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("archived_at", sa.DateTime(timezone=True)),
|
||||
sa.ForeignKeyConstraint(
|
||||
["profile_id", "profile_version"],
|
||||
["product_profiles.id", "product_profiles.version"],
|
||||
name="fk_buying_projects_profile",
|
||||
),
|
||||
sa.CheckConstraint("status IN ('ACTIVE', 'PURCHASED', 'ARCHIVED')", name="ck_buying_projects_status"),
|
||||
sa.CheckConstraint("shortlist_size > 0", name="ck_buying_projects_shortlist"),
|
||||
sa.CheckConstraint("schedule_interval_minutes > 0", name="ck_buying_projects_interval"),
|
||||
)
|
||||
op.create_table(
|
||||
"buying_project_criteria",
|
||||
sa.Column("project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), primary_key=True),
|
||||
sa.Column("version", sa.Integer(), primary_key=True),
|
||||
sa.Column("definition", postgresql.JSONB, nullable=False),
|
||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("version > 0", name="ck_project_criteria_version"),
|
||||
)
|
||||
op.create_table(
|
||||
"source_categories",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("external_category_key", sa.String(64), nullable=False),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("source_url", sa.Text(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("last_verified_at", sa.DateTime(timezone=True)),
|
||||
sa.UniqueConstraint("source_id", "external_category_key", name="uq_source_category"),
|
||||
)
|
||||
op.create_table(
|
||||
"source_filter_definitions",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
||||
sa.Column("label", sa.String(256), nullable=False),
|
||||
sa.Column("filter_type", sa.String(32), nullable=False),
|
||||
sa.Column("canonical_unit", sa.String(32)),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.UniqueConstraint("source_category_id", "external_filter_key", name="uq_source_filter"),
|
||||
)
|
||||
op.create_table(
|
||||
"source_filter_options",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), nullable=False),
|
||||
sa.Column("source_value", sa.String(256), nullable=False),
|
||||
sa.Column("normalized_value", postgresql.JSONB),
|
||||
sa.Column("operator", sa.String(16)),
|
||||
sa.Column("unit", sa.String(32)),
|
||||
sa.UniqueConstraint("filter_id", "source_value", name="uq_filter_option"),
|
||||
)
|
||||
op.create_table(
|
||||
"source_filter_ranges",
|
||||
sa.Column("filter_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_filter_definitions.id"), primary_key=True),
|
||||
sa.Column("minimum", sa.Numeric(12, 2)),
|
||||
sa.Column("maximum", sa.Numeric(12, 2)),
|
||||
sa.Column("step", sa.Numeric(12, 2)),
|
||||
sa.Column("unit", sa.String(32)),
|
||||
)
|
||||
op.create_table(
|
||||
"criterion_definitions",
|
||||
sa.Column("key", sa.String(128), primary_key=True),
|
||||
sa.Column("value_type", sa.String(32), nullable=False),
|
||||
sa.Column("unit", sa.String(32)),
|
||||
sa.Column("semantic", sa.String(128)),
|
||||
sa.Column("description", sa.Text()),
|
||||
)
|
||||
op.create_table(
|
||||
"criterion_mappings",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||
sa.Column("external_filter_key", sa.String(64), nullable=False),
|
||||
sa.Column("source_value", sa.String(256)),
|
||||
sa.Column("encoding", postgresql.JSONB, nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("definition_hash", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"discovery_searches",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
||||
sa.Column("mapping_version", sa.String(64), nullable=False),
|
||||
sa.Column("generated_url", sa.Text(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("broad_result_threshold", sa.Integer(), nullable=False, server_default="300"),
|
||||
sa.Column("next_run_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_successful_run_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"search_runs",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("discovery_search_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("discovery_searches.id"), nullable=False),
|
||||
sa.Column("criteria_version", sa.Integer(), nullable=False),
|
||||
sa.Column("mapping_version", sa.String(64), nullable=False),
|
||||
sa.Column("generated_url", sa.Text(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("result_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("http_status_summary", postgresql.JSONB, nullable=False),
|
||||
sa.Column("error_code", sa.String(64)),
|
||||
)
|
||||
op.create_table(
|
||||
"source_product_references",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id"), nullable=False),
|
||||
sa.Column("source_external_id", sa.String(256)),
|
||||
sa.Column("source_url", sa.Text()),
|
||||
sa.Column("source_title", sa.String(512), nullable=False),
|
||||
sa.Column("manufacturer", sa.String(256)),
|
||||
sa.Column("model", sa.String(256)),
|
||||
sa.Column("manufacturer_sku", sa.String(128)),
|
||||
sa.Column("gtin_ean", sa.String(32)),
|
||||
sa.Column("normalized_spec_fingerprint", sa.Text()),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("lifecycle_status", sa.String(32), nullable=False),
|
||||
sa.Column("match_status", sa.String(32), nullable=False),
|
||||
sa.Column("consecutive_missing_success_runs", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
||||
)
|
||||
op.create_table(
|
||||
"product_observations",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("raw_title", sa.String(512), nullable=False),
|
||||
sa.Column("best_price", sa.Numeric(12, 2)),
|
||||
sa.Column("currency", sa.String(3)),
|
||||
sa.Column("raw_attributes", postgresql.JSONB, nullable=False),
|
||||
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False),
|
||||
sa.Column("availability", sa.String(32)),
|
||||
sa.Column("seller_count", sa.Integer()),
|
||||
)
|
||||
op.create_table(
|
||||
"search_run_items",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("search_run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("search_runs.id"), nullable=False),
|
||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||
sa.Column("observation_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_observations.id")),
|
||||
sa.Column("diff_status", sa.String(32), nullable=False),
|
||||
sa.UniqueConstraint("search_run_id", "source_product_reference_id", name="uq_run_item"),
|
||||
)
|
||||
op.create_table(
|
||||
"project_candidates",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("buying_project_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("buying_projects.id"), nullable=False),
|
||||
sa.Column("source_product_reference_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_product_references.id"), nullable=False),
|
||||
sa.Column("product_variant_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("product_variants.id")),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("match_status", sa.String(32), nullable=False),
|
||||
sa.Column("product_eligibility", sa.String(16)),
|
||||
sa.Column("product_score", sa.Numeric(6, 2)),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("buying_project_id", "source_product_reference_id", name="uq_project_candidate"),
|
||||
)
|
||||
op.create_table(
|
||||
"candidate_status_history",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("candidate_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("project_candidates.id"), nullable=False),
|
||||
sa.Column("from_status", sa.String(32)),
|
||||
sa.Column("to_status", sa.String(32), nullable=False),
|
||||
sa.Column("reason", sa.String(128)),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_discovery_searches_next_run", "discovery_searches", ["next_run_at"])
|
||||
op.create_index("ix_search_runs_search", "search_runs", ["discovery_search_id", "started_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("candidate_status_history")
|
||||
op.drop_table("project_candidates")
|
||||
op.drop_table("search_run_items")
|
||||
op.drop_table("product_observations")
|
||||
op.drop_table("source_product_references")
|
||||
op.drop_table("search_runs")
|
||||
op.drop_table("discovery_searches")
|
||||
op.drop_table("criterion_mappings")
|
||||
op.drop_table("criterion_definitions")
|
||||
op.drop_table("source_filter_ranges")
|
||||
op.drop_table("source_filter_options")
|
||||
op.drop_table("source_filter_definitions")
|
||||
op.drop_table("source_categories")
|
||||
op.drop_table("buying_project_criteria")
|
||||
op.drop_table("buying_projects")
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Dynamic attribute and criterion value records.
|
||||
|
||||
Revision ID: 0004_dynamic_attributes
|
||||
Revises: 0003_discovery_schema
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0004_dynamic_attributes"
|
||||
down_revision: str | None = "0003_discovery_schema"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("criterion_definitions", sa.Column("category", sa.String(128)))
|
||||
op.add_column("source_categories", sa.Column("canonical_category", sa.String(128)))
|
||||
op.create_table(
|
||||
"project_criteria",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("version", sa.Integer(), nullable=False),
|
||||
sa.Column("item_id", sa.String(128), nullable=False),
|
||||
sa.Column("criterion_key", sa.String(128), sa.ForeignKey("criterion_definitions.key"), nullable=False),
|
||||
sa.Column("operator", sa.String(16), nullable=False),
|
||||
sa.Column("value", postgresql.JSONB),
|
||||
sa.Column("mandatory", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.ForeignKeyConstraint(
|
||||
["project_id", "version"],
|
||||
["buying_project_criteria.project_id", "buying_project_criteria.version"],
|
||||
name="fk_project_criteria_version",
|
||||
),
|
||||
sa.UniqueConstraint("project_id", "version", "item_id", name="uq_project_criterion_item"),
|
||||
)
|
||||
op.create_table(
|
||||
"product_observation_attributes",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column(
|
||||
"observation_id",
|
||||
postgresql.UUID(as_uuid=True),
|
||||
sa.ForeignKey("product_observations.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("attribute_key", sa.String(128), nullable=False),
|
||||
sa.Column("raw_value", postgresql.JSONB),
|
||||
sa.Column("normalized_value", postgresql.JSONB),
|
||||
sa.Column("unit", sa.String(32)),
|
||||
sa.UniqueConstraint("observation_id", "attribute_key", name="uq_observation_attribute"),
|
||||
)
|
||||
_migrate_criteria_json()
|
||||
_migrate_observation_json()
|
||||
op.drop_column("buying_project_criteria", "definition")
|
||||
op.drop_column("product_observations", "raw_attributes")
|
||||
op.drop_column("product_observations", "normalized_attributes")
|
||||
|
||||
|
||||
def _migrate_criteria_json() -> None:
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(sa.text("SELECT project_id, version, definition FROM buying_project_criteria")).mappings()
|
||||
for row in rows:
|
||||
items = (row["definition"] or {}).get("items") or []
|
||||
for raw in items:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO project_criteria
|
||||
(id, project_id, version, item_id, criterion_key, operator, value, mandatory)
|
||||
VALUES
|
||||
(gen_random_uuid(), :project_id, :version, :item_id, :criterion_key, :operator, :value, :mandatory)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"project_id": row["project_id"],
|
||||
"version": row["version"],
|
||||
"item_id": str(raw.get("id") or raw.get("criterion")),
|
||||
"criterion_key": str(raw.get("criterion")),
|
||||
"operator": str(raw.get("operator")),
|
||||
"value": raw.get("value"),
|
||||
"mandatory": bool(raw.get("mandatory", True)),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _migrate_observation_json() -> None:
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(
|
||||
sa.text("SELECT id, raw_attributes, normalized_attributes FROM product_observations")
|
||||
).mappings()
|
||||
for row in rows:
|
||||
raw = row["raw_attributes"] or {}
|
||||
normalized = row["normalized_attributes"] or {}
|
||||
keys = set(raw) | set(normalized)
|
||||
for key in keys:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO product_observation_attributes
|
||||
(id, observation_id, attribute_key, raw_value, normalized_value, unit)
|
||||
VALUES
|
||||
(gen_random_uuid(), :observation_id, :attribute_key, :raw_value, :normalized_value, NULL)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"observation_id": row["id"],
|
||||
"attribute_key": key,
|
||||
"raw_value": raw.get(key),
|
||||
"normalized_value": normalized.get(key),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("product_observations", sa.Column("raw_attributes", postgresql.JSONB, nullable=False, server_default="{}"))
|
||||
op.add_column(
|
||||
"product_observations",
|
||||
sa.Column("normalized_attributes", postgresql.JSONB, nullable=False, server_default="{}"),
|
||||
)
|
||||
op.add_column("buying_project_criteria", sa.Column("definition", postgresql.JSONB, nullable=False, server_default='{"items":[]}'))
|
||||
op.drop_table("product_observation_attributes")
|
||||
op.drop_table("project_criteria")
|
||||
op.drop_column("source_categories", "canonical_category")
|
||||
op.drop_column("criterion_definitions", "category")
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Frontend MVP v0.3 contracts: categories, optional templates, criterion UUIDs.
|
||||
|
||||
Revision ID: 0005_frontend_mvp_v03
|
||||
Revises: 0004_dynamic_attributes
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0005_frontend_mvp_v03"
|
||||
down_revision: str | None = "0004_dynamic_attributes"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"product_categories",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="ACTIVE"),
|
||||
)
|
||||
op.execute("INSERT INTO product_categories (id, name, status) VALUES ('notebook', 'Notebooks', 'ACTIVE')")
|
||||
|
||||
op.add_column("source_categories", sa.Column("product_category_id", sa.String(64)))
|
||||
op.create_foreign_key(
|
||||
"fk_source_categories_product_category",
|
||||
"source_categories",
|
||||
"product_categories",
|
||||
["product_category_id"],
|
||||
["id"],
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE source_categories
|
||||
SET product_category_id = canonical_category
|
||||
WHERE canonical_category IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM product_categories WHERE id = source_categories.canonical_category)
|
||||
"""
|
||||
)
|
||||
|
||||
op.add_column("criterion_definitions", sa.Column("id", postgresql.UUID(as_uuid=True)))
|
||||
op.execute("UPDATE criterion_definitions SET id = gen_random_uuid() WHERE id IS NULL")
|
||||
op.alter_column("criterion_definitions", "id", nullable=False)
|
||||
op.add_column(
|
||||
"criterion_definitions",
|
||||
sa.Column("origin", sa.String(16), nullable=False, server_default="SOURCE"),
|
||||
)
|
||||
op.add_column("criterion_definitions", sa.Column("source", sa.String(64)))
|
||||
op.add_column("criterion_definitions", sa.Column("product_category_id", sa.String(64)))
|
||||
op.add_column("criterion_definitions", sa.Column("label", sa.String(256)))
|
||||
op.add_column("criterion_definitions", sa.Column("scoring_attribute_key", sa.String(128)))
|
||||
op.execute("UPDATE criterion_definitions SET label = key WHERE label IS NULL")
|
||||
op.execute("UPDATE criterion_definitions SET scoring_attribute_key = key")
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE criterion_definitions
|
||||
SET product_category_id = category
|
||||
WHERE category IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM product_categories WHERE id = criterion_definitions.category)
|
||||
"""
|
||||
)
|
||||
op.alter_column("criterion_definitions", "label", nullable=False)
|
||||
op.create_foreign_key(
|
||||
"fk_criterion_definitions_product_category",
|
||||
"criterion_definitions",
|
||||
"product_categories",
|
||||
["product_category_id"],
|
||||
["id"],
|
||||
)
|
||||
|
||||
op.add_column("project_criteria", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
||||
op.add_column(
|
||||
"project_criteria",
|
||||
sa.Column("enabled_for_discovery", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
)
|
||||
op.add_column("project_criteria", sa.Column("created_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE project_criteria
|
||||
SET enabled_for_discovery = mandatory,
|
||||
created_at = timezone('utc', now())
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE project_criteria AS pc
|
||||
SET criterion_definition_id = cd.id
|
||||
FROM criterion_definitions AS cd
|
||||
WHERE pc.criterion_key = cd.key
|
||||
"""
|
||||
)
|
||||
op.alter_column("project_criteria", "criterion_definition_id", nullable=False)
|
||||
op.alter_column("project_criteria", "created_at", nullable=False)
|
||||
op.drop_constraint("project_criteria_criterion_key_fkey", "project_criteria", type_="foreignkey")
|
||||
|
||||
op.add_column("criterion_mappings", sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE criterion_mappings AS cm
|
||||
SET criterion_definition_id = cd.id
|
||||
FROM criterion_definitions AS cd
|
||||
WHERE cm.criterion_key = cd.key
|
||||
"""
|
||||
)
|
||||
op.alter_column("criterion_mappings", "criterion_definition_id", nullable=False)
|
||||
op.drop_constraint("criterion_mappings_criterion_key_fkey", "criterion_mappings", type_="foreignkey")
|
||||
|
||||
op.drop_constraint("criterion_definitions_pkey", "criterion_definitions", type_="primary")
|
||||
op.create_primary_key("criterion_definitions_pkey", "criterion_definitions", ["id"])
|
||||
op.create_unique_constraint("uq_criterion_definition_key", "criterion_definitions", ["key"])
|
||||
op.create_foreign_key(
|
||||
"fk_project_criteria_definition",
|
||||
"project_criteria",
|
||||
"criterion_definitions",
|
||||
["criterion_definition_id"],
|
||||
["id"],
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_criterion_mappings_definition",
|
||||
"criterion_mappings",
|
||||
"criterion_definitions",
|
||||
["criterion_definition_id"],
|
||||
["id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"source_attribute_mappings",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source_id", sa.String(64), sa.ForeignKey("sources.id"), nullable=False),
|
||||
sa.Column("source_category_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("source_categories.id")),
|
||||
sa.Column("raw_name", sa.String(256), nullable=False),
|
||||
sa.Column(
|
||||
"criterion_definition_id",
|
||||
postgresql.UUID(as_uuid=True),
|
||||
sa.ForeignKey("criterion_definitions.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
op.add_column("buying_projects", sa.Column("product_category_id", sa.String(64)))
|
||||
op.add_column("buying_projects", sa.Column("market", sa.String(8)))
|
||||
op.add_column("buying_projects", sa.Column("currency", sa.String(3)))
|
||||
op.add_column("buying_projects", sa.Column("target_budget", sa.Numeric(12, 2)))
|
||||
op.add_column("buying_projects", sa.Column("template_id", sa.String(128)))
|
||||
op.add_column("buying_projects", sa.Column("template_version", sa.Integer()))
|
||||
op.add_column("buying_projects", sa.Column("criteria_version", sa.Integer(), nullable=False, server_default="1"))
|
||||
op.add_column("buying_projects", sa.Column("updated_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE buying_projects
|
||||
SET template_id = profile_id,
|
||||
template_version = profile_version,
|
||||
product_category_id = 'notebook',
|
||||
market = 'DE',
|
||||
currency = 'EUR',
|
||||
updated_at = created_at
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE buying_projects AS bp
|
||||
SET criteria_version = COALESCE((
|
||||
SELECT MAX(version) FROM buying_project_criteria WHERE project_id = bp.id
|
||||
), 1)
|
||||
"""
|
||||
)
|
||||
op.alter_column("buying_projects", "product_category_id", nullable=False)
|
||||
op.alter_column("buying_projects", "market", nullable=False)
|
||||
op.alter_column("buying_projects", "currency", nullable=False)
|
||||
op.alter_column("buying_projects", "updated_at", nullable=False)
|
||||
op.create_foreign_key(
|
||||
"fk_buying_projects_product_category",
|
||||
"buying_projects",
|
||||
"product_categories",
|
||||
["product_category_id"],
|
||||
["id"],
|
||||
)
|
||||
op.drop_constraint("fk_buying_projects_profile", "buying_projects", type_="foreignkey")
|
||||
op.drop_constraint("ck_buying_projects_interval", "buying_projects", type_="check")
|
||||
op.drop_column("buying_projects", "profile_id")
|
||||
op.drop_column("buying_projects", "profile_version")
|
||||
op.drop_column("buying_projects", "schedule_interval_minutes")
|
||||
op.create_foreign_key(
|
||||
"fk_buying_projects_template",
|
||||
"buying_projects",
|
||||
"product_profiles",
|
||||
["template_id", "template_version"],
|
||||
["id", "version"],
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"discovery_searches",
|
||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||
)
|
||||
op.add_column("search_runs", sa.Column("new_count", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("search_runs", sa.Column("changed_count", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("search_runs", sa.Column("unchanged_count", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("search_runs", sa.Column("missing_count", sa.Integer(), nullable=False, server_default="0"))
|
||||
|
||||
op.add_column(
|
||||
"product_observation_attributes",
|
||||
sa.Column("criterion_definition_id", postgresql.UUID(as_uuid=True)),
|
||||
)
|
||||
op.add_column("product_observation_attributes", sa.Column("raw_name", sa.String(256)))
|
||||
op.create_foreign_key(
|
||||
"fk_observation_attributes_definition",
|
||||
"product_observation_attributes",
|
||||
"criterion_definitions",
|
||||
["criterion_definition_id"],
|
||||
["id"],
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE product_observation_attributes AS poa
|
||||
SET criterion_definition_id = cd.id,
|
||||
raw_name = poa.attribute_key
|
||||
FROM criterion_definitions AS cd
|
||||
WHERE poa.attribute_key = cd.scoring_attribute_key
|
||||
OR poa.attribute_key = cd.key
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint("fk_observation_attributes_definition", "product_observation_attributes", type_="foreignkey")
|
||||
op.drop_column("product_observation_attributes", "raw_name")
|
||||
op.drop_column("product_observation_attributes", "criterion_definition_id")
|
||||
op.drop_column("search_runs", "missing_count")
|
||||
op.drop_column("search_runs", "unchanged_count")
|
||||
op.drop_column("search_runs", "changed_count")
|
||||
op.drop_column("search_runs", "new_count")
|
||||
op.drop_column("discovery_searches", "schedule_interval_minutes")
|
||||
op.drop_constraint("fk_buying_projects_template", "buying_projects", type_="foreignkey")
|
||||
op.add_column("buying_projects", sa.Column("profile_id", sa.String(128)))
|
||||
op.add_column("buying_projects", sa.Column("profile_version", sa.Integer()))
|
||||
op.add_column(
|
||||
"buying_projects",
|
||||
sa.Column("schedule_interval_minutes", sa.Integer(), nullable=False, server_default="180"),
|
||||
)
|
||||
op.execute("UPDATE buying_projects SET profile_id = template_id, profile_version = template_version")
|
||||
op.alter_column("buying_projects", "profile_id", nullable=False)
|
||||
op.alter_column("buying_projects", "profile_version", nullable=False)
|
||||
op.drop_constraint("fk_buying_projects_product_category", "buying_projects", type_="foreignkey")
|
||||
op.drop_column("buying_projects", "updated_at")
|
||||
op.drop_column("buying_projects", "criteria_version")
|
||||
op.drop_column("buying_projects", "template_version")
|
||||
op.drop_column("buying_projects", "template_id")
|
||||
op.drop_column("buying_projects", "target_budget")
|
||||
op.drop_column("buying_projects", "currency")
|
||||
op.drop_column("buying_projects", "market")
|
||||
op.drop_column("buying_projects", "product_category_id")
|
||||
op.drop_table("source_attribute_mappings")
|
||||
raise NotImplementedError("full downgrade of criterion UUID primary key is not supported")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Seedable SourceCategory readiness for the notebook pilot.
|
||||
|
||||
Revision ID: 0006_source_category_readiness
|
||||
Revises: 0005_frontend_mvp_v03
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006_source_category_readiness"
|
||||
down_revision: str | None = "0005_frontend_mvp_v03"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source_categories",
|
||||
sa.Column("taxonomy_status", sa.String(32), nullable=False, server_default="NOT_LEARNED"),
|
||||
)
|
||||
op.add_column("source_categories", sa.Column("last_taxonomy_scan_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE source_categories
|
||||
SET taxonomy_status = 'READY',
|
||||
last_taxonomy_scan_at = last_verified_at
|
||||
WHERE last_verified_at IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source_categories", "last_taxonomy_scan_at")
|
||||
op.drop_column("source_categories", "taxonomy_status")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Persist source filter groups from the learned taxonomy.
|
||||
|
||||
Revision ID: 0007_filter_groups
|
||||
Revises: 0006_source_category_readiness
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007_filter_groups"
|
||||
down_revision: str | None = "0006_source_category_readiness"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("source_filter_definitions", sa.Column("group_label", sa.String(128)))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source_filter_definitions", "group_label")
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "mobile-development-notebook",
|
||||
"version": 1,
|
||||
"name": "Mobiler Entwicklungsrechner",
|
||||
"category": "notebook",
|
||||
"currency": "EUR",
|
||||
"market": "DE",
|
||||
"allowed_conditions": ["new"],
|
||||
"budget": {"target": "1500.00", "maximum": "2000.00"},
|
||||
"scoring_version": "generic-v1",
|
||||
"constraints": [
|
||||
{"id": "min_ram", "attribute": "ram_gb", "operator": ">=", "value": 16, "mandatory": true},
|
||||
{"id": "min_storage", "attribute": "storage_gb", "operator": ">=", "value": 512, "mandatory": true},
|
||||
{"id": "max_weight", "attribute": "weight_kg", "operator": "<=", "value": 1.6, "mandatory": true}
|
||||
],
|
||||
"preferences": [
|
||||
{"id": "ram", "attribute": "ram_gb", "strategy": "numeric_linear", "direction": "higher", "acceptable": 16, "ideal": 32, "weight": 0.30},
|
||||
{"id": "weight", "attribute": "weight_kg", "strategy": "numeric_linear", "direction": "lower", "acceptable": 1.6, "ideal": 1.0, "weight": 0.25},
|
||||
{"id": "battery", "attribute": "battery_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.20},
|
||||
{"id": "display", "attribute": "display_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.15},
|
||||
{"id": "cpu", "attribute": "cpu_score", "strategy": "numeric_linear", "direction": "higher", "acceptable": 60, "ideal": 100, "weight": 0.10}
|
||||
]
|
||||
}
|
||||
Loaded 100 of 172 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user