Schliesse das Notebook-Source-Mapping, damit Wizard und Taxonomie ohne leere Kategorie starten.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
b276758ee1
commit
d784248e28
13 files changed
+1505
-9
No files matched your search
@@ -35,6 +35,11 @@ def discover_taxonomy(source_id: str, category: str, session: Session = Depends(
|
||||
return TaxonomyService(session).discover(source_id, category)
|
||||
|
||||
|
||||
@router.get("/sources/{source_id}/categories/{category}/taxonomy-status")
|
||||
def taxonomy_status(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).taxonomy_status(source_id, category)
|
||||
|
||||
|
||||
@router.get("/sources/{source_id}/categories/{category}/filters")
|
||||
def list_filters(source_id: str, category: str, session: Session = Depends(get_session)):
|
||||
return TaxonomyService(session).filters(source_id, category)
|
||||
|
||||
@@ -7,6 +7,15 @@ class ProjectStatus(StrEnum):
|
||||
ARCHIVED = "ARCHIVED"
|
||||
|
||||
|
||||
class TaxonomyStatus(StrEnum):
|
||||
NOT_LEARNED = "NOT_LEARNED"
|
||||
LEARNING = "LEARNING"
|
||||
READY = "READY"
|
||||
DEGRADED = "DEGRADED"
|
||||
BROKEN = "BROKEN"
|
||||
BLOCKED = "BLOCKED"
|
||||
|
||||
|
||||
class MappingStatus(StrEnum):
|
||||
DISCOVERED = "DISCOVERED"
|
||||
LEARNED_UNVERIFIED = "LEARNED_UNVERIFIED"
|
||||
|
||||
@@ -105,6 +105,8 @@ class SourceCategoryRecord(Base):
|
||||
product_category_id: Mapped[str | None] = mapped_column(ForeignKey("product_categories.id"))
|
||||
canonical_category: Mapped[str | None] = mapped_column(String(128))
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
taxonomy_status: Mapped[str] = mapped_column(String(32), nullable=False, default="NOT_LEARNED")
|
||||
last_taxonomy_scan_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discovery.enums import FilterType, MappingStatus
|
||||
from app.discovery.enums import FilterType, MappingStatus, TaxonomyStatus
|
||||
from app.domain.serialize import jsonable
|
||||
from app.errors import AppError
|
||||
from app.persistence.discovery_models import (
|
||||
@@ -61,6 +62,7 @@ class CatalogService:
|
||||
"display_name": sources[row.source_id].name if row.source_id in sources else row.name,
|
||||
"external_category_key": row.external_category_key,
|
||||
"status": row.status,
|
||||
"taxonomy_status": row.taxonomy_status,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
@@ -82,6 +84,8 @@ class CatalogService:
|
||||
"name": row.name,
|
||||
"product_category_id": row.product_category_id,
|
||||
"status": row.status,
|
||||
"taxonomy_status": row.taxonomy_status,
|
||||
"source_url": row.source_url,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
@@ -96,6 +100,16 @@ class CatalogService:
|
||||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
if source_category is not None and source_category.taxonomy_status == TaxonomyStatus.NOT_LEARNED.value:
|
||||
from app.services.discovery_service import TaxonomyService
|
||||
|
||||
TaxonomyService(self.session).discover(source_id, source_category.external_category_key)
|
||||
source_category = self.session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.product_category_id == product_category_id,
|
||||
)
|
||||
).first()
|
||||
criteria: list[dict[str, Any]] = []
|
||||
if source_category is not None:
|
||||
filters = self.session.scalars(
|
||||
@@ -222,3 +236,43 @@ def ensure_product_category(session: Session, category_id: str, name: str | None
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def ensure_source_category(
|
||||
session: Session,
|
||||
*,
|
||||
source_id: str,
|
||||
external_category_key: str,
|
||||
product_category_id: str,
|
||||
name: str,
|
||||
source_url: str,
|
||||
) -> SourceCategoryRecord:
|
||||
row = session.scalars(
|
||||
select(SourceCategoryRecord).where(
|
||||
SourceCategoryRecord.source_id == source_id,
|
||||
SourceCategoryRecord.external_category_key == external_category_key,
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
row = SourceCategoryRecord(
|
||||
id=uuid4(),
|
||||
source_id=source_id,
|
||||
external_category_key=external_category_key,
|
||||
name=name,
|
||||
source_url=source_url,
|
||||
product_category_id=product_category_id,
|
||||
canonical_category=product_category_id,
|
||||
status="ACTIVE",
|
||||
taxonomy_status=TaxonomyStatus.NOT_LEARNED.value,
|
||||
last_taxonomy_scan_at=None,
|
||||
last_verified_at=None,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
if row.product_category_id is None:
|
||||
row.product_category_id = product_category_id
|
||||
if not row.source_url:
|
||||
row.source_url = source_url
|
||||
session.flush()
|
||||
return row
|
||||
@@ -21,6 +21,7 @@ from app.discovery.enums import (
|
||||
RankingStatus,
|
||||
RunStatus,
|
||||
SearchStatus,
|
||||
TaxonomyStatus,
|
||||
)
|
||||
from app.discovery.identity import IdentitySignals, decide_identity, spec_fingerprint
|
||||
from app.discovery.mapping import verify_roundtrip
|
||||
@@ -116,6 +117,8 @@ class TaxonomyService:
|
||||
canonical_category=payload.get("canonical_category"),
|
||||
product_category_id=_linked_product_category(self.session, payload.get("canonical_category")),
|
||||
status="ACTIVE",
|
||||
taxonomy_status=TaxonomyStatus.LEARNING.value,
|
||||
last_taxonomy_scan_at=now,
|
||||
last_verified_at=now,
|
||||
)
|
||||
self.session.add(category)
|
||||
@@ -126,6 +129,8 @@ class TaxonomyService:
|
||||
if payload.get("canonical_category"):
|
||||
category.canonical_category = payload.get("canonical_category")
|
||||
category.product_category_id = _linked_product_category(self.session, payload.get("canonical_category"))
|
||||
category.taxonomy_status = TaxonomyStatus.LEARNING.value
|
||||
category.last_taxonomy_scan_at = now
|
||||
category.last_verified_at = now
|
||||
for raw in payload.get("filters") or []:
|
||||
filt = self.session.scalars(
|
||||
@@ -156,9 +161,33 @@ class TaxonomyService:
|
||||
self._sync_options(filt, raw.get("options") or [])
|
||||
if raw.get("range"):
|
||||
self._sync_range(filt, raw["range"])
|
||||
category.taxonomy_status = (
|
||||
TaxonomyStatus.READY.value if payload.get("filters") else TaxonomyStatus.BROKEN.value
|
||||
)
|
||||
self.session.flush()
|
||||
return self.filters(source_id, category.external_category_key)
|
||||
|
||||
def taxonomy_status(self, source_id: str, category_key: str) -> dict[str, Any]:
|
||||
category = self._category(source_id, category_key)
|
||||
filters = self.session.scalars(
|
||||
select(SourceFilterDefinitionRecord).where(SourceFilterDefinitionRecord.source_category_id == category.id)
|
||||
).all()
|
||||
mappings = self.session.scalars(
|
||||
select(CriterionMappingRecord).where(CriterionMappingRecord.source_category_id == category.id)
|
||||
).all()
|
||||
return jsonable(
|
||||
{
|
||||
"source_category_status": category.taxonomy_status,
|
||||
"filters_discovered": len(filters),
|
||||
"filters_verified": sum(1 for row in mappings if row.status == MappingStatus.VERIFIED.value),
|
||||
"filters_unverified": sum(
|
||||
1 for row in mappings if row.status == MappingStatus.LEARNED_UNVERIFIED.value
|
||||
),
|
||||
"filters_broken": sum(1 for row in mappings if row.status == MappingStatus.BROKEN.value),
|
||||
"last_scan_at": category.last_taxonomy_scan_at,
|
||||
}
|
||||
)
|
||||
|
||||
def filters(self, source_id: str, category_key: str) -> dict[str, Any]:
|
||||
category = self._category(source_id, category_key)
|
||||
rows = self.session.scalars(
|
||||
@@ -207,7 +236,9 @@ class TaxonomyService:
|
||||
"name": category.name,
|
||||
"source_url": category.source_url,
|
||||
"canonical_category": category.canonical_category,
|
||||
"product_category_id": category.product_category_id,
|
||||
"status": category.status,
|
||||
"taxonomy_status": category.taxonomy_status,
|
||||
"last_verified_at": category.last_verified_at,
|
||||
"filters": filters,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Seedable SourceCategory readiness for the notebook pilot.
|
||||
|
||||
Revision ID: 0006_source_category_readiness
|
||||
Revises: 0005_frontend_mvp_v03
|
||||
Create Date: 2026-10-04
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006_source_category_readiness"
|
||||
down_revision: str | None = "0005_frontend_mvp_v03"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source_categories",
|
||||
sa.Column("taxonomy_status", sa.String(32), nullable=False, server_default="NOT_LEARNED"),
|
||||
)
|
||||
op.add_column("source_categories", sa.Column("last_taxonomy_scan_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE source_categories
|
||||
SET taxonomy_status = 'READY',
|
||||
last_taxonomy_scan_at = last_verified_at
|
||||
WHERE last_verified_at IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source_categories", "last_taxonomy_scan_at")
|
||||
op.drop_column("source_categories", "taxonomy_status")
|
||||
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
|
||||
from app.domain.validation import parse_scoring_model
|
||||
from app.persistence.db import SessionLocal
|
||||
from app.persistence.models import ScoringModelRecord, SourceRecord
|
||||
from app.services.catalog_service import ensure_product_category
|
||||
from app.services.catalog_service import ensure_product_category, ensure_source_category
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RULES = ROOT / "rules" / "generic-v1.json"
|
||||
@@ -19,6 +19,14 @@ def run_bootstrap(session: Session) -> None:
|
||||
_seed_source(session)
|
||||
_seed_scoring_model(session)
|
||||
ensure_product_category(session, "notebook", "Notebooks")
|
||||
ensure_source_category(
|
||||
session,
|
||||
source_id="geizhals",
|
||||
external_category_key="nb",
|
||||
product_category_id="notebook",
|
||||
name="Notebooks",
|
||||
source_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.discovery.definitions import get_definition_by_key, upsert_definition
|
||||
from app.persistence.models import AttributeDefinitionRecord
|
||||
from app.services.catalog_service import ensure_product_category
|
||||
from app.services.catalog_service import ensure_product_category, ensure_source_category
|
||||
|
||||
|
||||
NOTEBOOK_ATTRIBUTE_SEEDS = (
|
||||
@@ -72,6 +72,14 @@ NOTEBOOK_CRITERION_SEEDS = (
|
||||
|
||||
def seed_notebook_pilot(session: Session) -> None:
|
||||
ensure_product_category(session, "notebook", "Notebooks")
|
||||
ensure_source_category(
|
||||
session,
|
||||
source_id="geizhals",
|
||||
external_category_key="nb",
|
||||
product_category_id="notebook",
|
||||
name="Notebooks",
|
||||
source_url="https://geizhals.de/?cat=nb",
|
||||
)
|
||||
for item in NOTEBOOK_ATTRIBUTE_SEEDS:
|
||||
if session.get(AttributeDefinitionRecord, item["key"]) is None:
|
||||
session.add(AttributeDefinitionRecord(**item, description=None))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from app.discovery.criteria import parse_project_criteria
|
||||
from app.discovery.enums import TaxonomyStatus
|
||||
from app.settings import Settings
|
||||
|
||||
|
||||
@@ -22,6 +23,11 @@ def test_parse_accepts_definition_id_and_enabled_flag():
|
||||
assert parsed["items"][0]["enabled_for_discovery"] is True
|
||||
|
||||
|
||||
def test_source_category_readiness_values():
|
||||
assert TaxonomyStatus.NOT_LEARNED.value == "NOT_LEARNED"
|
||||
assert TaxonomyStatus.READY.value == "READY"
|
||||
|
||||
|
||||
def test_cors_origins_are_split_from_environment():
|
||||
settings = Settings(cors_allowed_origins="http://localhost:4173, http://127.0.0.1:4173")
|
||||
assert settings.cors_origins == ["http://localhost:4173", "http://127.0.0.1:4173"]
|
||||
Reference in new issue
Block a user