MVP 0.92
This commit is contained in:
1 parent
60f0d2830d
commit
53b1eff833
62 files changed
+6578
-355
No files matched your search
@@ -0,0 +1,193 @@
|
||||
"""Network-free OpenRouter catalog tests. Run from backend/: python tests/test_model_catalog.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-model-catalog-test.sqlite")
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from model_catalog import (
|
||||
ModelWindow,
|
||||
catalog_ttl,
|
||||
reset_catalog,
|
||||
resolve_generate_metadata,
|
||||
)
|
||||
from prompt_budget import ERROR_MODEL_METADATA_UNKNOWN, JournalBudgetError
|
||||
from providers import ProviderConfig
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload, status_code: int = 200, text: str = ""):
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
def json(self):
|
||||
if isinstance(self._payload, Exception):
|
||||
raise self._payload
|
||||
return self._payload
|
||||
|
||||
|
||||
def _openrouter_config(model: str = "openai/gpt-4o") -> ProviderConfig:
|
||||
return ProviderConfig(
|
||||
role="generate",
|
||||
name="openrouter",
|
||||
mode="http",
|
||||
url="https://openrouter.ai/api/v1/chat/completions",
|
||||
model=model,
|
||||
key="sk-test",
|
||||
local=False,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
|
||||
|
||||
def _clear_env(*names: str) -> dict[str, str | None]:
|
||||
previous = {}
|
||||
for name in names:
|
||||
previous[name] = os.environ.pop(name, None)
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_env(previous: dict[str, str | None]) -> None:
|
||||
for name, value in previous.items():
|
||||
if value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
|
||||
|
||||
def test_reads_context_and_completion() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"context_length": 128000,
|
||||
"top_provider": {"max_completion_tokens": 16384},
|
||||
}
|
||||
]
|
||||
}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
window = resolve_generate_metadata(_openrouter_config())
|
||||
expect(window.context_length == 128000, "catalog reads context_length")
|
||||
expect(window.max_completion_tokens == 16384, "catalog reads max completion")
|
||||
expect(window.source == "openrouter_models", "window is from the catalog")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_model_not_in_catalog() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 4096}}]}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
raise SystemExit("FAIL: missing catalog model should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "model not in catalog is fail-closed")
|
||||
expect((exc.diagnostics or {}).get("reason") == "model_not_in_catalog", "reason is model_not_in_catalog")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_incomplete_metadata() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "openai/gpt-4o", "top_provider": {"max_completion_tokens": 4096}}]}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config())
|
||||
raise SystemExit("FAIL: incomplete metadata should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "incomplete metadata is fail-closed")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_cache_hit_and_ttl_expiry() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_get(*args, **kwargs):
|
||||
calls["n"] += 1
|
||||
return FakeResponse(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"context_length": 64000 + calls["n"],
|
||||
"top_provider": {"max_completion_tokens": 4096},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
with patch("model_catalog.httpx.get", fake_get):
|
||||
first = resolve_generate_metadata(_openrouter_config(), now=1000.0)
|
||||
second = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() - 1)
|
||||
expect(calls["n"] == 1, "catalog is cached within TTL")
|
||||
expect(first.context_length == second.context_length, "cache hit returns the same window")
|
||||
expect(second.cached is True, "cached window is marked cached")
|
||||
third = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() + 1)
|
||||
expect(calls["n"] == 2, "catalog is refetched after TTL")
|
||||
expect(third.context_length != first.context_length, "refetch reads a new catalog row")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_env_fallback_and_fail_closed() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 2048}}]}
|
||||
os.environ["KANSHO_PROVIDER_CONTEXT_LENGTH"] = "65536"
|
||||
os.environ["KANSHO_PROVIDER_MAX_COMPLETION_TOKENS"] = "8192"
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
window = resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
expect(window.context_length == 65536, "documented env fallback supplies the window")
|
||||
expect(window.max_completion_tokens == 8192, "env fallback supplies completion limit")
|
||||
expect(window.source == "env_fallback", "fallback is labeled as env_fallback")
|
||||
|
||||
reset_catalog()
|
||||
os.environ.pop("KANSHO_PROVIDER_CONTEXT_LENGTH", None)
|
||||
os.environ.pop("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", None)
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
raise SystemExit("FAIL: no window should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "no window is fail-closed")
|
||||
|
||||
reset_catalog()
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse({"nope": True})):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config())
|
||||
raise SystemExit("FAIL: bad catalog shape should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "bad metadata shape is fail-closed")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_reads_context_and_completion()
|
||||
test_model_not_in_catalog()
|
||||
test_incomplete_metadata()
|
||||
test_cache_hit_and_ttl_expiry()
|
||||
test_env_fallback_and_fail_closed()
|
||||
print("model catalog tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user