feat: add database-backed verified models for dynamic model managemen… (#12833)

Co-authored-by: statxc <statxc@user.noreply.github.com>
Co-authored-by: bittoby <bittoby@users.noreply.github.com>
This commit is contained in:
BitToby
2026-02-26 15:17:18 +02:00
committed by GitHub
parent f93e3254d3
commit a92bfe6cc0
9 changed files with 701 additions and 37 deletions

View File

@@ -0,0 +1,92 @@
"""Create verified_models table.
Revision ID: 097
Revises: 096
Create Date: 2026-02-21 00:00:00.000000
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = '097'
down_revision: Union[str, None] = '096'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Create verified_models table and seed with current model list."""
op.create_table(
'verified_models',
sa.Column('id', sa.Integer, sa.Identity(), primary_key=True),
sa.Column('model_name', sa.String(255), nullable=False),
sa.Column('provider', sa.String(100), nullable=False),
sa.Column(
'is_enabled',
sa.Boolean(),
nullable=False,
server_default=sa.text('true'),
),
sa.Column(
'created_at',
sa.DateTime(),
nullable=False,
server_default=sa.text('CURRENT_TIMESTAMP'),
),
sa.Column(
'updated_at',
sa.DateTime(),
nullable=False,
server_default=sa.text('CURRENT_TIMESTAMP'),
),
sa.UniqueConstraint(
'model_name', 'provider', name='uq_verified_model_provider'
),
)
op.create_index(
'ix_verified_models_provider',
'verified_models',
['provider'],
)
op.create_index(
'ix_verified_models_is_enabled',
'verified_models',
['is_enabled'],
)
# Seed with current openhands provider models
models = [
('claude-opus-4-5-20251101', 'openhands'),
('claude-sonnet-4-5-20250929', 'openhands'),
('gpt-5.2-codex', 'openhands'),
('gpt-5.2', 'openhands'),
('minimax-m2.5', 'openhands'),
('gemini-3-pro-preview', 'openhands'),
('gemini-3-flash-preview', 'openhands'),
('deepseek-chat', 'openhands'),
('devstral-medium-2512', 'openhands'),
('kimi-k2-0711-preview', 'openhands'),
('qwen3-coder-480b', 'openhands'),
]
for model_name, provider in models:
op.execute(
sa.text(
"""
INSERT INTO verified_models (model_name, provider)
VALUES (:model_name, :provider)
"""
).bindparams(model_name=model_name, provider=provider)
)
def downgrade() -> None:
"""Drop verified_models table."""
op.drop_index('ix_verified_models_is_enabled', table_name='verified_models')
op.drop_index('ix_verified_models_provider', table_name='verified_models')
op.drop_table('verified_models')

View File

@@ -47,6 +47,9 @@ from server.routes.org_invitations import ( # noqa: E402
from server.routes.orgs import org_router # noqa: E402
from server.routes.readiness import readiness_router # noqa: E402
from server.routes.user import saas_user_router # noqa: E402
from server.routes.verified_models import ( # noqa: E402
api_router as verified_models_router,
)
from server.sharing.shared_conversation_router import ( # noqa: E402
router as shared_conversation_router,
)
@@ -105,6 +108,9 @@ if GITLAB_APP_CLIENT_ID:
base_app.include_router(api_keys_router) # Add routes for API key management
base_app.include_router(org_router) # Add routes for organization management
base_app.include_router(
verified_models_router
) # Add routes for verified models management
base_app.include_router(invitation_router) # Add routes for org invitation management
base_app.include_router(invitation_accept_router) # Add route for accepting invitations
add_github_proxy_routes(base_app)

View File

@@ -0,0 +1,184 @@
"""API routes for managing verified LLM models (admin only)."""
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import BaseModel, field_validator
from server.email_validation import get_admin_user_id
from storage.verified_model_store import VerifiedModelStore
from openhands.core.logger import openhands_logger as logger
api_router = APIRouter(prefix='/api/admin/verified-models', tags=['Verified Models'])
class VerifiedModelCreate(BaseModel):
model_name: str
provider: str
is_enabled: bool = True
@field_validator('model_name')
@classmethod
def validate_model_name(cls, v: str) -> str:
v = v.strip()
if not v or len(v) > 255:
raise ValueError('model_name must be 1-255 characters')
return v
@field_validator('provider')
@classmethod
def validate_provider(cls, v: str) -> str:
v = v.strip()
if not v or len(v) > 100:
raise ValueError('provider must be 1-100 characters')
return v
class VerifiedModelUpdate(BaseModel):
is_enabled: bool | None = None
class VerifiedModelResponse(BaseModel):
id: int
model_name: str
provider: str
is_enabled: bool
class VerifiedModelPage(BaseModel):
"""Paginated response model for verified model list."""
items: list[VerifiedModelResponse]
next_page_id: str | None = None
def _to_response(model) -> VerifiedModelResponse:
return VerifiedModelResponse(
id=model.id,
model_name=model.model_name,
provider=model.provider,
is_enabled=model.is_enabled,
)
@api_router.get('', response_model=VerifiedModelPage)
async def list_verified_models(
provider: str | None = None,
page_id: Annotated[
str | None,
Query(title='Optional next_page_id from the previously returned page'),
] = None,
limit: Annotated[
int, Query(title='The max number of results in the page', gt=0, le=100)
] = 100,
user_id: str = Depends(get_admin_user_id),
):
"""List all verified models, optionally filtered by provider."""
try:
if provider:
all_models = VerifiedModelStore.get_models_by_provider(provider)
else:
all_models = VerifiedModelStore.get_all_models()
try:
offset = int(page_id) if page_id else 0
except ValueError:
offset = 0
page = all_models[offset : offset + limit + 1]
has_more = len(page) > limit
if has_more:
page = page[:limit]
return VerifiedModelPage(
items=[_to_response(m) for m in page],
next_page_id=str(offset + limit) if has_more else None,
)
except Exception:
logger.exception('Error listing verified models')
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail='Failed to list verified models',
)
@api_router.post('', response_model=VerifiedModelResponse, status_code=201)
async def create_verified_model(
data: VerifiedModelCreate,
user_id: str = Depends(get_admin_user_id),
):
"""Create a new verified model."""
try:
model = VerifiedModelStore.create_model(
model_name=data.model_name,
provider=data.provider,
is_enabled=data.is_enabled,
)
return _to_response(model)
except ValueError as e:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=str(e),
)
except Exception:
logger.exception('Error creating verified model')
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail='Failed to create verified model',
)
@api_router.put('/{provider}/{model_name:path}', response_model=VerifiedModelResponse)
async def update_verified_model(
provider: str,
model_name: str,
data: VerifiedModelUpdate,
user_id: str = Depends(get_admin_user_id),
):
"""Update a verified model by provider and model name."""
try:
model = VerifiedModelStore.update_model(
model_name=model_name,
provider=provider,
is_enabled=data.is_enabled,
)
if not model:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f'Model {provider}/{model_name} not found',
)
return _to_response(model)
except HTTPException:
raise
except Exception:
logger.exception(f'Error updating verified model: {provider}/{model_name}')
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail='Failed to update verified model',
)
@api_router.delete('/{provider}/{model_name:path}')
async def delete_verified_model(
provider: str,
model_name: str,
user_id: str = Depends(get_admin_user_id),
):
"""Delete a verified model by provider and model name."""
try:
success = VerifiedModelStore.delete_model(
model_name=model_name, provider=provider
)
if not success:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f'Model {provider}/{model_name} not found',
)
return {'message': f'Model {provider}/{model_name} deleted'}
except HTTPException:
raise
except Exception:
logger.exception(f'Error deleting verified model: {provider}/{model_name}')
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail='Failed to delete verified model',
)

View File

@@ -0,0 +1,39 @@
"""SQLAlchemy model for verified LLM models."""
from sqlalchemy import (
Boolean,
Column,
DateTime,
Identity,
Integer,
String,
UniqueConstraint,
func,
text,
)
from storage.base import Base
class VerifiedModel(Base): # type: ignore
"""A verified LLM model available in the model selector.
The composite unique constraint on (model_name, provider) allows the same
model name to exist under different providers (e.g. 'claude-sonnet' under
both 'openhands' and 'anthropic').
"""
__tablename__ = 'verified_models'
__table_args__ = (
UniqueConstraint('model_name', 'provider', name='uq_verified_model_provider'),
)
id = Column(Integer, Identity(), primary_key=True)
model_name = Column(String(255), nullable=False)
provider = Column(String(100), nullable=False, index=True)
is_enabled = Column(
Boolean, nullable=False, default=True, server_default=text('true')
)
created_at = Column(DateTime, nullable=False, server_default=func.now())
updated_at = Column(
DateTime, nullable=False, server_default=func.now(), onupdate=func.now()
)

View File

@@ -0,0 +1,187 @@
"""Store for managing verified LLM models in the database."""
from sqlalchemy import and_
from storage.database import session_maker
from storage.verified_model import VerifiedModel
from openhands.core.logger import openhands_logger as logger
class VerifiedModelStore:
"""Store for CRUD operations on verified models.
Follows the project convention of static methods with session_maker()
(see UserStore, OrgMemberStore for reference).
"""
@staticmethod
def get_enabled_models() -> list[VerifiedModel]:
"""Get all enabled models.
Returns:
list[VerifiedModel]: All models where is_enabled is True
"""
with session_maker() as session:
return (
session.query(VerifiedModel)
.filter(VerifiedModel.is_enabled.is_(True))
.order_by(VerifiedModel.provider, VerifiedModel.model_name)
.all()
)
@staticmethod
def get_models_by_provider(provider: str) -> list[VerifiedModel]:
"""Get all enabled models for a specific provider.
Args:
provider: The provider name (e.g., 'openhands', 'anthropic')
"""
with session_maker() as session:
return (
session.query(VerifiedModel)
.filter(
and_(
VerifiedModel.provider == provider,
VerifiedModel.is_enabled.is_(True),
)
)
.order_by(VerifiedModel.model_name)
.all()
)
@staticmethod
def get_all_models() -> list[VerifiedModel]:
"""Get all models (including disabled)."""
with session_maker() as session:
return (
session.query(VerifiedModel)
.order_by(VerifiedModel.provider, VerifiedModel.model_name)
.all()
)
@staticmethod
def get_model(model_name: str, provider: str) -> VerifiedModel | None:
"""Get a model by its composite key (model_name, provider).
Args:
model_name: The model identifier
provider: The provider name
"""
with session_maker() as session:
return (
session.query(VerifiedModel)
.filter(
and_(
VerifiedModel.model_name == model_name,
VerifiedModel.provider == provider,
)
)
.first()
)
@staticmethod
def create_model(
model_name: str, provider: str, is_enabled: bool = True
) -> VerifiedModel:
"""Create a new verified model.
Args:
model_name: The model identifier
provider: The provider name
is_enabled: Whether the model is enabled (default True)
Raises:
ValueError: If a model with the same (model_name, provider) already exists
"""
with session_maker() as session:
existing = (
session.query(VerifiedModel)
.filter(
and_(
VerifiedModel.model_name == model_name,
VerifiedModel.provider == provider,
)
)
.first()
)
if existing:
raise ValueError(f'Model {provider}/{model_name} already exists')
model = VerifiedModel(
model_name=model_name,
provider=provider,
is_enabled=is_enabled,
)
session.add(model)
session.commit()
session.refresh(model)
logger.info(f'Created verified model: {provider}/{model_name}')
return model
@staticmethod
def update_model(
model_name: str,
provider: str,
is_enabled: bool | None = None,
) -> VerifiedModel | None:
"""Update an existing verified model.
Args:
model_name: The model name to update
provider: The provider name
is_enabled: New enabled state (optional)
Returns:
The updated model if found, None otherwise
"""
with session_maker() as session:
model = (
session.query(VerifiedModel)
.filter(
and_(
VerifiedModel.model_name == model_name,
VerifiedModel.provider == provider,
)
)
.first()
)
if not model:
return None
if is_enabled is not None:
model.is_enabled = is_enabled
session.commit()
session.refresh(model)
logger.info(f'Updated verified model: {provider}/{model_name}')
return model
@staticmethod
def delete_model(model_name: str, provider: str) -> bool:
"""Delete a verified model.
Args:
model_name: The model name to delete
provider: The provider name
Returns:
True if deleted, False if not found
"""
with session_maker() as session:
model = (
session.query(VerifiedModel)
.filter(
and_(
VerifiedModel.model_name == model_name,
VerifiedModel.provider == provider,
)
)
.first()
)
if not model:
return False
session.delete(model)
session.commit()
logger.info(f'Deleted verified model: {provider}/{model_name}')
return True

View File

@@ -25,6 +25,7 @@ from storage.stored_conversation_metadata_saas import (
from storage.stored_offline_token import StoredOfflineToken
from storage.stripe_customer import StripeCustomer
from storage.user import User
from storage.verified_model import VerifiedModel # noqa: F401
@pytest.fixture

View File

@@ -0,0 +1,123 @@
"""Unit tests for VerifiedModelStore."""
from unittest.mock import patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from storage.base import Base
from storage.verified_model_store import VerifiedModelStore
@pytest.fixture
def _mock_session_maker():
"""Create an in-memory SQLite database and patch session_maker."""
engine = create_engine('sqlite:///:memory:')
Base.metadata.create_all(engine)
session_factory = sessionmaker(bind=engine)
with patch(
'storage.verified_model_store.session_maker',
side_effect=lambda **kwargs: session_factory(**kwargs),
):
yield
Base.metadata.drop_all(engine)
@pytest.fixture
def _seed_models(_mock_session_maker):
"""Seed the database with test models."""
VerifiedModelStore.create_model(model_name='claude-sonnet', provider='openhands')
VerifiedModelStore.create_model(model_name='claude-sonnet', provider='anthropic')
VerifiedModelStore.create_model(
model_name='gpt-4o', provider='openhands', is_enabled=False
)
class TestCreateModel:
def test_create_model(self, _mock_session_maker):
model = VerifiedModelStore.create_model(
model_name='test-model', provider='test-provider'
)
assert model.model_name == 'test-model'
assert model.provider == 'test-provider'
assert model.is_enabled is True
assert model.id is not None
def test_create_duplicate_raises(self, _mock_session_maker):
VerifiedModelStore.create_model(model_name='test-model', provider='test')
with pytest.raises(ValueError, match='test/test-model already exists'):
VerifiedModelStore.create_model(model_name='test-model', provider='test')
def test_same_name_different_provider_allowed(self, _mock_session_maker):
VerifiedModelStore.create_model(model_name='claude', provider='openhands')
model = VerifiedModelStore.create_model(
model_name='claude', provider='anthropic'
)
assert model.provider == 'anthropic'
class TestGetModel:
def test_get_model(self, _seed_models):
model = VerifiedModelStore.get_model('claude-sonnet', 'openhands')
assert model is not None
assert model.provider == 'openhands'
def test_get_model_not_found(self, _seed_models):
assert VerifiedModelStore.get_model('nonexistent', 'openhands') is None
def test_get_model_wrong_provider(self, _seed_models):
assert VerifiedModelStore.get_model('claude-sonnet', 'openai') is None
class TestGetModels:
def test_get_all_models(self, _seed_models):
models = VerifiedModelStore.get_all_models()
assert len(models) == 3
def test_get_enabled_models(self, _seed_models):
models = VerifiedModelStore.get_enabled_models()
assert len(models) == 2
names = {m.model_name for m in models}
assert 'gpt-4o' not in names
def test_get_models_by_provider(self, _seed_models):
models = VerifiedModelStore.get_models_by_provider('openhands')
assert len(models) == 1
assert models[0].model_name == 'claude-sonnet'
class TestUpdateModel:
def test_update_model(self, _seed_models):
updated = VerifiedModelStore.update_model(
model_name='claude-sonnet', provider='openhands', is_enabled=False
)
assert updated is not None
assert updated.is_enabled is False
def test_update_not_found(self, _seed_models):
assert (
VerifiedModelStore.update_model(
model_name='nonexistent', provider='openhands', is_enabled=False
)
is None
)
def test_update_no_change(self, _seed_models):
updated = VerifiedModelStore.update_model(
model_name='claude-sonnet', provider='openhands'
)
assert updated is not None
assert updated.is_enabled is True
class TestDeleteModel:
def test_delete_model(self, _seed_models):
assert VerifiedModelStore.delete_model('claude-sonnet', 'openhands') is True
assert VerifiedModelStore.get_model('claude-sonnet', 'openhands') is None
# Other provider's version should still exist
assert VerifiedModelStore.get_model('claude-sonnet', 'anthropic') is not None
def test_delete_not_found(self, _seed_models):
assert VerifiedModelStore.delete_model('nonexistent', 'openhands') is False

View File

@@ -24,7 +24,8 @@ async def get_litellm_models() -> list[str]:
"""Get all models supported by LiteLLM.
This function combines models from litellm and Bedrock, removing any
error-prone Bedrock models.
error-prone Bedrock models. In SaaS mode, it uses database-backed
verified models for dynamic updates without code deployments.
To get the models:
```sh
@@ -34,7 +35,29 @@ async def get_litellm_models() -> list[str]:
Returns:
list[str]: A sorted list of unique model names.
"""
return get_supported_llm_models(config)
verified_models = _load_verified_models_from_db()
return get_supported_llm_models(config, verified_models)
def _load_verified_models_from_db() -> list[str] | None:
"""Try to load verified models from the database (SaaS mode only).
Returns:
List of model strings like 'provider/model_name' if available, None otherwise.
"""
try:
from storage.verified_model_store import VerifiedModelStore
except ImportError:
return None
try:
db_models = VerifiedModelStore.get_enabled_models()
return [f'{m.provider}/{m.model_name}' for m in db_models]
except Exception:
from openhands.core.logger import openhands_logger as logger
logger.exception('Failed to load verified models from database')
return None
@app.get('/agents', response_model=list[str])

View File

@@ -11,6 +11,38 @@ from openhands.core.config import LLMConfig, OpenHandsConfig
from openhands.core.logger import openhands_logger as logger
from openhands.llm import bedrock
# Hardcoded OpenHands provider models used in self-hosted mode.
# In SaaS mode these are loaded from the database instead.
OPENHANDS_MODELS = [
'openhands/claude-opus-4-5-20251101',
'openhands/claude-sonnet-4-5-20250929',
'openhands/gpt-5.2-codex',
'openhands/gpt-5.2',
'openhands/minimax-m2.5',
'openhands/gemini-3-pro-preview',
'openhands/gemini-3-flash-preview',
'openhands/deepseek-chat',
'openhands/devstral-medium-2512',
'openhands/kimi-k2-0711-preview',
'openhands/qwen3-coder-480b',
]
CLARIFAI_MODELS = [
'clarifai/openai.chat-completion.gpt-oss-120b',
'clarifai/openai.chat-completion.gpt-oss-20b',
'clarifai/openai.chat-completion.gpt-5',
'clarifai/openai.chat-completion.gpt-5-mini',
'clarifai/qwen.qwen3.qwen3-next-80B-A3B-Thinking',
'clarifai/qwen.qwenLM.Qwen3-30B-A3B-Instruct-2507',
'clarifai/qwen.qwenLM.Qwen3-30B-A3B-Thinking-2507',
'clarifai/qwen.qwenLM.Qwen3-14B',
'clarifai/qwen.qwenCoder.Qwen3-Coder-30B-A3B-Instruct',
'clarifai/deepseek-ai.deepseek-chat.DeepSeek-R1-0528-Qwen3-8B',
'clarifai/deepseek-ai.deepseek-chat.DeepSeek-V3_1',
'clarifai/zai.completion.GLM_4_5',
'clarifai/moonshotai.kimi.Kimi-K2-Instruct',
]
def is_openhands_model(model: str | None) -> bool:
"""Check if the model uses the OpenHands provider.
@@ -67,12 +99,20 @@ def get_provider_api_base(model: str) -> str | None:
return None
def get_supported_llm_models(config: OpenHandsConfig) -> list[str]:
def get_supported_llm_models(
config: OpenHandsConfig,
verified_models: list[str] | None = None,
) -> list[str]:
"""Get all models supported by LiteLLM.
This function combines models from litellm and Bedrock, removing any
error-prone Bedrock models.
Args:
config: The OpenHands configuration.
verified_models: Optional list of verified model strings from the database
(SaaS mode). When provided, these replace the hardcoded OPENHANDS_MODELS.
Returns:
list[str]: A sorted list of unique model names.
"""
@@ -109,39 +149,8 @@ def get_supported_llm_models(config: OpenHandsConfig) -> list[str]:
except httpx.HTTPError as e:
logger.error(f'Error getting OLLAMA models: {e}')
# Add OpenHands provider models
openhands_models = [
'openhands/claude-opus-4-5-20251101',
'openhands/claude-sonnet-4-5-20250929',
'openhands/gpt-5.2-codex',
'openhands/gpt-5.2',
'openhands/minimax-m2.5',
'openhands/gemini-3-pro-preview',
'openhands/gemini-3-flash-preview',
'openhands/deepseek-chat',
'openhands/devstral-medium-2512',
'openhands/kimi-k2-0711-preview',
'openhands/qwen3-coder-480b',
]
model_list = openhands_models + model_list
# Add Clarifai provider models (via OpenAI-compatible endpoint)
clarifai_models = [
# clarifai featured models
'clarifai/openai.chat-completion.gpt-oss-120b',
'clarifai/openai.chat-completion.gpt-oss-20b',
'clarifai/openai.chat-completion.gpt-5',
'clarifai/openai.chat-completion.gpt-5-mini',
'clarifai/qwen.qwen3.qwen3-next-80B-A3B-Thinking',
'clarifai/qwen.qwenLM.Qwen3-30B-A3B-Instruct-2507',
'clarifai/qwen.qwenLM.Qwen3-30B-A3B-Thinking-2507',
'clarifai/qwen.qwenLM.Qwen3-14B',
'clarifai/qwen.qwenCoder.Qwen3-Coder-30B-A3B-Instruct',
'clarifai/deepseek-ai.deepseek-chat.DeepSeek-R1-0528-Qwen3-8B',
'clarifai/deepseek-ai.deepseek-chat.DeepSeek-V3_1',
'clarifai/zai.completion.GLM_4_5',
'clarifai/moonshotai.kimi.Kimi-K2-Instruct',
]
model_list = clarifai_models + model_list
# Use database-backed models if provided (SaaS), otherwise use hardcoded list
openhands_models = verified_models if verified_models else OPENHANDS_MODELS
model_list = openhands_models + CLARIFAI_MODELS + model_list
return sorted(set(model_list))