Fix CLI displaying claude-2 as default model for anthropic provider (#9101)

Co-authored-by: openhands <openhands@all-hands.dev>

openhands/cli/settings.py

@@ -215,10 +215,18 @@ async def modify_llm_settings_basic(
             ]
             provider_models = VERIFIED_ANTHROPIC_MODELS + provider_models
 
-        # Set default model to the first model in the list (which will be a verified model if available)
-        default_model = (
-            provider_models[0] if provider_models else 'claude-sonnet-4-20250514'
-        )
+        # Set default model to the best verified model for the provider
+        if provider == 'anthropic' and VERIFIED_ANTHROPIC_MODELS:
+            # Use the first model in the VERIFIED_ANTHROPIC_MODELS list as it's the best/newest
+            default_model = VERIFIED_ANTHROPIC_MODELS[0]
+        elif provider == 'openai' and VERIFIED_OPENAI_MODELS:
+            # Use the first model in the VERIFIED_OPENAI_MODELS list as it's the best/newest
+            default_model = VERIFIED_OPENAI_MODELS[0]
+        else:
+            # For other providers, use the first model in the list
+            default_model = (
+                provider_models[0] if provider_models else 'claude-sonnet-4-20250514'
+            )
 
         # Show the default model but allow changing it
         print_formatted_text(

openhands/cli/utils.py

@@ -158,17 +158,17 @@ VERIFIED_OPENAI_MODELS = [
 ]
 
 VERIFIED_ANTHROPIC_MODELS = [
-    'claude-2',
-    'claude-2.1',
-    'claude-3-5-sonnet-20240620',
-    'claude-3-5-sonnet-20241022',
-    'claude-3-5-haiku-20241022',
-    'claude-3-haiku-20240307',
-    'claude-3-opus-20240229',
-    'claude-3-sonnet-20240229',
-    'claude-3-7-sonnet-20250219',
     'claude-sonnet-4-20250514',
     'claude-opus-4-20250514',
+    'claude-3-7-sonnet-20250219',
+    'claude-3-sonnet-20240229',
+    'claude-3-opus-20240229',
+    'claude-3-haiku-20240307',
+    'claude-3-5-haiku-20241022',
+    'claude-3-5-sonnet-20241022',
+    'claude-3-5-sonnet-20240620',
+    'claude-2.1',
+    'claude-2',
 ]
 
 

tests/unit/test_cli_default_model.py

@@ -0,0 +1,79 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from openhands.cli.settings import modify_llm_settings_basic
+from openhands.cli.utils import VERIFIED_ANTHROPIC_MODELS
+
+
+@pytest.mark.asyncio
+@patch('openhands.cli.settings.get_supported_llm_models')
+@patch('openhands.cli.settings.organize_models_and_providers')
+@patch('openhands.cli.settings.PromptSession')
+@patch('openhands.cli.settings.cli_confirm')
+@patch('openhands.cli.settings.print_formatted_text')
+async def test_anthropic_default_model_is_best_verified(
+    mock_print,
+    mock_confirm,
+    mock_session,
+    mock_organize,
+    mock_get_models,
+):
+    """Test that the default model for anthropic is the best verified model."""
+    # Setup mocks
+    mock_get_models.return_value = [
+        'anthropic/claude-sonnet-4-20250514',
+        'anthropic/claude-2',
+    ]
+    mock_organize.return_value = {
+        'anthropic': {
+            'models': ['claude-sonnet-4-20250514', 'claude-2'],
+            'separator': '/',
+        },
+    }
+
+    # Mock session to avoid actual user input
+    session_instance = MagicMock()
+    session_instance.prompt_async = AsyncMock(side_effect=KeyboardInterrupt())
+    mock_session.return_value = session_instance
+
+    # Mock config and settings store
+    app_config = MagicMock()
+    llm_config = MagicMock()
+    app_config.get_llm_config.return_value = llm_config
+    settings_store = AsyncMock()
+
+    # Mock cli_confirm to avoid actual user input
+    # We need enough values to handle all the calls in the function
+    mock_confirm.side_effect = [
+        0,
+        0,
+        0,
+    ]  # Use default provider, use default model, etc.
+
+    try:
+        # Call the function (it will exit early due to KeyboardInterrupt)
+        await modify_llm_settings_basic(app_config, settings_store)
+    except KeyboardInterrupt:
+        pass  # Expected exception
+
+    # Check that the default model displayed is the best verified model
+    best_verified_model = VERIFIED_ANTHROPIC_MODELS[
+        0
+    ]  # First model in the list is the best
+    default_model_displayed = False
+
+    for call in mock_print.call_args_list:
+        args, _ = call
+        if (
+            args
+            and hasattr(args[0], 'value')
+            and f'Default model: </grey><green>{best_verified_model}</green>'
+            in args[0].value
+        ):
+            default_model_displayed = True
+            break
+
+    assert default_model_displayed, (
+        f'Default model displayed was not {best_verified_model}'
+    )

tests/unit/test_cli_settings.py

@@ -391,7 +391,10 @@ class TestModifyLLMSettingsBasic:
         default_model_block = []
         in_default_model_block = False
         for line in source_lines:
-            if '# Set default model to the first model in the list' in line:
+            if (
+                '# Set default model to the best verified model for the provider'
+                in line
+            ):
                 in_default_model_block = True
                 default_model_block.append(line)
             elif in_default_model_block: