from unittest.mock import AsyncMock, MagicMock, patch import pytest from prompt_toolkit.formatted_text import HTML from pydantic import SecretStr from openhands.cli.settings import ( display_settings, modify_llm_settings_advanced, modify_llm_settings_basic, ) from openhands.cli.tui import UserCancelledError from openhands.core.config import OpenHandsConfig from openhands.storage.data_models.settings import Settings from openhands.storage.settings.file_settings_store import FileSettingsStore # Mock classes for condensers class MockLLMSummarizingCondenserConfig: def __init__(self, llm_config, type): self.llm_config = llm_config self.type = type class MockNoOpCondenserConfig: def __init__(self, type): self.type = type class TestDisplaySettings: @pytest.fixture def app_config(self): config = MagicMock(spec=OpenHandsConfig) llm_config = MagicMock() llm_config.base_url = None llm_config.model = 'openai/gpt-4' llm_config.api_key = SecretStr('test-api-key') config.get_llm_config.return_value = llm_config config.default_agent = 'test-agent' # Set up security as a separate mock security_mock = MagicMock() security_mock.confirmation_mode = True config.security = security_mock config.enable_default_condenser = True return config @pytest.fixture def advanced_app_config(self): config = MagicMock(spec=OpenHandsConfig) llm_config = MagicMock() llm_config.base_url = 'https://custom-api.com' llm_config.model = 'custom-model' llm_config.api_key = SecretStr('test-api-key') config.get_llm_config.return_value = llm_config config.default_agent = 'test-agent' # Set up security as a separate mock security_mock = MagicMock() security_mock.confirmation_mode = True config.security = security_mock config.enable_default_condenser = True return config @patch('openhands.cli.settings.print_container') def test_display_settings_standard_config(self, mock_print_container, app_config): display_settings(app_config) mock_print_container.assert_called_once() # Verify the container was created with the correct settings container = mock_print_container.call_args[0][0] text_area = container.body # Check that the text area contains expected labels and values settings_text = text_area.text assert 'LLM Provider:' in settings_text assert 'openai' in settings_text assert 'LLM Model:' in settings_text assert 'gpt-4' in settings_text assert 'API Key:' in settings_text assert '********' in settings_text assert 'Agent:' in settings_text assert 'test-agent' in settings_text assert 'Confirmation Mode:' in settings_text assert 'Enabled' in settings_text assert 'Memory Condensation:' in settings_text assert 'Enabled' in settings_text @patch('openhands.cli.settings.print_container') def test_display_settings_advanced_config( self, mock_print_container, advanced_app_config ): display_settings(advanced_app_config) mock_print_container.assert_called_once() # Verify the container was created with the correct settings container = mock_print_container.call_args[0][0] text_area = container.body # Check that the text area contains expected labels and values settings_text = text_area.text assert 'Custom Model:' in settings_text assert 'custom-model' in settings_text assert 'Base URL:' in settings_text assert 'https://custom-api.com' in settings_text assert 'API Key:' in settings_text assert '********' in settings_text assert 'Agent:' in settings_text assert 'test-agent' in settings_text class TestModifyLLMSettingsBasic: @pytest.fixture def app_config(self): config = MagicMock(spec=OpenHandsConfig) llm_config = MagicMock() llm_config.model = 'openai/gpt-4' llm_config.api_key = SecretStr('test-api-key') llm_config.base_url = None config.get_llm_config.return_value = llm_config config.set_llm_config = MagicMock() config.set_agent_config = MagicMock() agent_config = MagicMock() config.get_agent_config.return_value = agent_config # Set up security as a separate mock security_mock = MagicMock() security_mock.confirmation_mode = True config.security = security_mock return config @pytest.fixture def settings_store(self): store = MagicMock(spec=FileSettingsStore) store.load = AsyncMock(return_value=Settings()) store.store = AsyncMock() return store @pytest.mark.asyncio @patch('openhands.cli.settings.get_supported_llm_models') @patch('openhands.cli.settings.organize_models_and_providers') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) async def test_modify_llm_settings_basic_success( self, mock_confirm, mock_session, mock_organize, mock_get_models, app_config, settings_store, ): # Setup mocks mock_get_models.return_value = ['openai/gpt-4', 'anthropic/claude-3-opus'] mock_organize.return_value = { 'openai': {'models': ['gpt-4', 'gpt-3.5-turbo'], 'separator': '/'}, 'anthropic': { 'models': ['claude-3-opus', 'claude-3-sonnet'], 'separator': '/', }, } session_instance = MagicMock() session_instance.prompt_async = AsyncMock( side_effect=[ 'openai', # Provider 'gpt-4', # Model 'new-api-key', # API Key ] ) mock_session.return_value = session_instance # Mock cli_confirm to select the second option (change provider/model) for the first two calls # and then select the first option (save settings) for the last call mock_confirm.side_effect = [1, 1, 0] # Call the function await modify_llm_settings_basic(app_config, settings_store) # Verify LLM config was updated app_config.set_llm_config.assert_called_once() args, kwargs = app_config.set_llm_config.call_args # The model name might be different based on the default model in the list # Just check that it starts with 'openai/' assert args[0].model.startswith('openai/') assert args[0].api_key.get_secret_value() == 'new-api-key' assert args[0].base_url is None # Verify settings were saved settings_store.store.assert_called_once() args, kwargs = settings_store.store.call_args settings = args[0] # The model name might be different based on the default model in the list # Just check that it starts with openai/ assert settings.llm_model.startswith('openai/') assert settings.llm_api_key.get_secret_value() == 'new-api-key' assert settings.llm_base_url is None @pytest.mark.asyncio @patch('openhands.cli.settings.get_supported_llm_models') @patch('openhands.cli.settings.organize_models_and_providers') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) async def test_modify_llm_settings_basic_user_cancels( self, mock_confirm, mock_session, mock_organize, mock_get_models, app_config, settings_store, ): # Setup mocks mock_get_models.return_value = ['openai/gpt-4', 'anthropic/claude-3-opus'] mock_organize.return_value = { 'openai': {'models': ['gpt-4', 'gpt-3.5-turbo'], 'separator': '/'} } session_instance = MagicMock() session_instance.prompt_async = AsyncMock(side_effect=UserCancelledError()) mock_session.return_value = session_instance # Call the function await modify_llm_settings_basic(app_config, settings_store) # Verify settings were not changed app_config.set_llm_config.assert_not_called() settings_store.store.assert_not_called() @pytest.mark.asyncio @patch('openhands.cli.settings.get_supported_llm_models') @patch('openhands.cli.settings.organize_models_and_providers') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch('openhands.cli.settings.print_formatted_text') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) async def test_modify_llm_settings_basic_invalid_input( self, mock_print, mock_confirm, mock_session, mock_organize, mock_get_models, app_config, settings_store, ): # Setup mocks mock_get_models.return_value = ['openai/gpt-4', 'anthropic/claude-3-opus'] mock_organize.return_value = { 'openai': {'models': ['gpt-4', 'gpt-3.5-turbo'], 'separator': '/'} } session_instance = MagicMock() session_instance.prompt_async = AsyncMock( side_effect=[ 'invalid-provider', # First invalid provider 'openai', # Valid provider 'invalid-model', # Invalid model 'gpt-4', # Valid model 'new-api-key', # API key ] ) mock_session.return_value = session_instance # Mock cli_confirm to select the second option (change provider/model) for the first two calls # and then select the first option (save settings) for the last call mock_confirm.side_effect = [1, 1, 0] # Call the function await modify_llm_settings_basic(app_config, settings_store) # Verify error messages were shown for invalid inputs assert ( mock_print.call_count >= 2 ) # At least two error messages should be printed # Check for invalid provider error provider_error_found = False model_error_found = False for call in mock_print.call_args_list: args, _ = call if args and isinstance(args[0], HTML): if 'Invalid provider selected' in args[0].value: provider_error_found = True if 'Invalid model selected' in args[0].value: model_error_found = True assert provider_error_found, 'No error message for invalid provider' assert model_error_found, 'No error message for invalid model' # Verify LLM config was updated with correct values app_config.set_llm_config.assert_called_once() # Verify settings were saved settings_store.store.assert_called_once() args, kwargs = settings_store.store.call_args settings = args[0] assert settings.llm_model == 'openai/gpt-4' assert settings.llm_api_key.get_secret_value() == 'new-api-key' assert settings.llm_base_url is None class TestModifyLLMSettingsAdvanced: @pytest.fixture def app_config(self): config = MagicMock(spec=OpenHandsConfig) llm_config = MagicMock() llm_config.model = 'custom-model' llm_config.api_key = SecretStr('test-api-key') llm_config.base_url = 'https://custom-api.com' config.get_llm_config.return_value = llm_config config.set_llm_config = MagicMock() config.set_agent_config = MagicMock() agent_config = MagicMock() config.get_agent_config.return_value = agent_config # Set up security as a separate mock security_mock = MagicMock() security_mock.confirmation_mode = True config.security = security_mock return config @pytest.fixture def settings_store(self): store = MagicMock(spec=FileSettingsStore) store.load = AsyncMock(return_value=Settings()) store.store = AsyncMock() return store @pytest.mark.asyncio @patch('openhands.cli.settings.Agent.list_agents') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) @patch('openhands.cli.settings.NoOpCondenserConfig', MockNoOpCondenserConfig) async def test_modify_llm_settings_advanced_success( self, mock_confirm, mock_session, mock_list_agents, app_config, settings_store ): # Setup mocks mock_list_agents.return_value = ['default', 'test-agent'] session_instance = MagicMock() session_instance.prompt_async = AsyncMock( side_effect=[ 'new-model', # Custom model 'https://new-url', # Base URL 'new-api-key', # API key 'default', # Agent ] ) mock_session.return_value = session_instance # Mock user confirmations mock_confirm.side_effect = [ 0, # Enable confirmation mode 0, # Enable memory condensation 0, # Save settings ] # Call the function await modify_llm_settings_advanced(app_config, settings_store) # Verify LLM config was updated app_config.set_llm_config.assert_called_once() args, kwargs = app_config.set_llm_config.call_args assert args[0].model == 'new-model' assert args[0].api_key.get_secret_value() == 'new-api-key' assert args[0].base_url == 'https://new-url' # Verify settings were saved settings_store.store.assert_called_once() args, kwargs = settings_store.store.call_args settings = args[0] assert settings.llm_model == 'new-model' assert settings.llm_api_key.get_secret_value() == 'new-api-key' assert settings.llm_base_url == 'https://new-url' assert settings.agent == 'default' assert settings.confirmation_mode is True assert settings.enable_default_condenser is True @pytest.mark.asyncio @patch('openhands.cli.settings.Agent.list_agents') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) @patch('openhands.cli.settings.NoOpCondenserConfig', MockNoOpCondenserConfig) async def test_modify_llm_settings_advanced_user_cancels( self, mock_confirm, mock_session, mock_list_agents, app_config, settings_store ): # Setup mocks mock_list_agents.return_value = ['default', 'test-agent'] session_instance = MagicMock() session_instance.prompt_async = AsyncMock(side_effect=UserCancelledError()) mock_session.return_value = session_instance # Call the function await modify_llm_settings_advanced(app_config, settings_store) # Verify settings were not changed app_config.set_llm_config.assert_not_called() settings_store.store.assert_not_called() @pytest.mark.asyncio @patch('openhands.cli.settings.Agent.list_agents') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch('openhands.cli.settings.print_formatted_text') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) @patch('openhands.cli.settings.NoOpCondenserConfig', MockNoOpCondenserConfig) async def test_modify_llm_settings_advanced_invalid_agent( self, mock_print, mock_confirm, mock_session, mock_list_agents, app_config, settings_store, ): # Setup mocks mock_list_agents.return_value = ['default', 'test-agent'] session_instance = MagicMock() session_instance.prompt_async = AsyncMock( side_effect=[ 'new-model', # Custom model 'https://new-url', # Base URL 'new-api-key', # API key 'invalid-agent', # Invalid agent 'default', # Valid agent on retry ] ) mock_session.return_value = session_instance # Call the function await modify_llm_settings_advanced(app_config, settings_store) # Verify error message was shown assert ( mock_print.call_count == 3 ) # Called 3 times: empty line, error message, empty line error_message_call = mock_print.call_args_list[ 1 ] # The second call contains the error message args, kwargs = error_message_call assert isinstance(args[0], HTML) assert 'Invalid agent' in args[0].value # Verify settings were not changed app_config.set_llm_config.assert_not_called() settings_store.store.assert_not_called() @pytest.mark.asyncio @patch('openhands.cli.settings.Agent.list_agents') @patch('openhands.cli.settings.PromptSession') @patch('openhands.cli.settings.cli_confirm') @patch( 'openhands.cli.settings.LLMSummarizingCondenserConfig', MockLLMSummarizingCondenserConfig, ) @patch('openhands.cli.settings.NoOpCondenserConfig', MockNoOpCondenserConfig) async def test_modify_llm_settings_advanced_user_rejects_save( self, mock_confirm, mock_session, mock_list_agents, app_config, settings_store ): # Setup mocks mock_list_agents.return_value = ['default', 'test-agent'] session_instance = MagicMock() session_instance.prompt_async = AsyncMock( side_effect=[ 'new-model', # Custom model 'https://new-url', # Base URL 'new-api-key', # API key 'default', # Agent ] ) mock_session.return_value = session_instance # Mock user confirmations mock_confirm.side_effect = [ 0, # Enable confirmation mode 0, # Enable memory condensation 1, # Reject saving settings ] # Call the function await modify_llm_settings_advanced(app_config, settings_store) # Verify settings were not changed app_config.set_llm_config.assert_not_called() settings_store.store.assert_not_called()