from unittest.mock import MagicMock, patch import pytest import requests from litellm.exceptions import RateLimitError from openhands.core.config import LLMConfig from openhands.events.action.message import MessageAction from openhands.llm.llm import LLM from openhands.resolver.github_issue import GithubIssue from openhands.resolver.issue_definitions import IssueHandler, PRHandler @pytest.fixture(autouse=True) def mock_logger(monkeypatch): # suppress logging of completion data to file mock_logger = MagicMock() monkeypatch.setattr('openhands.llm.debug_mixin.llm_prompt_logger', mock_logger) monkeypatch.setattr('openhands.llm.debug_mixin.llm_response_logger', mock_logger) return mock_logger @pytest.fixture def default_config(): return LLMConfig( model='gpt-4o', api_key='test_key', num_retries=2, retry_min_wait=1, retry_max_wait=2, ) def test_handle_nonexistent_issue_reference(): llm_config = LLMConfig(model='test', api_key='test') handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config) # Mock the requests.get to simulate a 404 error mock_response = MagicMock() mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( '404 Client Error: Not Found' ) with patch('requests.get', return_value=mock_response): # Call the method with a non-existent issue reference result = handler._PRHandler__get_context_from_external_issues_references( closing_issues=[], closing_issue_numbers=[], issue_body='This references #999999', # Non-existent issue review_comments=[], review_threads=[], thread_comments=None, ) # The method should return an empty list since the referenced issue couldn't be fetched assert result == [] def test_handle_rate_limit_error(): llm_config = LLMConfig(model='test', api_key='test') handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config) # Mock the requests.get to simulate a rate limit error mock_response = MagicMock() mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( '403 Client Error: Rate Limit Exceeded' ) with patch('requests.get', return_value=mock_response): # Call the method with an issue reference result = handler._PRHandler__get_context_from_external_issues_references( closing_issues=[], closing_issue_numbers=[], issue_body='This references #123', review_comments=[], review_threads=[], thread_comments=None, ) # The method should return an empty list since the request was rate limited assert result == [] def test_handle_network_error(): llm_config = LLMConfig(model='test', api_key='test') handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config) # Mock the requests.get to simulate a network error with patch( 'requests.get', side_effect=requests.exceptions.ConnectionError('Network Error') ): # Call the method with an issue reference result = handler._PRHandler__get_context_from_external_issues_references( closing_issues=[], closing_issue_numbers=[], issue_body='This references #123', review_comments=[], review_threads=[], thread_comments=None, ) # The method should return an empty list since the network request failed assert result == [] def test_successful_issue_reference(): llm_config = LLMConfig(model='test', api_key='test') handler = PRHandler('test-owner', 'test-repo', 'test-token', llm_config) # Mock a successful response mock_response = MagicMock() mock_response.raise_for_status.return_value = None mock_response.json.return_value = {'body': 'This is the referenced issue body'} with patch('requests.get', return_value=mock_response): # Call the method with an issue reference result = handler._PRHandler__get_context_from_external_issues_references( closing_issues=[], closing_issue_numbers=[], issue_body='This references #123', review_comments=[], review_threads=[], thread_comments=None, ) # The method should return a list with the referenced issue body assert result == ['This is the referenced issue body'] class MockLLMResponse: """Mock LLM Response class to mimic the actual LLM response structure.""" class Choice: class Message: def __init__(self, content): self.content = content def __init__(self, content): self.message = self.Message(content) def __init__(self, content): self.choices = [self.Choice(content)] class DotDict(dict): """ A dictionary that supports dot notation access. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for key, value in self.items(): if isinstance(value, dict): self[key] = DotDict(value) elif isinstance(value, list): self[key] = [ DotDict(item) if isinstance(item, dict) else item for item in value ] def __getattr__(self, key): if key in self: return self[key] else: raise AttributeError( f"'{self.__class__.__name__}' object has no attribute '{key}'" ) def __setattr__(self, key, value): self[key] = value def __delattr__(self, key): if key in self: del self[key] else: raise AttributeError( f"'{self.__class__.__name__}' object has no attribute '{key}'" ) @patch('openhands.llm.llm.litellm_completion') def test_guess_success_rate_limit_wait_time(mock_litellm_completion, default_config): """Test that the retry mechanism in guess_success respects wait time between retries.""" with patch('time.sleep') as mock_sleep: # Simulate a rate limit error followed by a successful response mock_litellm_completion.side_effect = [ RateLimitError( 'Rate limit exceeded', llm_provider='test_provider', model='test_model' ), DotDict( { 'choices': [ { 'message': { 'content': '--- success\ntrue\n--- explanation\nRetry successful' } } ] } ), ] llm = LLM(config=default_config) handler = IssueHandler('test-owner', 'test-repo', 'test-token', default_config) handler.llm = llm # Mock issue and history issue = GithubIssue( owner='test-owner', repo='test-repo', number=1, title='Test Issue', body='This is a test issue.', thread_comments=['Please improve error handling'], ) history = [MessageAction(content='Fixed error handling.')] # Call guess_success success, _, explanation = handler.guess_success(issue, history) # Assertions assert success is True assert explanation == 'Retry successful' assert mock_litellm_completion.call_count == 2 # Two attempts made mock_sleep.assert_called_once() # Sleep called once between retries # Validate wait time wait_time = mock_sleep.call_args[0][0] assert ( default_config.retry_min_wait <= wait_time <= default_config.retry_max_wait ), f'Expected wait time between {default_config.retry_min_wait} and {default_config.retry_max_wait} seconds, but got {wait_time}' @patch('openhands.llm.llm.litellm_completion') def test_guess_success_exhausts_retries(mock_completion, default_config): """Test the retry mechanism in guess_success exhausts retries and raises an error.""" # Simulate persistent rate limit errors by always raising RateLimitError mock_completion.side_effect = RateLimitError( 'Rate limit exceeded', llm_provider='test_provider', model='test_model' ) # Initialize LLM and handler llm = LLM(config=default_config) handler = PRHandler('test-owner', 'test-repo', 'test-token', default_config) handler.llm = llm # Mock issue and history issue = GithubIssue( owner='test-owner', repo='test-repo', number=1, title='Test Issue', body='This is a test issue.', thread_comments=['Please improve error handling'], ) history = [MessageAction(content='Fixed error handling.')] # Call guess_success and expect it to raise an error after retries with pytest.raises(RateLimitError): handler.guess_success(issue, history) # Assertions assert ( mock_completion.call_count == default_config.num_retries ) # Initial call + retries