|
import hashlib
|
|
|
|
from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult
|
|
from openhands.events.action import (
|
|
AgentFinishAction,
|
|
FileWriteAction,
|
|
MessageAction,
|
|
)
|
|
from openhands.events.event import Event
|
|
from openhands.events.observation import AgentDelegateObservation
|
|
from openhands.runtime.base import Runtime
|
|
|
|
|
|
class Test(BaseIntegrationTest):
|
|
INSTRUCTION = 'Execute the python script /workspace/python_script.py with input "John" and "25" and tell me the secret number.'
|
|
SECRET_NUMBER = int(hashlib.sha256(str(25).encode()).hexdigest()[:8], 16) % 1000
|
|
|
|
@classmethod
|
|
def initialize_runtime(cls, runtime: Runtime) -> None:
|
|
from openhands.core.logger import openhands_logger as logger
|
|
|
|
action = FileWriteAction(
|
|
path='/workspace/python_script.py',
|
|
content=(
|
|
'name = input("Enter your name: "); age = input("Enter your age: "); '
|
|
'import hashlib; secret = int(hashlib.sha256(str(age).encode()).hexdigest()[:8], 16) % 1000; '
|
|
'print(f"Hello {name}, you are {age} years old. Tell you a secret number: {secret}")'
|
|
),
|
|
)
|
|
logger.info(action, extra={'msg_type': 'ACTION'})
|
|
observation = runtime.run_action(action)
|
|
logger.info(observation, extra={'msg_type': 'OBSERVATION'})
|
|
|
|
@classmethod
|
|
def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult:
|
|
from openhands.core.logger import openhands_logger as logger
|
|
|
|
|
|
message_actions = [
|
|
event
|
|
for event in histories
|
|
if isinstance(
|
|
event, (MessageAction, AgentFinishAction, AgentDelegateObservation)
|
|
)
|
|
]
|
|
logger.info(f'Total message-like events: {len(message_actions)}')
|
|
|
|
for event in message_actions:
|
|
try:
|
|
if isinstance(event, AgentDelegateObservation):
|
|
content = event.content
|
|
elif isinstance(event, AgentFinishAction):
|
|
content = event.outputs.get('content', '')
|
|
if event.thought:
|
|
content += f'\n\n{event.thought}'
|
|
elif isinstance(event, MessageAction):
|
|
content = event.content
|
|
else:
|
|
logger.warning(f'Unexpected event type: {type(event)}')
|
|
continue
|
|
|
|
if str(cls.SECRET_NUMBER) in content:
|
|
return TestResult(success=True)
|
|
except Exception as e:
|
|
logger.error(f'Error processing event: {e}')
|
|
|
|
logger.debug(
|
|
f'Total messages: {len(message_actions)}. Messages: {message_actions}'
|
|
)
|
|
return TestResult(
|
|
success=False,
|
|
reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.',
|
|
)
|
|
|
