Merge branch 'main' of https://huggingface.co/spaces/smolagents/computer-agent

app.py

@@ -355,6 +355,14 @@ def cleanup_sandboxes():
 
 def get_or_create_sandbox(session_hash):
     current_time = time.time()
+    print("======")
+    print(":=======")
+    print("Session hash:", session_hash)
+    print("Sandboxes:", SANDBOXES.keys())
+    print("Session hash in SANDBOXES:", session_hash in SANDBOXES)
+    print("Session hash in SANDBOX_METADATA:", session_hash in SANDBOX_METADATA)
+    if session_hash in SANDBOX_METADATA:
+        print("Session not timeout:", current_time - SANDBOX_METADATA[session_hash]['created_at'] < SANDBOX_TIMEOUT)
 
     # Check if sandbox exists and is still valid
     if (session_hash in SANDBOXES and 
@@ -553,7 +561,7 @@ with gr.Blocks(theme=theme, css=custom_css, js=custom_js) as demo:
         )
         with gr.Sidebar(position="left"):
             task_input = gr.Textbox(
-                value="Find picture of cute puppies",
+                value="Download a picture of a cute puppy",
                 label="Enter your task below:",
                 elem_classes="primary-color-label"
             )
@@ -565,7 +573,7 @@ with gr.Blocks(theme=theme, css=custom_css, js=custom_js) as demo:
                     "Check the commuting time between Bern and Zurich on Google maps",
                     "Write 'Hello World' in a text editor",
                     "Search a flight Paris - Berlin for tomorrow",
-                    "Could you head to Fontainebleau (France) in Google Maps then drag and drop to position the castle of Fontainebleau exactly in the center?",
+                    "Could you head to Fontainebleau (France) in Google Maps, and get me the name of the pond just south of the castle?",
                     "Download me a picture of a puppy from Google, then head to Hugging Face, find a Space dedicated to background removal, and use it to remove the puppy picture's background"
                 ],
                 inputs = task_input,

e2bqwen.py

@@ -22,6 +22,23 @@ from smolagents.agent_types import AgentImage
 from PIL import ImageDraw
 
 E2B_SYSTEM_PROMPT_TEMPLATE = """You are a desktop automation assistant that can control a remote desktop environment.
+<action process>
+You willbe given a task to solve in several steps. At each step you will perform an action.
+After each action, you'll receive an updated screenshot. 
+Then you will proceed as follows, with these sections: don't skip any!
+
+Short term goal: ...
+Where I am: ...
+What I see: ...
+Reflection: ...
+Action: ...
+Code:
+```python
+click(250, 300)
+```<end_code>
+</action_process>
+
+<tools>
 On top of performing computations in the Python code snippets that you create, you only have access to these tools to interact with the desktop, no additional ones:
 {%- for tool in tools.values() %}
 - {{ tool.name }}: {{ tool.description }}
@@ -30,68 +47,105 @@ On top of performing computations in the Python code snippets that you create, y
 {%- endfor %}
 
 The desktop has a resolution of <<resolution_x>>x<<resolution_y>>, take it into account to decide clicking coordinates.
+If you clicked somewhere in the previous action, a red crosshair will appear at the exact location of the previous click.
+The image might have change since then but the cross stays at the previous click. If your click seems to have changed nothing, check that this location is exactly where you intended to click. Otherwise correct the click coordinates.
+</tools>
 
-IMPORTANT:
-- Remember the tools that you have as those can save you time, for example open_url to enter a website rather than searching for the browser in the OS.
-- Whenever you click, MAKE SURE to click in the middle of the button, text, link or any other clickable element. Not under, not on the side. IN THE MIDDLE. In menus it is always better to click in the middle of the text rather than in the tiny icon. Calculate extremelly well the coordinates. A mistake here can make the full task fail.
-- To navigate the desktop you should open menus and click. Menus usually expand with more options, the tiny triangle next to some text in a menu means that menu expands. For example in Office in the Applications menu expands showing presentation or writing applications. 
-- Always analyze the latest screenshot carefully before performing actions. If you clicked somewhere in the previous action, a red crosshair will appear at the exact click location: if nothing happened, check that this location is exactly where you intended to click. Otherwise correct the click coordinates.
-
-You must proceed step by step:
-1. Understand the task thoroughly
-2. Break down the task into logical steps
-3. For each step:
-   a. Analyze the current screenshot to identify UI elements
-   b. Plan the appropriate action with precise coordinates
-   c. Execute ONE action at a time using the proper tool
-   d. Wait for the action to complete before proceeding
-
-After each action, you'll receive an updated screenshot. Review it carefully before your next action.
-
-COMMAND FORMAT:
-Always format your actions as Python code blocks. For example:
-
+<code_format>
+Always format your actions as Python code blocks, as shown below:
+Code:
 ```python
 click(250, 300)
 ```<end_code>
+</code_format>
 
-
-TASK EXAMPLE:
+<task_resolution_example>
 For a task like "Open a text editor and type 'Hello World'":
-1- First, analyze the screenshot to find the Applications menu and click on it being very precise, clicking in the middle of the text 'Applications':
+Step 1:
+Short term goal: I want to open a text editor.
+Where I am: I am on the homepage of my desktop.
+What I see: I see the applications
+Reflection: I think that a notes application would fit in the Applications menu, let's open it. 
+Action: I'll click it, carefully clicking in the middle of the text 'Applications'/
+Code:
 ```python
 click(50, 10) 
 ```<end_code>
-2- Remembering that menus are navigated through clicking, after analyzing the screenshot with the applications menu open we see that a notes application probably fits in the Accessories section (we see it is a section in the menu thanks to the tiny white triangle after the text accessories). We look for Accessories and click on it being very precise, clicking in the middle of the text 'Accessories'. DO NOT try to move through the menus with scroll, it won't work:
+
+Step 2:
+Short term goal: I want to open a text editor.
+Where I am: I am on the homepage of my desktop, with the applications menu open.
+What I see: I see an Accessories section, I see it is a section in the menu thanks to the tiny white triangle after the text accessories.
+Reflection: I think that a notes application would fit the Accessories section. I SHOULD NOT try to move through the menus with scroll, it won't work:.
+Action: I'll look for Accessories and click on it being very precise, clicking in the middle of the text 'Accessories'.
+Code:
 ```python
 click(76, 195) 
 ```<end_code>
-3- Remembering that menus are navigated through clicking, after analyzing the screenshot with the submenu Accessories open, look for 'Text Editor' and click on it being very precise, clicking in the middle of the text 'Text Editor':
+
+Step 3:
+Short term goal: I want to open a text editor.
+Where I am: I am under the Accessories menu.
+What I see: under the open submenu Accessories, I've found 'Text Editor'.
+Reflection: This must be my notes app. I remember that menus are navigated through clicking.
+Action: I will now click on it being very precise, clicking in the middle of the text 'Text Editor'.
+Code:
+```python
+click(251, 441) 
+```<end_code>
+
+Step 4:
+Short term goal: I want to open a text editor.
+Where I am: I am still under the Accessories menu.
+What I see: Nothing has changed compared to previous screenshot. Under the open submenu Accessories, I still see 'Text Editor'. The red crosshair is off from the element.
+Reflection: My last click must have been off. Let's correct this.
+Action: I will click the correct place, right in the middle of the element.
+Code:
 ```python
 click(241, 441) 
 ```<end_code>
-4- Once Notepad is open, type the requested text:
+
+Step 5:
+Short term goal: I want to type 'Hello World'.
+Where I am: I have opened a Notepad.
+What I see: The Notepad app is open on an empty page
+Reflection: Now Notepad is open as intended, time to type text.
+Action: I will type the requested text.
+Code:
 ```python
 type_text("Hello World")
 ```<end_code>
 
-5- Task is completed:
+Step 6:
+Short term goal: I want to type 'Hello World'.
+Where I am: I have opened a Notepad.
+What I see: The Notepad app displays 'Hello World'
+Reflection: Now that I've 1. Opened the notepad and 2. typed 'Hello World', and 3. the result seems correct, I think the Task is completed.
+Action: I will return a confirmation that the task is completed.
+Code:
 ```python
 final_answer("Done")
 ```<end_code>
-
-Remember to:
-Always wait for appropriate loading times
+</task_resolution_example>
+
+<click_guidelines>
+Look at elements on the screen to determine what to click or interact with.
+Use precise coordinates for mouse movements and clicks. When clicking an element, ALWAYS CLICK THE MIDDLE of that element, not UNDER OR ABOVE! Else you risk to miss it.
+Sometimes you may have missed a click, so never assume that you're on the right page, always make sure that your previous action worked. In the screenshot you can see if the mouse is out of the clickable area. Pay special attention to this.
+Remember the tools that you have as those can save you time, for example open_url to enter a website rather than searching for the browser in the OS.
+Whenever you click, MAKE SURE to click in the middle of the button, text, link or any other clickable element. Not under, not on the side. IN THE MIDDLE. In menus it is always better to click in the middle of the text rather than in the tiny icon. Calculate extremelly well the coordinates. A mistake here can make the full task fail.
+</click_guidelines>
+
+<general_guidelines>
+You can wait for appropriate loading times using the wait() tool. But don't wait forever, sometimes you've just misclicked and the process didn't launch.
 Use precise coordinates based on the current screenshot
-Execute one action at a time
+Execute one action at a time: don't try to pack a click and typing in one action.
 On each step, look at the last screenshot and action to validate if previous steps worked and decide the next action. If you repeated an action already without effect, it means that this action is useless: don't repeat it and try something else.
 Use click to move through menus on the desktop and scroll for web and specific applications.
-When clicking an element, always make sure to click THE MIDDLE of that element! Else you risk to miss it.
 Always analyze the latest screenshot carefully before performing actions. Make sure to:
-1. Look at elements on the screen to determine what to click or interact with
-2. Use precise coordinates for mouse movements and clicks
-3. You can wait for page loads or animations to complete using the wait() tool
-4. Sometimes you may have missed a click, so never assume that you're on the right page, always make sure that your previous action worked. In the screenshot you can see if the mouse is out of the clickable area. Pay special attention to this.
+To navigate the desktop you should open menus and click. Menus usually expand with more options, the tiny triangle next to some text in a menu means that menu expands. For example in Office in the Applications menu expands showing presentation or writing applications. 
+Always analyze the latest screenshot carefully before performing actions.
+</general_guidelines>
 """
 
 def draw_marker_on_image(image, click_coordinates):
@@ -162,9 +216,8 @@ class E2BVisionAgent(CodeAgent):
             """
             self.desktop.move_mouse(x, y)
             self.desktop.left_click()
+            self.click_coordinates = [x, y]
             self.logger.log(f"Clicked at coordinates ({x}, {y})")
-            self.memory.steps[-1].click_coordinates = [x, y]
-            print("FLAGG", self.memory.steps[-1])
             return f"Clicked at coordinates ({x}, {y})"
 
         @tool
@@ -177,8 +230,8 @@ class E2BVisionAgent(CodeAgent):
             """
             self.desktop.move_mouse(x, y)
             self.desktop.right_click()
+            self.click_coordinates = [x, y]
             self.logger.log(f"Right-clicked at coordinates ({x}, {y})")
-            self.memory.steps[-1].click_coordinates = [x, y]
             return f"Right-clicked at coordinates ({x}, {y})"
 
         @tool
@@ -191,8 +244,8 @@ class E2BVisionAgent(CodeAgent):
             """
             self.desktop.move_mouse(x, y)
             self.desktop.double_click()
+            self.click_coordinates = [x, y]
             self.logger.log(f"Double-clicked at coordinates ({x}, {y})")
-            self.memory.steps[-1].click_coordinates = [x, y]
             return f"Double-clicked at coordinates ({x}, {y})"
 
         @tool
@@ -320,6 +373,17 @@ class E2BVisionAgent(CodeAgent):
         screenshot_bytes = self.desktop.screenshot()
         image = Image.open(BytesIO(screenshot_bytes))
 
+        if getattr(self, "click_coordinates", None):
+            # If a click was performed in the last action, mark it on the image
+            x, y = self.click_coordinates
+            draw = ImageDraw.Draw(image)
+            cross_size, linewidth = 10, 3
+            # Draw red cross lines
+            draw.line((x - cross_size, y, x + cross_size, y), fill="red", width=linewidth)
+            draw.line((x, y - cross_size, x, y + cross_size), fill="red", width=linewidth)
+            # Add a circle around it for better visibility
+            draw.ellipse((x - cross_size * 2, y - cross_size * 2, x + cross_size * 2, y + cross_size * 2), outline="red", width=linewidth)
+
         # Create a filename with step number
         screenshot_path = os.path.join(self.data_dir, f"step_{current_step:03d}.png")
         image.save(screenshot_path)
@@ -334,22 +398,14 @@ class E2BVisionAgent(CodeAgent):
                 and previous_memory_step.step_number <= current_step - 2
             ):
                 previous_memory_step.observations_images = None
-            if (
-                isinstance(previous_memory_step, ActionStep)
-                and previous_memory_step.step_number  == current_step - 1
-                and hasattr(memory_step, "click_coordinates")
-            ):
-                print("Drawing cross on previous step image")
-                draw_marker_on_image(previous_memory_step.observations_images[0], memory_step.click_coordinates)
-
-        if hasattr(memory_step, "click_coordinates"):
-            draw_marker_on_image(image, memory_step.click_coordinates)
 
         # Add to the current memory step
         memory_step.observations_images = [image.copy()]
 
         # memory_step.observations_images = [screenshot_path] # IF YOU USE THIS INSTEAD OF ABOVE, LAUNCHING A SECOND TASK BREAKS
 
+        self.click_coordinates = None # Reset click marker
+
 
     def close(self):
         """Clean up resources"""