init

llava/eval/seeact/__init__.py

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

llava/eval/seeact/agent.py

@@ -1,945 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import os
-import traceback
-from datetime import datetime
-import json
-import toml
-import torch
-from playwright.async_api import async_playwright,Locator
-
-from .data_utils.format_prompt_utils import get_index_from_option_name, generate_new_query_prompt, \
-    generate_new_referring_prompt, format_options
-from .demo_utils.browser_helper import normal_launch_async, normal_new_context_async, \
-    get_interactive_elements_with_playwright, select_option, saveconfig, get_select_elements_with_playwright
-from .demo_utils.format_prompt import format_choices, postprocess_action_lmm
-from .demo_utils.inference_engine import engine_factory
-
-
-
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
-
-from PIL import Image, ImageDraw,ImageFont
-
-import numpy as np
-
-import asyncio
-
-
-def average_color(image, bbox):
-    """计算指定区域的平均颜色。"""
-    region = image.crop(bbox)
-    numpy_image = np.array(region)
-    avg_color = np.mean(numpy_image, axis=(0, 1))
-    return tuple(avg_color)
-
-
-def color_contrast(color1, color2):
-    """计算两种颜色之间的对比度。"""
-    l1 = 0.2126 * pow(color1[0]/255, 2.2) + \
-         0.7152 * pow(color1[1]/255, 2.2) + \
-         0.0722 * pow(color1[2]/255, 2.2)
-    l2 = 0.2126 * pow(color2[0]/255, 2.2) + \
-         0.7152 * pow(color2[1]/255, 2.2) + \
-         0.0722 * pow(color2[2]/255, 2.2)
-    if l1 > l2:
-        return (l1 + 0.05) / (l2 + 0.05)
-    else:
-        return (l2 + 0.05) / (l1 + 0.05)
-
-def text_color_for_background(background_color):
-    """选择最佳的文本颜色基于背景颜色。"""
-    red = (255, 0, 0)
-    blue = (0, 0, 255)
-    contrast_red = color_contrast(background_color, red)
-    contrast_blue = color_contrast(background_color, blue)
-    if contrast_red > contrast_blue:
-        return red
-    else:
-        return blue
-
-
-def draw_text(draw, text, position, font, max_width, image):
-    """在图像上绘制自动换行的文本，并根据背景色调整文本颜色。"""
-    x, y = position
-    words = text.split()
-    current_line = ""
-
-    # 使用一个空字符串来获取单行文字的高度
-    line_height = 40
-
-    for word in words:
-        test_line = f"{current_line} {word}".strip()
-        width, _ = 40,40  # 正确调用font对象的getsize方法
-        if width <= max_width:
-            current_line = test_line
-        else:
-            bbox = (x, y, x + width, y + line_height)
-            bg_color = average_color(image, bbox)
-            color = text_color_for_background(bg_color)
-            draw.text((x, y), current_line, font=font, fill=color)
-            y += line_height
-            current_line = word
-    if current_line:
-        bbox = (x, y, x + width, y + line_height)
-        bg_color = average_color(image, bbox)
-        color = text_color_for_background(bg_color)
-        draw.text((x, y), current_line, font=font, fill=color)
-
-
-class SeeActAgent:
-    def __init__(self,
-                 config_path=None,
-                 save_file_dir="seeact_agent_files",
-                 save_task_id=None,
-                 default_task='Search for the flight status for the flight AA 3942 leaving on Jun. 10"',
-                 default_website="https://www.aa.com/homePage.do",
-                 input_info=["screenshot"],
-                 grounding_strategy="text_choice",
-                 max_auto_op=50,
-                 max_continuous_no_op=5,
-                 highlight=False,
-                 headless=False,
-                 args=[],
-                 browser_app="chrome",
-                 persistant=False,
-                 persistant_user_path="",
-                 save_video=False,
-                 viewport={
-                     "width": 1280,
-                     "height": 960
-                 },
-                 tracing=False,
-                 trace={
-                     "screenshots": True,
-                     "snapshots": True,
-                     "sources": True
-                 },
-                 rate_limit=-1,
-                 model="gpt-4-turbo",
-                 temperature=0.9
-
-                 ):
-
-        try:
-            if config_path is not None:
-                with open(config_path,
-                          'r') as config:
-                    print(f"Configuration File Loaded - {config_path}")
-                    config = toml.load(config)
-            else:
-                config = {
-                    "basic": {
-                        "save_file_dir": save_file_dir,
-                        "default_task": default_task,
-                        "default_website": default_website
-                    },
-                    "agent": {
-                        "input_info": input_info,
-                        "grounding_strategy": grounding_strategy,
-                        "max_auto_op": max_auto_op,
-                        "max_continuous_no_op": max_continuous_no_op,
-                        "highlight": highlight
-                    },
-                    "openai": {
-                        "rate_limit": rate_limit,
-                        "model": model,
-                        "temperature": temperature
-                    }
-                }
-            config.update({
-                "browser": {
-                    "headless": headless,
-                    "args": args,
-                    "browser_app": browser_app,
-                    "persistant": persistant,
-                    "persistant_user_path": persistant_user_path,
-                    "save_video": save_video,
-                    "viewport": viewport,
-                    "tracing": tracing,
-                    "trace": trace
-                }
-            })
-
-        except FileNotFoundError:
-            print(f"Error: File '{os.path.abspath(config_path)}' not found.")
-        except toml.TomlDecodeError:
-            print(f"Error: File '{os.path.abspath(config_path)}' is not a valid TOML file.")
-
-        self.config = config
-        self.complete_flag = False
-        self.session_control = {
-            'active_page': None,
-            'context': None,
-            'browser': None
-        }
-        self.tasks = [self.config["basic"]["default_task"]]
-        if save_task_id:
-            self.main_path = os.path.join(self.config["basic"]["save_file_dir"],
-                                          save_task_id)
-
-
-        else:
-            self.main_path = os.path.join(self.config["basic"]["save_file_dir"], datetime.now().strftime("%Y%m%d_%H%M%S"))
-        if os.path.exists(self.main_path):
-            self.complete_flag=True
-
-        os.makedirs(self.main_path, exist_ok=True)
-        self.action_space = ["CLICK", "PRESS ENTER", "HOVER", "SCROLL UP", "SCROLL DOWN", "NEW TAB", "CLOSE TAB",
-                             "GO BACK", "GO FORWARD",
-                             "TERMINATE", "SELECT", "TYPE", "GOTO", "MEMORIZE"]  # Define the list of actions here
-
-        self.no_value_op = ["CLICK", "PRESS ENTER", "HOVER", "SCROLL UP", "SCROLL DOWN", "NEW TAB", "CLOSE TAB",
-                            "PRESS HOME", "PRESS END", "PRESS PAGEUP", "PRESS PAGEDOWN"
-                                                                       "GO BACK",
-                            "GO FORWARD",
-                            "TERMINATE", "NONE"]
-
-        self.with_value_op = ["SELECT", "TYPE", "GOTO", "MEMORIZE", "SAY"]
-
-        self.no_element_op = ["PRESS ENTER", "SCROLL UP", "SCROLL DOWN", "NEW TAB", "CLOSE TAB", "GO BACK", "GOTO",
-                              "PRESS HOME", "PRESS END", "PRESS PAGEUP", "PRESS PAGEDOWN",
-                              "GO FORWARD",
-                              "TERMINATE", "NONE", "MEMORIZE", "SAY"]
-
-        # Initialize the primary logger and the developer logger
-        self.logger = self._setup_logger(redirect_to_dev_log=False)
-        # self.dev_logger = self._setup_dev_logger()
-
-        # # Redirect primary logger messages to dev_logger as well
-        # for handler in self.logger.handlers:
-        #     self.dev_logger.addHandler(handler)
-
-        self.engine = engine_factory(**self.config['openai'])
-        self.taken_actions = []
-        self.prompts = self._initialize_prompts()
-        self.time_step = 0
-        self.valid_op = 0
-        # self.error=0
-        self.continuous_no_op = 0
-        self.predictions=[]
-
-        disable_torch_init()
-        self.pixui_model_path = os.path.expanduser(
-            "/fs/ess/PAS1576/boyu_gou/train_vlm/ui_llava_fine_tune/checkpoints/only-web/merged-llava-v1.5-vicuna-7b-16k-pad-no-fusion-web-80k")
-        self.pixui_model_name = get_model_name_from_path(self.pixui_model_path)
-        self.pixui_tokenizer, self.pixui_model, self.pixui_image_processor, self.pixui_context_len = load_pretrained_model(self.pixui_model_path, None, self.pixui_model_name)
-
-    def _initialize_prompts(self):
-        """Initialize prompt information including dynamic action space."""
-        action_format = f""  # Dynamically generate action_format based on self.action_space
-
-        return {
-            "system_prompt": '''You are assisting humans doing web navigation tasks step by step. At each stage, you can see the webpage by a screenshot and know the previous actions before the current step decided by yourself that have been executed for this task through recorded history. You need to decide on the first following action to take.''',
-
-            "action_space": '''
-Here are the descriptions of all allowed actions:
-
-No Value Operations:
-- CLICK: Click on a webpage element using the mouse.
-- PRESS ENTER: Press the Enter key, typically to submit a form or confirm an input.
-- SCROLL UP: Scroll the webpage upwards by half of the window height.
-- SCROLL DOWN: Scroll the webpage downwards by half of the window height.
-- PRESS HOME: Scroll to the top of the webpage.
-- PRESS END: Scroll to the bottom of the webpage.
-- PRESS PAGEUP: Scroll up by one window height.
-- PRESS PAGEDOWN: Scroll down by one window height.
-- GO BACK: Navigate to the previous page in the browser history.
-- GO FORWARD: Navigate to the next page in the browser history.
-- TERMINATE: End the current task, typically used when the task is considered complete or requires potentially harmful actions.
-- NONE: Indicates that no action is necessary at this stage. Used to skip an action or wait.
-
-With Value Operations:
-- SELECT: Choose an option from a dropdown menu or <select> element. The value indicates the option to select.
-- TYPE: Enter text into a text area or text box. The value is the text to be typed.
-''',
-
-            "question_description": '''The screenshot below shows the webpage you see. Think step by step before outlining the next action step at the current stage. Clearly outline which element in the webpage users will operate with as the first next target element, its detailed location, and the corresponding operation.
-
-To be successful, it is important to follow the following rules:
-1. You should only issue a valid action given the current observation.
-2. You should only issue one action at a time
-3. Unlike humans, since you are using playwright APIs, for typing (e.g., in text areas, text boxes) and selecting (e.g., from dropdown menus or <select> elements), you should try directly typing the input or selecting the choice, bypassing the need for an initial click.
-4. You should not attempt to create accounts, log in or do the final submission.
-5. Terminate when you deem the task complete or if it requires potentially harmful actions.
-6. Details of <select> elements will be provided, to help you figure out the exact choice text to be chosen if the action is a SELECT.
-7. If you find there are one or more failed actions in the most recent actions, you should change the description and make your descriptions more precise and concise (and at least do not repeat the latest description.).
-
-(Final Answer)
-Finally, conclude your answer using the format below. Ensure your answer is strictly adhering to the format provided below. Please do not leave any explanation in your answers of the final standardized format part, and this final part should be clear and certain. The element choice, action, and value should be in three separate lines.
-
-Format:
-
-ELEMENT: The description of the target element to locate the element, if the action involves a specific element. Otherwise write "None"
-
-ACTION: Choose an action from allowed actions.
-
-VALUE: Provide additional input based on ACTION. (If it doesn't involve a value, write "None"''',
-
-            "referring_description": f"""(Reiteration)
-First, reiterate your next target element, its detailed location, and the corresponding operation.""",
-
-            "element_format": '''''',
-
-            "action_format": action_format,  # Use the dynamically generated action_format
-
-            "value_format": ''''''
-        }
-
-    def update_action_space(self, new_actions):
-        """Update the action space and regenerate the action_format prompt."""
-        if isinstance(new_actions, list) and all(isinstance(item, str) for item in new_actions):
-            self.action_space = new_actions
-            self.prompts["action_format"] = f"ACTION: Choose an action from {{{', '.join(self.action_space)}}}."
-        else:
-            print("Invalid action space provided. It must be a list of strings.")
-
-    def _setup_logger(self, redirect_to_dev_log=False):
-        """Set up a logger to log to both file and console within the main_path."""
-        logger_name = 'SeeActAgent'
-        logger = logging.getLogger(logger_name)
-        logger.setLevel(logging.INFO)
-        if not logger.handlers:  # Avoid adding handlers multiple times
-            # Create a file handler for writing logs to a file
-            log_filename = 'agent.log'
-            f_handler = logging.FileHandler(os.path.join(self.main_path, log_filename))
-            f_handler.setLevel(logging.INFO)
-
-            # Create a console handler for printing logs to the terminal
-            c_handler = logging.StreamHandler()
-            c_handler.setLevel(logging.INFO)
-
-            # Create formatters for file and console handlers
-            file_formatter = logging.Formatter('%(asctime)s - %(message)s')
-            console_formatter = logging.Formatter('%(message)s')
-
-            # Set formatters for file and console handlers
-            f_handler.setFormatter(file_formatter)
-            c_handler.setFormatter(console_formatter)
-
-            # Add the handlers to the logger
-            logger.addHandler(f_handler)
-            if not redirect_to_dev_log:  # Only add console handler if not redirecting to dev log
-                logger.addHandler(c_handler)
-
-        return logger
-
-    # def _setup_dev_logger(self):
-    #     """Set up a developer logger to log only to a file within the main_path."""
-    #     dev_logger_name = 'SeeActAgentDev'
-    #     dev_logger = logging.getLogger(dev_logger_name)
-    #     dev_logger.setLevel(logging.INFO)
-    #     if not dev_logger.handlers:  # Avoid adding handlers multiple times
-    #         # Create a file handler for writing logs to a dev log file
-    #         dev_log_filename = 'dev_agent.log'
-    #         f_handler = logging.FileHandler(os.path.join(self.main_path, dev_log_filename))
-    #         f_handler.setLevel(logging.INFO)
-    #
-    #         # Create a formatter and add it to the handler
-    #         formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    #         f_handler.setFormatter(formatter)
-    #
-    #         # Add the file handler to the dev logger
-    #         dev_logger.addHandler(f_handler)
-    #
-    #     return dev_logger
-
-    async def page_on_close_handler(self):
-        # Corrected to use 'self' for accessing class attributes
-        if self.session_control['context']:
-            try:
-                await self.session_control['active_page'].title()
-            except:
-                self.logger.info(
-                    "The active tab was closed. Will switch to the last page (or open a new default google page)")
-                if self.session_control['context'].pages:
-                    self.session_control['active_page'] = self.session_control['context'].pages[-1]
-                    await self.session_control['active_page'].bring_to_front()
-                    self.logger.info(f"Switched the active tab to: {self.session_control['active_page'].url}")
-                else:
-                    self.session_control['active_page'] = await self.session_control['context'].new_page()
-                    try:
-                        await self.session_control['active_page'].goto("https://www.google.com/", wait_until="load")
-                    except Exception as e:
-                        self.logger.info(f"Failed to navigate to Google: {e}")
-                    self.logger.info(f"Switched the active tab to: {self.session_control['active_page'].url}")
-
-    def save_action_history(self, filename="action_history.txt"):
-        """Save the history of taken actions to a file in the main path."""
-        history_path = os.path.join(self.main_path, filename)
-        with open(history_path, 'w') as f:
-            for action in self.taken_actions:
-                f.write(action + '\n')
-        self.logger.info(f"Action history saved to: {history_path}")
-
-    async def page_on_navigation_handler(self, frame):
-        # Corrected to use 'self' for accessing class attributes
-        self.session_control['active_page'] = frame.page
-
-    async def page_on_crash_handler(self, page):
-        # Corrected logging method
-        self.logger.info(f"Page crashed: {page.url}")
-        self.logger.info("Try to reload")
-        await page.reload()
-
-    async def page_on_open_handler(self, page):
-        # Added 'self' to the handler functions to reference the current instance of the class
-        page.on("framenavigated", self.page_on_navigation_handler)
-        page.on("close", self.page_on_close_handler)
-        page.on("crash", self.page_on_crash_handler)
-        self.session_control['active_page'] = page
-        # Additional event listeners can be added here
-
-    async def start(self, headless=None, args=None, website=None):
-        self.playwright = await async_playwright().start()
-        self.session_control['browser'] = await normal_launch_async(self.playwright,
-                                                                    headless=self.config['browser'][
-                                                                        'headless'] if headless is None else headless,
-                                                                    args=self.config['browser'][
-                                                                        'args'] if args is None else args)
-        self.session_control['context'] = await normal_new_context_async(self.session_control['browser'],
-                                                                         viewport=self.config['browser'][
-                                                                             'viewport'])
-
-        self.session_control['context'].on("page", self.page_on_open_handler)
-        await self.session_control['context'].new_page()
-
-        try:
-            await self.session_control['active_page'].goto(
-                self.config['basic']['default_website'] if website is None else website,
-                wait_until="load")
-            self.logger.info(f"Loaded website: {self.config['basic']['default_website']}")
-        except Exception as e:
-            self.logger.info("Failed to fully load the webpage before timeout")
-            self.logger.info(e)
-
-            # await asyncio.sleep(2)
-
-    def update_prompt_part(self, part_name, new_text):
-        """Update the specified part of the prompt information."""
-        if part_name in self.prompts:
-            self.prompts[part_name] = new_text
-            return True
-        else:
-            print(f"Prompt part '{part_name}' not found.")
-            return False
-
-    def generate_prompt(self, task=None, previous=None, choices=None):
-
-        """Generate a prompt based on the current task, previous actions, and choices."""
-        # assert task is not None, "Please input the task."
-
-        prompt_list = []
-
-        system_prompt_input = self.prompts["system_prompt"]
-        action_space_input = self.prompts["action_space"]
-        question_description_input = self.prompts["question_description"]
-        referring_input = self.prompts["referring_description"]
-        element_format_input = self.prompts["element_format"]
-        action_format_input = self.prompts["action_format"]
-        value_format_input = self.prompts["value_format"]
-
-        # print(previous)
-
-        previous_ = self.taken_actions if self.taken_actions else None
-
-        # print(previous_)
-
-        prompt_list.extend(
-            generate_new_query_prompt(system_prompt=system_prompt_input + "\n" + action_space_input,
-                                      task=self.tasks[-1], previous_actions=previous_,
-                                      question_description=question_description_input,select_elements=choices))
-        prompt_list.append(
-            generate_new_referring_prompt(referring_description=referring_input, element_format=element_format_input,
-                                          action_format=action_format_input, value_format=value_format_input,
-                                          choices=None))
-
-        return prompt_list
-
-    async def perform_action(self, target_element=None, action_name=None, value=None, element_repr=""):
-        if target_element is not None:
-            selector = target_element['selector']
-            element_repr =target_element['description']
-        else:
-            selector = None
-
-        page = self.session_control['active_page']
-
-
-
-        if action_name == "CLICK" and selector:
-            # await selector.click(timeout=2000)
-            await self.session_control['active_page'].mouse.click(x=selector[0], y=selector[1])
-            self.logger.info(f"Clicked on element: {element_repr}")
-        elif action_name == "HOVER" and selector:
-            # await selector.hover(timeout=2000)
-            await self.session_control['active_page'].mouse.move(x=selector[0], y=selector[1])
-            self.logger.info(f"Hovered over element: {element_repr}")
-        elif action_name == "TYPE" and selector:
-            # await selector.fill(value)
-            # await selector.fill(value)
-            await self.session_control['active_page'].mouse.click(x=selector[0], y=selector[1])
-            await page.keyboard.type(value)
-
-            self.logger.info(f"Typed '{value}' into element: {element_repr}")
-        elif action_name == "SCROLL UP":
-            await page.evaluate(f"window.scrollBy(0, -{self.config['browser']['viewport']['height'] // 2});")
-            self.logger.info("Scrolled up")
-        elif action_name == "SCROLL DOWN":
-            await page.evaluate(f"window.scrollBy(0, {self.config['browser']['viewport']['height'] // 2});")
-            self.logger.info("Scrolled down")
-        elif action_name == "PRESS HOME":
-            await page.keyboard.press('Home')
-            self.logger.info("Pressed Home key")
-        elif action_name == "PRESS END":
-            await page.keyboard.press('End')
-            self.logger.info("Pressed End key")
-        elif action_name == "PRESS PAGEUP":
-            await page.keyboard.press('PageUp')
-            self.logger.info("Pressed PageUp key")
-        elif action_name == "PRESS PAGEDOWN":
-            await page.keyboard.press('PageDown')
-            self.logger.info("Pressed PageDown key")
-        elif action_name == "NEW TAB":
-            new_page = await self.session_control['context'].new_page()
-            # self.session_control['pages'].append(new_page)
-            self.logger.info("Opened a new tab")
-        elif action_name == "CLOSE TAB":
-            await page.close()
-            self.logger.info("Closed the current tab")
-        elif action_name == "GO BACK":
-            await page.go_back()
-            self.logger.info("Navigated back")
-        elif action_name == "GO FORWARD":
-            await page.go_forward()
-            self.logger.info("Navigated forward")
-        elif action_name == "GOTO" and value:
-            await page.goto(value, wait_until="load")
-            self.logger.info(f"Navigated to {value}")
-        # elif action_name == "PRESS ENTER" and selector:
-        #     await selector.press('Enter')
-        #     self.logger.info(f"Pressed Enter on element: {element_repr}")
-        elif action_name == "PRESS ENTER":
-            await page.keyboard.press('Enter')
-            self.logger.info(f"Pressed Enter on element: {element_repr}")
-        elif action_name == "SELECT" and selector:
-            await select_option(selector, value)
-            self.logger.info(f"Selected option '{value}' from element: {element_repr}")
-        elif action_name == "TERMINATE":
-            self.complete_flag = True
-            self.logger.info("Task has been marked as complete. Terminating...")
-        elif action_name in ["NONE"]:
-            self.logger.info("No action necessary at this stage. Skipped")
-        elif action_name in ["SAY"]:
-            self.logger.info(f"Say {value} to the user")
-        elif action_name in ["MEMORIZE"]:
-            self.logger.info(f"Keep {value} to the action history.")
-        else:
-            raise Exception(f"Unsupported or improperly specified action: {action_name}")
-        if action_name in self.no_element_op and target_element is None:
-            new_action = action_name
-        else:
-            # new_action = "[" + target_element['tag_with_role'] + "]" + " "
-            new_action = target_element['description'] + " -> " + action_name
-        if action_name in self.with_value_op:
-            new_action += ": " + value
-
-        # self.dev_logger.info(new_action)
-        return new_action
-
-    async def predict(self):
-
-        """
-        Generate a prediction for the next action based on the webpage elements and previous actions.
-        """
-
-        self.time_step += 1
-
-        try:
-            await self.session_control["active_page"].wait_for_load_state('load')
-        except Exception as e:
-            pass
-
-        # elements = await get_interactive_elements_with_playwright(self.session_control['active_page'],
-        #                                                           self.config['browser']['viewport'])
-        elements = None
-
-        '''
-             0: center_point =(x,y)
-             1: description
-             2: tag_with_role: tag_head with role and type # TODO: Consider adding more
-             3. box
-             4. selector
-             5. tag
-             '''
-
-        # elements = sorted(elements, key=lambda el: (
-        #     el["center_point"][1], el["center_point"][0]))  # Sorting by y and then x coordinate
-
-        # Generate choices for the prompt
-
-        # , self.config['basic']['default_task'], self.taken_actions
-        # choices = format_choices(elements)
-
-        select_elements = await get_select_elements_with_playwright(self.session_control['active_page'],
-                                                                           self.config['browser']['viewport'],
-                                                                           )
-
-        select_elements_formated=format_choices(select_elements)
-
-        # print("\n\n",choices)
-        prompt = self.generate_prompt(task=self.tasks[-1], previous=self.taken_actions, choices=select_elements_formated)
-        # print("\n\n",prompt)
-
-        # Logging prompt for debugging
-
-        # Capture a screenshot for the current state of the webpage, if required by the model
-        screenshot_path = os.path.join(self.main_path, 'screenshots', f'screen_{self.time_step}.png')
-        try:
-            await self.session_control['active_page'].screenshot(path=screenshot_path)
-        except Exception as e:
-            self.logger.info(f"Failed to take screenshot: {e}")
-
-        terminal_width = 10
-        self.logger.info(f"Step - {self.time_step}\n{'-'*terminal_width}\nAction Generation ➡️")
-        # for prompt_part in prompt:
-        self.logger.info("TASK: "+self.tasks[-1])
-        self.logger.info("Previous:")
-        for action in self.taken_actions:
-            self.logger.info(action)
-
-        output0 = self.engine.generate(prompt=prompt, image_path=screenshot_path, turn_number=0)
-
-        terminal_width = 10
-        self.logger.info("-" * terminal_width)
-        self.logger.info("🤖 Action Generation Output 🤖")
-
-        for line in output0.split('\n'):
-            self.logger.info(line)
-
-        terminal_width = 10
-        self.logger.info("-" * (terminal_width))
-
-        # choice_text = f"Action Grounding ➡️" + "\n" + format_options(
-        #     choices)
-        # choice_text = choice_text.replace("\n\n", "")
-        #
-        # for line in choice_text.split('\n'):
-        #     self.logger.info(line)
-
-        # output = self.engine.generate(prompt=prompt, image_path=screenshot_path, turn_number=1,
-        #                                      ouput_0=output0)
-
-        output=""
-        self.logger.info("🤖 Action Grounding Output 🤖")
-        for line in output.split('\n'):
-            self.logger.info(line)
-
-        pred_element_label, pred_action, pred_value = postprocess_action_lmm(output0)
-
-        # print(pred_element_label)
-        # print(pred_action)
-        # print(pred_value)
-        # exit()
-
-        # if len(pred_element_label) in [1, 2]:
-        #     element_id = get_index_from_option_name(pred_element_label)
-        # else:
-        #     element_id = None
-        pred_element = pred_element_label
-
-        def get_scale_factor(original_size):
-            original_width, original_height = original_size
-            new_width = min(nearest_multiple_of_224_at_least_224(original_width, ceiling=False), 1344)
-            scale_factor = new_width / original_width
-            return scale_factor
-
-        def nearest_multiple_of_224_at_least_224(num, ceiling=False):
-            if num <= 224:
-                return 224
-            division, remainder = divmod(num, 224)
-            if ceiling and remainder > 0:
-                return (division + 1) * 224
-            if remainder < 112:
-                return division * 224
-            else:
-                return (division + 1) * 224
-
-
-
-        image_file = screenshot_path
-
-        qs = f"In the screenshot, where are the pixel coordinates (x, y) of the element corresponding to \"{pred_element}\"?"
-        cur_prompt = qs
-        if self.pixui_model.config.mm_use_im_start_end:
-            qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
-        else:
-            qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
-
-        conv = conv_templates["llava_v1"].copy()
-        conv.append_message(conv.roles[0], qs)
-        conv.append_message(conv.roles[1], None)
-        prompt = conv.get_prompt()
-
-        input_ids = tokenizer_image_token(prompt, self.pixui_tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
-
-        image = Image.open(os.path.join(image_file)).convert('RGB')
-        # print("DEBUG",model.config)
-        image_tensor, image_new_size = process_images([image], self.pixui_image_processor, self.pixui_model.config)
-        # image_tensor,image_new_size = process_images([image], image_processor, model.config)[0]
-
-        with torch.inference_mode():
-            output_ids = self.pixui_model.generate(
-                input_ids,
-                # images=image_tensor.unsqueeze(0).half().cuda(),
-                images=image_tensor.half().cuda(),
-                image_sizes=[image_new_size],
-                do_sample=False,
-                temperature=0,
-                top_p=None,
-                num_beams=1,
-                # no_repeat_ngram_size=3,
-                max_new_tokens=16384,
-                use_cache=True)
-
-        outputs = self.pixui_tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
-
-
-        # print("predicted coordinate: ",outputs)
-
-
-        grounding_image=Image.open(screenshot_path)
-
-
-
-
-
-
-
-        scale=get_scale_factor(grounding_image.size)
-
-        coord = eval(outputs)
-        coord = tuple((i / scale for i in coord))
-
-        elements_at_point= await get_select_elements_with_playwright(self.session_control['active_page'],self.config['browser']['viewport'])
-
-
-        print(elements_at_point)
-
-        if pred_action=="SELECT":
-            import math
-            def euclidean_distance(coord1, coord2):
-                return math.sqrt((coord1[0] - coord2[0]) ** 2 + (coord1[1] - coord2[1]) ** 2)
-
-            min_distance = float('inf')
-            closest_element = None
-
-            # 遍历所有元素，找到最近的
-            for element in elements_at_point:
-                distance = euclidean_distance(coord, element['center_point'])
-                if distance < min_distance:
-                    min_distance = distance
-                    closest_element = element
-            pred_element=closest_element
-            pred_element['description']=pred_element_label
-
-
-
-        # print(scale)
-
-        # print(coord)
-
-        annotated_screenshot_path = os.path.join(self.main_path, 'screenshots', f'screen_{self.time_step}_grounding.png')
-        draw = ImageDraw.Draw(image)
-
-        # prompt = self.generate_prompt(task=self.tasks[-1], previous=self.taken_actions,
-        #                               choices=select_elements_formated)
-        #
-        # output = self.engine.generate(prompt=prompt, image_path=screenshot_path, turn_number=1,
-        #                                      ouput_0=output0)
-
-
-        # i=pred_element
-        # print(i["description"])
-        # print()
-        # box=i['box']
-        # left = box[0]
-        # top = box[1]
-        # right = box[0] + box[2]
-        # bottom = box[1] + box[3]
-        # # draw = ImageDraw.Draw(image2)
-        # # 绘制红色边界框
-        # draw.rectangle([left, top, right, bottom], outline="red", width=4)
-        x, y = coord
-        # 定义圆点的半径
-        radius = 7
-        # 绘制红色圆点
-        draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill="blue")
-        # 显示图片
-        # image2.show()
-
-        text = pred_element_label
-
-        # 选择字体和字号
-        font = ImageFont.truetype("/fs/ess/PAS1576/boyu_gou/train_vlm/ui_llava_fine_tune/llava/eval/Roboto-Medium.ttf", 36)
-
-        # 计算文本的大小
-        # text_width, text_height = draw.textsize(text, font=font)
-
-        # 设置文本的位置（左上角）
-        x = 0
-        y = 0
-
-        # 在图片上写入文本
-        max_width = image.width
-
-        draw_text(draw, text+str(coord), (0, 0), font, max_width, image)
-
-        image.save(fp=annotated_screenshot_path)
-        image.close()
-
-
-        # exit()
-
-        # Log the prediction result
-        self.logger.debug(f"Retrieved Answer")
-        self.logger.debug(f"Predicted Element: {pred_element}")
-        self.logger.debug(f"Action: {pred_action}")
-        self.logger.debug(f"Value: {pred_value}")
-
-        prediction={"action_generation": output0, "action_grounding": None, "element": {"center_point":coord,"description":pred_element_label,"tag_with_role":None,"box":None,"selector":coord,"tag":None},
-                "action": pred_action, "value": pred_value}
-
-        self.predictions.append(prediction)
-
-        return {"action_generation": output0, "action_grounding": None, "element": {"center_point":coord,"description":pred_element_label,"tag_with_role":None,"box":None,"selector":coord,"tag":None} if pred_action!="SELECT" else pred_element,
-                "action": pred_action, "value": pred_value}
-
-
-
-        # return output0,output,pred_element, pred_action, pred_value
-
-    async def execute(self, prediction_dict):
-        """
-        Execute the predicted action on the webpage.
-        """
-
-        pred_element = prediction_dict["element"]
-        pred_action = prediction_dict["action"]
-        pred_value = prediction_dict["value"]
-        try:
-            if (pred_action not in self.no_element_op) and pred_element == None:
-                # self.dev_logger.info
-                self.logger.info("DEBUG: WHAT IS PRED ACTION???:" + pred_action)
-                # self.dev_logger.info("DEBUG WHAT IS self.no_element_op???:"+ self.no_element_op)
-                pred_action = "NONE"
-            new_action = await self.perform_action(pred_element, pred_action, pred_value)
-            self.taken_actions.append(new_action)
-            if pred_action != "NONE":
-                self.valid_op += 1
-                self.continuous_no_op = 0
-            else:
-                self.continuous_no_op += 1
-            await asyncio.sleep(3)
-            return 0
-        except Exception as e:
-
-            new_action = f"Failed to perform {pred_action} on {pred_element['description']} with value '{pred_value}': {e}"
-
-
-            traceback_info = traceback.format_exc()
-            error_message = f"Error executing action {pred_action}: {str(e)}"
-            print(traceback_info)
-            # exit()
-            error_message_with_traceback = f"{error_message}\n\nTraceback:\n{traceback_info}"
-
-            self.logger.info(new_action)
-            self.taken_actions.append(new_action)
-            self.continuous_no_op += 1
-            await asyncio.sleep(3)
-            return 1
-
-    async def stop(self):
-
-        try:
-            close_context = self.session_control['context']
-            self.session_control['context'] = None
-            await close_context.close()
-            self.logger.info("Browser context closed.")
-        except Exception as e:
-            self.logger.info(e)
-
-        final_json = {"task": self.tasks, "website": self.config["basic"]["default_website"],
-                      "num_step": len(self.taken_actions), "action_history": self.taken_actions}
-
-        def locator_serializer(obj):
-            """Convert non-serializable objects to a serializable format."""
-            if isinstance(obj, Locator):
-                # Assuming Locator has attributes 'frame' and 'selector' you want to serialize
-                return str(obj)
-            raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable")
-
-        # Using the custom default function in json.dump
-        with open(os.path.join(self.main_path, 'all_predictions.json'), 'w', encoding='utf-8') as f:
-            json.dump(self.predictions, f, default=locator_serializer, indent=4)
-
-
-        with open(os.path.join(self.main_path, 'result.json'), 'w', encoding='utf-8') as file:
-            json.dump(final_json, file, indent=4)
-        self.logger.info("Agent stopped.")
-
-        saveconfig(self.config, os.path.join(self.main_path, 'config.toml'))
-
-    def clear_action_history(self):
-        """
-        Clears the history of actions taken by the agent.
-        """
-        self.taken_actions.clear()
-        self.logger.info("Cleared action history.")
-
-    def reset_comlete_flag(self, flag=False):
-        self.complete_flag = flag
-
-    def change_task(self, new_task, clear_history=False):
-        """
-        Changes the task requirement for the agent.
-
-        Parameters:
-        - new_task: The new task requirement as a string.
-        """
-        if new_task and isinstance(new_task, str):
-
-            self.logger.info(f"Changed task from {self.tasks[-1]} to: {new_task}")
-            self.tasks.append(new_task)
-            # Optionally clear action history when changing task
-            if clear_history:
-                self.clear_action_history()
-            else:
-                self.taken_actions.append(f"Changed task from {self.tasks[-2]} to: {new_task}")
-
-        else:
-            self.logger.info("Invalid new task. It must be a non-empty string.")
-
-        # Optionally, you can save the taken_actions to a file or database for record-keeping
-
-    # ADD no op count and op count, add limit to op
-
-    # decompose run to predict and execute.

llava/eval/seeact/data_utils/__init__.py

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

llava/eval/seeact/data_utils/format_prompt_utils.py

@@ -1,129 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import string
-
-def generate_new_query_prompt(system_prompt="", task="", previous_actions=None, question_description="",select_elements=None):
-    """
-    Generate the first phase prompt to ask model to generate general descriptions about {environment, high-level plans, next step action}
-    Each experiment will have a similar prompt in this phase
-    This prompt is used to generate models' thoughts without disrupt of formatting/referring prompts
-    """
-    sys_role=""+system_prompt
-    query_text = ""
-
-    # System Prompt
-    query_text += "You are asked to complete the following task: "
-
-    # Task Description
-    query_text += task
-    query_text += "\n\n"
-
-    # Previous Actions
-    previous_action_text = "Previous Actions:\n"
-    if previous_actions is None:
-        previous_actions = []
-    for action_text in previous_actions:
-        previous_action_text += action_text
-        previous_action_text += "\n"
-    query_text += previous_action_text
-    query_text += "\n"
-
-    # Question Description
-    query_text += question_description
-
-    if select_elements:
-        query_text += "\n"
-        for element in select_elements:
-            query_text+=element+'\n'
-    return [sys_role,query_text]
-
-
-
-def generate_new_referring_prompt(referring_description="", element_format="", action_format="", value_format="",
-                              choices=None,split="4"):
-    referring_prompt = ""
-
-    # Add description about how to format output
-    if referring_description != "":
-        referring_prompt += referring_description
-        referring_prompt += "\n\n"
-
-    # Add element prediction format and choices
-
-
-    # Prepare Option texts
-    # For exp {1, 2, 4}, generate option
-    # For element_atttribute, set options field at None
-    # if choices:
-    #     choice_text = format_options(choices)
-    #     referring_prompt += choice_text
-
-    if element_format != "":
-        referring_prompt += element_format
-        referring_prompt += "\n\n"
-
-    # Format Action Prediction
-    if action_format != "":
-        referring_prompt += action_format
-        referring_prompt += "\n\n"
-
-    # Format Value Prediction
-    if value_format != "":
-        referring_prompt += value_format
-        referring_prompt += ""
-
-    return referring_prompt
-
-def format_options(choices):
-    option_text = ""
-    abcd = ''
-    non_abcd = ''
-
-    multi_choice = ''
-    for multichoice_idx, choice in enumerate(choices):
-        multi_choice += f"{generate_option_name(multichoice_idx)}. {choice}\n"
-        abcd += f"{generate_option_name(multichoice_idx)}, "
-
-        non_abcd = generate_option_name(multichoice_idx + 1)
-
-    multi_choice += f"{non_abcd}. None of the other options match the correct element or the action doesn't involve an element."
-    # option_text += abcd
-    option_text += f"If none of these elements match your target element or your target action doesn't involve an element, please select {non_abcd}.\n"
-    option_text += (multi_choice + '\n\n')
-    return option_text
-
-
-def generate_option_name(index):
-    if index < 26:
-        return string.ascii_uppercase[index]
-    else:
-        first_letter_index = (index - 26) // 26
-        second_letter_index = (index - 26) % 26
-        first_letter = string.ascii_uppercase[first_letter_index]
-        second_letter = string.ascii_uppercase[second_letter_index]
-        return f"{first_letter}{second_letter}"
-
-def get_index_from_option_name(name):
-    if len(name) == 1:
-        return string.ascii_uppercase.index(name)
-    elif len(name) == 2:
-        first_letter_index = string.ascii_uppercase.index(name[0])
-        second_letter_index = string.ascii_uppercase.index(name[1])
-        return 26 + first_letter_index * 26 + second_letter_index
-    else:
-        raise Exception("The string should be either 1 or 2 characters long")
-
-

llava/eval/seeact/data_utils/prompts.py

@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from .format_prompt_utils import generate_new_referring_prompt, generate_new_query_prompt
-
-##### SeeAct Online Prompts
-
-seeact_online_sys_prompt = '''Imagine that you are imitating humans doing web navigation for a task step by step. At each stage, you can see the webpage like humans by a screenshot and know the previous actions before the current step decided by yourself through recorded history. You need to decide on the first following action to take. You can click on an element with the mouse, select an option, type text or press Enter with the keyboard. (For your understanding, they are like the click(), select_option() type() and keyboard.press('Enter') functions in playwright respectively) One next step means one operation within the four. Unlike humans, for typing (e.g., in text areas, text boxes) and selecting (e.g., from dropdown menus or <select> elements), you should try directly typing the input or selecting the choice, bypassing the need for an initial click. You should not attempt to create accounts, log in or do the final submission. Terminate when you deem the task complete or if it requires potentially harmful actions.'''
-
-seeact_online_question_description_new_exp4 = '''The screenshot below shows the webpage you see. Follow the following guidance to think step by step before outlining the next action step at the current stage:
-
-(Current Webpage Identification)
-Firstly, think about what the current webpage is.
-
-(Previous Action Analysis)
-Secondly, combined with the screenshot, analyze each step of the previous action history and their intention one by one. Particularly, pay more attention to the last step, which may be more related to what you should do now as the next step. Specifically, if the last action involved a TYPE, always evaluate whether it necessitates a confirmation step, because typically a single TYPE action does not make effect. (often, simply pressing 'Enter', assuming the default element involved in the last action, unless other clear elements are present for operation).
-
-(Screenshot Details Analysis)
-Closely examine the screenshot to check the status of every part of the webpage to understand what you can operate with and what has been set or completed. You should closely examine the screenshot details to see what steps have been completed by previous actions even though you are given the textual previous actions. Because the textual history may not clearly and sufficiently record some effects of previous actions, you should closely evaluate the status of every part of the webpage to understand what you have done.
-
-(Next Action Based on Webpage and Analysis)
-Then, based on your analysis, in conjunction with human web browsing habits and the logic of web design, decide on the following action. And clearly outline which element in the webpage users will operate with as the first next target element, its detailed location, and the corresponding operation.
-
-To be successful, it is important to follow the following rules: 
-1. You should only issue a valid action given the current observation. 
-2. You should only issue one action at a time
-3. For handling the select dropdown elements on the webpage, it's not necessary for you to provide completely accurate options right now. The full list of options for these elements will be supplied later.'''
-
-seeact_online_action_format = "ACTION: Choose an action from {CLICK, SELECT, TYPE, PRESS ENTER, TERMINATE, NONE}."
-
-seeact_online_value_format = "VALUE: Provide additional input based on ACTION.\n\nThe VALUE means:\nIf ACTION == TYPE, specify the " \
-               "text to be typed.\nIf ACTION == SELECT, indicate the option to be chosen. Revise the selection value to align with the available options within the element.\nIf ACTION == CLICK, PRESS ENTER, TERMINATE or NONE, " \
-               "write \"None\"."
-
-seeact_choice_prompt_dict = {
-    "system_prompt": seeact_online_sys_prompt,
-
-    "question_description": seeact_online_question_description_new_exp4,
-
-    "referring_description": f"""(Reiteration)
-First, reiterate your next target element, its detailed location, and the corresponding operation.
-
-(Multichoice Question)
-Below is a multi-choice question, where the choices are elements in the webpage. All elements are arranged in the order based on their height on the webpage, from top to bottom (and from left to right). This arrangement can be used to locate them. From the screenshot, find out where and what each one is on the webpage, taking into account both their text content and HTML details. Then, determine whether one matches your target element. Please examine the choices one by one. Choose the matching one. If multiple options match your answer, choose the most likely one by re-examining the screenshot, the choices, and your further reasoning.""",
-
-    "element_format": """(Final Answer)
-Finally, conclude your answer using the format below. Ensure your answer is strictly adhering to the format provided below. Please do not leave any explanation in your answers of the final standardized format part, and this final part should be clear and certain. The element choice, action, and value should be in three separate lines.
-
-Format:
-
-ELEMENT: The uppercase letter of your choice. (No need for PRESS ENTER)""",
-
-    "action_format": f"{seeact_online_action_format}",
-
-    "value_format": f"{seeact_online_value_format}"
-}
-
-
-def generate_prompt(experiment_split, task=None, previous=None, choices=None):
-    assert experiment_split != None, "Please specify the experiment split."
-    assert task != None, "Please input the task."
-    assert previous != None, "Please input the previous actions."
-
-    if False:
-        raise Exception("Please change experiment type to seeact")
-    elif experiment_split in ["seeact_online","online","seeact","SeeAct"]:
-        system_prompt_input = seeact_choice_prompt_dict["system_prompt"]
-        question_description_input = seeact_choice_prompt_dict["question_description"]
-        referring_input = seeact_choice_prompt_dict["referring_description"]
-        element_format_input = seeact_choice_prompt_dict["element_format"]
-        action_format_input = seeact_choice_prompt_dict["action_format"]
-        value_format_input = seeact_choice_prompt_dict["value_format"]
-        prompt_list = []
-
-        prompt_list.extend(
-            generate_new_query_prompt(system_prompt=system_prompt_input, task=task, previous_actions=previous,
-                                      question_description=question_description_input))
-        prompt_list.append(
-            generate_new_referring_prompt(referring_description=referring_input, element_format=element_format_input,
-                                          action_format=action_format_input, value_format=value_format_input,
-                                          choices=choices))
-        return prompt_list
-

llava/eval/seeact/demo_utils/browser_helper.py

@@ -1,457 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import asyncio
-from difflib import SequenceMatcher
-from playwright.sync_api import Playwright, expect, sync_playwright
-# from playwright.async_api import async_playwright
-from pathlib import Path
-import toml
-import os
-import traceback
-
-async def normal_launch_async(playwright: Playwright,headless=False,args=None):
-    browser = await playwright.chromium.launch(
-        traces_dir=None,
-        headless=False,
-        args=args,
-        # ignore_default_args=ignore_args,
-        # chromium_sandbox=False,
-    )
-    return browser
-
-
-
-async def normal_new_context_async(
-        browser,
-        storage_state=None,
-        har_path=None,
-        video_path=None,
-        tracing=False,
-        trace_screenshots=False,
-        trace_snapshots=False,
-        trace_sources=False,
-        locale=None,
-        geolocation=None,
-        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-        viewport: dict = {"width": 1280, "height": 720},
-):
-    context = await browser.new_context(
-        storage_state=storage_state,
-        user_agent=user_agent,
-        viewport=viewport,
-        locale=locale,
-        record_har_path=har_path,
-        record_video_dir=video_path,
-        geolocation=geolocation,
-    )
-
-    if tracing:
-        await context.tracing.start(screenshots=trace_screenshots, snapshots=trace_snapshots, sources=trace_sources)
-    return context
-
-#
-# def persistent_launch(playwright: Playwright, user_data_dir: str = ""):
-#     context = playwright.chromium.launch_persistent_context(
-#         user_data_dir=user_data_dir,
-#         headless=False,
-#         args=["--no-default-browser-check",
-#               "--no_sandbox",
-#               "--disable-blink-features=AutomationControlled",
-#               ],
-#         ignore_default_args=ignore_args,
-#         user_agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-#         viewport={"width": 1280, "height": 720},
-#         bypass_csp=True,
-#         slow_mo=1000,
-#         chromium_sandbox=True,
-#         channel="chrome-dev"
-#     )
-#     return context
-
-#
-# async def persistent_launch_async(playwright: Playwright, user_data_dir: str = "", record_video_dir="video"):
-#     context = await playwright.chromium.launch_persistent_context(
-#         user_data_dir=user_data_dir,
-#         headless=False,
-#         args=[
-#             "--disable-blink-features=AutomationControlled",
-#         ],
-#         ignore_default_args=ignore_args,
-#         user_agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-#         # viewport={"width": 1280, "height": 720},
-#         record_video_dir=record_video_dir,
-#         channel="chrome-dev"
-#         # slow_mo=1000,
-#     )
-#     return context
-
-
-
-def remove_extra_eol(text):
-    # Replace EOL symbols
-    text = text.replace('\n', ' ')
-    return re.sub(r'\s{2,}', ' ', text)
-
-
-def get_first_line(s):
-    first_line = s.split('\n')[0]
-    tokens = first_line.split()
-    if len(tokens) > 8:
-        return ' '.join(tokens[:8]) + '...'
-    else:
-        return first_line
-
-async def get_element_description(element, tag_name, role_value, type_value):
-    '''
-         Asynchronously generates a descriptive text for a web element based on its tag type.
-         Handles various HTML elements like 'select', 'input', and 'textarea', extracting attributes and content relevant to accessibility and interaction.
-    '''
-    # text_content = await element.inner_text(timeout=0)
-    # text = (text_content or '').strip()
-    #
-    # print(text)
-    salient_attributes = [
-        "alt",
-        "aria-describedby",
-        "aria-label",
-        "aria-role",
-        "input-checked",
-        # "input-value",
-        "label",
-        "name",
-        "option_selected",
-        "placeholder",
-        "readonly",
-        "text-value",
-        "title",
-        "value",
-    ]
-
-    parent_value = "parent_node: "
-    parent_locator = element.locator('xpath=..')
-    num_parents = await parent_locator.count()
-    if num_parents > 0:
-        # only will be zero or one parent node
-        parent_text = (await parent_locator.inner_text(timeout=0) or "").strip()
-        if parent_text:
-            parent_value += parent_text
-    parent_value = remove_extra_eol(get_first_line(parent_value)).strip()
-    if parent_value == "parent_node:":
-        parent_value = ""
-    else:
-        parent_value += " "
-
-    if tag_name == "select":
-        text1 = "Selected Options: "
-        text3 = " - Options: "
-
-        text2 = await element.evaluate(
-            "select => select.options[select.selectedIndex].textContent", timeout=0
-        )
-
-        if text2:
-            options = await element.evaluate("select => Array.from(select.options).map(option => option.text)",
-                                             timeout=0)
-            text4 = " | ".join(options)
-
-            if not text4:
-                text4 = await element.text_content(timeout=0)
-                if not text4:
-                    text4 = await element.inner_text(timeout=0)
-
-
-            return parent_value+text1 + remove_extra_eol(text2.strip()) + text3 + text4
-
-    input_value = ""
-
-    none_input_type = ["submit", "reset", "checkbox", "radio", "button", "file"]
-
-    if tag_name == "input" or tag_name == "textarea":
-        if role_value not in none_input_type and type_value not in none_input_type:
-            text1 = "input value="
-            text2 = await element.input_value(timeout=0)
-            if text2:
-                input_value = text1 + "\"" + text2 + "\"" + " "
-
-    text_content = await element.text_content(timeout=0)
-    text = (text_content or '').strip()
-
-    # print(text)
-    if text:
-        text = remove_extra_eol(text)
-        if len(text) > 80:
-            text_content_in = await element.inner_text(timeout=0)
-            text_in = (text_content_in or '').strip()
-            if text_in:
-                return input_value + remove_extra_eol(text_in)
-        else:
-            return input_value + text
-
-    # get salient_attributes
-    text1 = ""
-    for attr in salient_attributes:
-        attribute_value = await element.get_attribute(attr, timeout=0)
-        if attribute_value:
-            text1 += f"{attr}=" + "\"" + attribute_value.strip() + "\"" + " "
-
-    text = (parent_value + text1).strip()
-    if text:
-        return input_value + remove_extra_eol(text.strip())
-
-
-    # try to get from the first child node
-    first_child_locator = element.locator('xpath=./child::*[1]')
-
-    num_childs = await first_child_locator.count()
-    if num_childs>0:
-        for attr in salient_attributes:
-            attribute_value = await first_child_locator.get_attribute(attr, timeout=0)
-            if attribute_value:
-                text1 += f"{attr}=" + "\"" + attribute_value.strip() + "\"" + " "
-
-        text = (parent_value + text1).strip()
-        if text:
-            return input_value + remove_extra_eol(text.strip())
-
-    return None
-
-
-async def get_element_data(element, tag_name,viewport_size,seen_elements=[],coordinates=None):
-    try:
-        tag_name_list = ['a', 'button',
-                         'input',
-                         'select', 'textarea', 'adc-tab']
-
-
-
-
-
-
-        rect = await element.bounding_box() or {'x': -1, 'y': -1, 'width': 0, 'height': 0}
-
-        if rect['x']<0 or rect['y']<0 or rect['width']<=4 or rect['height']<=4 or rect['y']+rect['height']>viewport_size["height"] or rect['x']+ rect['width']>viewport_size["width"]:
-            return None
-
-
-
-        if coordinates is not None:
-            if coordinates[0]>=rect['x'] and coordinates[0]<=rect['x']+rect['width'] and coordinates[1]>=rect['y'] and coordinates[1]<=rect['y']+rect['height']:
-                print(coordinates)
-                print(rect)
-            else:
-                return None
-
-
-        box_model = [rect['x'], rect['y'], rect['x'] + rect['width'], rect['y'] + rect['height']]
-        center_point = (round((box_model[0] + box_model[2]) / 2 , 3),
-                        round((box_model[1] + box_model[3]) / 2 , 3))
-
-
-
-
-
-        if await element.is_hidden(timeout=0) or await element.is_disabled(timeout=0):
-            return None
-
-        if center_point in seen_elements:
-            return None
-
-        # await aprint(element,tag_name)
-
-        if tag_name in tag_name_list:
-            tag_head = tag_name
-            real_tag_name = tag_name
-        else:
-            real_tag_name = await element.evaluate("element => element.tagName.toLowerCase()", timeout=0)
-            if real_tag_name in tag_name_list:
-                # already detected
-                return None
-            else:
-                tag_head = real_tag_name
-
-        text_element = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'td', "div","em","center","strong","b","i","small","mark","abbr","cite","q","blockquote","span","nobr"]
-
-        if real_tag_name in text_element:
-            return None
-
-        role_value = await element.get_attribute('role', timeout=0)
-        type_value = await element.get_attribute('type', timeout=0)
-        # await aprint("start to get element description",element,tag_name )
-        description = await get_element_description(element, real_tag_name, role_value, type_value)
-        # print(description)
-        if not description:
-            return None
-
-        if role_value:
-            tag_head += " role=" + "\"" + role_value + "\""
-        if type_value:
-            tag_head += " type=" + "\"" + type_value + "\""
-
-        '''
-                     0: center_point =(x,y)
-                     1: description
-                     2: tag_with_role: tag_head with role and type # TODO: Consider adding more
-                     3. box
-                     4. selector
-                     5. tag
-                     '''
-        selector = element
-
-
-        if coordinates is not None:
-            if coordinates[0]>=rect['x'] and coordinates[0]<=rect['x']+rect['width'] and coordinates[1]>=rect['y'] and coordinates[1]<=rect['y']+rect['height']:
-                print(tag_head)
-                print(description)
-                print(box_model)
-            else:
-
-
-                return None
-
-        return {"center_point":center_point,"description":description,"tag_with_role":tag_head,"box":box_model,"selector":selector,"tag":real_tag_name}
-        # return [center_point, description, tag_head, box_model, selector, real_tag_name]
-    except Exception as e:
-        print(traceback.format_exc())
-        print(e)
-        return None
-
-
-async def get_interactive_elements_with_playwright(page,viewport_size,coordinates=None):
-
-    print("Get Interactive elements around: ", coordinates)
-    interactive_elements_selectors = [
-        'a', 'button',
-        'input',
-        'select', 'textarea',
-    ]
-
-    seen_elements = set()
-    tasks = []
-
-
-    for selector in interactive_elements_selectors:
-        locator = page.locator(selector)
-        element_count = await locator.count()
-        for index in range(element_count):
-            element = locator.nth(index)
-            tag_name = selector
-            task = get_element_data(element, tag_name,viewport_size,seen_elements=[],coordinates=coordinates)
-
-            tasks.append(task)
-
-    results = await asyncio.gather(*tasks)
-
-    interactive_elements = []
-    for i in results:
-        if i:
-            if i["center_point"] in seen_elements:
-                continue
-            else:
-                seen_elements.add(i["center_point"])
-                interactive_elements.append(i)
-
-    # interactive_elements_selectors = [
-    #     '*'
-    # ]
-    # tasks = []
-    #
-    # for selector in interactive_elements_selectors:
-    #     locator = page.locator(selector)
-    #     element_count = await locator.count()
-    #     for index in range(element_count):
-    #         element = locator.nth(index)
-    #         tag_name = selector
-    #         task = get_element_data(element, tag_name, viewport_size,seen_elements,coordinates)
-    #
-    #         tasks.append(task)
-    #
-    # results = await asyncio.gather(*tasks)
-    #
-    #
-    # for i in results:
-    #     if i:
-    #         if i["center_point"] in seen_elements:
-    #             continue
-    #         else:
-    #             seen_elements.add(i["center_point"])
-    #             interactive_elements.append(i)
-
-    return interactive_elements
-
-
-
-async def get_select_elements_with_playwright(page,viewport_size):
-
-    interactive_elements_selectors = [
-        'select'
-    ]
-
-    seen_elements = set()
-    tasks = []
-
-
-    for selector in interactive_elements_selectors:
-        locator = page.locator(selector)
-        element_count = await locator.count()
-        for index in range(element_count):
-            element = locator.nth(index)
-            tag_name = selector
-            task = get_element_data(element, tag_name,viewport_size,seen_elements=[],coordinates=None)
-
-            tasks.append(task)
-
-    results = await asyncio.gather(*tasks)
-
-    interactive_elements = []
-    for i in results:
-        if i:
-            if i["center_point"] in seen_elements:
-                continue
-            else:
-                seen_elements.add(i["center_point"])
-                interactive_elements.append(i)
-
-    return interactive_elements
-
-
-async def select_option(selector, value):
-    best_option = [-1, "", -1]
-    for i in range(await selector.locator("option").count()):
-        option = await selector.locator("option").nth(i).inner_text()
-        similarity = SequenceMatcher(None, option, value).ratio()
-        if similarity > best_option[2]:
-            best_option = [i, option, similarity]
-    await selector.select_option(index=best_option[0], timeout=10000)
-    return remove_extra_eol(best_option[1]).strip()
-
-
-def saveconfig(config, save_file):
-    """
-    config is a dictionary.
-    save_path: saving path include file name.
-    """
-
-
-    if isinstance(save_file, str):
-        save_file = Path(save_file)
-    if isinstance(config, dict):
-        with open(save_file, 'w') as f:
-            config_without_key = config
-            config_without_key["openai"]["api_key"] = "Your API key here"
-            toml.dump(config_without_key, f)
-    else:
-        os.system(" ".join(["cp", str(config), str(save_file)]))

llava/eval/seeact/demo_utils/format_prompt.py

@@ -1,133 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import re
-
-
-def format_choices(elements):
-
-    converted_elements = [
-                    f'<{element["tag_with_role"]}>'
-                    + (
-                        element["description"]
-                    )
-                    + f"</{element['tag']}>"
-                    for i, element in enumerate(elements)
-                ]
-
-    return converted_elements
-
-def postprocess_action_lmm(text):
-    text = text.strip()
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:\n\n",
-        "")
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:\n",
-        "")
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:",
-        "")
-    text = text.replace("The uppercase letter of your choice based on your analysis is:\n\n", "")
-    text = text.replace("The uppercase letter of your choice based on your analysis is:\n", "")
-    text = text.replace("The uppercase letter of your choice based on your analysis is:", "")
-    text = text.replace("The uppercase letter of my choice is \n\n", "")
-    text = text.replace("The uppercase letter of my choice is \n", "")
-    text = text.replace("The uppercase letter of my choice is ", "")
-    text = text.replace("The uppercase letter of your choice is \n\n", "")
-    text = text.replace("The uppercase letter of your choice is \n", "")
-    text = text.replace("The uppercase letter of your choice is ", "")
-    text = text.replace("The uppercase letter of your choice.\n\n", "")
-    text = text.replace("The uppercase letter of your choice.\n", "")
-    text = text.replace("The uppercase letter of your choice.", "")
-    text = text.replace("The uppercase letter of your choice based on my analysis is:\n\n", "")
-    text = text.replace("The uppercase letter of your choice based on my analysis is:\n", "")
-    text = text.replace("The uppercase letter of your choice based on my analysis is:", "")
-    text = text.replace("The correct choice based on the analysis would be:\n\n", "")
-    text = text.replace("The correct choice based on the analysis would be:\n", "")
-    text = text.replace("The correct choice based on the analysis would be :", "")
-    text = text.replace("The correct choice based on the analysis would be ", "")
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:\n\n",
-        "")
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:\n",
-        "")
-    text = text.replace(
-        "The uppercase letter of your choice. Choose one of the following elements if it matches the target element based on your analysis:",
-        "")
-    text = text.replace("The uppercase letter of your choice.\n\n", "")
-    text = text.replace("The uppercase letter of your choice.\n", "")
-    text = text.replace("The uppercase letter of your choice based on the analysis is:\n\n", "")
-    text = text.replace("The uppercase letter of your choice based on the analysis is:\n", "")
-    text = text.replace("The uppercase letter of your choice based on the analysis is:", "")
-    text = text.replace("The uppercase letter of your choice based on the analysis is ", "")
-    text = text.replace("The uppercase letter of my choice based on the analysis is:\n\n", "")
-    text = text.replace("The uppercase letter of my choice based on the analysis is:\n", "")
-    text = text.replace("The uppercase letter of my choice based on the analysis is:", "")
-    text = text.replace("The uppercase letter of my choice based on the analysis is ", "")
-    text = text.replace("The correct element to select would be:\n\n", "")
-    text = text.replace("The correct element to select would be:\n", "")
-    text = text.replace("The correct element to select would be:", "")
-    text = text.replace("The correct element to select would be ", "")
-    text = text.replace("The uppercase letter of my choice is:\n\n", "")
-    text = text.replace("The uppercase letter of my choice is:\n", "")
-    text = text.replace("The uppercase letter of my choice is:", "")
-    text = text.replace("The uppercase letter of my choice is ", "")
-    text = text.replace("Choose an action from {CLICK, TYPE, SELECT}.\n\n", "")
-    text = text.replace("Choose an action from {CLICK, TYPE, SELECT}.\n", "")
-    text = text.replace("Choose an action from {CLICK, TYPE, SELECT}.", "")
-    text = text.replace("Provide additional input based on ACTION.\n\n", "")
-    text = text.replace("Provide additional input based on ACTION.\n", "")
-    text = text.replace("Provide additional input based on ACTION.", "")
-
-    def extract_element_description(text):
-        pattern = r'ELEMENT:\s*(.*?)\s*ACTION:'
-        match = re.search(pattern, text)
-        if match:
-            return match.group(1)
-        else:
-            return None
-
-    description = extract_element_description(text)
-    action = re.search(
-        r"ACTION: (CLICK|SELECT|TYPE|HOVER|PRESS ENTER|SCROLL UP|SCROLL DOWN|PRESS HOME|PRESS END|PRESS PAGEUP|PRESS PAGEDOWN|NEW TAB|CLOSE TAB|GO BACK|GO FORWARD|TERMINATE|NONE|GOTO|SAY|MEMORIZE)",
-        text
-    )
-
-
-    if action:
-        action = action.group(1)
-    else:
-        action = "None"
-
-    value = re.search(r"VALUE: (.*)$", text, re.MULTILINE)
-    value = value.group(1) if value is not None else ""
-    return description, action.strip(), process_string(process_string(value.strip()))
-
-def process_string(input_string):
-    if input_string.startswith('"') and input_string.endswith('"'):
-        input_string = input_string[1:-1]
-    if input_string.endswith('.'):
-        input_string = input_string[:-1]
-    return input_string
-
-
-
-
-
-
-
-
-

llava/eval/seeact/demo_utils/inference_engine.py

@@ -1,314 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 OSU Natural Language Processing Group
-#
-# Licensed under the OpenRAIL-S License;
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.licenses.ai/ai-pubs-open-rails-vz1
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import time
-
-import backoff
-import openai
-from openai import (
-    APIConnectionError,
-    APIError,
-    RateLimitError,
-)
-import requests
-from dotenv import load_dotenv
-import litellm
-import base64
-
-EMPTY_API_KEY="Your API KEY Here"
-
-def load_openai_api_key():
-    load_dotenv()
-    assert (
-            os.getenv("OPENAI_API_KEY") is not None and
-            os.getenv("OPENAI_API_KEY") != EMPTY_API_KEY
-    ), "must pass on the api_key or set OPENAI_API_KEY in the environment"
-    return os.getenv("OPENAI_API_KEY")
-
-
-def load_gemini_api_key():
-    load_dotenv()
-    assert (
-            os.getenv("GEMINI_API_KEY") is not None and
-            os.getenv("GEMINI_API_KEY") != EMPTY_API_KEY
-    ), "must pass on the api_key or set GEMINI_API_KEY in the environment"
-    return os.getenv("GEMINI_API_KEY")
-
-def encode_image(image_path):
-    with open(image_path, "rb") as image_file:
-        return base64.b64encode(image_file.read()).decode('utf-8')
-
-
-def engine_factory(api_key=None, model=None, **kwargs):
-    model = model.lower()
-    if model in ["gpt-4-vision-preview", "gpt-4-turbo", "gpt-4o"]:
-        if api_key and api_key != EMPTY_API_KEY:
-            os.environ["OPENAI_API_KEY"] = api_key
-        else:
-            load_openai_api_key()
-        return OpenAIEngine(model=model, **kwargs)
-    elif model in ["gemini-1.5-pro-latest", "gemini-1.5-flash"]:
-        if api_key and api_key != EMPTY_API_KEY:
-            os.environ["GEMINI_API_KEY"] = api_key
-        else:
-            load_gemini_api_key()
-        model=f"gemini/{model}"
-        return GeminiEngine(model=model, **kwargs)
-    elif model == "llava":
-        model="llava"
-        return OllamaEngine(model=model, **kwargs)
-    raise Exception(f"Unsupported model: {model}, currently supported models: \
-                    gpt-4-vision-preview, gpt-4-turbo, gemini-1.5-pro-latest, llava")
-
-class Engine:
-    def __init__(
-            self,
-            stop=["\n\n"],
-            rate_limit=-1,
-            model=None,
-            temperature=0,
-            **kwargs,
-    ) -> None:
-        """
-            Base class to init an engine
-
-        Args:
-            api_key (_type_, optional): Auth key from OpenAI. Defaults to None.
-            stop (list, optional): Tokens indicate stop of sequence. Defaults to ["\n"].
-            rate_limit (int, optional): Max number of requests per minute. Defaults to -1.
-            model (_type_, optional): Model family. Defaults to None.
-        """
-        self.time_slots = [0]
-        self.stop = stop
-        self.temperature = temperature
-        self.model = model
-        # convert rate limit to minmum request interval
-        self.request_interval = 0 if rate_limit == -1 else 60.0 / rate_limit
-        self.next_avil_time = [0] * len(self.time_slots)
-        self.current_key_idx = 0
-        print(f"Initializing model {self.model}")        
-
-    def tokenize(self, input):
-        return self.tokenizer(input)
-
-
-class OllamaEngine(Engine):
-    def __init__(self, **kwargs) -> None:
-        """
-            Init an Ollama engine
-            To use Ollama, dowload and install Ollama from https://ollama.com/
-            After Ollama start, pull llava with command: ollama pull llava
-        """
-        super().__init__(**kwargs)
-        self.api_url = "http://localhost:11434/api/chat"
-
-
-    def generate(self, prompt: list = None, max_new_tokens=4096, temperature=None, model=None, image_path=None,
-                 ouput_0=None, turn_number=0, **kwargs):
-        self.current_key_idx = (self.current_key_idx + 1) % len(self.time_slots)
-        start_time = time.time()
-        if (
-                self.request_interval > 0
-                and start_time < self.next_avil_time[self.current_key_idx]
-        ):
-            wait_time = self.next_avil_time[self.current_key_idx] - start_time
-            print(f"Wait {wait_time} for rate limitting")
-            time.sleep(wait_time)
-        prompt0, prompt1, prompt2 = prompt
-
-        base64_image = encode_image(image_path)
-        if turn_number == 0:
-            # Assume one turn dialogue
-            prompt_input = [
-                {"role": "assistant", "content": prompt0},
-                {"role": "user", "content": prompt1, "images": [f"{base64_image}"]},
-            ]
-        elif turn_number == 1:
-            prompt_input = [
-                {"role": "assistant", "content": prompt0},
-                {"role": "user", "content": prompt1, "images": [f"{base64_image}"]},
-                {"role": "assistant", "content": f"\n\n{ouput_0}"},
-                {"role": "user", "content": prompt2}, 
-            ]
-
-        options = {"temperature": self.temperature, "num_predict": max_new_tokens}
-        data = {
-            "model": self.model,
-            "messages": prompt_input,
-            "options": options,
-            "stream": False,
-        }
-        _request = {
-            "url": f"{self.api_url}",
-            "json": data,
-        }
-        response = requests.post(**_request)  # type: ignore
-        if response.status_code != 200:
-            raise Exception(f"Ollama API Error: {response.status_code}, {response.text}")
-        response_json = response.json()
-        return response_json["message"]["content"]
-
-
-class GeminiEngine(Engine):
-    def __init__(self, **kwargs) -> None:
-        """
-            Init a Gemini engine
-            To use this engine, please provide the GEMINI_API_KEY in the environment
-            Supported Model             Rate Limit
-            gemini-1.5-pro-latest    	2 queries per minute, 1000 queries per day
-        """
-        super().__init__(**kwargs)
-
-
-    def generate(self, prompt: list = None, max_new_tokens=4096, temperature=None, model=None, image_path=None,
-                 ouput_0=None, turn_number=0, **kwargs):
-        self.current_key_idx = (self.current_key_idx + 1) % len(self.time_slots)
-        start_time = time.time()
-        if (
-                self.request_interval > 0
-                and start_time < self.next_avil_time[self.current_key_idx]
-        ):
-            wait_time = self.next_avil_time[self.current_key_idx] - start_time
-            print(f"Wait {wait_time} for rate limitting")
-        prompt0, prompt1, prompt2 = prompt
-        litellm.set_verbose=True
-
-        base64_image = encode_image(image_path)
-        if turn_number == 0:
-            # Assume one turn dialogue
-            prompt_input = [
-                {"role": "system", "content": prompt0},
-                {"role": "user",
-                 "content": [{"type": "text", "text": prompt1}, {"type": "image_url", "image_url": {"url": image_path,
-                                                                                                    "detail": "high"},
-                                                                }]},
-            ]
-        elif turn_number == 1:
-            prompt_input = [
-                {"role": "system", "content": prompt0},
-                {"role": "user",
-                 "content": [{"type": "text", "text": prompt1}, {"type": "image_url", "image_url": {"url": image_path,
-                                                                                                    "detail": "high"}, 
-                                                                }]},
-                {"role": "assistant", "content": [{"type": "text", "text": f"\n\n{ouput_0}"}]},
-                {"role": "user", "content": [{"type": "text", "text": prompt2}]}, 
-            ]
-        response = litellm.completion(
-            model=model if model else self.model,
-            messages=prompt_input,
-            max_tokens=max_new_tokens if max_new_tokens else 4096,
-            temperature=temperature if temperature else self.temperature,
-            **kwargs,
-        )
-        return [choice["message"]["content"] for choice in response.choices][0]
-
-
-class OpenAIEngine(Engine):
-    def __init__(self, **kwargs) -> None:
-        """
-            Init an OpenAI GPT/Codex engine
-            To find your OpenAI API key, visit https://platform.openai.com/api-keys
-        """
-        super().__init__(**kwargs)
-
-    @backoff.on_exception(
-        backoff.expo,
-        (APIError, RateLimitError, APIConnectionError),
-    )
-    def generate(self, prompt: list = None, max_new_tokens=4096, temperature=None, model=None, image_path=None,
-                 ouput_0=None, turn_number=0, **kwargs):
-        self.current_key_idx = (self.current_key_idx + 1) % len(self.time_slots)
-        start_time = time.time()
-        if (
-                self.request_interval > 0
-                and start_time < self.next_avil_time[self.current_key_idx]
-        ):
-            time.sleep(self.next_avil_time[self.current_key_idx] - start_time)
-        prompt0, prompt1, prompt2 = prompt
-        # litellm.set_verbose=True
-
-        base64_image = encode_image(image_path)
-        if turn_number == 0:
-            # Assume one turn dialogue
-            prompt_input = [
-                {"role": "system", "content": [{"type": "text", "text": prompt0}]},
-                {"role": "user",
-                 "content": [{"type": "text", "text": prompt1}, {"type": "image_url", "image_url": {"url":
-                                                                                                        f"data:image/jpeg;base64,{base64_image}",
-                                                                                                    "detail": "high"},
-                                                                 }]},
-            ]
-        elif turn_number == 1:
-            prompt_input = [
-                {"role": "system", "content": [{"type": "text", "text": prompt0}]},
-                {"role": "user",
-                 "content": [{"type": "text", "text": prompt1}, {"type": "image_url", "image_url": {"url":
-                                                                                                        f"data:image/jpeg;base64,{base64_image}",
-                                                                                                    "detail": "high"}, }]},
-                {"role": "assistant", "content": [{"type": "text", "text": f"\n\n{ouput_0}"}]},
-                {"role": "user", "content": [{"type": "text", "text": prompt2}]}, 
-            ]
-        response = litellm.completion(
-            model=model if model else self.model,
-            messages=prompt_input,
-            max_tokens=max_new_tokens if max_new_tokens else 4096,
-            temperature=temperature if temperature else self.temperature,
-            **kwargs,
-        )
-        return [choice["message"]["content"] for choice in response.choices][0]
-
-
-class OpenaiEngine_MindAct(Engine):
-    def __init__(self, **kwargs) -> None:
-        """Init an OpenAI GPT/Codex engine
-
-        Args:
-            api_key (_type_, optional): Auth key from OpenAI. Defaults to None.
-            stop (list, optional): Tokens indicate stop of sequence. Defaults to ["\n"].
-            rate_limit (int, optional): Max number of requests per minute. Defaults to -1.
-            model (_type_, optional): Model family. Defaults to None.
-        """
-        super().__init__(**kwargs)
-    #
-    @backoff.on_exception(
-        backoff.expo,
-        (APIError, RateLimitError, APIConnectionError),
-    )
-    def generate(self, prompt, max_new_tokens=50, temperature=0, model=None, **kwargs):
-        self.current_key_idx = (self.current_key_idx + 1) % len(self.time_slots)
-        start_time = time.time()
-        if (
-                self.request_interval > 0
-                and start_time < self.next_avil_time[self.current_key_idx]
-        ):
-            time.sleep(self.next_avil_time[self.current_key_idx] - start_time)
-        if isinstance(prompt, str):
-            # Assume one turn dialogue
-            prompt = [
-                {"role": "user", "content": prompt},
-            ]
-        response = litellm.completion(
-            model=model if model else self.model,
-            messages=prompt,
-            max_tokens=max_new_tokens,
-            temperature=temperature,
-            **kwargs,
-        )
-        if self.request_interval > 0:
-            self.next_avil_time[self.current_key_idx] = (
-                    max(start_time, self.next_avil_time[self.current_key_idx])
-                    + self.request_interval
-            )
-        return [choice["message"]["content"] for choice in response["choices"]]

llava/eval/seeact_run.py

@@ -1,118 +0,0 @@
-import asyncio
-import os
-import random
-
-from seeact.agent import SeeActAgent
-
-# Setup your API Key here, or pass through environment
-os.environ["OPENAI_API_KEY"] = "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"
-# os.environ["GEMINI_API_KEY"] = "Your API KEY Here"
-
-
-async def run_agent():
-    agent = SeeActAgent(model="gpt-4o")
-    await agent.start()
-    while not agent.complete_flag:
-        prediction_dict = await agent.predict()
-        await agent.execute(prediction_dict)
-    await agent.stop()
-
-if __name__ == "__main__":
-    asyncio.run(run_agent())
-#
-# async def run_agent():
-#
-#     import json
-#
-#     # Define the input file path
-#     input_file = "tasks.json"
-#
-#     # Load the data from the input file
-#     with open(input_file, 'r', encoding='utf-8') as infile:
-#         data = json.load(infile)
-#
-#     # Iterate through each data point and print the required fields
-#
-#     data=[i for i in data]
-#     random.shuffle(data)
-#     for item in data:
-#
-#
-#         website = item.get("website", "N/A")
-#         task_id = item.get("task_id", "N/A")
-#         confirmed_task = item.get("confirmed_task", "N/A")
-#
-#         agent = SeeActAgent(model="gpt-4o",default_task=confirmed_task,default_website=website,save_task_id=task_id)
-#         await agent.start()
-#         count = 0
-#         try:
-#             while not agent.complete_flag and count < 20:
-#                 prediction_dict = await agent.predict()
-#                 await agent.execute(prediction_dict)
-#                 count += 1
-#         except Exception as e:
-#             print(e)
-#         try:
-#             await agent.stop()
-#         except Exception as e:
-#             print(e)
-#
-#
-#
-#
-# if __name__ == "__main__":
-#     asyncio.run(run_agent())
-
-
-
-
-
-
-
-# import asyncio
-# import os
-# from seeact.agent import SeeActAgent
-#
-# # Setup your API Key here, or pass through environment
-# os.environ["OPENAI_API_KEY"] = "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"
-# # os.environ["GEMINI_API_KEY"] = "Your API KEY Here"
-#
-# async def process_task(item):
-#     website = item.get("website", "N/A")
-#     task_id = item.get("task_id", "N/A")
-#     confirmed_task = item.get("confirmed_task", "N/A")
-#
-#     agent = SeeActAgent(model="gpt-4o", default_task=confirmed_task, default_website=website, save_task_id=task_id,save_file_dir="mind2web-online90")
-#     await agent.start()
-#     count = 0
-#     while not agent.complete_flag and count < 20:
-#         prediction_dict = await agent.predict()
-#         await agent.execute(prediction_dict)
-#         count += 1
-#     await agent.stop()
-#
-# async def run_agent():
-#     import json
-#
-#     # Define the input file path
-#     input_file = "tasks.json"
-#
-#     # Load the data from the input file
-#     with open(input_file, 'r', encoding='utf-8') as infile:
-#         data = json.load(infile)
-#
-#     # Use a semaphore to limit the number of concurrent tasks
-#     semaphore = asyncio.Semaphore(1)
-#
-#     async def sem_task(item):
-#         async with semaphore:
-#             await process_task(item)
-#
-#     # Schedule all the tasks
-#     tasks = [sem_task(item) for item in data]
-#     await asyncio.gather(*tasks)
-#
-# if __name__ == "__main__":
-#     asyncio.run(run_agent())
-
-