Update

app.py

@@ -1,57 +1,24 @@
-import gradio as gr
 import os
-import datetime
+import sys
+import json
+import argparse
+import time
+import io
+import uuid
 from PIL import Image
+from typing import List, Dict, Any, Iterator
+import gradio as gr
+
+# Add the project root to the Python path
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))
+sys.path.insert(0, project_root)
 
-# Create a directory for uploaded images
-os.makedirs("uploaded_images", exist_ok=True)
-
-def save_image(image):
-    # Generate a timestamped filename
-    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-    file_path = f"uploaded_images/image_{timestamp}.png"
-    
-    # Save the image
-    image.save(file_path)
-
-    # Check if saved
-    print(f"Image saved to: {file_path} | Exists: {os.path.exists(file_path)}")
-
-    # Open image and print dims
-    img = Image.open(file_path)
-    print(f"Image dimensions: {img.size}")
-
-    return f"Image saved to: {file_path}"
-
-demo = gr.Interface(
-    fn=save_image,
-    inputs=gr.Image(type="pil"),  # Accepts PIL Image objects
-    outputs="text"
-)
-
-demo.launch()
-
-# import os
-# import sys
-# import json
-# import argparse
-# import time
-# import io
-# import uuid
-# from PIL import Image
-# from typing import List, Dict, Any, Iterator
-# import gradio as gr
-
-# # Add the project root to the Python path
-# current_dir = os.path.dirname(os.path.abspath(__file__))
-# project_root = os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))
-# sys.path.insert(0, project_root)
-
-# from opentools.models.initializer import Initializer
-# from opentools.models.planner import Planner
-# from opentools.models.memory import Memory
-# from opentools.models.executor import Executor
-# from opentools.models.utlis import make_json_serializable
+from .opentools.models.initializer import Initializer
+from .opentools.models.planner import Planner
+from .opentools.models.memory import Memory
+from .opentools.models.executor import Executor
+from .opentools.models.utlis import make_json_serializable
 
 # solver = None
 

opentools/engine/base.py

@@ -0,0 +1,43 @@
+import hashlib
+import diskcache as dc
+from abc import ABC, abstractmethod
+
+class EngineLM(ABC):
+    system_prompt: str = "You are a helpful, creative, and smart assistant."
+    model_string: str
+    @abstractmethod
+    def generate(self, prompt, system_prompt=None, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        pass
+
+
+class CachedEngine:
+    def __init__(self, cache_path):
+        super().__init__()
+        self.cache_path = cache_path
+        self.cache = dc.Cache(cache_path)
+
+    def _hash_prompt(self, prompt: str):
+        return hashlib.sha256(f"{prompt}".encode()).hexdigest()
+
+    def _check_cache(self, prompt: str):
+        if prompt in self.cache:
+            return self.cache[prompt]
+        else:
+            return None
+
+    def _save_cache(self, prompt: str, response: str):
+        self.cache[prompt] = response
+
+    def __getstate__(self):
+        # Remove the cache from the state before pickling
+        state = self.__dict__.copy()
+        del state['cache']
+        return state
+
+    def __setstate__(self, state):
+        # Restore the cache after unpickling
+        self.__dict__.update(state)
+        self.cache = dc.Cache(self.cache_path)

opentools/engine/openai.py

@@ -0,0 +1,268 @@
+try:
+    from openai import OpenAI
+except ImportError:
+    raise ImportError("If you'd like to use OpenAI models, please install the openai package by running `pip install openai`, and add 'OPENAI_API_KEY' to your environment variables.")
+
+import os
+import json
+import base64
+import platformdirs
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from typing import List, Union
+
+from .base import EngineLM, CachedEngine
+
+import openai
+
+from dotenv import load_dotenv
+load_dotenv()
+
+# Define global constant for structured models
+# https://platform.openai.com/docs/guides/structured-outputs
+# https://cookbook.openai.com/examples/structured_outputs_intro
+from pydantic import BaseModel
+
+class DefaultFormat(BaseModel):
+    response: str
+
+# Define global constant for structured models
+OPENAI_STRUCTURED_MODELS = ['gpt-4o', 'gpt-4o-2024-08-06','gpt-4o-mini',  'gpt-4o-mini-2024-07-18']
+
+
+class ChatOpenAI(EngineLM, CachedEngine):
+    DEFAULT_SYSTEM_PROMPT = "You are a helpful, creative, and smart assistant."
+
+    def __init__(
+        self,
+        model_string="gpt-4o-mini-2024-07-18",
+        system_prompt=DEFAULT_SYSTEM_PROMPT,
+        is_multimodal: bool=False,
+        # enable_cache: bool=True,
+        enable_cache: bool=False, # NOTE: disable cache for now
+        **kwargs):
+        """
+        :param model_string:
+        :param system_prompt:
+        :param is_multimodal:
+        """
+        if enable_cache:
+            root = platformdirs.user_cache_dir("opentools")
+            cache_path = os.path.join(root, f"cache_openai_{model_string}.db")
+            # For example, cache_path = /root/.cache/opentools/cache_openai_gpt-4o-mini.db
+            # print(f"Cache path: {cache_path}")
+            
+            self.image_cache_dir = os.path.join(root, "image_cache")
+            os.makedirs(self.image_cache_dir, exist_ok=True)
+
+            super().__init__(cache_path=cache_path)
+
+        self.system_prompt = system_prompt
+        if os.getenv("OPENAI_API_KEY") is None:
+            raise ValueError("Please set the OPENAI_API_KEY environment variable if you'd like to use OpenAI models.")
+        
+        self.client = OpenAI(
+            api_key=os.getenv("OPENAI_API_KEY"),
+        )
+        self.model_string = model_string
+        self.is_multimodal = is_multimodal
+        self.enable_cache = enable_cache
+
+        if enable_cache:
+            print(f"!! Cache enabled for model: {self.model_string}")
+        else:
+            print(f"!! Cache disabled for model: {self.model_string}")
+
+    @retry(wait=wait_random_exponential(min=1, max=5), stop=stop_after_attempt(5))
+    def generate(self, content: Union[str, List[Union[str, bytes]]], system_prompt=None, **kwargs):
+        try:
+            # Print retry attempt information
+            attempt_number = self.generate.retry.statistics.get('attempt_number', 0) + 1
+            if attempt_number > 1:
+                print(f"Attempt {attempt_number} of 5")
+
+            if isinstance(content, str):
+                return self._generate_text(content, system_prompt=system_prompt, **kwargs)
+            
+            elif isinstance(content, list):
+                if (not self.is_multimodal):
+                    raise NotImplementedError("Multimodal generation is only supported for GPT-4 models.")
+                
+                return self._generate_multimodal(content, system_prompt=system_prompt, **kwargs)
+
+        except openai.LengthFinishReasonError as e:
+            print(f"Token limit exceeded: {str(e)}")
+            print(f"Tokens used - Completion: {e.completion.usage.completion_tokens}, Prompt: {e.completion.usage.prompt_tokens}, Total: {e.completion.usage.total_tokens}")
+            return {
+                "error": "token_limit_exceeded",
+                "message": str(e),
+                "details": {
+                    "completion_tokens": e.completion.usage.completion_tokens,
+                    "prompt_tokens": e.completion.usage.prompt_tokens,
+                    "total_tokens": e.completion.usage.total_tokens
+                }
+            }
+        except openai.RateLimitError as e:
+            print(f"Rate limit error encountered: {str(e)}")
+            return {
+                "error": "rate_limit",
+                "message": str(e),
+                "details": getattr(e, 'args', None)
+            }
+        except Exception as e:
+            print(f"Error in generate method: {str(e)}")
+            print(f"Error type: {type(e).__name__}")
+            print(f"Error details: {e.args}")
+            return {
+                "error": type(e).__name__,
+                "message": str(e),
+                "details": getattr(e, 'args', None)
+            }
+        
+    def _generate_text(
+        self, prompt, system_prompt=None, temperature=0, max_tokens=4000, top_p=0.99, response_format=None
+    ):
+
+        sys_prompt_arg = system_prompt if system_prompt else self.system_prompt
+
+        if self.enable_cache:
+            cache_key = sys_prompt_arg + prompt
+            cache_or_none = self._check_cache(cache_key)
+            if cache_or_none is not None:
+                return cache_or_none
+
+        if self.model_string in ['o1', 'o1-mini']: # only supports base response currently
+            # print(f"Using structured model: {self.model_string}")
+            response = self.client.beta.chat.completions.parse(
+                model=self.model_string,
+                messages=[
+                    {"role": "user", "content": prompt},
+                ],
+                max_completion_tokens=max_tokens
+            )
+            if response.choices[0].finishreason == "length":
+                response = "Token limit exceeded"
+            else:
+                response = response.choices[0].message.parsed
+        elif self.model_string in OPENAI_STRUCTURED_MODELS and response_format is not None:
+            # print(f"Using structured model: {self.model_string}")
+            response = self.client.beta.chat.completions.parse(
+                model=self.model_string,
+                messages=[
+                    {"role": "system", "content": sys_prompt_arg},
+                    {"role": "user", "content": prompt},
+                ],
+                frequency_penalty=0,
+                presence_penalty=0,
+                stop=None,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                top_p=top_p,
+                response_format=response_format
+            )
+            response = response.choices[0].message.parsed
+        else:
+            # print(f"Using non-structured model: {self.model_string}")
+            response = self.client.chat.completions.create(
+                model=self.model_string,
+                messages=[
+                    {"role": "system", "content": sys_prompt_arg},
+                    {"role": "user", "content": prompt},
+                ],
+                frequency_penalty=0,
+                presence_penalty=0,
+                stop=None,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                top_p=top_p,
+            )
+            response = response.choices[0].message.content
+
+        if self.enable_cache:
+            self._save_cache(cache_key, response)
+        return response
+
+    def __call__(self, prompt, **kwargs):
+        return self.generate(prompt, **kwargs)
+
+    def _format_content(self, content: List[Union[str, bytes]]) -> List[dict]:
+        formatted_content = []
+        for item in content:
+            if isinstance(item, bytes):
+                base64_image = base64.b64encode(item).decode('utf-8')
+                formatted_content.append({
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{base64_image}"
+                    }
+                })
+            elif isinstance(item, str):
+                formatted_content.append({
+                    "type": "text",
+                    "text": item
+                })
+            else:
+                raise ValueError(f"Unsupported input type: {type(item)}")
+        return formatted_content
+
+    def _generate_multimodal(
+        self, content: List[Union[str, bytes]], system_prompt=None, temperature=0, max_tokens=4000, top_p=0.99, response_format=None
+    ):
+        sys_prompt_arg = system_prompt if system_prompt else self.system_prompt
+        formatted_content = self._format_content(content)
+
+        if self.enable_cache:
+            cache_key = sys_prompt_arg + json.dumps(formatted_content)
+            cache_or_none = self._check_cache(cache_key)
+            if cache_or_none is not None:
+                # print(f"Cache hit for prompt: {cache_key[:200]}")
+                return cache_or_none
+
+        if self.model_string in ['o1', 'o1-mini']: # only supports base response currently
+            # print(f"Using structured model: {self.model_string}")
+            print(f'Max tokens: {max_tokens}')
+            response = self.client.chat.completions.create(
+                model=self.model_string,
+                messages=[
+                    {"role": "user", "content": formatted_content},
+                ],
+                max_completion_tokens=max_tokens
+            )
+            if response.choices[0].finish_reason == "length":
+                response_text = "Token limit exceeded"
+            else:
+                response_text = response.choices[0].message.content
+        elif self.model_string in OPENAI_STRUCTURED_MODELS and response_format is not None:
+            # print(f"Using structured model: {self.model_string}")
+            response = self.client.beta.chat.completions.parse(
+                model=self.model_string,
+                messages=[
+                    {"role": "system", "content": sys_prompt_arg},
+                    {"role": "user", "content": formatted_content},
+                ],
+                temperature=temperature,
+                max_tokens=max_tokens,
+                top_p=top_p,
+                response_format=response_format
+            )
+            response_text = response.choices[0].message.parsed
+        else:
+            # print(f"Using non-structured model: {self.model_string}")
+            response = self.client.chat.completions.create(
+                model=self.model_string,
+                messages=[
+                    {"role": "system", "content": sys_prompt_arg},
+                    {"role": "user", "content": formatted_content},
+                ],
+                temperature=temperature,
+                max_tokens=max_tokens,
+                top_p=top_p,
+            )
+            response_text = response.choices[0].message.content
+
+        if self.enable_cache:
+            self._save_cache(cache_key, response_text)
+        return response_text

opentools/models/executor.py

@@ -0,0 +1,240 @@
+import os
+# import sys
+import importlib
+import re
+from typing import Dict, Any, List
+from datetime import datetime
+
+from opentools.engine.openai import ChatOpenAI 
+from opentools.models.formatters import ToolCommand
+
+import signal
+from typing import Dict, Any, List, Optional
+
+class TimeoutError(Exception):
+    pass
+
+def timeout_handler(signum, frame):
+    raise TimeoutError("Function execution timed out")
+
+class Executor:
+    def __init__(self, llm_engine_name: str, root_cache_dir: str = "solver_cache",  num_threads: int = 1, max_time: int = 120, max_output_length: int = 100000, enable_signal: bool = True):
+        self.llm_engine_name = llm_engine_name
+        self.root_cache_dir = root_cache_dir
+        self.num_threads = num_threads
+        self.max_time = max_time
+        self.max_output_length = max_output_length
+        self.enable_signal = enable_signal
+
+    def set_query_cache_dir(self, query_cache_dir):
+        if query_cache_dir:
+            self.query_cache_dir = query_cache_dir
+        else:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            self.query_cache_dir = os.path.join(self.root_cache_dir, timestamp)
+        os.makedirs(self.query_cache_dir, exist_ok=True)
+    
+    def generate_tool_command(self, question: str, image: str, context: str, sub_goal: str, tool_name: str, tool_metadata: Dict[str, Any], bytes_mode:bool = False) -> ToolCommand:
+        prompt_generate_tool_command = f"""
+Task: Generate a precise command to execute the selected tool based on the given information.
+
+Query: {question}
+Image: {image if not bytes_mode else 'image.jpg'}
+Context: {context}
+Sub-Goal: {sub_goal}
+Selected Tool: {tool_name}
+Tool Metadata: {tool_metadata}
+
+Instructions:
+1. Carefully review all provided information: the query, image path, context, sub-goal, selected tool, and tool metadata.
+2. Analyze the tool's input_types from the metadata to understand required and optional parameters.
+3. Construct a command or series of commands that aligns with the tool's usage pattern and addresses the sub-goal.
+4. Ensure all required parameters are included and properly formatted.
+5. Use appropriate values for parameters based on the given context, particularly the `Context` field which may contain relevant information from previous steps.
+6. If multiple steps are needed to prepare data for the tool, include them in the command construction.
+
+Output Format:
+<analysis>: a step-by-step analysis of the context, sub-goal, and selected tool to guide the command construction.
+<explanation>: a detailed explanation of the constructed command(s) and their parameters.
+<command>: the Python code to execute the tool, which can be one of the following types:
+    a. A single line command with `execution = tool.execute()`.
+    b. A multi-line command with complex data preparation, ending with `execution = tool.execute()`.
+    c. Multiple lines of `execution = tool.execute()` calls for processing multiple items.
+```python
+<your command here>
+```
+
+Rules:
+1. The command MUST be valid Python code and include at least one call to `tool.execute()`.
+2. Each `tool.execute()` call MUST be assigned to the 'execution' variable in the format `execution = tool.execute(...)`.
+3. For multiple executions, use separate `execution = tool.execute()` calls for each execution.
+4. The final output MUST be assigned to the 'execution' variable, either directly from `tool.execute()` or as a processed form of multiple executions.
+5. Use the exact parameter names as specified in the tool's input_types.
+6. Enclose string values in quotes, use appropriate data types for other values (e.g., lists, numbers).
+7. Do not include any code or text that is not part of the actual command.
+8. Ensure the command directly addresses the sub-goal and query.
+9. Include ALL required parameters, data, and paths to execute the tool in the command itself.
+10. If preparation steps are needed, include them as separate Python statements before the `tool.execute()` calls.
+
+Examples (Not to use directly unless relevant):
+
+Example 1 (Single line command):
+<analysis>: The tool requires an image path and a list of labels for object detection.
+<explanation>: We pass the image path and a list containing "baseball" as the label to detect.
+<command>:
+```python
+execution = tool.execute(image="path/to/image", labels=["baseball"])
+```
+
+Example 2 (Multi-line command with data preparation):
+<analysis>: The tool requires an image path, multiple labels, and a threshold for object detection.
+<explanation>: We prepare the data by defining variables for the image path, labels, and threshold, then pass these to the tool.execute() function.
+<command>:
+```python
+image = "path/to/image"
+labels = ["baseball", "football", "basketball"]
+threshold = 0.5
+execution = tool.execute(image=image, labels=labels, threshold=threshold)
+```
+
+Example 3 (Multiple executions):
+<analysis>: We need to process multiple images for baseball detection.
+<explanation>: We call the tool for each image path, using the same label and threshold for all.
+<command>:
+```python
+execution = tool.execute(image="path/to/image1", labels=["baseball"], threshold=0.5)
+execution = tool.execute(image="path/to/image2", labels=["baseball"], threshold=0.5)
+execution = tool.execute(image="path/to/image3", labels=["baseball"], threshold=0.5)
+```
+
+Some Wrong Examples:
+<command>:
+```python
+execution1 = tool.execute(query="...")
+execution2 = tool.execute(query="...")
+```
+Reason: only `execution = tool.execute` is allowed, not `execution1` or `execution2`.
+
+<command>:
+```python
+urls = [
+    "https://example.com/article1",
+    "https://example.com/article2"
+]
+
+execution = tool.execute(url=urls[0])
+execution = tool.execute(url=urls[1])
+```
+Reason: The command should process multiple items in a single execution, not separate executions for each item.
+
+Remember: Your <command> field MUST be valid Python code including any necessary data preparation steps and one or more `execution = tool.execute(` calls, without any additional explanatory text. The format `execution = tool.execute` must be strictly followed, and the last line must begin with `execution = tool.execute` to capture the final output.
+"""
+
+        llm_generate_tool_command = ChatOpenAI(model_string=self.llm_engine_name, is_multimodal=False)
+        tool_command = llm_generate_tool_command(prompt_generate_tool_command, response_format=ToolCommand)
+
+        return tool_command
+
+    # def extract_explanation_and_command(self, text: str) -> tuple:
+    #     # Extract explanation
+    #     explanation_pattern = r"Command Explanation:(.*?)Generated Command:"
+    #     explanation_match = re.search(explanation_pattern, text, re.DOTALL)
+    #     explanation = explanation_match.group(1).strip() if explanation_match else "No explanation found."
+    #     # Extract command
+    #     command_pattern = r"Generated Command:.*?```python\n(.*?)```"
+    #     command_match = re.search(command_pattern, text, re.DOTALL)
+    #     command = command_match.group(1).strip() if command_match else "No command found."
+
+    def extract_explanation_and_command(self, response: ToolCommand) -> tuple:
+        def normarlize_code(code: str) -> str:
+            # Remove leading and trailing whitespace and triple backticks
+            return re.sub(r'^```python\s*', '', code).rstrip('```').strip()
+        
+        explanation = response.explanation.strip()
+        command = normarlize_code(response.command.strip())
+        return explanation, command
+
+    def execute_tool_command(self, tool_name: str, command: str) -> Any:
+        """
+        Execute a tool command with timeout protection. If execution exceeds max_time seconds,
+        the function will be interrupted and return a timeout message.
+        
+        Args:
+            tool_name (str): Name of the tool to execute
+            command (str): Command string containing tool.execute() calls
+            
+        Returns:
+            Any: List of execution results or error message
+        """
+        
+        def split_commands(command: str) -> List[str]:
+            # Use regex to find all tool.execute() commands and their surrounding code
+            pattern = r'.*?execution\s*=\s*tool\.execute\([^\n]*\)\s*(?:\n|$)'
+            blocks = re.findall(pattern, command, re.DOTALL)
+            return [block.strip() for block in blocks if block.strip()]
+        
+        def execute_with_timeout(block: str, local_context: dict) -> Optional[str]:
+            if self.enable_signal:
+                # Set up the timeout handler
+                signal.signal(signal.SIGALRM, timeout_handler)
+                signal.alarm(self.max_time)
+            
+            try:
+                # Execute the block in the local context
+                exec(block, globals(), local_context)
+                result = local_context.get('execution')
+                if self.enable_signal:
+                    signal.alarm(0)  # Disable the alarm
+                return result
+            except TimeoutError:
+                return f"Execution timed out after {self.max_time} seconds"
+            finally:
+                if self.enable_signal:
+                    signal.alarm(0)  # Ensure alarm is disabled even if other exceptions occur
+
+        # Import the tool module and instantiate it
+        module_name = f"tools.{tool_name.lower().replace('_tool', '')}.tool"
+        
+        # print(f"Attempting to import module: {module_name}")
+        # print(f"Current sys.path: {sys.path}")
+
+        try:
+            # Dynamically import the module
+            module = importlib.import_module(module_name)
+
+            # Get the tool class
+            tool_class = getattr(module, tool_name)
+
+            # Check if the tool requires an LLM engine
+            # NOTE FIXME may need to refine base.py and tool.py to handle this better
+            if getattr(tool_class, 'require_llm_engine', False):
+                # Instantiate the tool with the model_string
+                tool = tool_class(model_string=self.llm_engine_name)
+            else:
+                # Instantiate the tool without model_string for tools that don't require it
+                tool = tool_class()
+            
+            # Set the custom output directory
+            # NOTE FIXME: May have a better way to handle this
+            tool.set_custom_output_dir(self.query_cache_dir)
+
+            # Split the command into blocks, execute each one and store execution results
+            command_blocks = split_commands(command)
+            executions = []
+
+            for block in command_blocks:
+                # Create a local context to safely execute the block
+                local_context = {'tool': tool}
+
+                # Execute the block with timeout protection
+                result = execute_with_timeout(block, local_context)
+                
+                if result is not None:
+                    executions.append(result)
+                else:
+                    executions.append(f"No execution captured from block: {block}")
+
+            # Return all the execution results
+            return executions
+        except Exception as e:
+            return f"Error in execute_tool_command: {str(e)}"

opentools/models/formatters.py

@@ -0,0 +1,40 @@
+from pydantic import BaseModel
+
+# Planner: QueryAnalysis
+class QueryAnalysis(BaseModel):
+    consice_summary: str
+    required_skills: str
+    relevant_tools: str
+    additional_considerations: str
+
+    def __str__(self):
+        return f"""
+Consice Summary: {self.consice_summary}
+
+Required Skills:
+{self.required_skills}
+
+Relevant Tools:
+{self.relevant_tools}
+
+Additional Considerations:
+{self.additional_considerations}
+"""
+
+# Planner: NextStep
+class NextStep(BaseModel):
+    justification: str
+    context: str
+    sub_goal: str
+    tool_name: str
+
+# Executor: MemoryVerification
+class MemoryVerification(BaseModel):
+    analysis: str
+    stop_signal: bool
+
+# Executor: ToolCommand
+class ToolCommand(BaseModel):
+    analysis: str
+    explanation: str
+    command: str

opentools/models/initializer.py

@@ -0,0 +1,179 @@
+import os
+import sys
+import importlib
+import inspect
+import traceback
+from typing import Dict, Any, List, Tuple
+
+
+class Initializer:
+    def __init__(self, enabled_tools: List[str] = [], model_string: str = None):
+        self.toolbox_metadata = {}
+        self.available_tools = []
+        self.enabled_tools = enabled_tools
+        self.model_string = model_string # llm model string
+
+        print("\nInitializing OpenTools...")
+        print(f"Enabled tools: {self.enabled_tools}")
+        print(f"LLM model string: {self.model_string}")
+        self._set_up_tools()
+
+    def get_project_root(self):
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        while current_dir != '/':
+            if os.path.exists(os.path.join(current_dir, 'opentools')):
+                return os.path.join(current_dir, 'opentools')
+            current_dir = os.path.dirname(current_dir)
+        raise Exception("Could not find project root")
+        
+    def load_tools_and_get_metadata(self) -> Dict[str, Any]:
+        # Implementation of load_tools_and_get_metadata function
+        print("Loading tools and getting metadata...")
+        self.toolbox_metadata = {}
+        opentools_dir = self.get_project_root()
+        tools_dir = os.path.join(opentools_dir, 'tools')
+        
+        print(f"OpenTools directory: {opentools_dir}")
+        print(f"Tools directory: {tools_dir}")
+        
+        # Add the OpenTools directory and its parent to the Python path
+        sys.path.insert(0, opentools_dir)
+        sys.path.insert(0, os.path.dirname(opentools_dir))
+        print(f"Updated Python path: {sys.path}")
+        
+        if not os.path.exists(tools_dir):
+            print(f"Error: Tools directory does not exist: {tools_dir}")
+            return self.toolbox_metadata
+
+        for root, dirs, files in os.walk(tools_dir):
+            # print(f"\nScanning directory: {root}")
+            if 'tool.py' in files and os.path.basename(root) in self.available_tools:
+                file = 'tool.py'
+                module_path = os.path.join(root, file)
+                module_name = os.path.splitext(file)[0]
+                relative_path = os.path.relpath(module_path, opentools_dir)
+                import_path = '.'.join(os.path.split(relative_path)).replace(os.sep, '.')[:-3]
+
+                print(f"\nAttempting to import: {import_path}")
+                try:
+                    module = importlib.import_module(import_path)
+                    for name, obj in inspect.getmembers(module):
+                        if inspect.isclass(obj) and name.endswith('Tool') and name != 'BaseTool':
+                            print(f"Found tool class: {name}")
+                            # print(f"Class attributes: {dir(obj)}")
+                            # print(f"Class __dict__: {obj.__dict__}")
+                            try:
+                                # Check if the tool requires an LLM engine
+                                if hasattr(obj, 'require_llm_engine') and obj.require_llm_engine:
+                                    tool_instance = obj(model_string=self.model_string)
+                                else:
+                                    tool_instance = obj()
+
+                                # print(f"\nInstance attributes: {dir(tool_instance)}")
+                                # print(f"\nInstance __dict__: {tool_instance.__dict__}")
+                                
+                                self.toolbox_metadata[name] = {
+                                    'tool_name': getattr(tool_instance, 'tool_name', 'Unknown'),
+                                    'tool_description': getattr(tool_instance, 'tool_description', 'No description'),
+                                    'tool_version': getattr(tool_instance, 'tool_version', 'Unknown'),
+                                    'input_types': getattr(tool_instance, 'input_types', {}),
+                                    'output_type': getattr(tool_instance, 'output_type', 'Unknown'),
+                                    'demo_commands': getattr(tool_instance, 'demo_commands', []),
+                                    'user_metadata': getattr(tool_instance, 'user_metadata', {}), # NOTE: This is a placeholder for user-defined metadata
+                                    'require_llm_engine': getattr(obj, 'require_llm_engine', False),
+                                }
+                                print(f"\nMetadata for {name}: {self.toolbox_metadata[name]}")
+                            except Exception as e:
+                                print(f"Error instantiating {name}: {str(e)}")
+                except Exception as e:
+                    print(f"Error loading module {module_name}: {str(e)}")
+                    
+        print(f"\nTotal number of tools loaded: {len(self.toolbox_metadata)}")
+
+        return self.toolbox_metadata
+
+    def run_demo_commands(self) -> List[str]:
+        print("\nRunning demo commands for each tool...")
+        self.available_tools = []
+
+        for tool_name, tool_data in self.toolbox_metadata.items():
+            print(f"\nChecking availability of {tool_name}...")
+
+            try:
+                # Import the tool module
+                module_name = f"tools.{tool_name.lower().replace('_tool', '')}.tool"
+                module = importlib.import_module(module_name)
+
+                # Get the tool class
+                tool_class = getattr(module, tool_name)
+
+                # Instantiate the tool
+                tool_instance = tool_class()
+
+                # FIXME This is a temporary workaround to avoid running demo commands
+                self.available_tools.append(tool_name)
+
+                # # TODO Run the first demo command if available
+                # demo_commands = tool_data.get('demo_commands', [])
+                # if demo_commands:
+                #     print(f"Running demo command: {demo_commands[0]['command']}")
+                #     # Extract the arguments from the demo command
+                #     command = demo_commands[0]['command']
+                #     args_start = command.index('(') + 1
+                #     args_end = command.rindex(')')
+                #     args_str = command[args_start:args_end]
+
+                #     # Create a dictionary of arguments
+                #     args_dict = eval(f"dict({args_str})")
+
+                #     # Execute the demo command
+                #     result = tool_instance.execute(**args_dict)
+                #     print(f"Demo command executed successfully. Result: {result}")
+
+                #     self.available_tools.append(tool_name)
+                # else:
+                #     print(f"No demo commands available for {tool_name}")
+                #     # If no demo commands, we'll assume the tool is available
+                #     self.available_tools.append(tool_name)
+
+            except Exception as e:
+                print(f"Error checking availability of {tool_name}: {str(e)}")
+                print(traceback.format_exc())
+
+        # update the toolmetadata with the available tools
+        self.toolbox_metadata = {tool: self.toolbox_metadata[tool] for tool in self.available_tools}
+        print(f"\nUpdated total number of available tools: {len(self.toolbox_metadata)}")
+        print(f"\nAvailable tools: {self.available_tools}")
+
+        return self.available_tools
+    
+    def _set_up_tools(self) -> None:
+        print("Setting up tools...")
+
+        # Keep enabled tools
+        self.available_tools = [tool.lower().replace('_tool', '') for tool in self.enabled_tools]
+        
+        # Load tools and get metadata
+        self.load_tools_and_get_metadata()
+        
+        # Run demo commands to determine available tools
+        self.run_demo_commands()
+        
+        # Filter toolbox_metadata to include only available tools
+        self.toolbox_metadata = {tool: self.toolbox_metadata[tool] for tool in self.available_tools}
+        
+        print(f"\nTotal number of available tools: {len(self.available_tools)}")
+        print(f"Available tools: {self.available_tools}")
+        print(f"Enabled tools: {self.enabled_tools}")
+
+
+if __name__ == "__main__":
+    enabled_tools = ["Generalist_Solution_Generator_Tool"]
+    initializer = Initializer(enabled_tools=enabled_tools)
+
+    print("\nAvailable tools:")
+    print(initializer.available_tools)
+
+    print("\nToolbox metadata for available tools:")
+    print(initializer.toolbox_metadata)
+    

opentools/models/memory.py

@@ -0,0 +1,84 @@
+from typing import Dict, Any, List, Union, Optional
+import os
+
+class Memory:
+    # TODO Need to fix this to support multiple data sources (e.g. images, pdf, txt, etc.)
+    
+    def __init__(self):
+        self.query: Optional[str] = None
+        self.files: List[Dict[str, str]] = []
+        self.actions: Dict[str, Dict[str, Any]] = {}
+        self._init_file_types()
+
+    def set_query(self, query: str) -> None:
+        if not isinstance(query, str):
+            raise TypeError("Query must be a string")
+        self.query = query
+
+    def _init_file_types(self):
+        self.file_types = {
+            'image': ['.jpg', '.jpeg', '.png', '.gif', '.bmp'],
+            'text': ['.txt', '.md'],
+            'document': ['.pdf', '.doc', '.docx'],
+            'code': ['.py', '.js', '.java', '.cpp', '.h'],
+            'data': ['.json', '.csv', '.xml'],
+            'spreadsheet': ['.xlsx', '.xls'],
+            'presentation': ['.ppt', '.pptx'],
+        }
+        self.file_type_descriptions = {
+            'image': "An image file ({ext} format) provided as context for the query",
+            'text': "A text file ({ext} format) containing additional information related to the query",
+            'document': "A document ({ext} format) with content relevant to the query",
+            'code': "A source code file ({ext} format) potentially related to the query",
+            'data': "A data file ({ext} format) containing structured data pertinent to the query",
+            'spreadsheet': "A spreadsheet file ({ext} format) with tabular data relevant to the query",
+            'presentation': "A presentation file ({ext} format) with slides related to the query",
+        }
+
+    def _get_default_description(self, file_name: str) -> str:
+        _, ext = os.path.splitext(file_name)
+        ext = ext.lower()
+
+        for file_type, extensions in self.file_types.items():
+            if ext in extensions:
+                return self.file_type_descriptions[file_type].format(ext=ext[1:])
+
+        return f"A file with {ext[1:]} extension, provided as context for the query"
+    
+    def add_file(self, file_name: Union[str, List[str]], description: Union[str, List[str], None] = None) -> None:
+        if isinstance(file_name, str):
+            file_name = [file_name]
+        
+        if description is None:
+            description = [self._get_default_description(fname) for fname in file_name]
+        elif isinstance(description, str):
+            description = [description]
+        
+        if len(file_name) != len(description):
+            raise ValueError("The number of files and descriptions must match.")
+        
+        for fname, desc in zip(file_name, description):
+            self.files.append({
+                'file_name': fname,
+                'description': desc
+            })
+
+    def add_action(self, step_count: int, tool_name: str, sub_goal: str, command: str, result: Any) -> None:
+        action = {
+            'tool_name': tool_name,
+            'sub_goal': sub_goal,
+            'command': command,
+            'result': result,
+        }
+        step_name = f"Action Step {step_count}"
+        self.actions[step_name] = action
+
+    def get_query(self) -> Optional[str]:
+        return self.query
+
+    def get_files(self) -> List[Dict[str, str]]:
+        return self.files
+    
+    def get_actions(self) -> Dict[str, Dict[str, Any]]:
+        return self.actions
+    

opentools/models/planner.py

@@ -0,0 +1,368 @@
+import os
+import re
+from PIL import Image
+from io import BytesIO
+from typing import Dict, Any, List, Tuple
+
+from opentools.engine.openai import ChatOpenAI
+from opentools.models.memory import Memory
+from opentools.models.formatters import QueryAnalysis, NextStep, MemoryVerification
+
+class Planner:
+    def __init__(self, llm_engine_name: str, toolbox_metadata: dict = None, available_tools: List = None):
+        self.llm_engine_name = llm_engine_name
+        self.llm_engine_mm = ChatOpenAI(model_string=llm_engine_name, is_multimodal=True)
+        self.llm_engine = ChatOpenAI(model_string=llm_engine_name, is_multimodal=False)
+        self.toolbox_metadata = toolbox_metadata if toolbox_metadata is not None else {}
+        self.available_tools = available_tools if available_tools is not None else []
+
+    def get_image_info(self, image_path: str) -> Dict[str, Any]:
+        image_info = {}
+        if image_path and os.path.isfile(image_path):
+            image_info["image_path"] = image_path
+            try:
+                with Image.open(image_path) as img:
+                    width, height = img.size
+                image_info.update({
+                    "width": width,
+                    "height": height
+                })
+            except Exception as e:
+                print(f"Error processing image file: {str(e)}")
+        return image_info
+    
+    def get_image_info_bytes(self, bytes: str) -> Dict[str, Any]:
+        image_info = {}
+        if bytes:
+            try:
+                with Image.open(BytesIO(bytes)) as img:
+                    width, height = img.size
+                image_info.update({
+                    "image_path": 'image.jpg', # generic image name
+                    "width": width,
+                    "height": height
+                })
+            except Exception as e:
+                print(f"Error processing image bytes: {str(e)}")
+        return image_info
+
+    def generate_base_response(self, question: str, image: str, max_tokens: str = 4000, bytes_mode: bool = False) -> str:
+        if bytes_mode:
+            image_info = self.get_image_info_bytes(image)
+        else:
+            image_info = self.get_image_info(image)
+
+        input_data = [question]
+        if image_info and "image_path" in image_info and not bytes_mode:
+            try:
+                with open(image_info["image_path"], 'rb') as file:
+                    image_bytes = file.read()
+                input_data.append(image_bytes)
+            except Exception as e:
+                print(f"Error reading image file: {str(e)}")
+
+        self.base_response = self.llm_engine_mm(input_data, max_tokens=max_tokens)
+
+        return self.base_response
+
+    def analyze_query(self, question: str, image: str, bytes_mode: bool = False) -> str:
+        if bytes_mode:
+            image_info = self.get_image_info_bytes(image)
+        else:
+            image_info = self.get_image_info(image)
+        print("image_info: ", image_info)
+
+        query_prompt = f"""
+Task: Analyze the given query with accompanying inputs and determine the skills and tools needed to address it effectively.
+
+Available tools: {self.available_tools}
+
+Metadata for the tools: {self.toolbox_metadata}
+
+Image: {image_info}
+
+Query: {question}
+
+Instructions:
+1. Carefully read and understand the query and any accompanying inputs.
+2. Identify the main objectives or tasks within the query.
+3. List the specific skills that would be necessary to address the query comprehensively.
+4. Examine the available tools in the toolbox and determine which ones might relevant and useful for addressing the query. Make sure to consider the user metadata for each tool, including limitations and potential applications (if available).
+5. Provide a brief explanation for each skill and tool you've identified, describing how it would contribute to answering the query.
+
+Your response should include:
+1. A concise summary of the query's main points and objectives, as well as content in any accompanying inputs.
+2. A list of required skills, with a brief explanation for each.
+3. A list of relevant tools from the toolbox, with a brief explanation of how each tool would be utilized and its potential limitations.
+4. Any additional considerations that might be important for addressing the query effectively.
+
+Please present your analysis in a clear, structured format.
+"""
+
+        input_data = [query_prompt]
+        if bytes_mode:
+            image_bytes = image
+        else:
+            try:
+                with open(image_info["image_path"], 'rb') as file:
+                    image_bytes = file.read()
+                input_data.append(image_bytes)
+            except Exception as e:
+                print(f"Error reading image file: {str(e)}")
+
+        self.query_analysis = self.llm_engine_mm(input_data, response_format=QueryAnalysis)
+
+        return str(self.query_analysis).strip()
+
+    def extract_context_subgoal_and_tool(self, response: NextStep) -> Tuple[str, str, str]:
+
+        def normalize_tool_name(tool_name: str) -> str:
+            # Normalize the tool name to match the available tools
+            for tool in self.available_tools:
+                if tool.lower() in tool_name.lower():
+                    return tool
+            return "No matched tool given: " + tool_name
+        
+        try:
+            context = response.context.strip()
+            sub_goal = response.sub_goal.strip()
+            tool_name = normalize_tool_name(response.tool_name.strip())
+            return context, sub_goal, tool_name
+        except Exception as e:
+            print(f"Error extracting context, sub-goal, and tool name: {str(e)}")
+            return None, None, None
+        
+    def generate_next_step(self, question: str, image: str, query_analysis: str, memory: Memory, step_count: int, max_step_count: int, bytes_mode: bool = False) -> NextStep:
+        prompt_generate_next_step = f"""
+Task: Determine the optimal next step to address the given query based on the provided analysis, available tools, and previous steps taken.
+
+Context:
+Query: {question}
+Image: {image if not bytes_mode else 'image.jpg'}
+Query Analysis: {query_analysis}
+
+Available Tools:
+{self.available_tools}
+
+Tool Metadata:
+{self.toolbox_metadata}
+
+Previous Steps and Their Results:
+{memory.get_actions()}
+
+Current Step: {step_count} in {max_step_count} steps
+Remaining Steps: {max_step_count - step_count}
+
+Instructions:
+1. Analyze the context thoroughly, including the query, its analysis, any image, available tools and their metadata, and previous steps taken.
+
+2. Determine the most appropriate next step by considering:
+   - Key objectives from the query analysis
+   - Capabilities of available tools
+   - Logical progression of problem-solving
+   - Outcomes from previous steps
+   - Current step count and remaining steps
+
+3. Select ONE tool best suited for the next step, keeping in mind the limited number of remaining steps.
+
+4. Formulate a specific, achievable sub-goal for the selected tool that maximizes progress towards answering the query.
+
+Output Format:
+<justification>: detailed explanation of why the selected tool is the best choice for the next step, considering the context and previous outcomes.
+<context>: MUST include ALL necessary information for the tool to function, structured as follows:
+    * Relevant data from previous steps
+    * File names or paths created or used in previous steps (list EACH ONE individually)
+    * Variable names and their values from previous steps' results
+    * Any other context-specific information required by the tool
+<sub_goal>: a specific, achievable objective for the tool, based on its metadata and previous outcomes. It MUST contain any involved data, file names, and variables from Previous Steps and Their Results that the tool can act upon.
+<tool_name>: MUST be the exact name of a tool from the available tools list.
+
+Rules:
+- Select only ONE tool for this step.
+- The sub-goal MUST directly address the query and be achievable by the selected tool.
+- The Context section MUST include ALL necessary information for the tool to function, including ALL relevant file paths, data, and variables from previous steps.
+- The tool name MUST exactly match one from the available tools list: {self.available_tools}.
+- Avoid redundancy by considering previous steps and building on prior results.
+
+Example (do not copy, use only as reference):
+<justification>: [Your detailed explanation here]
+<context>: Image path: "example/image.jpg", Previous detection results: [list of objects]
+<sub_goal>: Detect and count the number of specific objects in the image "example/image.jpg"
+<tool_name>: Object_Detector_Tool
+"""
+        next_step = self.llm_engine(prompt_generate_next_step, response_format=NextStep)
+        return next_step
+
+    def verificate_memory(self, question: str, image: str, query_analysis: str, memory: Memory, bytes_mode: bool = False) -> MemoryVerification:
+        if bytes_mode:
+            image_info = self.get_image_info_bytes(image)
+        else:
+            image_info = self.get_image_info(image)
+
+        prompt_memory_verification = f"""
+Task: Thoroughly evaluate the completeness and accuracy of the memory for fulfilling the given query, considering the potential need for additional tool usage.
+
+Context:
+Query: {question}
+Image: {image_info}
+Available Tools: {self.available_tools}
+Toolbox Metadata: {self.toolbox_metadata}
+Initial Analysis: {query_analysis}
+Memory (tools used and results): {memory.get_actions()}
+
+Detailed Instructions:
+1. Carefully analyze the query, initial analysis, and image (if provided):
+   - Identify the main objectives of the query.
+   - Note any specific requirements or constraints mentioned.
+   - If an image is provided, consider its relevance and what information it contributes.
+
+2. Review the available tools and their metadata:
+   - Understand the capabilities and limitations and best practices of each tool.
+   - Consider how each tool might be applicable to the query.
+
+3. Examine the memory content in detail:
+   - Review each tool used and its execution results.
+   - Assess how well each tool's output contributes to answering the query.
+
+4. Critical Evaluation (address each point explicitly):
+   a) Completeness: Does the memory fully address all aspects of the query?
+      - Identify any parts of the query that remain unanswered.
+      - Consider if all relevant information has been extracted from the image (if applicable).
+
+   b) Unused Tools: Are there any unused tools that could provide additional relevant information?
+      - Specify which unused tools might be helpful and why.
+
+   c) Inconsistencies: Are there any contradictions or conflicts in the information provided?
+      - If yes, explain the inconsistencies and suggest how they might be resolved.
+
+   d) Verification Needs: Is there any information that requires further verification due to tool limitations?
+      - Identify specific pieces of information that need verification and explain why.
+
+   e) Ambiguities: Are there any unclear or ambiguous results that could be clarified by using another tool?
+      - Point out specific ambiguities and suggest which tools could help clarify them.
+
+5. Final Determination:
+   Based on your thorough analysis, decide if the memory is complete and accurate enough to generate the final output, or if additional tool usage is necessary.
+
+Response Format:
+<analysis>: Provide a detailed analysis of why the memory is sufficient. Reference specific information from the memory and explain its relevance to each aspect of the task. Address how each main point of the query has been satisfied.
+<stop_signal>: Whether to stop the problem solving process and proceed to generating the final output.
+    * "True": if the memory is sufficient for addressing the query to proceed and no additional available tools need to be used. If ONLY manual verification without tools is needed, choose "True".
+    * "False": if the memory is insufficient and needs more information from additional tool usage.
+"""
+
+        input_data = [prompt_memory_verification]
+        if image_info:
+            try:
+                with open(image_info["image_path"], 'rb') as file:
+                    image_bytes = file.read()
+                input_data.append(image_bytes)
+            except Exception as e:
+                print(f"Error reading image file: {str(e)}")
+
+        stop_verification = self.llm_engine_mm(input_data, response_format=MemoryVerification)
+
+        return stop_verification
+
+    def extract_conclusion(self, response: MemoryVerification) -> str:
+        if response.stop_signal:
+            return 'STOP'
+        else:
+            return 'CONTINUE'
+
+    def generate_final_output(self, question: str, image: str, memory: Memory, bytes_mode: bool = False) -> str:
+        if bytes_mode:
+            image_info = self.get_image_info_bytes(image)
+        else:
+            image_info = self.get_image_info(image)
+
+        prompt_generate_final_output = f"""
+Task: Generate the final output based on the query, image, and tools used in the process.
+
+Context:
+Query: {question}
+Image: {image_info}
+Actions Taken:
+{memory.get_actions()}
+
+Instructions:
+1. Review the query, image, and all actions taken during the process.
+2. Consider the results obtained from each tool execution.
+3. Incorporate the relevant information from the memory to generate the step-by-step final output.
+4. The final output should be consistent and coherent using the results from the tools.
+
+Output Structure:
+Your response should be well-organized and include the following sections:
+
+1. Summary:
+   - Provide a brief overview of the query and the main findings.
+
+2. Detailed Analysis:
+   - Break down the process of answering the query step-by-step.
+   - For each step, mention the tool used, its purpose, and the key results obtained.
+   - Explain how each step contributed to addressing the query.
+
+3. Key Findings:
+   - List the most important discoveries or insights gained from the analysis.
+   - Highlight any unexpected or particularly interesting results.
+
+4. Answer to the Query:
+   - Directly address the original question with a clear and concise answer.
+   - If the query has multiple parts, ensure each part is answered separately.
+
+5. Additional Insights (if applicable):
+   - Provide any relevant information or insights that go beyond the direct answer to the query.
+   - Discuss any limitations or areas of uncertainty in the analysis.
+
+6. Conclusion:
+   - Summarize the main points and reinforce the answer to the query.
+   - If appropriate, suggest potential next steps or areas for further investigation.
+"""
+
+        input_data = [prompt_generate_final_output]
+        if image_info:
+            try:
+                with open(image_info["image_path"], 'rb') as file:
+                    image_bytes = file.read()
+                input_data.append(image_bytes)
+            except Exception as e:
+                print(f"Error reading image file: {str(e)}")
+
+        final_output = self.llm_engine_mm(input_data)
+
+        return final_output
+
+
+    def generate_direct_output(self, question: str, image: str, memory: Memory, bytes_mode: bool = False) -> str:
+        if bytes_mode:
+            image_info = self.get_image_info_bytes(image)
+        else:
+            image_info = self.get_image_info(image)
+
+        prompt_generate_final_output = f"""
+Context:
+Query: {question}
+Image: {image_info}
+Initial Analysis:
+{self.query_analysis}
+Actions Taken:
+{memory.get_actions()}
+
+Please generate the concise output based on the query, image information, initial analysis, and actions taken. Break down the process into clear, logical, and conherent steps. Conclude with a precise and direct answer to the query.
+
+Answer:
+"""
+
+        input_data = [prompt_generate_final_output]
+        if image_info:
+            try:
+                with open(image_info["image_path"], 'rb') as file:
+                    image_bytes = file.read()
+                input_data.append(image_bytes)
+            except Exception as e:
+                print(f"Error reading image file: {str(e)}")
+
+        final_output = self.llm_engine_mm(input_data)
+
+        return final_output
+    

opentools/models/utlis.py

@@ -0,0 +1,73 @@
+# import json
+
+# def truncate_result(result, max_length: int = 100000, truncation_indicator: str = "...") -> str:
+#     """
+#     Truncate the result to specified length while preserving JSON structure when possible.
+    
+#     Args:
+#         result: The result to truncate (can be str, list, dict, or other types)
+#         max_length: Maximum length of the output string (default: 1000)
+#         truncation_indicator: String to indicate truncation (default: "...")
+        
+#     Returns:
+#         str: Truncated string representation of the result
+#     """
+#     if isinstance(result, (dict, list)):
+#         try:
+#             result_str = json.dumps(result, ensure_ascii=False)
+#         except:
+#             result_str = str(result)
+#     else:
+#         result_str = str(result)
+    
+#     indicator_length = len(truncation_indicator)
+    
+#     if len(result_str) > max_length:
+#         # For JSON-like strings, try to find the last complete structure
+#         if result_str.startswith('{') or result_str.startswith('['):
+#             # Find last complete element
+#             pos = max_length - indicator_length
+#             while pos > 0 and not (
+#                 result_str[pos] in ',]}' and 
+#                 result_str[pos:].count('"') % 2 == 0
+#             ):
+#                 pos -= 1
+#             if pos > 0:
+#                 return result_str[:pos + 1] + truncation_indicator
+        
+#         # Default truncation if not JSON or no suitable truncation point found
+#         return result_str[:max_length - indicator_length] + truncation_indicator
+    
+#     return result_str
+
+def make_json_serializable(obj):
+    if isinstance(obj, (str, int, float, bool, type(None))):
+        return obj
+    elif isinstance(obj, dict):
+        return {make_json_serializable(key): make_json_serializable(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [make_json_serializable(element) for element in obj]
+    elif hasattr(obj, '__dict__'):
+        return make_json_serializable(obj.__dict__)
+    else:
+        return str(obj)
+    
+
+def make_json_serializable_truncated(obj, max_length: int = 100000):
+    if isinstance(obj, (int, float, bool, type(None))):
+        if isinstance(obj, (int, float)) and len(str(obj)) > max_length:
+            return str(obj)[:max_length - 3] + "..."
+        return obj
+    elif isinstance(obj, str):
+        return obj if len(obj) <= max_length else obj[:max_length - 3] + "..."
+    elif isinstance(obj, dict):
+        return {make_json_serializable_truncated(key, max_length): make_json_serializable_truncated(value, max_length) 
+                for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [make_json_serializable_truncated(element, max_length) for element in obj]
+    elif hasattr(obj, '__dict__'):
+        return make_json_serializable_truncated(obj.__dict__, max_length)
+    else:
+        result = str(obj)
+        return result if len(result) <= max_length else result[:max_length - 3] + "..."
+    

opentools/tools/README.md

@@ -0,0 +1,44 @@
+
+## Testing the Tools
+
+To test the text detection tool, follow these steps:
+
+1. **Navigate to the Project Directory:**
+
+   Change your current directory to where the tools are located. Replace `your_path` with the actual path to your project directory.
+
+   ```sh
+   cd your_path/toolbox-agent/opentools
+   ```
+
+2. **Run the Text Detection Tool:**
+
+   ```sh
+   cd toolbox-agent
+   export PYTHONPATH=$(pwd)
+   ```
+
+
+   Execute the tool using the following command:
+
+   ```sh
+   python tools/text_detector/tool.py
+
+   python tools/object_detector/tool.py
+
+   ```
+
+## File Structure
+
+The project is organized as follows:
+
+```sh
+├── __init__.py              # Initializes the tools package and possibly exposes submodules
+├── base.py                  # Base class for tools, providing common functionality
+├── text_detector/           # Directory for the text detection tool
+│   ├── readme.md            # Documentation for the text detection tool
+│   └── tool.py              # Implementation of the text detection tool
+├── object_detector/         # Directory for the object detection tool
+│   ├── readme.md            # Documentation for the object detection tool
+│   └── tool.py              # Implementation of the object detection tool
+```

opentools/tools/base.py

@@ -0,0 +1,103 @@
+# opentools/tools/base.py
+
+from opentools.engine.openai import ChatOpenAI
+
+class BaseTool:
+    """
+    A base class for building tool classes that perform specific tasks, such as image processing or text detection.
+    """
+
+    require_llm_engine = False  # Default is False, tools that need LLM should set this to True
+
+    def __init__(self, tool_name=None, tool_description=None, tool_version=None, input_types=None, output_type=None, demo_commands=None, output_dir=None, user_metadata=None, model_string=None):
+        """
+        Initialize the base tool with optional metadata.
+
+        Parameters:
+            tool_name (str): The name of the tool.
+            tool_description (str): A description of the tool.
+            tool_version (str): The version of the tool.
+            input_types (dict): The expected input types for the tool.
+            output_type (str): The expected output type for the tool.
+            demo_commands (list): A list of example commands for using the tool.
+            output_dir (str): The directory where the tool should save its output (optional).
+            user_metadata (dict): Additional metadata specific to user needs (optional).
+            model_string (str): The model string for the LLM engine (optional, only used if require_llm_engine is True).
+        """
+        self.tool_name = tool_name
+        self.tool_description = tool_description
+        self.tool_version = tool_version
+        self.input_types = input_types
+        self.output_type = output_type
+        self.demo_commands = demo_commands
+        self.output_dir = output_dir
+        self.user_metadata = user_metadata
+        self.model_string = model_string
+
+    def set_metadata(self, tool_name, tool_description, tool_version, input_types, output_type, demo_commands, user_metadata=None):
+        """
+        Set the metadata for the tool.
+
+        Parameters:
+            tool_name (str): The name of the tool.
+            tool_description (str): A description of the tool.
+            tool_version (str): The version of the tool.
+            input_types (dict): The expected input types for the tool.
+            output_type (str): The expected output type for the tool.
+            demo_commands (list): A list of example commands for using the tool.
+            user_metadata (dict): Additional metadata specific to user needs (optional).
+        """
+        self.tool_name = tool_name
+        self.tool_description = tool_description
+        self.tool_version = tool_version
+        self.input_types = input_types
+        self.output_type = output_type
+        self.demo_commands = demo_commands
+        self.user_metadata = user_metadata
+
+    def get_metadata(self):
+        """
+        Returns the metadata for the tool.
+
+        Returns:
+            dict: A dictionary containing the tool's metadata.
+        """
+        metadata = {
+            "tool_name": self.tool_name,
+            "tool_description": self.tool_description,
+            "tool_version": self.tool_version,
+            "input_types": self.input_types,
+            "output_type": self.output_type,
+            "demo_commands": self.demo_commands,
+            "require_llm_engine": self.require_llm_engine,
+        }
+        if self.user_metadata:
+            metadata["user_metadata"] = self.user_metadata
+        return metadata
+
+    def set_custom_output_dir(self, output_dir):
+        """
+        Set a custom output directory for the tool.
+
+        Parameters:
+            output_dir (str): The new output directory path.
+        """
+        self.output_dir = output_dir
+
+    def set_llm_engine(self, model_string):
+        """
+        Set the LLM engine for the tool.
+
+        Parameters:
+            model_string (str): The model string for the LLM engine.
+        """
+        self.model_string = model_string
+
+    def execute(self, *args, **kwargs):
+        """
+        Execute the tool's main functionality. This method should be overridden by subclasses.
+
+        Raises:
+            NotImplementedError: If the subclass does not implement this method.
+        """
+        raise NotImplementedError("Subclasses must implement the execute method.")

opentools/tools/generalist_solution_generator/tool.py

@@ -0,0 +1,142 @@
+import os
+from opentools.tools.base import BaseTool
+from opentools.engine.openai import ChatOpenAI
+
+class Generalist_Solution_Generator_Tool(BaseTool):
+    require_llm_engine = True
+
+    def __init__(self, model_string="gpt-4o-mini"):
+        super().__init__(
+            tool_name="Generalist_Solution_Generator_Tool",
+            tool_description="A generalized tool that takes query from the user as prompt, and answers the question step by step to the best of its ability. It can also accept an image.",
+            tool_version="1.0.0",
+            input_types={
+                "prompt": "str - The prompt that includes query from the user to guide the agent to generate response (Examples: 'Describe this image in detail').",
+                "image": "str - The path to the image file if applicable (default: None).",
+            },
+            output_type="str - The generated response to the original query prompt",
+            demo_commands=[
+                {
+                    "command": 'execution = tool.execute(prompt="Summarize the following text in a few lines")',
+                    "description": "Generate a short summary given the prompt from the user."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Explain the mood of this scene.", image="path/to/image1.png")',
+                    "description": "Generate a caption focusing on the mood using a specific prompt and image."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Give your best coordinate estimate for the pacemaker in the image and return (x1, y1, x2, y2)", image="path/to/image2.png")',
+                    "description": "Generate bounding box coordinates given the image and prompt from the user. The format should be (x1, y1, x2, y2)."
+                },
+                {
+                    "command": 'execution = tool.execute(prompt="Is the number of tiny objects that are behind the small metal jet less than the number of tiny things left of the tiny sedan?", image="path/to/image2.png")',
+                    "description": "Answer a question step by step given the image."
+                }
+            ],
+            # # vesion 0 (bowen) (Generalist: %; 6 Tools: %; Generalist + 6 Tools: %)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge. For optimal results: 1) Provide clear, specific prompts. 2) Use it as a starting point for complex tasks, then refine with specialized tools. 3) Verify important information from its responses. 4) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+            # vesion 2 (Generalist: 68%; 6 Tools: 66%; Generalist + 6 Tools: 54%)
+            user_metadata = {
+                "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+                "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+                "1) Provide clear, specific prompts.\n"
+                "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+                "3) For complex queries, break them down into subtasks and use the tool multiple times.\n"
+                "4) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+                "5) Verify important information from its responses.\n"
+                "6) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            }
+            # # vesion 6 (Generalist: 70%; 6 Tools: 66%; Generalist + 6 Tools: 60%)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+            #     "1) Provide clear, specific prompts.\n"
+            #     "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+            #     "3) For complex queries, break them down into smaller, focused sub-tasks and use the tool multiple times.\n"
+            #     "4) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+            #     "5) Verify important information from its responses.\n"
+            #     "6) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+            # # vesion 8 (Generalist: 68%; 6 Tools: 66%; Generalist + 6 Tools: 60%)
+            # user_metadata = {
+            #     "limitation": "The Generalist_Solution_Generator_Tool may provide hallucinated or incorrect responses.",
+            #     "best_practice": "Use the Generalist_Solution_Generator_Tool for general queries or tasks that don't require specialized knowledge or specific tools in the toolbox. For optimal results:\n\n"
+            #     "1) Provide clear, specific prompts.\n"
+            #     "2) Use it to answer the original query through step by step reasoning for tasks without complex or multi-step reasoning.\n"
+            #     "3) Use it as a starting point for complex tasks, then refine with specialized tools.\n"
+            #     "4) Verify important information from its responses.\n"
+            #     "5) For image-related tasks, ensure the image path is correct and the prompt is relevant to the image content."
+            # }
+        )
+        self.model_string = model_string  
+
+    def execute(self, prompt, image=None):
+
+        print(f"\nInitializing Generalist Tool with model: {self.model_string}")
+        multimodal = True if image else False
+        llm_engine = ChatOpenAI(model_string=self.model_string, is_multimodal=multimodal)
+
+        try:
+            input_data = [prompt]
+            if multimodal:
+                if not os.path.isfile(image):
+                    return "Error: Invalid image file path."
+                try:
+                    with open(image, 'rb') as file:
+                        image_bytes = file.read()
+                    input_data.append(image_bytes)
+                except Exception as e:
+                    return f"Error reading image file: {str(e)}"
+
+                response = llm_engine(input_data)
+            else:
+                response = llm_engine(input_data[0])
+            return response
+        except Exception as e:
+            return f"Error generating response: {str(e)}"
+
+    def get_metadata(self):
+        metadata = super().get_metadata()
+        return metadata
+
+if __name__ == "__main__":
+    # Test command:
+    """
+    Run the following commands in the terminal to test the script:
+    
+    cd opentools
+    python tools/default/tool.py
+    """
+
+    # Get the directory of the current script
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    print(f"Script directory: {script_dir}")
+
+    # Example usage of the Generalist_Tool
+    tool = Generalist_Solution_Generator_Tool()
+    # tool = Generalist_Solution_Generator_Tool(model_string="gpt-4o-mini")
+    # tool = Generalist_Solution_Generator_Tool(model_string="gpt-4o")
+
+    # Get tool metadata
+    metadata = tool.get_metadata()
+    print(metadata)
+
+    # Construct the full path to the image using the script's directory
+    relative_image_path = "../../tasks/minitoolbench/data/mathvista_113.png"
+    relative_image_path = "examples/mathvista_113.png"
+    image_path = os.path.join(script_dir, relative_image_path)
+    prompt = "Describe the image in detail."
+
+    # Execute the tool with default prompt
+    try:
+        execution = tool.execute(prompt=prompt, image=image_path)
+        # execution = tool.execute(prompt=prompt)
+        print("Generated Response:")
+        print(execution)
+    except Exception as e: 
+        print(f"Execution failed: {e}")
+
+    print("Done!")

requirements.txt

@@ -0,0 +1,13 @@
+easyocr==1.7.1
+openai==1.58.1
+python-dotenv==1.0.1
+wikipedia==1.4.0
+pillow==10.4.0
+platformdirs==4.2.2
+sympy==1.13.2
+tenacity==9.0.0
+diskcache==5.6.3
+transformers==4.44.2
+pymed==0.8.9
+metapub==0.5.12
+-e ./opentools

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='opentools',
+    version='0.1.0',
+    # description='A flexible and versatile toolbox agent framework for complex tasks in both general and scientific scenarios.',
+    # long_description=open('README.md').read(),
+    # long_description_content_type='text/markdown',
+    # author='Pan Lu, Bowen Chen, Sheng Liu',
+    # author_email='[email protected]',
+    # url='',  # You can add a GitHub or project URL here
+    packages=find_packages(),
+    # install_requires=open('requirements.txt').read().splitlines(),
+    # classifiers=[
+    #     'Programming Language :: Python :: 3',
+    #     'License :: OSI Approved :: MIT License',
+    #     'Operating System :: OS Independent',
+    # ],
+    # python_requires='>=3.10',
+)