update prompt n tools

app.py

@@ -6,7 +6,6 @@ import pandas as pd
 from typing import TypedDict, Annotated
 from langchain_core.messages import AnyMessage
 from langgraph.graph.message import add_messages
-from langchain_hyperbrowser import HyperbrowserBrowserUseTool
 from langgraph.graph import START, StateGraph
 from langgraph.prebuilt import ToolNode, tools_condition
 from langchain_core.messages import HumanMessage, SystemMessage
@@ -22,7 +21,10 @@ import os
 import shutil
 import uuid
 from langchain_tavily import TavilySearch
-
+import numpy as np
+from markdownify import markdownify
+import re
+from io import StringIO
 
 # set_debug(True)
 
@@ -82,6 +84,7 @@ def interpret_image(image_name: str, question: str):
             }
         ])]
         response = vision_llm.invoke(messages)
+        print(response.content)
         return response.content
     except Exception as e:
         print(str(e))
@@ -94,6 +97,7 @@ def transcribe_audio(file_name: str):
     
     model = whisper.load_model("small")
     result = model.transcribe(file_name)
+    print(result["text"])
     return result["text"]
 
 def download_youtube_video(url: str):
@@ -105,7 +109,7 @@ def download_youtube_video(url: str):
     ydl_opts = {
         'format': 'bestvideo+bestaudio/best',
         'outtmpl': output_path,
-        'merge_output_format': 'mp4',  # Use mp4 as the final output format
+        'merge_output_format': 'mp4', 
         'quiet': True,
     }
 
@@ -174,12 +178,49 @@ def watch_video(file_name: str):
     print(captions)
     return captions
         
-        
-browser_tool = HyperbrowserBrowserUseTool()
+
+def add_tool(numbers: list):
+    """
+    Calculate sum of numbers.
+    """
+    numbers = np.array(numbers)
+    return np.sum(numbers, dtype=float)
+
+def visit_web_page(url: str):
+    """
+    Visit a webpage.
+    """
+    response = requests.get(url, timeout=20)
+    response.raise_for_status()
+    markdown_content = markdownify(response.text).strip()
+    markdown_content = re.sub(r"\n{3, }", "\n\n", markdown_content)
+    if len(markdown_content <= 20000):
+        return markdown_content
+    else:
+        print(markdown_content[:20000//2] + "\nThe content has been truncated to stay below 20000 characters.\n" + markdown_content[-20000//2:])
+        return markdown_content[:20000//2] + "\nThe content has been truncated to stay below 20000 characters.\n" + markdown_content[-20000//2:] # - to count from the end
+
+def final_answer(text: str):
+    """
+    Extract the final answer.
+    """
+    text = text.split("FINAL ANSWER:")
+    return text[-1]
+
+def markdown(content: str):
+    """
+    Interpret markdown representation of a table.
+    """
+    clean_content = "\n".join([line for i, line in enumerate(content.strip().splitlines()) if i!=1])
+    df = pd.read_csv(StringIO(clean_content), sep="|", engine="python")
+    df = df.drop(columns=[""])
+    print(df.to_string())
+    return df.to_string()
+    
 # search_tool = DuckDuckGoSearchRun()
 search_tool = TavilySearch()
 code_executor_tool = ExecPython()
-tools = [search_tool, code_executor_tool, interpret_image, get_file, transcribe_audio, download_youtube_video, read_file, watch_video, read_excel]
+tools = [search_tool, interpret_image, get_file, transcribe_audio, download_youtube_video, read_file, read_excel, add_tool, visit_web_page, markdown]
 llm = ChatOpenAI(model="gpt-4o", temperature=0)
 llm_with_tools = llm.bind_tools(tools)
 
@@ -264,41 +305,75 @@ def assistant(state:AgentState):
         Returns:
             A list of captions for each frame.
     """
+    
+    add_tool_description = """
+    math_tool(numbers: list) -> float:
+            Calculate sum of numbers.
+            
+            Args:
+                list: List of numbers to perform an operation on.
+            
+            Returns:
+                The sum of the numbers.
+    """
+    
+    visit_web_page_tool_description = """
+    visit_web_page(url: str) -> str:
+        Visit a web page.
+    
+        Args:
+            url: The URL of the web page to visit as string.
+        
+        Returns:
+            Markdown representation of the HTML content of the web page.
+    """
+    
+    markdown_tool_description = """
+    markdown(content: str) -> str:
+        Interpret markdown representation of a table.
+        
+        Args:
+            content: Markdown table as string.
+        
+        Returns:
+            String representation of the extracted tabled.
+    """
 
     search_tool_description = search_tool.description
-    code_executor_tool_description = code_executor_tool.description
     
     has_file = state["has_file"]
     
     system_message = SystemMessage(content=f"""
                                    You are a general AI assistant. I will ask you a question.
                                    
-                                   Your response should be a number, OR as few words as possible, OR a comma-separated list of numbers and/or strings. 
-                                   You SHOULD NOT provide explanations in your response.
-                                   If you are asked for a number, don't use a comma to write your number, neither use symbols such as $ or % unless specified otherwise.
-                                   If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities). 
-                                   If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
-                                   If you are including text from the question in your response, make sure to include the text exactly as it appears in the question (e.g. with adjective).
-                                   Do NOT end your response with a period.
-                                   Do NOT write numbers as text.
-                                   
-				                   You have access to the following tools, which you can use as needed to answer a question:
-                                   - File downloading tool: {download_file_tool_description}
-                                   - Image interpretation tool: {image_tool_description}
+                                   You have access to the following tools, which you can use as needed to answer a question:
+                                   - File downloader: {download_file_tool_description}
+                                   - Image interpretation: {image_tool_description}
                                    - YouTube video downloader: {download_youtube_video_description}
-                                   - Audio transcription tool: {audio_tool_description}
-                                   - Text-based file reading tool: {read_file_tool_description}
-                                   - Internet search tool: {search_tool_description}
-                                   - Code execution tool: {code_executor_tool_description}
-                                   - Watch video tool: {watch_video_tool_description}
-                                   - Read Excel file tool: {excel_tool_description}
+                                   - Audio transcription: {audio_tool_description}
+                                   - Read text-based file: {read_file_tool_description}
+                                   - Internet search: {search_tool_description}
+                                   - Read Excel file: {excel_tool_description}
+                                   - Math: {add_tool_description}
+                                   - Visit web page: {visit_web_page_tool_description}
+                                   - Markdown table interpretation: {markdown_tool_description}
                                    
                                    You may download a file for a given task ONLY if it has a file by using its associated task ID.
                                    Always ensure you have downloaded a file before using a relevant tool.
-                                   You MUST use the name of a particular downloaded file in your tool call. Do NOT use a file name mentioned in the question.
-                                   When asked about a YouTube video, you can watch it and/or hear it.
-                                   When writing code, avoid excess formatting and keep it clean.
-                                   Do NOT make up answers, instead use a tool to answer the question.
+                                   You MUST use the name of a particular downloaded file in your tool call. DO NOT use a file name mentioned in the question.
+                                   When asked about a YouTube video, you can hear it and/or check its description.
+                                   Use a tool only when needed and never re-do a tool call that you previously did with the exact same arguments.
+                                   If a tool call fails, try using another tool to reach an answer.
+                                   Avoid returning your response directly, instead verify your response with a tool when available.
+                                   
+                                   Your response should be a number, OR as few words as possible, OR a comma-separated list of numbers and/or strings. 
+                                   You SHOULD NOT provide explanations in your response.
+                                   If you are asked for a number, don't use a comma to write your number, nor use symbols such as $ or % unless specified otherwise.
+                                   If you are asked for a string, don't use articles, nor abbreviations (e.g., for cities). 
+                                   If you are asked for a comma-separated list, apply the above rules depending on whether the element to be put in the list is a number or a string.
+                                   When including a phrase in your response from the input, always include the complete phrase with the adjective. For example, if the input contains the phrase "natural spring water", your response should include "fresh lemon juice", not just "lemon juice". 
+                                   DO NOT end your response with a period.
+                                   DO NOT write numbers as text.
                                    
                                    The current task ID is {task_id}.
                                    The current task has a file: {has_file}
@@ -335,6 +410,7 @@ class BasicAgent:
         answer = app.invoke(state)
         answer = answer["messages"][-1].content
         # print(f"Agent returning fixed answer: {fixed_answer}")
+        answer = final_answer(answer)
         print(f"Agent returning answer: {answer}")
         return answer
 
@@ -501,7 +577,7 @@ if __name__ == "__main__":
     # Check for SPACE_HOST and SPACE_ID at startup for information
     space_host_startup = os.getenv("SPACE_HOST")
     space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
-
+    
     if space_host_startup:
         print(f"✅ SPACE_HOST found: {space_host_startup}")
         print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
@@ -521,22 +597,25 @@ if __name__ == "__main__":
 
     print("-"*(60 + len(" App Starting ")) + "\n")
    
-    # try:
+    try:
         # random_url = f"{DEFAULT_API_URL}/random-question"
         # response = requests.get(random_url, timeout=20)
         # response.raise_for_status()
         # question = response.json()
         # print(question)
-        # agent = BasicAgent()
+        agent = BasicAgent()
         # print(question.get("question"))
         # print(question.get("task_id"))
         # has_file=False
         # if question.get("file_name"):
         #     has_file=True
         # print(agent(question.get("question"), question.get("task_id"), has_file))
+        x=(agent("How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?", "3f57289b-8c60-48be-bd80-01f8099ca449", False))
+        print(x)
+        # print(final_answer(x))
 
-    # except Exception as e:
-    #     print(str(e))
+    except Exception as e:
+        print(str(e))
         
     print("Launching Gradio Interface for Basic Agent Evaluation...")
     demo.launch(debug=True, share=False)

requirements.txt

@@ -9,3 +9,4 @@ yt-dlp
 rizaio
 langchain-openai
 langchain-tavily
+markdownify