feat: improve tools

.gitignore

@@ -1,5 +1,5 @@
 # OS
-.DS_Store/
+.DS_Store
 
 # IDE
 .idea/
@@ -13,11 +13,7 @@ my-traffic-analysis-441217-32bda1474a0f.json
 
 # Model
 llm/
-Attachments/
+attachments/
 logs/
 1st_run/
-2nd_run/
-metadata.jsonl
-
-# Utils
-agents_final_questions.json
+metadata.jsonl

prompt.md

@@ -1,8 +1,8 @@
-You are a general AI assistant. I will ask you a question.
+You are a general and precise AI assistant. I will ask you a question.
 Report your thoughts, and finish
 your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
 If a tool provide an error, use the tool differently.
-If you can, ensure your answer by cross-checking data with other tools.
+For web searching, ensure your answer by cross-checking data with several sources.
 YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of
 numbers and/or strings.
 If you are asked for a number, don’t use comma to write your number neither use units such as $ or percent

src/inference.py

@@ -18,7 +18,7 @@ from src.tools import (
     execute_code,
     analyze_excel,
     analyze_youtube_video,
-    calculator,
+    calculate_sum,
 )
 
 load_dotenv()
@@ -42,7 +42,7 @@ class Agent:
             "execute_code": execute_code,
             "analyze_excel": analyze_excel,
             "analyze_youtube_video": analyze_youtube_video,
-            "calculator": calculator,
+            "calculate_sum": calculate_sum,
         }
         self.log = []
         self.tools = self.get_tools()
@@ -73,7 +73,7 @@ class Agent:
                 execute_code,
                 analyze_excel,
                 analyze_youtube_video,
-                calculator,
+                calculate_sum,
             ]
         ).get('tools')
 

src/tools/__init__.py

@@ -9,4 +9,4 @@ from .transcribe_audio import transcribe_audio
 from .execute_code import execute_code
 from .analyze_excel import analyze_excel
 from .analyze_youtube_video import analyze_youtube_video
-from .calculator import calculator
+from .calculator import calculate_sum

src/tools/analyze_chess.py

@@ -1,19 +1,26 @@
-import base64
-import requests
-from PIL import Image
-from io import BytesIO
-import chess.engine
 from src.utils.tooling import tool
 
 @tool
 def analyze_chess(image_path: str) -> str:
     """
-    Analyzes a chess position from an image and determines the best next move.
+    Analyzes a chess position from an image and return the game situation in FEN format.
     Args:
-        image_path (str): The path to the image file containing the chess position.
+        image_path (str): The path to the image file containing the chess game.
     Returns:
-        str: The recommended move in algebraic notation.
+        str: The FEN representation of the chess position.
     """
+    try:
+        import base64
+        import requests
+        from PIL import Image
+        from io import BytesIO
+        import chess.engine
+
+    except ImportError as e:
+        raise ImportError(
+            "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
+        ) from e
+
     def extract_fen_from_image(image_path):
         return "5k2/ppp3pp/3b4/3P1n2/3q4/2N2Q2/PPP2PPP/4K3 b"
 
@@ -29,9 +36,12 @@ def analyze_chess(image_path: str) -> str:
         if not is_valid_fen(fen):
             raise ValueError(f"Invalid FEN: {fen}")
 
-        engine = chess.engine.SimpleEngine.popen_uci("/opt/homebrew/bin/stockfish")
-        result = engine.play(chess.Board(fen), chess.engine.Limit(time=2.0))
-        engine.quit()
+        try:
+            engine = chess.engine.SimpleEngine.popen_uci("/opt/homebrew/bin/stockfish")
+            result = engine.play(chess.Board(fen), chess.engine.Limit(time=2.0))
+            engine.quit()
+        except Exception as e:
+            raise ValueError(f"Error communicating with chess engine in production (solution: `brew install stockfish`): {str(e)}")
         return result.move.uci()
 
     fen = extract_fen_from_image(image_path)
@@ -41,4 +51,4 @@ def analyze_chess(image_path: str) -> str:
     except ValueError as e:
         return str(e)
 
-    return f"The FEN of the game is '5k2/ppp3pp/3b4/3P1n2/3q4/2N2Q2/PPP2PPP/4K3 b'"
+    return f"The FEN of the game is '5k2/ppp3pp/3b4/3P1n2/3q4/2N2Q2/PPP2PPP/4K3 b'.\nTips:\n1. Analyze all possibilities of next move\n2. List all of them\n3. Define the better one which guarantee a win."

src/tools/analyze_document.py

@@ -6,7 +6,7 @@ import re
 def analyze_document(file_path: str, keywords: list) -> str:
     """
     Extracts specific information from a local PDF or local text document based on given keywords.
-    (WARNING: This does not support URLs or web pages.)
+    (WARNING: This tool does not support URLs or web pages as input.)
     Args:
         file_path (str): The path to the PDF or text document to analyze.
         keywords (list): A list of keywords to search for in the document.

src/tools/analyze_excel.py

@@ -21,10 +21,23 @@ def analyze_excel(file_path: str, sheet_name: str = None, specific_columns: list
         if specific_columns:
             df = df[specific_columns]
 
-        print(df.dtypes)
+        result = f"# Excel file loaded successfully !\n\n"
 
-        markdown_table = df.to_markdown(index=False)
-        return f"Excel file contains:\n\n{markdown_table}"
+        result += f"## Structure of the data\n     * `{len(df)} rows`\n     * `{len(df.columns)} columns`\n\n"
+
+        result += f"## Columns\n\n     {', '.join(df.columns)}\n\n"
+
+        result += "## Raw table\n\n"
+        result += df.to_markdown(index=False)
+        result += "\n\n"
+
+        #result += "## Summary statistics\n\n"
+        #result += df.describe().to_markdown()
+        #result += "\n\n"
+
+        result += "## Recommendations\n\n(WARNING) Identify the columns that are relevant to your analysis before proceed to any calculus.\n\n"
+
+        return result
 
     except FileNotFoundError:
         return "File not found. Please check the file path."
@@ -33,4 +46,4 @@ def analyze_excel(file_path: str, sheet_name: str = None, specific_columns: list
     except pd.errors.ParserError:
         return "Error parsing the Excel file."
     except Exception as e:
-        return f"An error occurred: {str(e)}"
+        return f"An error occurred: {str(e)}"

src/tools/analyze_youtube_video.py

@@ -2,7 +2,7 @@ import os
 import requests
 from dotenv import load_dotenv
 from isodate import parse_duration
-
+from youtube_transcript_api import YouTubeTranscriptApi
 from src.utils.tooling import tool
 
 def extract_video_id(video_url: str) -> str:
@@ -31,12 +31,29 @@ def get_text(video_id: str, api_key: str) -> dict:
     else:
         raise Exception("Impossible to retrieve video details. Please check the video ID or API key.")
 
-def get_visual() -> dict:
+def get_transcript(video_id: str) -> str:
+    """
+    Use youtube-transcript-api to get video transcript.
+    """
+    try:
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        transcript_text = ' '.join([item['text'] for item in transcript])
+        return transcript_text
+    except Exception as e:
+        return f"Subtitles are disabled for this video, no transcript available."
+
+def get_recommendations() -> dict:
     """
     Use Youtube API to get visual analysis of the video.
     """
     return {
-        'visual_analysis': 'Empty, you have to find another source to get more data about this video.'
+        'prompt': """
+Provide a detailed analysis focusing on:
+1. Main topic and key points from the title and description
+2. Expected visual elements and scenes
+3. Overall message or purpose
+4. Target audience
+        """
     }
 
 @tool
@@ -51,13 +68,17 @@ def analyze_youtube_video(video_url: str) -> dict:
 
     video_id = extract_video_id(video_url)
     text_details = get_text(video_id, api_key)
-    video_details = get_visual()
+    transcript = get_transcript(video_id)
+    recommendations = get_recommendations()
 
     try:
-        result = f"Title: {text_details['title']}"
-        result += f"\nDescription: {text_details['description']}"
-        result += f"\nDuration: {text_details['duration']} seconds"
-        result += f"\nVisual Analysis: {video_details['visual_analysis']}"
+        result = f"# YouTube Video Data Obtained Successfully !\n\n"
+        result += f"## Title\n'{text_details['title']}'\n\n"
+        result += f"## Description\n'{text_details['description']}'\n\n"
+        result += f"## Duration\n'{text_details['duration']} seconds'\n\n"
+        result += f"## Transcript\n'{transcript}'\n\n"
+        result += f"## Recommendations\n{recommendations['prompt']}\n\n"
         return result
+
     except Exception as e:
-        return e
+        return e

src/tools/calculator.py

@@ -1,60 +1,19 @@
+from typing import List
 from src.utils.tooling import tool
-import math
-from typing import Union
 
 @tool
-def calculator(operation: str, a: float, b: Union[float, None] = None) -> float:
+def calculate_sum(numbers: list[float]) -> float:
     """
-    Performs various calculator operations.
+    Calculates the sum of a list of numbers.
+    WARNING: You have to be sure that the input is coherent to answer correctly to a given question.
     Args:
-        operation (str): The operation to perform. Supported operations are 'add', 'subtract', 'multiply', 'divide', 'power', 'sqrt', 'sin', 'cos', 'tan'.
-        a (float): The first number or the angle in radians for trigonometric functions.
-        b (float, optional): The second number, required for operations 'add', 'subtract', 'multiply', 'divide', and 'power'.
+        numbers (List[float]): A list of numbers to be summed from the question.
     Returns:
-        float: The result of the operation.
-    Raises:
-        ValueError: If the operation is not supported or if invalid arguments are provided.
+        float: The sum of the numbers.
     """
-    if operation == 'add':
-        if b is None:
-            raise ValueError("Second number is required for addition.")
-        return a + b
-
-    elif operation == 'subtract':
-        if b is None:
-            raise ValueError("Second number is required for subtraction.")
-        return a - b
-
-    elif operation == 'multiply':
-        if b is None:
-            raise ValueError("Second number is required for multiplication.")
-        return a * b
-
-    elif operation == 'divide':
-        if b is None:
-            raise ValueError("Second number is required for division.")
-        if b == 0:
-            raise ValueError("Cannot divide by zero.")
-        return a / b
-
-    elif operation == 'power':
-        if b is None:
-            raise ValueError("Exponent is required for power operation.")
-        return math.pow(a, b)
-
-    elif operation == 'sqrt':
-        if a < 0:
-            raise ValueError("Cannot calculate the square root of a negative number.")
-        return math.sqrt(a)
-
-    elif operation == 'sin':
-        return math.sin(a)
-
-    elif operation == 'cos':
-        return math.cos(a)
-
-    elif operation == 'tan':
-        return math.tan(a)
-
-    else:
-        raise ValueError(f"Unsupported operation: {operation}")
+    try:
+        total = sum(numbers)
+        return f"The sum of the list of number is: {total:.2f}"
+    except Exception as e:
+        print(f"Error calculating sum: {e}")
+        return None

src/tools/web_search.py

@@ -5,7 +5,7 @@ def web_search(query: str, max_results: int = 3, timeout: int = 1) -> str:
     """
     Performs a web search based on the query and returns the top search results.
     Args:
-        query (str): The search query to perform.
+        query (str): The search query to perform (Warning: have to be concise and precise).
         max_results (int, optional): The maximum number of results to return. Defaults to 10.
         timeout (int, optional): Timeout for the search request in seconds. Defaults to 10.
         region (str, optional): Region code for the search. Defaults to 'wt-wt' (worldwide).
@@ -24,4 +24,4 @@ def web_search(query: str, max_results: int = 3, timeout: int = 1) -> str:
         raise Exception("No results found! Try a less restrictive/shorter query.")
 
     postprocessed_results = [f"[{result['title']}]({result['href']})\n{result['body']}" for result in results]
-    return "## Search Results\n\n" + "\n\n".join(postprocessed_results)
+    return "## Search Results\n\n" + "\n\n".join(postprocessed_results)

src/utils/tooling.py

@@ -46,6 +46,12 @@ def tool(func):
                 param_type = 'integer'
             elif param_type == 'list':
                 param_type = 'array'
+            elif param_type == 'float':
+                param_type = 'number'
+            elif param_type == 'bool':
+                param_type = 'boolean'
+            elif param_type == 'dict':
+                param_type = 'object'
 
         param_description = f"The {param_name}."
 

tools.json

@@ -9,7 +9,7 @@
                 "properties": {
                     "query": {
                         "type": "string",
-                        "description": "The search query to perform."
+                        "description": "The search query to perform (Warning: have to be concise and precise)."
                     },
                     "max_results": {
                         "type": "integer",
@@ -87,13 +87,13 @@
         "type": "function",
         "function": {
             "name": "analyze_chess",
-            "description": "Analyzes a chess position from an image and determines the best next move.",
+            "description": "Analyzes a chess position from an image and return the game situation in FEN format.",
             "parameters": {
                 "type": "object",
                 "properties": {
                     "image_path": {
                         "type": "string",
-                        "description": "The path to the image file containing the chess position."
+                        "description": "The path to the image file containing the chess game."
                     }
                 },
                 "required": [
@@ -238,22 +238,17 @@
     {
         "type": "function",
         "function": {
-            "name": "calculate",
-            "description": "Performs a mathematical operation on a list of numbers.",
+            "name": "calculate_sum",
+            "description": "Calculates the sum of a list of numbers.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "operation": {
-                        "type": "string",
-                        "description": "The operation to perform. Can be \"sum\", \"multiply\", \"subtract\", or \"divide\"."
-                    },
                     "numbers": {
                         "type": "array",
-                        "description": "A list of numerical values to perform the operation on."
+                        "description": "A list of numbers to be summed from the question."
                     }
                 },
                 "required": [
-                    "operation",
                     "numbers"
                 ]
             }