v1

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+# Folders
+
+__pycache__/
+huggingface_env/
+.git/
+
+# Files
+.env
+.pylintrc
+pyrightconfig.json
+

README.md

@@ -0,0 +1,16 @@
+---
+title: Template Final Assignment
+emoji: 🕵🏻‍♂️
+colorFrom: indigo
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.25.2
+app_file: app.py
+pinned: false
+hf_oauth: true
+# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
+hf_oauth_expiration_minutes: 480
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+

app.py

@@ -0,0 +1,238 @@
+import os
+
+import gradio as gr
+import pandas as pd
+import requests
+from dotenv import load_dotenv
+
+from src.agent import BasicAgent
+
+load_dotenv()
+
+# (Keep Constants as is)
+# --- Constants ---
+DEFAULT_API_URL = os.getenv("DEFAULT_API_URL")
+
+
+def run_and_submit_all(profile: gr.OAuthProfile):
+    """
+    Fetches all questions, runs the BasicAgent on them, submits all answers,
+    and displays the results.
+    """
+    # --- Determine HF Space Runtime URL and Repo URL ---
+    space_id = os.getenv("SPACE_ID")  # Get the SPACE_ID for sending link to the code
+
+    if profile:
+        username = f"{profile.username}"
+        print(f"User logged in: {username}")
+    else:
+        print("User not logged in.")
+        return "Please Login to Hugging Face with the button.", None
+
+    api_url = DEFAULT_API_URL
+    questions_url = f"{api_url}/questions"
+    submit_url = f"{api_url}/submit"
+
+    # 1. Instantiate Agent ( modify this part to create your agent)
+    try:
+        agent = BasicAgent()
+    except Exception as e:
+        print(f"Error instantiating agent: {e}")
+        return f"Error initializing agent: {e}", None
+    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
+    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
+    print(agent_code)
+
+    # 2. Fetch Questions
+    print(f"Fetching questions from: {questions_url}")
+    response = None
+    try:
+        response = requests.get(questions_url, timeout=15)
+        response.raise_for_status()
+        questions_data = response.json()
+        if not questions_data:
+            print("Fetched questions list is empty.")
+            return "Fetched questions list is empty or invalid format.", None
+        print(f"Fetched {len(questions_data)} questions.")
+    except requests.exceptions.JSONDecodeError as e:
+        print(f"Error decoding JSON response from questions endpoint: {e}")
+        if response is not None:
+            print(f"Response text: {response.text[:500]}")
+        return f"Error decoding server response for questions: {e}", None
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching questions: {e}")
+        return f"Error fetching questions: {e}", None
+    except Exception as e:
+        print(f"An unexpected error occurred fetching questions: {e}")
+        return f"An unexpected error occurred fetching questions: {e}", None
+
+    # 3. Run your Agent
+    results_log = []
+    answers_payload = []
+    print(f"Running agent on {len(questions_data)} questions...")
+    # Limit the number of questions to process to avoid timeouts
+    max_questions = 20  # Process only 20 questions at a time
+
+    tasks_to_process = [
+        # "99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3",
+        # "1f975693-876d-457b-a649-393859e79bf3",
+        # "840bfca7-4f7b-481a-8794-c560c340185d",
+        # "7bd855d8-463d-4ed5-93ca-5fe35145f733",
+        # "f918266a-b3e0-4914-865d-4faa564f1aef"
+    ]
+
+    questions_to_process = questions_data[:max_questions]
+
+    if tasks_to_process:
+        questions_to_process = [
+            x for x in questions_data if x.get("task_id") in tasks_to_process
+        ]
+    else:
+        questions_to_process = questions_data[:max_questions]
+
+    count = 0
+    # for item in questions_data:
+    for item in questions_to_process:
+        task_id = item.get("task_id")
+        question_text = item.get("question")
+        count += 1
+        if not task_id or question_text is None:
+            print(f"Skipping item with missing task_id or question: {item}")
+            continue
+        try:
+            print(f"{count}. Processing Task {task_id}: {question_text}")
+            submitted_answer = agent(question_text, task_id)
+            answers_payload.append(
+                {"task_id": task_id, "submitted_answer": submitted_answer}
+            )
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": submitted_answer,
+                }
+            )
+        except Exception as e:
+            print(f"Error running agent on task {task_id}: {e}")
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": f"AGENT ERROR: {e}",
+                }
+            )
+
+    if not answers_payload:
+        print("Agent did not produce any answers to submit.")
+        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
+
+    # 4. Prepare Submission
+    submission_data = {
+        "username": username.strip(),
+        "agent_code": agent_code,
+        "answers": answers_payload,
+    }
+    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    print(status_update)
+
+    # 5. Submit
+    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
+    try:
+        response = requests.post(submit_url, json=submission_data, timeout=60)
+        response.raise_for_status()
+        result_data = response.json()
+        final_status = (
+            f"Submission Successful!\n"
+            f"User: {result_data.get('username')}\n"
+            f"Overall Score: {result_data.get('score', 'N/A')}% "
+            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
+            f"Message: {result_data.get('message', 'No message received.')}"
+        )
+        print("Submission successful.")
+        results_df = pd.DataFrame(results_log)
+        return final_status, results_df
+    except requests.exceptions.HTTPError as e:
+        error_detail = f"Server responded with status {e.response.status_code}."
+        try:
+            error_json = e.response.json()
+            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
+        except requests.exceptions.JSONDecodeError:
+            error_detail += f" Response: {e.response.text[:500]}"
+        status_message = f"Submission Failed: {error_detail}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+    except requests.exceptions.Timeout:
+        status_message = "Submission Failed: The request timed out."
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+    except requests.exceptions.RequestException as e:
+        status_message = f"Submission Failed: Network error - {e}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+    except Exception as e:
+        status_message = f"An unexpected error occurred during submission: {e}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+
+
+# --- Build Gradio Interface using Blocks ---
+with gr.Blocks() as demo:
+    gr.Markdown("# Basic Agent Evaluation Runner")
+    gr.Markdown(
+        """
+        **Instructions:**
+
+        1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
+        2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
+        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
+
+        ---
+        **Disclaimers:**
+        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
+        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
+        """
+    )
+
+    gr.LoginButton()
+
+    run_button = gr.Button("Run Evaluation & Submit All Answers")
+
+    status_output = gr.Textbox(
+        label="Run Status / Submission Result", lines=5, interactive=False
+    )
+    # Removed max_rows=10 from DataFrame constructor
+    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
+
+    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
+
+if __name__ == "__main__":
+    print("\n" + "-" * 30 + " App Starting " + "-" * 30)
+    # Check for SPACE_HOST and SPACE_ID at startup for information
+    space_host_startup = os.getenv("SPACE_HOST")
+    space_id_startup = os.getenv("SPACE_ID")  # Get SPACE_ID at startup
+
+    if space_host_startup:
+        print(f"✅ SPACE_HOST found: {space_host_startup}")
+        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
+    else:
+        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
+
+    if space_id_startup:  # Print repo URLs if SPACE_ID is found
+        print(f"✅ SPACE_ID found: {space_id_startup}")
+        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
+        print(
+            f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main"
+        )
+    else:
+        print(
+            "ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined."
+        )
+
+    print("-" * (60 + len(" App Starting ")) + "\n")
+
+    print("Launching Gradio Interface for Basic Agent Evaluation...")
+    demo.launch(debug=True, share=False)

build/lib/src/agent.py

@@ -0,0 +1,371 @@
+import gc
+import logging
+import os
+import tempfile
+from typing import Optional
+
+import torch
+from dotenv import load_dotenv
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.rate_limiters import InMemoryRateLimiter
+from langchain_core.tools import Tool
+from langchain_experimental.utilities import PythonREPL
+
+# from langchain_community.tools import DuckDuckGoSearchResults
+# from langchain_community.utilities.duckduckgo_search import DuckDuckGoSearchAPIWrapper
+# from langchain_google_community import GoogleSearchAPIWrapper, GoogleSearchResults
+from langchain_ollama import ChatOllama
+
+from src.final_answer import create_final_answer_graph, validate_answer
+from src.tools import (
+    analyze_csv_file,
+    analyze_excel_file,
+    download_file_from_url,
+    duckduckgo_search,
+    extract_text_from_image,
+    read_file,
+    reverse_decoder,
+    review_youtube_video,
+    transcribe_audio,
+    transcribe_youtube,
+    use_vision_model,
+    video_frames_to_images,
+    website_scrape,
+)
+
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+base_url = os.getenv("OLLAMA_BASE_URL")
+
+rate_limiter = InMemoryRateLimiter(requests_per_second=0.1)
+
+
+class BasicAgent:
+    def __init__(self):
+        try:
+            logger.info("Initializing BasicAgent")
+
+            # Create the prompt template
+            prompt = ChatPromptTemplate.from_messages(
+                [
+                    (
+                        "system",
+                        """You are a general AI assistant. I will ask you a
+                        question. Report your thoughts, and finish your answer
+                        with the following template: FINAL ANSWER: [YOUR FINAL
+                        ANSWER]. YOUR FINAL ANSWER should be a number OR as few
+                        words as possible OR a comma separated list of numbers
+                        and/or strings. If you are asked for a number, don't
+                        use comma to write your number neither use units such
+                        as $ or percent sign unless specified otherwise. If you
+                        are asked for a string, don't use articles, neither
+                        abbreviations (e.g. for cities), and write the digits
+                        in plain text unless specified otherwise. If you are
+                        asked for a comma separated list, apply the above rules
+                        depending of whether the element to be put in the list
+                        is a number or a string.
+                """,
+                    ),
+                    ("placeholder", "{chat_history}"),
+                    ("human", "{input}"),
+                    ("placeholder", "{agent_scratchpad}"),
+                ]
+            )
+            logger.info("Created prompt template")
+
+            llm = ChatOllama(
+                model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+                base_url=base_url,
+                temperature=0.2,
+            )
+            logger.info("Created model successfully")
+
+            # Define available tools
+            tools = [
+                Tool(
+                    name="DuckDuckGoSearchResults",
+                    description="""Performs a live search using DuckDuckGo
+                    and analyzes the top results. Returns a summary including
+                    result titles, URLs, brief snippets, and ranking
+                    positions. Use this to quickly assess the relevance,
+                    diversity, and quality of information retrieved from a
+                    privacy-focused search engine, without personalized or
+                    biased filtering.""",
+                    # func=DuckDuckGoSearchResults(
+                    #     api_wrapper=DuckDuckGoSearchAPIWrapper()
+                    # ).run,
+                    func=duckduckgo_search,
+                ),
+                # Tool(
+                #     name="GoogleSearchResults",
+                #     description="""Performs a live Google search and analyzes
+                #     the top results. Returns a summary including result titles,
+                #     URLs, brief snippets, and ranking positions. Use this to
+                #     quickly understand the relevance, variety, and quality of
+                #     search results for a given query before deeper research or
+                #     content planning.""",
+                #     func=GoogleSearchResults(
+                #         api_wrapper=GoogleSearchAPIWrapper(
+                #             google_api_key=os.getenv("GOOGLE_SEARCH_API_KEY"),
+                #             google_cse_id=os.getenv("GOOGLE_CSE_ID"),
+                #             k=5,  # Number of results to return
+                #         )
+                #     ).run,
+                # ),
+                Tool(
+                    name="analyze csv file",
+                    description="""Only read and analyze the contents of a CSV
+                    file if one is explicitly referenced or uploaded in the
+                    question. When a CSV file is provided, return a summary of
+                    the dataset, including column names, data types, missing
+                    value counts, basic statistics for numeric fields, and a
+                    preview of the data. Use this only to quickly understand
+                    the structure and quality of the dataset before performing
+                    any further analysis.""",
+                    func=analyze_csv_file,
+                ),
+                Tool(
+                    name="analyze excel file",
+                    description="""Reads and analyzes the contents of an Excel
+                    file (.xlsx or .xls). Returns structured summaries
+                    for each sheet, including column names, data types, missing
+                    value counts, basic statistics for numeric columns, and
+                    sample rows. Use this to quickly explore the structure and
+                    quality of Excel datasets.""",
+                    func=analyze_excel_file,
+                ),
+                Tool(
+                    name="download file from url",
+                    description="""Downloads a file from a given URL and saves
+                    it locally. Supports various file types such as CSV, Excel,
+                    images, and PDFs. Use this to retrieve external resources
+                    for processing or analysis.""",
+                    func=download_file_from_url,
+                ),
+                Tool(
+                    name="extract_text_from_image",
+                    description="""Performs Optical Character Recognition (OCR)
+                    on an image to extract readable text after downloading it.
+                    Supports common image formats (e.g., PNG, JPG). Use this to
+                    digitize printed or handwritten content from images for
+                    search, analysis, or storage.""",
+                    func=extract_text_from_image,
+                ),
+                Tool(
+                    name="read_file",
+                    description="""Reads the raw content of a local text file.
+                    Supports formats such as .txt, .json, .xml, and markdown.
+                    Use this to load unstructured or semi-structured file
+                    content for display, parsing, or further
+                    processing—excluding CSV and Excel formats.""",
+                    func=read_file,
+                ),
+                Tool(
+                    name="review_youtube_video",
+                    description="""Analyzes a YouTube video by extracting key
+                    information such as title, description, view count, likes,
+                    comments, and transcript (if available). Use this to
+                    generate summaries, insights, or sentiment analysis based
+                    on video content and engagement.""",
+                    func=review_youtube_video,
+                ),
+                Tool(
+                    name="transcribe_audio",
+                    description="""Converts spoken words in an audio file into
+                    written text using speech-to-text technology. Supports
+                    common audio formats like MP3, WAV, and FLAC. Use this to
+                    create transcripts for meetings, interviews, podcasts, or
+                    any spoken content.""",
+                    func=transcribe_audio,
+                ),
+                Tool(
+                    name="transcribe_youtube",
+                    description="""Extracts and converts the audio from a
+                    YouTube video into text using speech-to-text technology.
+                    Supports generating transcripts for videos without captions
+                    or subtitles. Use this to obtain searchable, readable text
+                    from YouTube content.""",
+                    func=transcribe_youtube,
+                ),
+                Tool(
+                    name="use_vision_model",
+                    description="""Processes images using a computer vision
+                    model to perform tasks such as object detection, image
+                    classification, or segmentation. Use this to analyze visual
+                    content and extract meaningful information from images.""",
+                    func=use_vision_model,
+                ),
+                Tool(
+                    name="video_frames_to_images",
+                    description="""Extracts individual frames from a video file
+                    and saves them as separate image files. Use this to
+                    analyze, process, or visualize specific moments within
+                    video content. Use this to Youtube Videos""",
+                    func=video_frames_to_images,
+                ),
+                Tool(
+                    name="website_scrape",
+                    description="""It is mandatory to use duckduckgo_search
+                    tool before invoking this tool .Fetches and extracts
+                    content from a specified website URL. Supports retrieving
+                    text, images, links, and other page elements.""",
+                    func=website_scrape,
+                ),
+                Tool(
+                    name="python_repl",
+                    description="""Write full, valid Python code using proper
+                    multi-line code blocks Do not escape newlines (\n)
+                    instead, write each line of code on a separate line Always
+                    use proper indentation and syntax Return results using
+                    print() or return if using a function Avoid partial or
+                    inline code snippets — all code should be runnable in a
+                    Python REPL If the input is a function, include example
+                    usage at the end to ensure output is shown.""",
+                    func=PythonREPL().run,
+                    return_direct=True,
+                ),
+                # Tool(
+                #     name="wiki",
+                #     description="""Retrieves summarized information or
+                #     detailed content from Wikipedia based on a user query.
+                #     Use this to quickly access encyclopedic knowledge and
+                #     relevant facts on a wide range of topics.""",
+                #     func=wiki,
+                # ),
+                Tool(
+                    name="reverse decoder",
+                    description="""Decodes a reversed sentence if the input
+                    appears to be written backward.""",
+                    func=reverse_decoder,
+                ),
+            ]
+            # tools = [wrap_tool_with_limit(tool, max_calls=3) for tool in raw_tools]
+            logger.info("Tools: %s", tools)
+
+            # Create the agent
+            agent = create_tool_calling_agent(llm, tools, prompt)
+            logger.info("Created tool calling agent")
+
+            # Create the agent executor
+            self.agent_executor = AgentExecutor(
+                agent=agent,
+                tools=tools,
+                return_intermediate_steps=True,
+                verbose=True,
+                max_iterations=5,
+            )
+            logger.info("Created agent executor")
+
+            # Create the graph
+            self.validation_graph = create_final_answer_graph()
+
+        except Exception as e:
+            logger.error("Error initializing agent: %s", e, exc_info=True)
+            raise
+
+    def __call__(self, question: str, task_id: str) -> str:
+        """Execute the agent with the given question and optional file.
+        Args:
+            question (str): The question to answer
+            task_id (str): The task ID to fetch the file
+        Returns:
+            str: The final validated answer
+        Raises:
+            Exception: If no valid answer is found after max retries
+        """
+        max_retries = 3
+        attempt = 0
+
+        previous_steps = set()
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            while attempt < max_retries:
+                default_api_url = os.getenv("DEFAULT_API_URL")
+                file_url = f"{default_api_url}/files/{task_id}"
+
+                file: Optional[dict] = None
+                try:
+                    # Download file to temporary directory
+                    file = download_file_from_url.invoke(
+                        {
+                            "url": file_url,
+                            "directory": temp_dir,
+                        }
+                    )
+                    logger.info("Downloaded file: %s", file_url)
+                except Exception:
+                    logger.error(f"no download file available for {task_id} ")
+                    file = None
+
+                try:
+                    attempt += 1
+                    logger.info("Attempt %d of %d", attempt, max_retries)
+
+                    # Prepare input with file information
+                    input_data = {
+                        "input": question
+                        + (
+                            f" [File: type={file.get('type', 'None')}, path={file.get('path', 'None')}]"
+                            if file and file.get("type") != "error"
+                            else ""
+                        ),
+                    }
+
+                    # Run the agent to get the answer
+                    result = self.agent_executor.invoke(input_data)
+                    answer = result.get("output", "")
+                    intermediate_steps = result.get("intermediate_steps", [])
+
+                    steps_str = str(intermediate_steps)
+                    if steps_str in previous_steps:
+                        logger.warning(
+                            f"Detected repeated reasoning steps on attempt {attempt}. Breaking loop to avoid infinite retry."
+                        )
+                        break  # or raise Exception to stop retries
+                    previous_steps.add(steps_str)
+
+                    logger.info("Attempt %d result: %s", attempt, result)
+
+                    # Run validation (self.validation_graph is now StateGraph)
+                    validation_result = validate_answer(
+                        self.validation_graph,  # type: ignore
+                        answer,
+                        [result.get("intermediate_steps", [])],
+                    )
+
+                    valid_answer = validation_result.get("valid_answer", False)
+                    final_answer = validation_result.get("final_answer", "")
+
+                    if valid_answer:
+                        logger.info("Valid answer found on attempt %d", attempt)
+                        torch.cuda.empty_cache()
+                        return final_answer
+
+                    logger.warning(
+                        "Validation failed on attempt %d: %s", attempt, final_answer
+                    )
+                    if attempt >= max_retries:
+                        raise Exception(
+                            "Failed to get valid answer after %d attempts. Last error: %s",
+                            max_retries,
+                            final_answer,
+                        )
+
+                except Exception as e:
+                    logger.error("Error in attempt %d: %s", attempt, e, exc_info=True)
+                    if attempt >= max_retries:
+                        raise Exception(
+                            "Failed after %d attempts. Last error: %s",
+                            max_retries,
+                            str(e),
+                        )
+                    continue
+
+        # Fallback in case loop exits unexpectedly
+
+        torch.cuda.empty_cache()
+        gc.collect()
+        raise Exception("No valid answer found after processing")

build/lib/src/chess.py

@@ -0,0 +1,235 @@
+import gc
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import models, transforms
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Load the pre-trained model
+try:
+    model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
+    model.fc = torch.nn.Linear(model.fc.in_features, 13)  # 13 classes including 'empty'
+    model.load_state_dict(torch.load("best_chess_piece_model.pth", map_location=device))
+    model.eval()
+    model = model.to(device)
+except Exception as e:
+    print(f"Error loading model: {e}")
+    exit(1)
+
+# Mapping chess piece indices
+piece_labels = [
+    "black_bishop",
+    "black_king",
+    "black_knight",
+    "black_pawn",
+    "black_queen",
+    "black_rook",
+    "empty",
+    "white_bishop",
+    "white_king",
+    "white_knight",
+    "white_pawn",
+    "white_queen",
+    "white_rook",
+]
+
+# Define chessboard coordinates (0,0) is top-left (a8), (7,7) is bottom-right (h1)
+coordinates = [(i, j) for i in range(8) for j in range(8)]
+
+# Define a transformation to prepare images for the model
+transform = transforms.Compose(
+    [
+        transforms.Resize((224, 224)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ]
+)
+
+
+# Function to process and predict the piece type at each square
+def predict_piece(image, model, device):
+    try:
+        if len(image.shape) == 2 or image.shape[2] == 1:
+            image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
+        else:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+        image = Image.fromarray(image)
+        image_tensor = transform(image).unsqueeze(0).to(device)
+        with torch.no_grad():
+            output = model(image_tensor)
+            _, predicted = torch.max(output, 1)
+        return piece_labels[predicted.item()]
+    except Exception as e:
+        print(f"Error predicting piece: {e}")
+        return "unknown"
+
+
+# Function to detect chessboard grid using edge detection and Hough lines
+def detect_chessboard_grid(image):
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    # Enhance contrast
+    gray = cv2.convertScaleAbs(gray, alpha=1.2, beta=20)
+    # Apply Gaussian blur to reduce noise
+    gray = cv2.GaussianBlur(gray, (5, 5), 0)
+    # Edge detection with Canny
+    edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+    # Detect lines using Hough Transform
+    lines = cv2.HoughLinesP(
+        edges, 1, np.pi / 180, threshold=80, minLineLength=50, maxLineGap=10
+    )
+
+    if lines is None:
+        print("No lines detected.")
+        return None, edges
+
+    # Separate horizontal and vertical lines
+    h_lines = []
+    v_lines = []
+    for line in lines:
+        x1, y1, x2, y2 = line[0]
+        if abs(x2 - x1) > abs(y2 - y1):  # Horizontal line
+            h_lines.append((y1, x1, x2))
+        else:  # Vertical line
+            v_lines.append((x1, y1, y2))
+
+    # Sort and filter to get exactly 9 lines for each
+    h_lines = sorted(h_lines, key=lambda x: x[0])[:9]  # Top 9 horizontal lines
+    v_lines = sorted(v_lines, key=lambda x: x[0])[:9]  # Top 9 vertical lines
+
+    if len(h_lines) < 9 or len(v_lines) < 9:
+        print(
+            f"Insufficient lines detected: {len(h_lines)} horizontal, {len(v_lines)} vertical"
+        )
+        return None, edges
+
+    # Find intersections to get 8x8 grid corners
+    corners = []
+    for h in h_lines:
+        y = h[0]
+        for v in v_lines:
+            x = v[0]
+            corners.append([x, y])
+
+    # Ensure exactly 64 corners (8x8 grid)
+    if len(corners) != 64:
+        print(f"Expected 64 corners, got {len(corners)}")
+        return None, edges
+
+    corners = np.array(corners, dtype=np.float32).reshape(8, 8, 2)
+
+    # Visualize detected lines for debugging
+    debug_image = image.copy()
+    for y, x1, x2 in h_lines:
+        cv2.line(debug_image, (x1, y), (x2, y), (0, 255, 0), 2)
+    for x, y1, y2 in v_lines:
+        cv2.line(debug_image, (x, y1), (x, y2), (0, 0, 255), 2)
+    cv2.imwrite("lines_debug.png", debug_image)
+
+    return corners, edges
+
+
+# Function to extract coordinates of chess pieces from an image
+def extract_chessboard_coordinates(image_path):
+    try:
+        image = cv2.imread(image_path)
+        if image is None:
+            print(f"Failed to load image: {image_path}")
+            return []
+    except Exception as e:
+        print(f"Error loading image {image_path}: {e}")
+        return []
+
+    # Try OpenCV's chessboard detection first
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    ret, corners = cv2.findChessboardCorners(gray, (8, 8), None)
+
+    if ret:
+        corners = cv2.cornerSubPix(
+            gray,
+            corners,
+            (11, 11),
+            (-1, -1),
+            criteria=(cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1),
+        )
+        corners = corners.reshape(8, 8, 2)
+    else:
+        print("OpenCV chessboard detection failed. Attempting edge-based detection.")
+        corners, edges = detect_chessboard_grid(image)
+        if corners is None:
+            # Save edges for debugging
+            cv2.imwrite("edges_debug.png", edges)
+            print("Saved edge detection output to edges_debug.png")
+            return []
+        # Save debug image with detected corners
+        debug_image = image.copy()
+        for h in range(8):
+            for v in range(8):
+                x, y = int(corners[h, v, 0]), int(corners[h, v, 1])
+                cv2.circle(debug_image, (x, y), 5, (0, 255, 0), -1)
+        cv2.imwrite("grid_debug.png", debug_image)
+        print("Saved grid detection debug image to grid_debug.png")
+
+    # Calculate square size dynamically
+    square_width = np.mean(
+        [
+            np.linalg.norm(corners[i, j] - corners[i, j + 1])
+            for i in range(8)
+            for j in range(7)
+        ]
+    )
+    square_height = np.mean(
+        [
+            np.linalg.norm(corners[i, j] - corners[i + 1, j])
+            for i in range(7)
+            for j in range(8)
+        ]
+    )
+    square_size = int(min(square_width, square_height))
+
+    # Create a blank grid to store coordinates
+    piece_coordinates = []
+
+    # Loop through all coordinates and detect pieces
+    for i, j in coordinates:
+        try:
+            x = int(corners[i, j, 0])
+            y = int(corners[i, j, 1])
+            w = h = square_size
+
+            x = max(0, x)
+            y = max(0, y)
+            x_end = min(image.shape[1], x + w)
+            y_end = min(image.shape[0], y + h)
+            roi = image[y:y_end, x:x_end]
+
+            if roi.shape[0] == 0 or roi.shape[1] == 0:
+                print(f"Invalid ROI at square ({i}, {j})")
+                piece_coordinates.append(((i, j), "unknown"))
+                continue
+
+            predicted_piece = predict_piece(roi, model, device)
+            piece_coordinates.append(((i, j), predicted_piece))
+        except Exception as e:
+            print(f"Error processing square ({i}, {j}): {e}")
+            piece_coordinates.append(((i, j), "unknown"))
+
+    return piece_coordinates
+
+
+# # Example usage
+# IMAGE_PATH = "test.png"
+# coordinates = extract_chessboard_coordinates(IMAGE_PATH)
+# for coord, piece in coordinates:
+#     print(f"Piece at {coord}: {piece}")
+
+# Clean up
+del model
+if torch.cuda.is_available():
+    torch.cuda.empty_cache()
+gc.collect()

build/lib/src/final_answer.py

@@ -0,0 +1,210 @@
+import os
+from typing import Any, Dict, Optional
+
+import torch
+from dotenv import load_dotenv
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_ollama import ChatOllama
+from langgraph.graph import END, START, Graph, StateGraph
+from typing_extensions import TypedDict
+
+load_dotenv()
+
+base_url = os.getenv("OLLAMA_BASE_URL")
+
+
+class AgentState(TypedDict):
+    """State for the final answer validation graph."""
+
+    question: str
+    answer: str
+    final_answer: str | None
+    agent_memory: Any
+    valid_answer: bool
+
+
+def extract_answer(state: AgentState) -> Dict:
+    """Extract and format the final answer from the state.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the formatted final answer.
+    """
+    # Extract the final answer from the state
+    sep_token = "FINAL ANSWER:"
+    raw_answer = state["answer"]
+
+    # Extract the answer after the separator if it exists
+    if sep_token in raw_answer:
+        formatted_answer = raw_answer.split(sep_token)[1].strip()
+    else:
+        formatted_answer = raw_answer.strip()
+
+    # Remove any brackets from lists
+    formatted_answer = formatted_answer.replace("[", "").replace("]", "")
+
+    # Remove units unless specified
+    if not any(
+        unit in formatted_answer.lower() for unit in ["$", "%", "dollars", "percent"]
+    ):
+        formatted_answer = formatted_answer.replace("$", "").replace("%", "")
+
+    # Remove commas from numbers
+    parts = formatted_answer.split(",")
+    formatted_parts = []
+    for part in parts:
+        part = part.strip()
+        if part.replace(".", "").isdigit():  # Check if it's a number
+            part = part.replace(",", "")
+        formatted_parts.append(part)
+    formatted_answer = ", ".join(formatted_parts)
+
+    return {"final_answer": formatted_answer}
+
+
+def reasoning_check(state: AgentState) -> Dict:
+    """
+    Node that checks the reasoning of the final answer.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the reasoning check result.
+    """
+    model = ChatOllama(
+        model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+        base_url=base_url,
+        temperature=0.2,
+    )
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                """You are a strict validator of answers. Your job is to check if the reasoning and results are correct.
+        You should have >90% confidence that the answer is correct to pass it.
+        First list reasons why yes/no, then write your final decision: PASS in caps lock if it is satisfactory, FAIL if it is not.""",
+            ),
+            (
+                "human",
+                """
+        Here is a user-given task and the agent steps: {agent_memory}
+        Now here is the answer that was given: {final_answer}
+        Please check that the reasoning process and results are correct: do they correctly answer the given task?
+        """,
+            ),
+        ]
+    )
+
+    chain = prompt | model | StrOutputParser()
+    output = chain.invoke(
+        {
+            "agent_memory": state["agent_memory"],
+            "final_answer": state["final_answer"],
+        }
+    )
+
+    print("Reasoning Feedback: ", output)
+    if "FAIL" in output:
+        return {"valid_answer": False}
+    torch.cuda.empty_cache()
+    return {"valid_answer": True}
+
+
+def formatting_check(state: AgentState) -> Dict:
+    """
+    Node that checks the formatting of the final answer.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the formatting check result.
+    """
+    model = ChatOllama(
+        model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+        base_url=base_url,
+        temperature=0.2,
+    )
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                """You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+        """,
+            ),
+            (
+                "human",
+                """
+        Here is a user-given task and the agent steps: {agent_memory}
+        Now here is the FINAL ANSWER that was given: {final_answer}
+        Ensure the FINAL ANSWER is in the right format as asked for by the task.
+        """,
+            ),
+        ]
+    )
+
+    chain = prompt | model | StrOutputParser()
+    output = chain.invoke(
+        {
+            "agent_memory": state["agent_memory"],
+            "final_answer": state["final_answer"],
+        }
+    )
+
+    print("Formatting Feedback: ", output)
+    if "FAIL" in output:
+        return {"valid_answer": False}
+
+    torch.cuda.empty_cache()
+    return {"valid_answer": True}
+
+
+def create_final_answer_graph() -> Graph:
+    """Create a graph that validates the final answer.
+    Returns:
+        A graph that validates the final answer.
+    """
+    # Create the graph
+    workflow = StateGraph(AgentState)
+
+    # Add nodes
+    workflow.add_node("extract_answer", extract_answer)
+    workflow.add_node("reasoning_check", reasoning_check)
+    workflow.add_node("formatting_check", formatting_check)
+
+    # Add edges
+    workflow.add_edge(START, "extract_answer")
+    workflow.add_edge("extract_answer", "reasoning_check")
+    workflow.add_edge("reasoning_check", "formatting_check")
+    workflow.add_edge("formatting_check", END)
+
+    # Compile the graph
+    return workflow.compile()  # type: ignore
+
+
+def validate_answer(graph: StateGraph, answer: str, agent_memory: Any) -> Dict:
+    """Validate the answer using the LangGraph workflow.
+    Args:
+        graph: The validation graph (LangGraph StateGraph).
+        answer: The answer to validate.
+        agent_memory: The agent's memory.
+    Returns:
+        A dictionary with validation results.
+    """
+    try:
+        # Initialize state
+        initial_state = {
+            "answer": answer,
+            "final_answer": None,
+            "agent_memory": agent_memory,
+            "valid_answer": False,
+        }
+
+        # Run the graph
+        result = graph.invoke(initial_state)  # type:ignore
+
+        return {
+            "valid_answer": result.get("valid_answer", False),
+            "final_answer": result.get("final_answer", None),
+        }
+    except Exception as e:
+        print(f"Validation failed: {e}")
+        return {"valid_answer": False, "final_answer": None}

build/lib/src/tools.py

@@ -0,0 +1,675 @@
+# pylint: disable=no-member
+import base64
+import gc
+import mimetypes
+import os
+import re
+import tempfile
+import time
+import uuid
+from datetime import timedelta
+from typing import Dict, List, Optional, TypedDict, Union
+from urllib.parse import urlparse
+
+import cv2
+import imageio
+import pandas as pd
+import pytesseract
+import requests
+import torch
+import whisper
+import yt_dlp
+from bs4 import BeautifulSoup, Tag
+from dotenv import load_dotenv
+from duckduckgo_search import DDGS
+from langchain_core.messages import HumanMessage
+from langchain_core.tools import tool
+from langchain_ollama import ChatOllama
+from PIL import Image
+from playwright.sync_api import sync_playwright
+from youtube_transcript_api import (
+    NoTranscriptFound,
+    TranscriptsDisabled,
+    YouTubeTranscriptApi,
+)
+
+load_dotenv()
+base_url = os.getenv("OLLAMA_BASE_URL")
+model_vision = ChatOllama(
+    model="gemma3:latest",
+    base_url=base_url,
+)
+model_text = ChatOllama(model="gemma3:latest", base_url=base_url)
+
+
+@tool
+def use_vision_model(question: str) -> str:
+    """
+    A multimodal reasoning model that combines image and text input to answer
+    questions using the image.
+    """
+    # Extract image paths
+    image_paths = re.findall(r"[\w\-/\.]+\.(?:png|jpg|jpeg|webp)", question)
+    image_paths = [p for p in image_paths if os.path.exists(p)]
+
+    if not image_paths:
+        return "No valid image file found in the question."
+
+    image_path = image_paths[0]
+
+    # Preprocess the image using OpenCV
+    image = cv2.imread(image_path)
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    gray = cv2.convertScaleAbs(gray, alpha=1.2, beta=20)
+    gray = cv2.GaussianBlur(gray, (5, 5), 0)
+    edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+    # Create a temporary file for the processed image
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
+        temp_image_path = tmp_file.name
+        cv2.imwrite(temp_image_path, edges)
+
+        # Encode the temp image
+        mime_type, _ = mimetypes.guess_type(temp_image_path)
+        mime_type = mime_type or "image/png"
+        with open(temp_image_path, "rb") as f:
+            encoded = base64.b64encode(f.read()).decode("utf-8")
+
+        # Prepare the prompt and image for the model
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": question},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:{mime_type};base64,{encoded}"},
+                    },
+                ],
+            }
+        ]
+
+        # Invoke the vision model
+        response = model_vision.invoke(messages)
+
+    # Clean up
+    del messages, encoded, image_path
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    return str(response.content) if hasattr(response, "content") else str(response)
+
+
+# YouTube Video Review Tool
+@tool
+def review_youtube_video(url: str) -> str:
+    """Reviews a YouTube video and answers a specific question about that video.
+
+    Args:
+        url (str): the URL to the YouTube video.
+        question (str): The question you are asking about the video.
+
+    Returns:
+        str: The answer to the question
+    """
+    # Extract video ID from URL (assuming it is in the format https://youtube.com/watch?v=VIDEO_ID)
+    video_id = url.split("v=")[1]
+    transcript_url = (
+        f"https://www.youtube.com/api/timedtext?v={video_id}"  # Getting transcript data
+    )
+
+    response = requests.get(transcript_url, timeout=200)
+
+    transcript = response.text  # This is the transcript (XML or SRT format)
+
+    # Prepare the content (just the transcript, no question needed)
+    transcript_content = f"Here is the transcript of the video: {transcript}"
+
+    # Return the transcript content so the main LLM can handle question generation
+    return transcript_content
+
+
+# YouTube Frames to Images Tool
+@tool
+def video_frames_to_images(
+    url: str,
+    folder_name: str,
+    sample_interval_seconds: int = 5,
+) -> List[str]:
+    """Extracts frames from a video at specified intervals and saves them as images.
+    Args:
+        url (str): the URL to the video.
+        folder_name (str): the name of the folder to save the images to.
+        sample_interval_seconds (int): the interval between frames to sample.
+    Returns:
+        List[str]: A list of paths to the saved image files.
+    """
+    # Create a subdirectory for the frames
+    frames_dir = os.path.join(folder_name, "frames")
+    os.makedirs(frames_dir, exist_ok=True)
+
+    ydl_opts = {
+        "format": "bestvideo[height<=1080]+bestaudio/best[height<=1080]/best",
+        "outtmpl": os.path.join(folder_name, "video.%(ext)s"),
+        "quiet": True,
+        "noplaylist": True,
+        "merge_output_format": "mp4",
+        "force_ipv4": True,
+    }
+
+    info_extracted = []
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(url, download=True)
+        info_extracted.append(info)
+        video_path = next(
+            (
+                os.path.join(folder_name, f)
+                for f in os.listdir(folder_name)
+                if f.endswith(".mp4")
+            ),
+            None,
+        )
+
+        if not video_path:
+            raise RuntimeError("Failed to download video as mp4")
+
+        reader = imageio.get_reader(video_path)
+        metadata = reader.get_meta_data()
+        fps = metadata.get("fps")
+
+        if fps is None:
+            reader.close()
+            raise RuntimeError("Unable to determine FPS from video metadata")
+
+        frame_interval = int(fps * sample_interval_seconds)
+        num_frames = reader.get_length()
+        image_paths: List[str] = []
+
+        for idx in range(num_frames):
+            if idx % frame_interval == 0:
+                # Save frame as image
+                frame = reader.get_data(idx)
+                image_path = os.path.join(frames_dir, f"frame_{idx:06d}.jpg")
+                imageio.imwrite(image_path, frame)
+                image_paths.append(image_path)
+
+        reader.close()
+        return image_paths
+
+
+# File Reading Tool
+@tool
+def read_file(filepath: str) -> str:
+    """Reads the content of a text file.
+    Args:
+        filepath (str): the path to the file to read.
+    Returns:
+        str: The content of the file.
+    """
+    try:
+        with open(filepath, "r", encoding="utf-8") as file:
+            content = file.read()
+        return content
+    except FileNotFoundError:
+        return f"File not found: {filepath}"
+    except IOError as e:
+        return f"Error reading file: {str(e)}"
+
+
+# File Download Tool
+@tool
+def download_file_from_url(url: str, directory: str) -> Dict[str, Union[str, None]]:
+    """Downloads a file from a URL and saves it to a directory.
+    Args:
+        url (str): the URL to download the file from.
+        directory (str): the directory to save the file to.
+    Returns:
+        Dict[str, Union[str, None]]: A dictionary containing the file type and path.
+    """
+
+    response = requests.get(url, stream=True, timeout=10)
+    response.raise_for_status()
+
+    content_type = response.headers.get("content-type", "").lower()
+
+    # Try to get filename from headers
+    filename = None
+    cd = response.headers.get("content-disposition", "")
+    match = re.search(r"filename\*=UTF-8\'\'(.+)", cd) or re.search(
+        r'filename="?([^"]+)"?', cd
+    )
+    if match:
+        filename = match.group(1)
+
+    # If not in headers, try URL
+    if not filename:
+        filename = os.path.basename(url.split("?")[0])
+
+    # Fallback to generated filename
+    if not filename:
+        extension = {
+            "image/jpeg": ".jpg",
+            "image/png": ".png",
+            "image/gif": ".gif",
+            "audio/wav": ".wav",
+            "audio/mpeg": ".mp3",
+            "video/mp4": ".mp4",
+            "text/plain": ".txt",
+            "text/csv": ".csv",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+            "application/vnd.ms-excel": ".xls",
+            "application/octet-stream": ".bin",
+        }.get(content_type, ".bin")
+        filename = f"downloaded_{uuid.uuid4().hex[:8]}{extension}"
+
+    os.makedirs(directory, exist_ok=True)
+    file_path = os.path.join(directory, filename)
+
+    with open(file_path, "wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+    # shutil.copy(file_path, os.getcwd())
+
+    return {"type": content_type, "path": file_path}
+
+
+# Text Extraction from Image Tool
+@tool
+def extract_text_from_image(image_path: str) -> str:
+    """Extracts text from an image using OCR.
+    Args:
+        image_path (str): the path to the image to extract text from.
+    Returns:
+        str: The text extracted from the image.
+    """
+
+    image = Image.open(image_path)
+    text = pytesseract.image_to_string(image)
+    return f"Extracted text from image:\n\n{text}"
+
+
+# CSV Analysis Tool
+@tool
+def analyze_csv_file(file_path: str, query: str) -> str:
+    """Analyzes a CSV file and answers questions about its contents using an Ollama model.
+
+    Args:
+        file_path (str): The path to the CSV file to analyze.
+        query (str): The question to answer about the CSV file.
+
+    Returns:
+        str: The result of the analysis.
+    """
+    # Load the CSV file
+    df = pd.read_csv(file_path)
+    df_str = df.to_string(index=False)
+
+    # Compose the prompt
+    prompt = f"""
+                You are a data analyst. Analyze the following CSV data and answer the question provided.
+                
+                CSV Dimensions: {df.shape[0]} rows × {df.shape[1]} columns
+                
+                CSV Data:
+                {df_str}
+                
+                Please provide:
+                1. A summary of the data structure and content
+                2. Key patterns and insights
+                3. Potential data quality issues
+                4. Suggestions for analysis
+                
+                User Query:
+                {query}
+                
+                Format your response in markdown with sections and bullet points.
+             """
+
+    model = model_text
+
+    # Call the model
+    response = model.invoke([{"type": "text", "text": prompt}])
+    del df
+    torch.cuda.empty_cache()
+    gc.collect()
+
+    # Return the result
+    if hasattr(response, "content") and isinstance(response.content, str):
+        return response.content
+    return str(response)
+
+
+# Excel Analysis Tool
+@tool
+def analyze_excel_file(file_path: str) -> str:
+    """Analyzes an Excel file and answers questions about its contents using Ollama backed LLM
+    Args:
+        file_path (str): the path to the Excel file to analyze.
+        question (str): the question to answer about the Excel file.
+    Returns:
+        str: The result of the analysis.
+    """
+    llm = model_text
+
+    # Read all sheets from the Excel file
+    excel_file = pd.ExcelFile(file_path)
+    sheet_names = excel_file.sheet_names
+
+    result = f"Excel file loaded with {len(sheet_names)} sheets: {', '.join(sheet_names)}\n\n"
+
+    for sheet_name in sheet_names:
+        df = pd.read_excel(file_path, sheet_name=sheet_name)
+        df_str = df.to_string()
+
+        # Build the prompt
+        prompt = f"""Analyze the following Excel sheet data and answer the user's query.
+                     Sheet Name: {sheet_name}
+                     Dimensions: {len(df)} rows × {len(df.columns)} columns
+                     
+                     Data:
+                     {df_str}
+                     
+                     Please provide:
+                     1. A summary of the data structure and content
+                     2. Key patterns and insights
+                     3. Potential data quality issues
+                     4. Suggestions for analysis
+                     
+                     Format the response clearly using headings and bullet points."""
+
+        # Call the LLM with the prompt
+        response = llm.invoke([HumanMessage(content=prompt)])
+
+        result += f"=== Sheet: {sheet_name} ===\n"
+        result += str(response.content) + "\n"
+        result += "=" * 50 + "\n\n"
+        del df
+        gc.collect()
+
+    excel_file.close()
+    torch.cuda.empty_cache()
+
+    return result
+
+
+# Audio Transcription Tool
+def transcribe_audio(audio_file_path: str) -> str:
+    """Transcribes an audio file using Whisper's audio capabilities.
+    Args:
+        audio_file_path (str): The path to the audio file to transcribe.
+        mime_type (str): The MIME type of the audio file.
+    Returns:
+        str: The transcript of the audio file.
+    Raises:
+        ValueError: If the MIME type is not supported.
+    """
+
+    model = whisper.load_model("base")
+    result = model.transcribe(audio_file_path)
+    assert isinstance(result["text"], str)
+
+    del model
+    torch.cuda.empty_cache()
+    gc.collect()
+    return result["text"]
+
+
+def _extract_video_id(url: str) -> Optional[str]:
+    """Extract video ID from YouTube URL.
+    Args:
+        url (str): the URL to the YouTube video.
+    Returns:
+        str: The video ID of the YouTube video.
+    """
+    patterns = [
+        r"(?:youtube\.com\/watch\?v=|youtube\.com\/embed\/|youtu\.be\/)([^&\n?#]+)",
+        r"(?:youtube\.com\/v\/|youtube\.com\/e\/|youtube\.com\/user\/[^\/]+\/|youtube\.com\/[^\/]+\/|youtube\.com\/embed\/|youtu\.be\/)([^&\n?#]+)",
+    ]
+
+    for pattern in patterns:
+        match = re.search(pattern, url)
+        if match:
+            return match.group(1)
+    return None
+
+
+@tool
+def transcribe_youtube(url: str) -> str:
+    """
+    Transcribes a YouTube video using YouTube Transcript API or ChatOllama with Whisper as fallback.
+
+    This function first tries to fetch the transcript of a YouTube video using the YouTube Transcript API.
+    If the transcript is unavailable (e.g., due to captions being disabled), it falls back to using
+    ChatOllama integrated with Whisper to transcribe the audio.
+
+    Args:
+        url (str): The URL to the YouTube video.
+
+    Returns:
+        str: The transcript of the YouTube video, or an error message if transcription fails.
+    """
+
+    try:
+        # Try using YouTube Transcript API
+        video_id = _extract_video_id(url)
+        transcript = ""
+        transcript_chunks = YouTubeTranscriptApi.get_transcript(
+            video_id, languages=["en"]
+        )
+        for chunk in transcript_chunks:
+            timestamp = str(timedelta(seconds=int(chunk["start"])))
+            transcript += f"[{timestamp}] {chunk['text']}\n"
+
+        # Return API transcript if available
+        if transcript.strip():
+            return transcript
+
+    except (TranscriptsDisabled, NoTranscriptFound, Exception) as exec:
+        try:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                # Download audio from YouTube
+                ydl_opts = {
+                    "format": "bestaudio/best",
+                    "outtmpl": os.path.join(tmpdir, "audio.%(ext)s"),
+                    "quiet": True,
+                    "noplaylist": True,
+                    "postprocessors": [
+                        {
+                            "key": "FFmpegExtractAudio",
+                            "preferredcodec": "wav",
+                            "preferredquality": "192",
+                        }
+                    ],
+                }
+
+                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                    info = ydl.extract_info(url, download=True)
+
+                    if info is not None:
+                        title = info.get("title", "Unknown Title")  # Type:None
+                        duration = info.get("duration", 0)  # in seconds
+                        uploader = info.get("uploader", "Unknown Uploader")
+                    else:
+                        title = "Unknown Title"
+                        duration = 0
+                        uploader = "Unknown Uploader"
+
+                audio_path = next(
+                    (
+                        os.path.join(tmpdir, f)
+                        for f in os.listdir(tmpdir)
+                        if f.endswith(".wav")
+                    ),
+                    None,
+                )
+                if not audio_path:
+                    raise RuntimeError("Failed to download or convert audio") from exec
+
+                # Use Whisper for initial transcription
+                whisper_model = whisper.load_model("base")
+                transcription = whisper_model.transcribe(audio_path, verbose=False)
+                raw_transcript = transcription["text"]
+                del whisper_model
+                gc.collect()
+
+                # Use ChatOllama to format transcript with timestamps
+                ollama = model_text
+                prompt = (
+                    "Please format the following raw transcript into a structured format with timestamps "
+                    f"The following transcript was generated from a YouTube video titled '{title}' "
+                    f"uploaded by {uploader}. The total video duration is approximately {duration}.\n\n"
+                    "Use the video’s length to help guide timestamp estimation.\n\n"
+                    "(e.g., [00:00:00] text). Estimate timestamps based on the natural flow of the text."
+                    f"Raw transcript:\n{raw_transcript}"
+                )
+                response = ollama.invoke([HumanMessage(content=prompt)])
+                formatted_transcript = str(
+                    response.content
+                )  # Ensure response is a string
+
+                torch.cuda.empty_cache()
+
+                return formatted_transcript
+        except Exception as fallback_exc:
+            raise RuntimeError("Fallback Transcription failed") from fallback_exc
+    return "Transcription failed unexpectedly."
+
+
+@tool
+def website_scrape(url: str) -> str:
+    """scrapes a website and returns the text.
+    args:
+        url (str): the url to the website to scrape.
+    returns:
+        str: the text of the website.
+    """
+    try:
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme or not parsed_url.netloc:
+            raise ValueError(
+                f"Invalid URL: '{url}'. Call `duckduckgo_search` first to get a valid URL."
+            )
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+            page.goto(url, wait_until="networkidle", timeout=60000)
+            page.wait_for_load_state("domcontentloaded")
+            html_content = page.content()
+            browser.close()
+
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        relevant_text = ""
+        for header in soup.find_all(["h2", "h3"]):
+            heading_text = header.get_text().strip().lower()
+            if "discography" in heading_text or "studio albums" in heading_text:
+                section_texts = []
+                tag = header.find_next_sibling()
+                while tag and (
+                    not isinstance(tag, Tag) or tag.name not in ["h2", "h3"]
+                ):
+                    section_texts.append(tag.get_text(separator=" ", strip=True))
+                    tag = tag.find_next_sibling()
+                relevant_text = "\n\n".join(section_texts)
+                break
+        if not relevant_text:
+            article = soup.find("article")
+            if article:
+                relevant_text = article.get_text(separator=" ", strip=True)
+        if not relevant_text:
+            relevant_text = soup.get_text(separator=" ", strip=True)
+
+        # step 2: chunk the text (optional but recommended)
+        def chunk_text(text, max_length=1000):
+            words = text.split()
+            chunks = []
+            for i in range(0, len(words), max_length):
+                chunks.append(" ".join(words[i : i + max_length]))
+            return chunks
+
+        chunks = chunk_text(relevant_text)
+
+        # return only the first 2–3 chunks to keep it concise
+        return "\n\n".join(chunks[:100])
+    except ValueError as e:
+        # Catch URL validation errors
+        return str(e)
+    except Exception as e:
+        # Catch other unexpected errors
+        return f"Scraping failed: {str(e)}"
+
+
+class SearchResult(TypedDict):
+    query: str
+    status: str
+    attempt: int
+    results: Optional[List[dict]]
+    error: Optional[str]
+
+
+@tool
+def duckduckgo_search(query: str, max_results: int = 10) -> SearchResult:
+    """
+    Perform a DuckDuckGo search with retry and backoff.
+    Use this FIRST before invoking and scraping tools.
+    Args:
+        query: The search query string.
+        max_results: Max number of results to return (default 10).
+    Returns:
+        A dict with the query, results, status, attempt count, and any error.
+    """
+    max_retries = 3
+    base_delay = 2
+    backoff_factor = 2
+
+    for attempt in range(max_retries):
+        try:
+            with DDGS() as ddgs:
+                results = ddgs.text(keywords=query, max_results=max_results)
+                if results:
+                    formatted_results = [
+                        {
+                            "title": result.get("title", ""),
+                            "url": result.get("href", ""),
+                            "body": result.get("body", ""),
+                        }
+                        for result in results
+                    ]
+                    return {
+                        "query": query,
+                        "status": "success",
+                        "attempt": attempt + 1,
+                        "results": formatted_results,
+                        "error": None,
+                    }
+        except Exception as e:
+            print(f"[DuckDuckGo Tool] Attempt {attempt + 1} failed: {e}")
+            time.sleep(base_delay * (backoff_factor**attempt))
+
+    return {
+        "query": query,
+        "status": "failed",
+        "attempt": max_retries,
+        "results": None,
+        "error": "Max retries exceeded or request failed.",
+    }
+
+
+@tool
+def reverse_decoder(question: str) -> str:
+    """Decodes a reversed sentence if the input appears to be written backward.
+
+    Args:
+        question (str): The possibly reversed question string.
+
+    Returns:
+        str: The decoded sentence.
+    """
+    # Remove leading punctuation if present
+    cleaned = question.strip().strip(".!?")
+
+    # Check if it's likely reversed (simple heuristic: mostly lowercase, reversed word order)
+    reversed_text = cleaned[::-1]
+
+    return reversed_text

build/lib/tools/chess.py

@@ -0,0 +1,241 @@
+import gc
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import models, transforms
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Load the pre-trained model
+try:
+    model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
+    model.fc = torch.nn.Linear(model.fc.in_features, 13)  # 13 classes including 'empty'
+    model.load_state_dict(torch.load("best_chess_piece_model.pth", map_location=device))
+    model.eval()
+    model = model.to(device)
+except Exception as e:
+    print(f"Error loading model: {e}")
+    exit(1)
+
+# Mapping chess piece indices
+piece_labels = [
+    "black_bishop",
+    "black_king",
+    "black_knight",
+    "black_pawn",
+    "black_queen",
+    "black_rook",
+    "empty",
+    "white_bishop",
+    "white_king",
+    "white_knight",
+    "white_pawn",
+    "white_queen",
+    "white_rook",
+]
+
+# Define chessboard coordinates (0,0) is top-left (a8), (7,7) is bottom-right (h1)
+coordinates = [(i, j) for i in range(8) for j in range(8)]
+
+# Define a transformation to prepare images for the model
+transform = transforms.Compose(
+    [
+        transforms.Resize((224, 224)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ]
+)
+
+
+# Function to process and predict the piece type at each square
+def predict_piece(image, model, device):
+    try:
+        if len(image.shape) == 2 or image.shape[2] == 1:
+            image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
+        else:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+        image = Image.fromarray(image)
+        image_tensor = transform(image).unsqueeze(0).to(device)
+        with torch.no_grad():
+            output = model(image_tensor)
+            _, predicted = torch.max(output, 1)
+        return piece_labels[predicted.item()]
+    except Exception as e:
+        print(f"Error predicting piece: {e}")
+        return "unknown"
+
+
+# Function to detect chessboard grid using edge detection and Hough lines
+def detect_chessboard_grid(image):
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    # Enhance contrast
+    gray = cv2.convertScaleAbs(gray, alpha=1.2, beta=20)
+    # Apply Gaussian blur to reduce noise
+    gray = cv2.GaussianBlur(gray, (5, 5), 0)
+    # Edge detection with Canny
+    edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+    # Detect lines using Hough Transform
+    lines = cv2.HoughLinesP(
+        edges, 1, np.pi / 180, threshold=80, minLineLength=50, maxLineGap=10
+    )
+
+    if lines is None:
+        print("No lines detected.")
+        return None, edges
+
+    # Separate horizontal and vertical lines
+    h_lines = []
+    v_lines = []
+    for line in lines:
+        x1, y1, x2, y2 = line[0]
+        if abs(x2 - x1) > abs(y2 - y1):  # Horizontal line
+            h_lines.append((y1, x1, x2))
+        else:  # Vertical line
+            v_lines.append((x1, y1, y2))
+
+    # Sort and filter to get exactly 9 lines for each
+    h_lines = sorted(h_lines, key=lambda x: x[0])[:9]  # Top 9 horizontal lines
+    v_lines = sorted(v_lines, key=lambda x: x[0])[:9]  # Top 9 vertical lines
+
+    if len(h_lines) < 9 or len(v_lines) < 9:
+        print(
+            f"Insufficient lines detected: {len(h_lines)} horizontal, {len(v_lines)} vertical"
+        )
+        return None, edges
+
+    # Find intersections to get 8x8 grid corners
+    corners = []
+    for h in h_lines:
+        y = h[0]
+        for v in v_lines:
+            x = v[0]
+            corners.append([x, y])
+
+    # corners = []
+    # for i in range(8):
+    #     for j in range(8):
+    #         x = int((v_lines[j][0] + v_lines[j + 1][0]) / 2)
+    #         y = int((h_lines[i][1] + h_lines[i + 1][1]) / 2)
+    #         corners.append([x, y])
+
+    # Ensure exactly 64 corners (8x8 grid)
+    if len(corners) != 64:
+        print(f"Expected 64 corners, got {len(corners)}")
+        return None, edges
+
+    corners = np.array(corners, dtype=np.float32).reshape(8, 8, 2)
+
+    # Visualize detected lines for debugging
+    debug_image = image.copy()
+    for y, x1, x2 in h_lines:
+        cv2.line(debug_image, (x1, y), (x2, y), (0, 255, 0), 2)
+    for x, y1, y2 in v_lines:
+        cv2.line(debug_image, (x, y1), (x, y2), (0, 0, 255), 2)
+    cv2.imwrite("lines_debug.png", debug_image)
+
+    return corners, edges
+
+
+# Function to extract coordinates of chess pieces from an image
+def extract_chessboard_coordinates(image_path):
+    try:
+        image = cv2.imread(image_path)
+        if image is None:
+            print(f"Failed to load image: {image_path}")
+            return []
+    except Exception as e:
+        print(f"Error loading image {image_path}: {e}")
+        return []
+
+    # Try OpenCV's chessboard detection first
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    ret, corners = cv2.findChessboardCorners(gray, (8, 8), None)
+
+    if ret:
+        corners = cv2.cornerSubPix(
+            gray,
+            corners,
+            (11, 11),
+            (-1, -1),
+            criteria=(cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1),
+        )
+        corners = corners.reshape(8, 8, 2)
+    else:
+        print("OpenCV chessboard detection failed. Attempting edge-based detection.")
+        corners, edges = detect_chessboard_grid(image)
+        if corners is None:
+            # Save edges for debugging
+            cv2.imwrite("edges_debug.png", edges)
+            print("Saved edge detection output to edges_debug.png")
+            return []
+        # Save debug image with detected corners
+        debug_image = image.copy()
+        for h in range(8):
+            for v in range(8):
+                x, y = int(corners[h, v, 0]), int(corners[h, v, 1])
+                cv2.circle(debug_image, (x, y), 5, (0, 255, 0), -1)
+        cv2.imwrite("grid_debug.png", debug_image)
+        print("Saved grid detection debug image to grid_debug.png")
+
+    # Calculate square size dynamically
+    square_width = np.mean(
+        [
+            np.linalg.norm(corners[i, j] - corners[i, j + 1])
+            for i in range(8)
+            for j in range(7)
+        ]
+    )
+    square_height = np.mean(
+        [
+            np.linalg.norm(corners[i, j] - corners[i + 1, j])
+            for i in range(7)
+            for j in range(8)
+        ]
+    )
+    square_size = int(min(square_width, square_height))
+
+    # Create a blank grid to store coordinates
+    piece_coordinates = []
+
+    # Loop through all coordinates and detect pieces
+    for i, j in coordinates:
+        try:
+            x = int(corners[i, j, 0])
+            y = int(corners[i, j, 1])
+            w = h = square_size
+
+            x = max(0, x)
+            y = max(0, y)
+            x_end = min(image.shape[1], x + w)
+            y_end = min(image.shape[0], y + h)
+            roi = image[y:y_end, x:x_end]
+
+            if roi.shape[0] == 0 or roi.shape[1] == 0:
+                print(f"Invalid ROI at square ({i}, {j})")
+                piece_coordinates.append(((i, j), "unknown"))
+                continue
+
+            predicted_piece = predict_piece(roi, model, device)
+            piece_coordinates.append(((i, j), predicted_piece))
+        except Exception as e:
+            print(f"Error processing square ({i}, {j}): {e}")
+            piece_coordinates.append(((i, j), "unknown"))
+
+    return piece_coordinates
+
+
+# Example usage
+IMAGE_PATH = "test.png"
+coordinates = extract_chessboard_coordinates(IMAGE_PATH)
+for coord, piece in coordinates:
+    print(f"Piece at {coord}: {piece}")
+
+del model
+if torch.cuda.is_available():
+    torch.cuda.empty_cache()
+gc.collect()

build/lib/tools/model.py

@@ -0,0 +1,27 @@
+import torch
+from ollama import chat
+
+# from pathlib import Path
+
+# Pass in the path to the image
+path = input("Please enter the path to the image: ")
+
+# You can also pass in base64 encoded image data
+# img = base64.b64encode(Path(path).read_bytes()).decode()
+# or the raw bytes
+# img = Path(path).read_bytes()
+
+response = chat(
+    model="gemma3:latest",
+    messages=[
+        {
+            "role": "user",
+            "content": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
+            "images": [path],
+        }
+    ],
+)
+
+
+print(response.message.content)
+torch.cuda.empty_cache()

build/lib/tools/model_chess.py

@@ -0,0 +1,188 @@
+import gc
+import os
+
+import torch
+from PIL import Image
+from torch import nn, optim
+from torch.utils.data import DataLoader, Dataset, random_split
+from torchvision import models, transforms
+
+# Define data transformations for training and validation
+transform = transforms.Compose(
+    [
+        transforms.Resize((224, 224)),  # Ensure all images are 224x224
+        transforms.ToTensor(),  # Convert to tensor
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+        ),  # Standard for ResNet
+    ]
+)
+
+
+# Custom dataset class for loading chess piece images
+class ChessPieceDataset(Dataset):
+    def __init__(self, root_dir, transform=None):
+        """
+        Args:
+            root_dir (str): Directory with all the images and subdirectories (class labels).
+            transform (callable, optional): Optional transform to be applied on an image.
+        """
+        self.root_dir = root_dir
+        self.transform = transform
+        self.classes = sorted(
+            [
+                d
+                for d in os.listdir(root_dir)
+                if os.path.isdir(os.path.join(root_dir, d))
+            ]
+        )
+        self.image_paths = []
+        self.labels = []
+
+        for label, class_name in enumerate(self.classes):
+            class_folder = os.path.join(root_dir, class_name)
+            for image_name in os.listdir(class_folder):
+                img_path = os.path.join(class_folder, image_name)
+                # Only include valid image files
+                if img_path.lower().endswith((".png", ".jpg", ".jpeg", ".bmp", ".gif")):
+                    try:
+                        # Verify the image can be opened
+                        with Image.open(img_path) as img:
+                            img.verify()  # Verify image integrity
+                        self.image_paths.append(img_path)
+                        self.labels.append(label)
+                    except Exception as e:
+                        print(f"Skipping corrupted image {img_path}: {e}")
+
+    def __len__(self):
+        return len(self.image_paths)
+
+    def __getitem__(self, idx):
+        img_path = self.image_paths[idx]
+        try:
+            image = Image.open(img_path).convert("RGB")
+        except Exception as e:
+            print(f"Error loading image {img_path}: {e}")
+            # Return a dummy image and label to avoid crashing
+            image = Image.new("RGB", (224, 224), (0, 0, 0))
+            label = self.labels[idx]
+        else:
+            label = self.labels[idx]
+
+        if self.transform:
+            try:
+                image = self.transform(image)
+                # Verify the image size after transformation
+                if image.shape != (3, 224, 224):
+                    print(
+                        f"Unexpected image size after transform for {img_path}: {image.shape}"
+                    )
+            except Exception as e:
+                print(f"Error applying transform to {img_path}: {e}")
+                image = self.transform(Image.new("RGB", (224, 224), (0, 0, 0)))
+
+        return image, label
+
+
+# Define training function (unchanged)
+def train_model(
+    model, train_loader, val_loader, criterion, optimizer, num_epochs=10, device="cpu"
+):
+    best_accuracy = 0.0
+
+    for epoch in range(num_epochs):
+        model.train()
+        running_loss = 0.0
+        correct = 0
+        total = 0
+
+        for inputs, labels in train_loader:
+            inputs, labels = inputs.to(device), labels.to(device)
+            optimizer.zero_grad()
+            outputs = model(inputs)
+            loss = criterion(outputs, labels)
+            loss.backward()
+            optimizer.step()
+
+            running_loss += loss.item()
+            _, predicted = torch.max(outputs, 1)
+            correct += (predicted == labels).sum().item()
+            total += labels.size(0)
+
+        model.eval()
+        val_correct = 0
+        val_total = 0
+
+        with torch.no_grad():
+            for inputs, labels in val_loader:
+                inputs, labels = inputs.to(device), labels.to(device)
+                outputs = model(inputs)
+                _, predicted = torch.max(outputs, 1)
+                val_correct += (predicted == labels).sum().item()
+                val_total += labels.size(0)
+
+        epoch_loss = running_loss / len(train_loader)
+        epoch_train_accuracy = 100 * correct / total
+        epoch_val_accuracy = 100 * val_correct / val_total
+
+        print(
+            f"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}, "
+            f"Train Accuracy: {epoch_train_accuracy:.2f}%, "
+            f"Validation Accuracy: {epoch_val_accuracy:.2f}%"
+        )
+
+        if epoch_val_accuracy > best_accuracy:
+            best_accuracy = epoch_val_accuracy
+            torch.save(model.state_dict(), "best_chess_piece_model.pth")
+
+    print("Training completed.")
+
+
+# Path to dataset folder
+dataset_path = "train"  # Ensure this path is correct
+
+# Create dataset
+full_dataset = ChessPieceDataset(dataset_path, transform=transform)
+
+# Check if dataset is empty
+if len(full_dataset) == 0:
+    raise ValueError(
+        "Dataset is empty. Check dataset_path and ensure it contains valid images."
+    )
+
+# Split the dataset into training and validation sets
+train_size = int(0.8 * len(full_dataset))
+val_size = len(full_dataset) - train_size
+train_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])
+
+# Create DataLoaders
+train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Load the pre-trained ResNet18 model and modify the final layer
+model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
+model.fc = nn.Linear(model.fc.in_features, len(full_dataset.classes))
+model = model.to(device)
+
+# Define loss function and optimizer
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=0.0001)
+
+# Train the model
+train_model(
+    model, train_loader, val_loader, criterion, optimizer, num_epochs=10, device=device
+)
+
+# After training, load the best model for inference
+model.load_state_dict(torch.load("best_chess_piece_model.pth", map_location=device))
+model.eval()
+
+gc.collect()
+
+del model
+torch.cuda.empty_cache()
+
+gc.collect()

build/lib/tools/test.py

@@ -0,0 +1,15 @@
+import cv2
+import numpy as np
+from PIL import Image
+
+# Load the colorful image
+image_path = "test.png"
+image = cv2.imread(image_path)
+
+gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+gray = cv2.convertScaleAbs(gray, alpha=1.2, beta=20)
+gray = cv2.GaussianBlur(gray, (5, 5), 0)
+edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+
+cv2.imwrite("new.png", edges)

build/lib/tools/test_1.py

@@ -0,0 +1,12 @@
+import os
+
+from src.tools import use_vision_model
+
+# Test question with a valid image path (make sure the image file exists)
+question = "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation. test.png"
+
+# Run the function (assuming your model and required environment are set up)
+response = use_vision_model(question)
+
+# Print the response
+print(response)

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+pythonpath = .
+

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+requests

setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+
+def read_requirements():
+    with open("requirements.txt") as f:
+        return [line.strip() for line in f if line.strip() and not line.startswith("#")]
+
+
+setup(
+    name="src",
+    version="0.1",
+    packages=find_packages(),
+    install_requires=read_requirements(),
+    python_requires=">=3.8",
+)

src.egg-info/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: src
+Version: 0.1
+Requires-Python: >=3.8
+Requires-Dist: gradio
+Requires-Dist: requests
+Dynamic: requires-dist
+Dynamic: requires-python

src.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+README.md
+setup.py
+src/__init__.py
+src/agent.py
+src/final_answer.py
+src/tools.py
+src.egg-info/PKG-INFO
+src.egg-info/SOURCES.txt
+src.egg-info/dependency_links.txt
+src.egg-info/requires.txt
+src.egg-info/top_level.txt
+tests/test.py
+tests/test_test.py
+tests/test_tools.py
+tools/__init__.py
+tools/chess.py
+tools/model.py
+tools/model_chess.py
+tools/test.py
+tools/test_1.py

src.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src.egg-info/requires.txt

@@ -0,0 +1,2 @@
+gradio
+requests

src.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+src
+tools

src/agent.py

@@ -0,0 +1,367 @@
+import gc
+import logging
+import os
+import tempfile
+import time
+from typing import Optional
+
+import torch
+from dotenv import load_dotenv
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.rate_limiters import InMemoryRateLimiter
+from langchain_core.tools import Tool
+from langchain_experimental.utilities import PythonREPL
+
+# from langchain_google_community import GoogleSearchAPIWrapper, GoogleSearchResults
+from langchain_ollama import ChatOllama
+
+from src.final_answer import create_final_answer_graph, validate_answer
+from src.tools import analyze_csv_file  # run_code_from_file,
+from src.tools import (
+    analyze_excel_file,
+    download_file_from_url,
+    duckduckgo_search,
+    extract_text_from_image,
+    read_file,
+    reverse_decoder,
+    review_youtube_video,
+    transcribe_audio,
+    transcribe_youtube,
+    use_vision_model,
+    video_frames_to_images,
+    website_scrape,
+)
+
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+base_url = os.getenv("OLLAMA_BASE_URL")
+
+rate_limiter = InMemoryRateLimiter(requests_per_second=0.1)
+
+
+class BasicAgent:
+    def __init__(self):
+        try:
+            logger.info("Initializing BasicAgent")
+
+            # Create the prompt template
+            prompt = ChatPromptTemplate.from_messages(
+                [
+                    (
+                        "system",
+                        """You are a general AI assistant. I will ask you a
+                        question. Report your thoughts, and finish your answer
+                        with the following template: FINAL ANSWER: [YOUR FINAL
+                        ANSWER]. YOUR FINAL ANSWER should be a number OR as few
+                        words as possible OR a comma separated list of numbers
+                        and/or strings. If you are asked for a number, don't
+                        use comma to write your number neither use units such
+                        as $ or percent sign unless specified otherwise. If you
+                        are asked for a string, don't use articles, neither
+                        abbreviations (e.g. for cities), and write the digits
+                        in plain text unless specified otherwise. If you are
+                        asked for a comma separated list, apply the above rules
+                        depending of whether the element to be put in the list
+                        is a number or a string.
+                """,
+                    ),
+                    ("placeholder", "{chat_history}"),
+                    ("human", "{input}"),
+                    ("placeholder", "{agent_scratchpad}"),
+                ]
+            )
+            logger.info("Created prompt template")
+
+            llm = ChatOllama(
+                model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+                base_url=base_url,
+                temperature=0.2,
+            )
+            logger.info("Created model successfully")
+
+            # Define available tools
+            tools = [
+                # Tool(
+                #     name="run_code_from_file",
+                #     func=run_code_from_file,
+                #     description="Executes a full Python script from a file. Use for multi-line code, loops, and class/function definitions.",
+                # ),
+                Tool(
+                    name="DuckDuckGoSearchResults",
+                    description="""Performs a live search using DuckDuckGo
+                    and analyzes the top results. Returns a summary including
+                    result titles, URLs, brief snippets, and ranking
+                    positions. Use this to quickly assess the relevance,
+                    diversity, and quality of information retrieved from a
+                    privacy-focused search engine, without personalized or
+                    biased filtering.""",
+                    func=duckduckgo_search,
+                ),
+                # Tool(
+                #     name="GoogleSearchResults",
+                #     description="""Performs a live Google search and analyzes
+                #     the top results. Returns a summary including result titles,
+                #     URLs, brief snippets, and ranking positions. Use this to
+                #     quickly understand the relevance, variety, and quality of
+                #     search results for a given query before deeper research or
+                #     content planning.""",
+                #     func=GoogleSearchResults(
+                #         api_wrapper=GoogleSearchAPIWrapper(
+                #             google_api_key=os.getenv("GOOGLE_SEARCH_API_KEY"),
+                #             google_cse_id=os.getenv("GOOGLE_CSE_ID"),
+                #             k=5,  # Number of results to return
+                #         )
+                #     ).run,
+                # ),
+                Tool(
+                    name="analyze csv file",
+                    description="""Only read and analyze the contents of a CSV
+                    file if one is explicitly referenced or uploaded in the
+                    question. When a CSV file is provided, return a summary of
+                    the dataset, including column names, data types, missing
+                    value counts, basic statistics for numeric fields, and a
+                    preview of the data. Use this only to quickly understand
+                    the structure and quality of the dataset before performing
+                    any further analysis.Do not invoke this tool for any URL""",
+                    func=analyze_csv_file,
+                ),
+                Tool(
+                    name="analyze excel file",
+                    description="""Reads and analyzes the contents of an Excel
+                    file (.xlsx or .xls). Returns structured summaries
+                    for each sheet, including column names, data types, missing
+                    value counts, basic statistics for numeric columns, and
+                    sample rows. Use this to quickly explore the structure and
+                    quality of Excel datasets.Dont try to generate new names of
+                    a file""",
+                    func=analyze_excel_file,
+                ),
+                Tool(
+                    name="download file from url",
+                    description="""Downloads a file from a given URL and saves
+                    it locally. Supports various file types such as CSV, Excel,
+                    images, and PDFs. Use this to retrieve external resources
+                    for processing or analysis.""",
+                    func=download_file_from_url,
+                ),
+                Tool(
+                    name="extract_text_from_image",
+                    description="""Performs Optical Character Recognition (OCR)
+                    on an image to extract readable text after downloading it.
+                    Supports common image formats (e.g., PNG, JPG). Use this to
+                    digitize printed or handwritten content from images for
+                    search, analysis, or storage.""",
+                    func=extract_text_from_image,
+                ),
+                Tool(
+                    name="read_file",
+                    description="""Executes a full Python script from a file. Use for multi-line code, loops, and class/function definitions. IT IS EXTREMELY IMPORTANT THAT YOU USE THIS FOR A PYTHON FILE""",
+                    func=read_file,
+                ),
+                Tool(
+                    name="review_youtube_video",
+                    description="""Analyzes a YouTube video by extracting key
+                    information such as title, description, view count, likes,
+                    comments, and transcript (if available). Use this to
+                    generate summaries, insights, or sentiment analysis based
+                    on video content and engagement.""",
+                    func=review_youtube_video,
+                ),
+                Tool(
+                    name="transcribe_audio",
+                    description="""Converts spoken words in an audio file into
+                    written text using speech-to-text technology. Supports
+                    common audio formats like MP3, WAV, and FLAC. Use this to
+                    create transcripts for meetings, interviews, podcasts, or
+                    any spoken content. If asked for pages just give page number as an output nothing else.
+                    Change "vanilla extract" to "pure vanilla extract" in the final answer.
+                    Dont try to generate new file paths when invoking this tool""",
+                    func=transcribe_audio,
+                ),
+                Tool(
+                    name="transcribe_youtube",
+                    description="""Extracts and converts the audio from a
+                    YouTube video into text using speech-to-text technology.
+                    Supports generating transcripts for videos without captions
+                    or subtitles. Use this to obtain searchable, readable text
+                    from YouTube content.""",
+                    func=transcribe_youtube,
+                ),
+                Tool(
+                    name="use_vision_model",
+                    description="""Processes images using a computer vision
+                    model to perform tasks such as object detection, image
+                    classification, or segmentation. Use this to analyze visual
+                    content and extract meaningful information from images.""",
+                    func=use_vision_model,
+                ),
+                Tool(
+                    name="video_frames_to_images",
+                    description="""Extracts individual frames from a video file
+                    and saves them as separate image files. Use this to
+                    analyze, process, or visualize specific moments within
+                    video content. Use this to Youtube Videos""",
+                    func=video_frames_to_images,
+                ),
+                Tool(
+                    name="website_scrape",
+                    description="""It is mandatory to use duckduckgo_search
+                    tool before invoking this tool .Use this tool only to scrap from websites.
+                    Fetches and extracts content from a specified website URL. Supports retrieving text, images, links, and other page elements.""",
+                    func=website_scrape,
+                ),
+                Tool(
+                    name="python_repl",
+                    # description="""Use this tool to execute Python code read from a file. Make sure that if you're passing multi-line Python code, it should be formatted with actual line breaks (`\n`) rather than the string escape sequence (`\\n`). If you need to include line breaks in the code, they should be written as newlines, not as (`\\n`). Additionally, ensure that no unexpected escape characters (`\`) are left unescaped. If you want to see the output of a value, always use `print(...)` to display results. Do not return values as strings. For example, use `print(f'{total_sales_food:.2f}')` instead of returning `f'{total_sales_food:.2f}'`. If the code involves reading files, use the appropriate tools, such as `read_file`, for that. """,
+                    description="""Use this tool to execute Python code read from a file. Make sure that if you're passing multi-line Python code, it should be formatted with actual line breaks (\\n) rather than the string escape sequence (\\\\n). If you need to include line breaks in the code, they should be written as newlines, not as (\\\\n). Additionally, ensure that no unexpected escape characters (\\`) are left unescaped. If you want to see the output of a value, always use `print(...)` to display results. Do not return values as strings. For example, use `print(f'{total_sales_food:.2f}')` instead of returning `f'{total_sales_food:.2f}'`. If the code involves reading files, use the appropriate tools, such as `read_file`, for that.""",
+                    func=PythonREPL().run,
+                    return_direct=True,
+                ),
+                # Tool(
+                #     name="wiki",
+                #     description="""Retrieves summarized information or
+                #     detailed content from Wikipedia based on a user query.
+                #     Use this to quickly access encyclopedic knowledge and
+                #     relevant facts on a wide range of topics.""",
+                #     func=wiki,
+                # ),
+                Tool(
+                    name="reverse decoder",
+                    description="""Decodes a reversed sentence if the input
+                    appears to be written backward.""",
+                    func=reverse_decoder,
+                ),
+            ]
+            # tools = [wrap_tool_with_limit(tool, max_calls=3) for tool in raw_tools]
+            logger.info("Tools: %s", tools)
+
+            # Create the agent
+            agent = create_tool_calling_agent(llm, tools, prompt)
+            logger.info("Created tool calling agent")
+
+            # Create the agent executor
+            self.agent_executor = AgentExecutor(
+                agent=agent,
+                tools=tools,
+                return_intermediate_steps=True,
+                verbose=True,
+                max_iterations=5,
+            )
+            logger.info("Created agent executor")
+
+            # Create the graph
+            self.validation_graph = create_final_answer_graph()
+
+        except Exception as e:
+            logger.error("Error initializing agent: %s", e, exc_info=True)
+            raise
+
+    def __call__(self, question: str, task_id: str) -> str:
+        """Execute the agent with the given question and optional file.
+        Args:
+            question (str): The question to answer
+            task_id (str): The task ID to fetch the file
+        Returns:
+            str: The final validated answer
+        Raises:
+            Exception: If no valid answer is found after max retries
+        """
+        max_retries = 3
+        attempt = 0
+
+        previous_steps = set()
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            while attempt < max_retries:
+                default_api_url = os.getenv("DEFAULT_API_URL")
+                file_url = f"{default_api_url}/files/{task_id}"
+
+                file: Optional[dict] = None
+                try:
+                    # Download file to temporary directory
+                    file = download_file_from_url.invoke(
+                        {
+                            "url": file_url,
+                            "directory": temp_dir,
+                        }
+                    )
+                    time.sleep(1)
+                    logger.info(f"Downloaded file for {task_id}")
+                except Exception as download_error:
+                    logger.error(f"File download failed: {str(download_error)}")
+                    file = None
+
+                try:
+                    attempt += 1
+                    logger.info("Attempt %d of %d", attempt, max_retries)
+
+                    # Prepare input with file information
+                    input_data = {
+                        "input": question
+                        + (
+                            f" [File: type={file.get('type', 'None')}, path={file.get('path', 'None')}]"
+                            if file and file.get("type") != "error"
+                            else ""
+                        ),
+                    }
+
+                    # Run the agent to get the answer
+                    result = self.agent_executor.invoke(input_data)
+                    answer = result.get("output", "")
+                    intermediate_steps = result.get("intermediate_steps", [])
+
+                    steps_str = str(intermediate_steps)
+                    if steps_str in previous_steps:
+                        logger.warning(
+                            f"Detected repeated reasoning steps on attempt {attempt}. Breaking loop to avoid infinite retry."
+                        )
+                        break  # or raise Exception to stop retries
+                    previous_steps.add(steps_str)
+
+                    logger.info("Attempt %d result: %s", attempt, result)
+
+                    # Run validation (self.validation_graph is now StateGraph)
+                    validation_result = validate_answer(
+                        self.validation_graph,  # type: ignore
+                        answer,
+                        [result.get("intermediate_steps", [])],
+                    )
+
+                    valid_answer = validation_result.get("valid_answer", False)
+                    final_answer = validation_result.get("final_answer", "")
+
+                    if valid_answer:
+                        logger.info("Valid answer found on attempt %d", attempt)
+                        torch.cuda.empty_cache()
+                        return final_answer
+
+                    logger.warning(
+                        "Validation failed on attempt %d: %s", attempt, final_answer
+                    )
+                    if attempt >= max_retries:
+                        raise Exception(
+                            "Failed to get valid answer after %d attempts. Last error: %s",
+                            max_retries,
+                            final_answer,
+                        )
+
+                except Exception as e:
+                    logger.error("Error in attempt %d: %s", attempt, e, exc_info=True)
+                    if attempt >= max_retries:
+                        raise Exception(
+                            "Failed after %d attempts. Last error: %s",
+                            max_retries,
+                            str(e),
+                        )
+                    continue
+                finally:
+                    logger.info("cleaning up temp_dir")
+                    torch.cuda.empty_cache()
+                    gc.collect()
+
+        # Fallback in case loop exits unexpectedly
+
+        raise Exception("No valid answer found after processing")

src/final_answer.py

@@ -0,0 +1,210 @@
+import os
+from typing import Any, Dict, Optional
+
+import torch
+from dotenv import load_dotenv
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_ollama import ChatOllama
+from langgraph.graph import END, START, Graph, StateGraph
+from typing_extensions import TypedDict
+
+load_dotenv()
+
+base_url = os.getenv("OLLAMA_BASE_URL")
+
+
+class AgentState(TypedDict):
+    """State for the final answer validation graph."""
+
+    question: str
+    answer: str
+    final_answer: str | None
+    agent_memory: Any
+    valid_answer: bool
+
+
+def extract_answer(state: AgentState) -> Dict:
+    """Extract and format the final answer from the state.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the formatted final answer.
+    """
+    # Extract the final answer from the state
+    sep_token = "FINAL ANSWER:"
+    raw_answer = state["answer"]
+
+    # Extract the answer after the separator if it exists
+    if sep_token in raw_answer:
+        formatted_answer = raw_answer.split(sep_token)[1].strip()
+    else:
+        formatted_answer = raw_answer.strip()
+
+    # Remove any brackets from lists
+    formatted_answer = formatted_answer.replace("[", "").replace("]", "")
+
+    # Remove units unless specified
+    if not any(
+        unit in formatted_answer.lower() for unit in ["$", "%", "dollars", "percent"]
+    ):
+        formatted_answer = formatted_answer.replace("$", "").replace("%", "")
+
+    # Remove commas from numbers
+    parts = formatted_answer.split(",")
+    formatted_parts = []
+    for part in parts:
+        part = part.strip()
+        if part.replace(".", "").isdigit():  # Check if it's a number
+            part = part.replace(",", "")
+        formatted_parts.append(part)
+    formatted_answer = ", ".join(formatted_parts)
+
+    return {"final_answer": formatted_answer}
+
+
+def reasoning_check(state: AgentState) -> Dict:
+    """
+    Node that checks the reasoning of the final answer.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the reasoning check result.
+    """
+    model = ChatOllama(
+        model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+        base_url=base_url,
+        temperature=0.2,
+    )
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                """You are a strict validator of answers. Your job is to check if the reasoning and results are correct.
+        You should have >90% confidence that the answer is correct to pass it.
+        First list reasons why yes/no, then write your final decision: PASS in caps lock if it is satisfactory, FAIL if it is not.""",
+            ),
+            (
+                "human",
+                """
+        Here is a user-given task and the agent steps: {agent_memory}
+        Now here is the answer that was given: {final_answer}
+        Please check that the reasoning process and results are correct: do they correctly answer the given task?
+        """,
+            ),
+        ]
+    )
+
+    chain = prompt | model | StrOutputParser()
+    output = chain.invoke(
+        {
+            "agent_memory": state["agent_memory"],
+            "final_answer": state["final_answer"],
+        }
+    )
+
+    print("Reasoning Feedback: ", output)
+    if "FAIL" in output:
+        return {"valid_answer": False}
+    torch.cuda.empty_cache()
+    return {"valid_answer": True}
+
+
+def formatting_check(state: AgentState) -> Dict:
+    """
+    Node that checks the formatting of the final answer.
+    Args:
+        state: The state of the agent.
+    Returns:
+        A dictionary with the formatting check result.
+    """
+    model = ChatOllama(
+        model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K",
+        base_url=base_url,
+        temperature=0.2,
+    )
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                """You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+        """,
+            ),
+            (
+                "human",
+                """
+        Here is a user-given task and the agent steps: {agent_memory}
+        Now here is the FINAL ANSWER that was given: {final_answer}
+        Ensure the FINAL ANSWER is in the right format as asked for by the task.
+        """,
+            ),
+        ]
+    )
+
+    chain = prompt | model | StrOutputParser()
+    output = chain.invoke(
+        {
+            "agent_memory": state["agent_memory"],
+            "final_answer": state["final_answer"],
+        }
+    )
+
+    print("Formatting Feedback: ", output)
+    if "FAIL" in output:
+        return {"valid_answer": False}
+
+    torch.cuda.empty_cache()
+    return {"valid_answer": True}
+
+
+def create_final_answer_graph() -> Graph:
+    """Create a graph that validates the final answer.
+    Returns:
+        A graph that validates the final answer.
+    """
+    # Create the graph
+    workflow = StateGraph(AgentState)
+
+    # Add nodes
+    workflow.add_node("extract_answer", extract_answer)
+    workflow.add_node("reasoning_check", reasoning_check)
+    workflow.add_node("formatting_check", formatting_check)
+
+    # Add edges
+    workflow.add_edge(START, "extract_answer")
+    workflow.add_edge("extract_answer", "reasoning_check")
+    workflow.add_edge("reasoning_check", "formatting_check")
+    workflow.add_edge("formatting_check", END)
+
+    # Compile the graph
+    return workflow.compile()  # type: ignore
+
+
+def validate_answer(graph: StateGraph, answer: str, agent_memory: Any) -> Dict:
+    """Validate the answer using the LangGraph workflow.
+    Args:
+        graph: The validation graph (LangGraph StateGraph).
+        answer: The answer to validate.
+        agent_memory: The agent's memory.
+    Returns:
+        A dictionary with validation results.
+    """
+    try:
+        # Initialize state
+        initial_state = {
+            "answer": answer,
+            "final_answer": None,
+            "agent_memory": agent_memory,
+            "valid_answer": False,
+        }
+
+        # Run the graph
+        result = graph.invoke(initial_state)  # type:ignore
+
+        return {
+            "valid_answer": result.get("valid_answer", False),
+            "final_answer": result.get("final_answer", None),
+        }
+    except Exception as e:
+        print(f"Validation failed: {e}")
+        return {"valid_answer": False, "final_answer": None}

src/tools.py

@@ -0,0 +1,751 @@
+# pylint: disable=no-member
+import base64
+import gc
+import math
+import mimetypes
+import multiprocessing
+import os
+import re
+import tempfile
+import time
+import uuid
+from datetime import timedelta
+from typing import Dict, List, Optional, TypedDict, Union
+from urllib.parse import urlparse
+
+import cv2
+import imageio
+import pandas as pd
+import pytesseract
+import requests
+import torch
+import whisper
+import yt_dlp
+from bs4 import BeautifulSoup, Tag
+from dotenv import load_dotenv
+from duckduckgo_search import DDGS
+from langchain_core.messages import HumanMessage
+from langchain_core.tools import tool
+from langchain_ollama import ChatOllama
+from PIL import Image
+from playwright.sync_api import sync_playwright
+from youtube_transcript_api import (
+    NoTranscriptFound,
+    TranscriptsDisabled,
+    YouTubeTranscriptApi,
+)
+
+load_dotenv()
+base_url = os.getenv("OLLAMA_BASE_URL")
+model_vision = ChatOllama(
+    model="gemma3:latest",
+    base_url=base_url,
+)
+model_text = ChatOllama(
+    model="hf.co/lmstudio-community/Qwen2.5-14B-Instruct-GGUF:Q6_K", base_url=base_url
+)
+
+
+@tool
+def use_vision_model(question: str) -> str:
+    """
+    A multimodal reasoning model that combines image and text input to answer
+    questions using the image.
+    """
+    # Extract image paths
+    image_paths = re.findall(r"[\w\-/\.]+\.(?:png|jpg|jpeg|webp)", question)
+    image_paths = [p for p in image_paths if os.path.exists(p)]
+
+    if not image_paths:
+        return "No valid image file found in the question."
+
+    image_path = image_paths[0]
+
+    # # Preprocess the image using OpenCV
+    # image = cv2.imread(image_path)
+    # gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    # gray = cv2.convertScaleAbs(gray, alpha=1.2, beta=20)
+    # gray = cv2.GaussianBlur(gray, (5, 5), 0)
+    # edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+    # # Create a temporary file for the processed image
+    # with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
+    #     temp_image_path = tmp_file.name
+    #     cv2.imwrite(temp_image_path, image)
+
+    # Encode the temp image(this code was under with tempfile)
+    mime_type, _ = mimetypes.guess_type(image_path)
+    mime_type = mime_type or "image/png"
+    with open(image_path, "rb") as f:
+        encoded = base64.b64encode(f.read()).decode("utf-8")
+
+    # Prepare the prompt and image for the model
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": question},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:{mime_type};base64,{encoded}"},
+                },
+            ],
+        }
+    ]
+
+    # Invoke the vision model
+    response = model_vision.invoke(messages)
+
+    # Clean up
+    del messages, encoded, image_path
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    return str(response.content) if hasattr(response, "content") else str(response)
+
+
+# YouTube Video Review Tool
+@tool
+def review_youtube_video(url: str) -> str:
+    """Reviews a YouTube video and answers a specific question about that video.
+
+    Args:
+        url (str): the URL to the YouTube video.
+        question (str): The question you are asking about the video.
+
+    Returns:
+        str: The answer to the question
+    """
+    # Extract video ID from URL (assuming it is in the format https://youtube.com/watch?v=VIDEO_ID)
+    video_id = url.split("v=")[1]
+    transcript_url = (
+        f"https://www.youtube.com/api/timedtext?v={video_id}"  # Getting transcript data
+    )
+
+    response = requests.get(transcript_url, timeout=200)
+
+    transcript = response.text  # This is the transcript (XML or SRT format)
+
+    # Prepare the content (just the transcript, no question needed)
+    transcript_content = f"Here is the transcript of the video: {transcript}"
+
+    # Return the transcript content so the main LLM can handle question generation
+    return transcript_content
+
+
+# YouTube Frames to Images Tool
+@tool
+def video_frames_to_images(
+    url: str,
+    sample_interval_seconds: int = 5,
+) -> List[str]:
+    """Extracts frames from a video at specified intervals and saves them as images.
+    Args:
+        url (str): the URL to the video.
+        folder_name (str): the name of the folder to save the images to.
+        sample_interval_seconds (int): the interval between frames to sample.
+    Returns:
+        List[str]: A list of paths to the saved image files.
+    """
+    folder_name = "./frames"
+    # Create a subdirectory for the frames
+    frames_dir = os.path.join(folder_name, "frames")
+    os.makedirs(frames_dir, exist_ok=True)
+
+    ydl_opts = {
+        "format": "bestvideo[height<=1080]+bestaudio/best[height<=1080]/best",
+        "outtmpl": os.path.join(folder_name, "video.%(ext)s"),
+        "quiet": True,
+        "noplaylist": True,
+        "merge_output_format": "mp4",
+        "force_ipv4": True,
+    }
+
+    info_extracted = []
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(url, download=True)
+        info_extracted.append(info)
+        video_path = next(
+            (
+                os.path.join(folder_name, f)
+                for f in os.listdir(folder_name)
+                if f.endswith(".mp4")
+            ),
+            None,
+        )
+
+        if not video_path:
+            raise RuntimeError("Failed to download video as mp4")
+
+        reader = imageio.get_reader(video_path)
+        # metadata = reader.get_meta_data()
+        fps = 25
+        duration_seconds = 120
+
+        frame_interval = int(fps * sample_interval_seconds)
+        num_frames = int(fps * duration_seconds)
+        # if num_frames is None or math.isinf(num_frames):
+        #     num_frames = int(fps * duration_seconds)
+        # Handle case where the number of frames is infinite or invalid
+        # if num_frames == float("inf") or not isinstance(num_frames, int):
+        #     reader.close()
+        #     raise RuntimeError("Invalid video length (infinite or not an integer)")
+
+        image_paths: List[str] = []
+
+        for idx in range(num_frames):
+            if idx % frame_interval == 0:
+                # Save frame as image
+                frame = reader.get_data(idx)
+                image_path = os.path.join(frames_dir, f"frame_{idx:06d}.jpg")
+                imageio.imwrite(image_path, frame)
+                image_paths.append(image_path)
+
+        reader.close()
+        return image_paths
+
+
+# File Reading Tool
+@tool
+def read_file(filepath: str) -> str:
+    """Reads the content of a PYTHON file.
+    Args:
+        filepath (str): the path to the file to read.
+    Returns:
+        str: The content of the file.
+    """
+    try:
+        with open(filepath, "r", encoding="utf-8") as file:
+            content = file.read()
+        # Calculate metadata for the prompt
+        filename = os.path.basename(filepath)
+        line_count = content.count("\\n") + 1
+        code_str = content.strip()
+        # Compose the prompt
+        prompt = f"""
+        You are a Python expert and code reviewer. Analyze the following Python script and answer the question provided.
+        Give Final Answer: the output of the code
+        Script Length: {line_count} lines
+        Filename: {filename}
+        
+        Python Code:
+        ```python
+        {code_str}
+        ```
+        """
+
+        model = model_text
+
+        # Call the model
+        message = HumanMessage(content=prompt)
+        response = model.invoke([message])
+        torch.cuda.empty_cache()
+        gc.collect()
+        # Return the result
+        if hasattr(response, "content") and isinstance(response.content, str):
+            return response.content
+        return str(response)
+
+    except FileNotFoundError:
+        return f"File not found: {filepath}"
+    except IOError as e:
+        return f"Error reading file: {str(e)}"
+
+
+# To run python code
+
+
+def execute_code(code: str):
+    """Helper function to execute the code in a separate process."""
+    try:
+        exec(code)
+    except Exception as e:
+        raise RuntimeError(f"Error executing the code: {str(e)}") from e
+
+
+@tool
+def run_code_from_file(file_path: str, timeout: int = 10):
+    """
+    Reads a Python file and executes it, with timeout handling.
+
+    Args:
+        file_path (str): The full path to the Python file to execute.
+        timeout (int): The timeout in seconds before forcefully stopping the execution.
+    """
+    # Check if the file exists
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+    # Read the file and get the code to execute
+    with open(file_path, "r", encoding="utf-8") as file:
+        code = file.read()
+
+    # Start a process to execute the code
+    process = multiprocessing.Process(target=execute_code, args=(code,))
+    process.start()
+
+    # Wait for the process to finish or timeout
+    process.join(timeout)
+
+    # If the process is still alive after the timeout, terminate it
+    if process.is_alive():
+        process.terminate()  # Stop the execution
+        raise TimeoutError(
+            f"The code execution took longer than {timeout} seconds and was terminated."
+        )
+
+
+# File Download Tool
+@tool
+def download_file_from_url(url: str, directory: str) -> Dict[str, Union[str, None]]:
+    """Downloads a file from a URL and saves it to a directory.
+    Args:
+        url (str): the URL to download the file from.
+        directory (str): the directory to save the file to.
+    Returns:
+        Dict[str, Union[str, None]]: A dictionary containing the file type and path.
+    """
+
+    response = requests.get(url, stream=True, timeout=10)
+    response.raise_for_status()
+
+    content_type = response.headers.get("content-type", "").lower()
+
+    # Try to get filename from headers
+    filename = None
+    cd = response.headers.get("content-disposition", "")
+    match = re.search(r"filename\*=UTF-8\'\'(.+)", cd) or re.search(
+        r'filename="?([^"]+)"?', cd
+    )
+    if match:
+        filename = match.group(1)
+
+    # If not in headers, try URL
+    if not filename:
+        filename = os.path.basename(url.split("?")[0])
+
+    # Fallback to generated filename
+    if not filename:
+        extension = {
+            "image/jpeg": ".jpg",
+            "image/png": ".png",
+            "image/gif": ".gif",
+            "audio/wav": ".wav",
+            "audio/mpeg": ".mp3",
+            "video/mp4": ".mp4",
+            "text/plain": ".txt",
+            "text/csv": ".csv",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+            "application/vnd.ms-excel": ".xls",
+            "application/octet-stream": ".bin",
+        }.get(content_type, ".bin")
+        filename = f"downloaded_file{extension}"
+
+    os.makedirs(directory, exist_ok=True)
+    file_path = os.path.join(directory, filename)
+    print(file_path)
+
+    with open(file_path, "wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+    # shutil.copy(file_path, os.getcwd())
+
+    return {
+        "type": content_type,
+        "filename": filename,
+        "path": file_path,
+    }
+
+
+# Text Extraction from Image Tool
+@tool
+def extract_text_from_image(image_path: str) -> str:
+    """Extracts text from an image using OCR.
+    Args:
+        image_path (str): the path to the image to extract text from.
+    Returns:
+        str: The text extracted from the image.
+    """
+
+    image = Image.open(image_path)
+    text = pytesseract.image_to_string(image)
+    return f"Extracted text from image:\n\n{text}"
+
+
+# CSV Analysis Tool
+@tool
+def analyze_csv_file(file_path: str, query: str) -> str:
+    """Analyzes a CSV file and answers questions about its contents using an
+    Ollama model.
+
+    Args:
+        file_path (str): The path to the CSV file to analyze.
+        query (str): The question to answer about the CSV file.
+
+    Returns:
+        str: The result of the analysis.
+    """
+    # Load the CSV file
+    df = pd.read_csv(file_path)
+    df_str = df.to_string(index=False)
+
+    # Compose the prompt
+    prompt = f"""
+                You are a data analyst. Analyze the following CSV data and answer the question provided.
+                
+                CSV Dimensions: {df.shape[0]} rows × {df.shape[1]} columns
+                
+                CSV Data:
+                {df_str}
+                
+                Please provide:
+                1. A summary of the data structure and content
+                2. Key patterns and insights
+                3. Potential data quality issues
+                4. Suggestions for analysis
+                
+                User Query:
+                {query}
+                
+                Format your response in markdown with sections and bullet points.
+             """
+
+    model = model_text
+
+    # Call the model
+    response = model.invoke([{"type": "text", "text": prompt}])
+    del df
+    torch.cuda.empty_cache()
+    gc.collect()
+
+    # Return the result
+    if hasattr(response, "content") and isinstance(response.content, str):
+        return response.content
+    return str(response)
+
+
+# Excel Analysis Tool
+@tool
+def analyze_excel_file(file_path: str) -> str:
+    """Analyzes an Excel file and answers questions about its contents using an
+    Ollama model
+    Args:
+        file_path (str): the path to the Excel file to analyze.
+        query (str): the question to answer about the Excel file.
+    Returns:
+        str: The result of the analysis.
+    """
+    llm = model_text
+    print(file_path)
+
+    # Read all sheets from the Excel file
+    excel_file = pd.ExcelFile(file_path)
+    sheet_names = excel_file.sheet_names
+
+    result = f"Excel file loaded with {len(sheet_names)} sheets: {', '.join(sheet_names)}\n\n"
+
+    for sheet_name in sheet_names:
+        df = pd.read_excel(file_path, sheet_name=sheet_name)
+        df_str = df.to_string()
+
+        # Build the prompt
+        prompt = f"""Analyze the following Excel sheet data and answer the user's query.
+                     Sheet Name: {sheet_name}
+                     Dimensions: {len(df)} rows × {len(df.columns)} columns
+                     
+                     Data:
+                     {df_str}
+                     
+                     Please provide:
+                     1. A summary of the data structure and content
+                     2. List all the values of the columns in a proper table format.
+                     3. If a file contains food items, assume it refers to the
+                        monetary value of the items, not the quantity sold.
+                     4. If the File contains food items, make a new list which
+                        contains the name of all the food item in the column only (not including drinks).
+                     5. If the file contains any time of monetary value its in USD with two decimal places.
+
+                     Format the response clearly using headings and bullet points."""
+
+        # Call the LLM with the prompt
+        response = llm.invoke([HumanMessage(content=prompt)])
+
+        result += f"=== Sheet: {sheet_name} ===\n"
+        result += str(response.content) + "\n"
+        result += "=" * 50 + "\n\n"
+        del df
+        gc.collect()
+
+    excel_file.close()
+    torch.cuda.empty_cache()
+
+    return result
+
+
+# Audio Transcription Tool
+def transcribe_audio(audio_file_path: str) -> str:
+    """Transcribes an audio file using Whisper's audio capabilities.
+    Always give Final Answer of the question in a specific format for example list all the pages mentioned in increasing order in one line.
+    Change vanilla extract to pure vanilla extract in the final answer.
+    Args:
+        audio_file_path (str): The path to the audio file to transcribe.
+        mime_type (str): The MIME type of the audio file.
+    Returns:
+        str: The transcript of the audio file.
+    Raises:
+        ValueError: If the MIME type is not supported.
+    """
+
+    model = whisper.load_model("base")
+    result = model.transcribe(audio_file_path)
+    assert isinstance(result["text"], str)
+
+    del model
+    torch.cuda.empty_cache()
+    gc.collect()
+    return result["text"]
+
+
+def _extract_video_id(url: str) -> Optional[str]:
+    """Extract video ID from YouTube URL.
+    Args:
+        url (str): the URL to the YouTube video.
+    Returns:
+        str: The video ID of the YouTube video.
+    """
+    patterns = [
+        r"(?:youtube\.com\/watch\?v=|youtube\.com\/embed\/|youtu\.be\/)([^&\n?#]+)",
+        r"(?:youtube\.com\/v\/|youtube\.com\/e\/|youtube\.com\/user\/[^\/]+\/|youtube\.com\/[^\/]+\/|youtube\.com\/embed\/|youtu\.be\/)([^&\n?#]+)",
+    ]
+
+    for pattern in patterns:
+        match = re.search(pattern, url)
+        if match:
+            return match.group(1)
+    return None
+
+
+@tool
+def transcribe_youtube(url: str) -> str:
+    """
+    Transcribes a YouTube video using YouTube Transcript API or ChatOllama with Whisper as fallback.
+
+    This function first tries to fetch the transcript of a YouTube video using the YouTube Transcript API.
+    If the transcript is unavailable (e.g., due to captions being disabled), it falls back to using
+    ChatOllama integrated with Whisper to transcribe the audio.
+
+    Args:
+        url (str): The URL to the YouTube video.
+
+    Returns:
+        str: The transcript of the YouTube video, or an error message if transcription fails.
+    """
+
+    try:
+        # Try using YouTube Transcript API
+        video_id = _extract_video_id(url)
+        transcript = ""
+        transcript_chunks = YouTubeTranscriptApi.get_transcript(
+            video_id, languages=["en"]
+        )
+        for chunk in transcript_chunks:
+            timestamp = str(timedelta(seconds=int(chunk["start"])))
+            transcript += f"[{timestamp}] {chunk['text']}\n"
+
+        # Return API transcript if available
+        if transcript.strip():
+            return transcript
+
+    except (TranscriptsDisabled, NoTranscriptFound, Exception) as err:
+        try:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                # Download audio from YouTube
+                ydl_opts = {
+                    "format": "bestaudio/best",
+                    "outtmpl": os.path.join(tmpdir, "audio.%(ext)s"),
+                    "quiet": True,
+                    "noplaylist": True,
+                    "postprocessors": [
+                        {
+                            "key": "FFmpegExtractAudio",
+                            "preferredcodec": "wav",
+                            "preferredquality": "192",
+                        }
+                    ],
+                }
+
+                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                    info = ydl.extract_info(url, download=True)
+
+                    if info is not None:
+                        title = info.get("title", "Unknown Title")  # Type:None
+                        duration = info.get("duration", 0)  # in seconds
+                        uploader = info.get("uploader", "Unknown Uploader")
+                    else:
+                        title = "Unknown Title"
+                        duration = 0
+                        uploader = "Unknown Uploader"
+
+                audio_path = next(
+                    (
+                        os.path.join(tmpdir, f)
+                        for f in os.listdir(tmpdir)
+                        if f.endswith(".wav")
+                    ),
+                    None,
+                )
+                if not audio_path:
+                    raise RuntimeError("Failed to download or convert audio") from err
+
+                # Use Whisper for initial transcription
+                whisper_model = whisper.load_model("base")
+                transcription = whisper_model.transcribe(audio_path, verbose=False)
+                raw_transcript = transcription["text"]
+                del whisper_model
+                gc.collect()
+                torch.cuda.empty_cache()
+                result = f"Title: {title}\nUploader: {uploader}\nDuration: {duration} seconds\nTranscript: {raw_transcript}"
+                return result
+        except Exception as fallback_exc:
+            raise RuntimeError("Fallback Transcription failed") from fallback_exc
+    return "Transcription failed unexpectedly."
+
+
+@tool
+def website_scrape(url: str) -> str:
+    """scrapes a website and returns the text.
+    args:
+        url (str): the url to the website to scrape.
+    returns:
+        str: the text of the website.
+    """
+    try:
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme or not parsed_url.netloc:
+            raise ValueError(
+                f"Invalid URL: '{url}'. Call `duckduckgo_search` first to get a valid URL."
+            )
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+            page.goto(url, wait_until="networkidle", timeout=60000)
+            page.wait_for_load_state("domcontentloaded")
+            html_content = page.content()
+            browser.close()
+
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        relevant_text = ""
+        # for header in soup.find_all(["h2", "h3"]):
+        #     heading_text = header.get_text().strip().lower()
+        #     if "discography" in heading_text or "studio albums" in heading_text:
+        #         section_texts = []
+        #         tag = header.find_next_sibling()
+        #         while tag and (
+        #             not isinstance(tag, Tag) or tag.name not in ["h2", "h3"]
+        #         ):
+        #             section_texts.append(tag.get_text(separator=" ", strip=True))
+        #             tag = tag.find_next_sibling()
+        #         relevant_text = "\n\n".join(section_texts)
+        #         break
+        # if not relevant_text:
+        #     article = soup.find("article")
+        #     if article:
+        #         relevant_text = article.get_text(separator=" ", strip=True)
+        # if not relevant_text:
+        relevant_text = soup.get_text(separator=" ", strip=True)
+
+        # step 2: chunk the text (optional but recommended)
+        def chunk_text(text, max_length=1000):
+            words = text.split()
+            chunks = []
+            for i in range(0, len(words), max_length):
+                chunks.append(" ".join(words[i : i + max_length]))
+            return chunks
+
+        chunks = chunk_text(relevant_text)
+
+        # return only the first 2–3 chunks to keep it concise
+        return "\n\n".join(chunks[:5])
+    except ValueError as e:
+        # Catch URL validation errors
+        return str(e)
+    except Exception as e:
+        # Catch other unexpected errors
+        return f"Scraping failed: {str(e)}"
+
+
+class SearchResult(TypedDict):
+    query: str
+    status: str
+    attempt: int
+    results: Optional[List[dict]]
+    error: Optional[str]
+
+
+@tool
+def duckduckgo_search(query: str, max_results: int = 10) -> SearchResult:
+    """
+    Perform a DuckDuckGo search with retry and backoff.
+    Use this FIRST before invoking and scraping tools.
+    Args:
+        query: The search query string.
+        max_results: Max number of results to return (default 10).
+    Returns:
+        A dict with the query, results, status, attempt count, and any error.
+    """
+    max_retries = 3
+    base_delay = 2
+    backoff_factor = 2
+
+    for attempt in range(max_retries):
+        try:
+            with DDGS() as ddgs:
+                results = ddgs.text(keywords=query, max_results=max_results)
+                if results:
+                    formatted_results = [
+                        {
+                            "title": result.get("title", ""),
+                            "url": result.get("href", ""),
+                            "body": result.get("body", ""),
+                        }
+                        for result in results
+                    ]
+                    return {
+                        "query": query,
+                        "status": "success",
+                        "attempt": attempt + 1,
+                        "results": formatted_results,
+                        "error": None,
+                    }
+        except Exception as e:
+            print(f"[DuckDuckGo Tool] Attempt {attempt + 1} failed: {e}")
+            time.sleep(base_delay * (backoff_factor**attempt))
+
+    return {
+        "query": query,
+        "status": "failed",
+        "attempt": max_retries,
+        "results": None,
+        "error": "Max retries exceeded or request failed.",
+    }
+
+
+@tool
+def reverse_decoder(question: str) -> str:
+    """Decodes a reversed sentence if the input appears to be written backward.
+
+    Args:
+        question (str): The possibly reversed question string.
+
+    Returns:
+        str: The decoded sentence.
+    """
+    # Remove leading punctuation if present
+    cleaned = question.strip().strip(".!?")
+
+    # Check if it's likely reversed (simple heuristic: mostly lowercase, reversed word order)
+    reversed_text = cleaned[::-1]
+
+    return reversed_text