artyomboyko commited on
Commit
a46eb40
·
verified ·
1 Parent(s): 3e70275

Пробуем запустить space.

Browse files
Files changed (7) hide show
  1. .gitattributes +35 -0
  2. agent.py +72 -0
  3. app.py +213 -0
  4. gaia_dataset.py +36 -0
  5. packages.txt +19 -0
  6. requirements.txt +32 -0
  7. tools.py +397 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
agent.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from smolagents import CodeAgent, ToolCallingAgent, TransformersModel
2
+ from tools import available_tools
3
+ import torch
4
+ import json
5
+ import os
6
+
7
+
8
+ def instantiate_agent(executor_type : str="local", agent_type: str ="tool_calling", tools = available_tools) -> CodeAgent:
9
+
10
+ # Локальный агент с моделью кодером
11
+ if executor_type == "local" and agent_type == "code":
12
+
13
+ code_system_prompt = os.getenv("CODE_AGENT_SYSTEM_PROMPT")
14
+
15
+ print(code_system_prompt)
16
+
17
+ hf_model = "Qwen/Qwen2.5-Coder-32B-Instruct"
18
+ # hf_model = "Qwen/Qwen2.5-Coder-7B-Instruct"
19
+ # hf_model = "Qwen/Qwen2.5-Coder-3B-Instruct" # For debug purpose
20
+
21
+ llm = TransformersModel(model_id=hf_model,
22
+ device_map="cuda",
23
+ torch_dtype=torch.bfloat16,
24
+ )
25
+
26
+ agent = CodeAgent(tools=available_tools,
27
+ model=llm,
28
+ additional_authorized_imports=['pandas','numpy', 'numpy.*', 'csv', 'markdownify', 'requests'],
29
+ prompt_templates=({'system_prompt': code_system_prompt}),
30
+ max_steps=20,
31
+ )
32
+ return agent
33
+
34
+ elif executor_type == "local" and agent_type == "tool_calling":
35
+
36
+ tool_call_system_prompt = os.getenv("TOOL_CALLING_SYSTEM_PROMPT")
37
+
38
+ print(tool_call_system_prompt)
39
+
40
+ hf_model = "Qwen/Qwen2.5-7B-Instruct"
41
+ # hf_model = "Qwen/Qwen2.5-3B-Instruct" # For debug purpose
42
+ # hf_model = "google/gemma-2-2b-it"
43
+ # hf_model = "Qwen/Qwen2.5-7B-Instruct-1M"
44
+
45
+ llm = TransformersModel(model_id=hf_model, device_map="cuda", torch_dtype=torch.bfloat16)
46
+
47
+ agent = ToolCallingAgent(tools=available_tools, model=llm, max_steps=3, planning_interval=1)
48
+ agent.prompt_templates["system_prompt"] = agent.prompt_templates["system_prompt"] + tool_call_system_prompt
49
+
50
+
51
+
52
+ return agent
53
+
54
+ else:
55
+ raise ValueError(f"Unsupported executor type: {executor_type} or agent type: {agent_type}")
56
+
57
+
58
+ if __name__ == "__main__":
59
+
60
+ agent = instantiate_agent()
61
+
62
+ question = """
63
+ Question: Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.
64
+
65
+ Tools required:
66
+ 1. search engine
67
+
68
+ Approximately, the problem can be solved as follows::
69
+ 1. Search "Kuznetzov Nedoshivina 2010"
70
+ 2. Find the 2010 paper "A catalogue of type specimens of the Tortricidae described by V. I. Kuznetzov from Vietnam and deposited in the Zoological Institute, St. Petersburg"
71
+ """
72
+ agent.run(question)
app.py ADDED
@@ -0,0 +1,213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ import requests
4
+ import inspect
5
+ import pandas as pd
6
+
7
+ from huggingface_hub import login
8
+ from datasets import load_dataset
9
+ from dotenv import load_dotenv
10
+
11
+ from agent import instantiate_agent
12
+ from gaia_dataset import gaia_dataset, get_question
13
+
14
+ # (Сохраните константы как есть)
15
+ # --- Константы ---
16
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
17
+
18
+ class BasicAgent:
19
+
20
+ def __init__(self):
21
+ print("BasicAgent initialized.")
22
+ self.agent = instantiate_agent()
23
+
24
+ print("Agent initialized successfully.")
25
+
26
+ def __call__(self, question: str) -> str:
27
+
28
+ print(f"Agent received question (first 50 chars): {question[:50]}...")
29
+ fixed_answer = self.agent.run(question)
30
+ print(f"Agent returning fixed answer: {fixed_answer}")
31
+ return fixed_answer
32
+
33
+ def run_and_submit_all( profile: gr.OAuthProfile | None):
34
+ """
35
+ Fetches all questions, runs the BasicAgent on them, submits all answers,
36
+ and displays the results.
37
+ """
38
+ # --- Determine HF Space Runtime URL and Repo URL ---
39
+ space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
40
+ # space_id = "artyomboyko/Final_Assignment_Template" # Local inference only!
41
+
42
+ if profile:
43
+ username= f"{profile.username}"
44
+ print(f"User logged in: {username}")
45
+ else:
46
+ print("User not logged in.")
47
+ return "Please Login to Hugging Face with the button.", None
48
+
49
+ api_url = DEFAULT_API_URL
50
+ questions_url = f"{api_url}/questions"
51
+ submit_url = f"{api_url}/submit"
52
+
53
+ # 1. Instantiate Agent ( modify this part to create your agent)
54
+ try:
55
+ agent = BasicAgent()
56
+ except Exception as e:
57
+ print(f"Error instantiating agent: {e}")
58
+ return f"Error initializing agent: {e}", None
59
+ # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
60
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
61
+ print(agent_code)
62
+
63
+ # 2. Fetch Questions
64
+ print(f"Fetching questions from: {questions_url}")
65
+ try:
66
+ response = requests.get(questions_url, timeout=15)
67
+ response.raise_for_status()
68
+ questions_data = response.json()
69
+ if not questions_data:
70
+ print("Fetched questions list is empty.")
71
+ return "Fetched questions list is empty or invalid format.", None
72
+ print(f"Fetched {len(questions_data)} questions.")
73
+ except requests.exceptions.RequestException as e:
74
+ print(f"Error fetching questions: {e}")
75
+ return f"Error fetching questions: {e}", None
76
+ except requests.exceptions.JSONDecodeError as e:
77
+ print(f"Error decoding JSON response from questions endpoint: {e}")
78
+ print(f"Response text: {response.text[:500]}")
79
+ return f"Error decoding server response for questions: {e}", None
80
+ except Exception as e:
81
+ print(f"An unexpected error occurred fetching questions: {e}")
82
+ return f"An unexpected error occurred fetching questions: {e}", None
83
+
84
+ # 3. Run your Agent
85
+ results_log = []
86
+ answers_payload = []
87
+ print(f"Running agent on {len(questions_data)} questions...")
88
+ for item in questions_data:
89
+ task_id = item.get("task_id")
90
+ question_text = get_question(task_id)
91
+
92
+ if not task_id or question_text is None:
93
+ print(f"Skipping item with missing task_id or question: {item}")
94
+ continue
95
+ try:
96
+
97
+ print("CURRENT QUESTION: ", task_id, question_text)
98
+ submitted_answer = agent(question_text)
99
+
100
+
101
+ answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
102
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
103
+ except Exception as e:
104
+ print(f"Error running agent on task {task_id}: {e}")
105
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
106
+
107
+ if not answers_payload:
108
+ print("Agent did not produce any answers to submit.")
109
+ return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
110
+
111
+ # 4. Prepare Submission
112
+ submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
113
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
114
+ print(status_update)
115
+
116
+ # 5. Submit
117
+ print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
118
+ try:
119
+ response = requests.post(submit_url, json=submission_data, timeout=60)
120
+ response.raise_for_status()
121
+ result_data = response.json()
122
+ final_status = (
123
+ f"Submission Successful!\n"
124
+ f"User: {result_data.get('username')}\n"
125
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
126
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
127
+ f"Message: {result_data.get('message', 'No message received.')}"
128
+ )
129
+ print("Submission successful.")
130
+ results_df = pd.DataFrame(results_log)
131
+ return final_status, results_df
132
+ except requests.exceptions.HTTPError as e:
133
+ error_detail = f"Server responded with status {e.response.status_code}."
134
+ try:
135
+ error_json = e.response.json()
136
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
137
+ except requests.exceptions.JSONDecodeError:
138
+ error_detail += f" Response: {e.response.text[:500]}"
139
+ status_message = f"Submission Failed: {error_detail}"
140
+ print(status_message)
141
+ results_df = pd.DataFrame(results_log)
142
+ return status_message, results_df
143
+ except requests.exceptions.Timeout:
144
+ status_message = "Submission Failed: The request timed out."
145
+ print(status_message)
146
+ results_df = pd.DataFrame(results_log)
147
+ return status_message, results_df
148
+ except requests.exceptions.RequestException as e:
149
+ status_message = f"Submission Failed: Network error - {e}"
150
+ print(status_message)
151
+ results_df = pd.DataFrame(results_log)
152
+ return status_message, results_df
153
+ except Exception as e:
154
+ status_message = f"An unexpected error occurred during submission: {e}"
155
+ print(status_message)
156
+ results_df = pd.DataFrame(results_log)
157
+ return status_message, results_df
158
+
159
+
160
+ # --- Build Gradio Interface using Blocks ---
161
+ with gr.Blocks() as demo:
162
+ gr.Markdown("# Basic Agent Evaluation Runner")
163
+ gr.Markdown(
164
+ """
165
+ **Instructions:**
166
+ 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
167
+ 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
168
+ 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
169
+ ---
170
+ **Disclaimers:**
171
+ Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
172
+ This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
173
+ """
174
+ )
175
+
176
+ gr.LoginButton()
177
+
178
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
179
+
180
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
181
+ # Removed max_rows=10 from DataFrame constructor
182
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
183
+
184
+ run_button.click(
185
+ fn=run_and_submit_all,
186
+ outputs=[status_output, results_table]
187
+ )
188
+
189
+ if __name__ == "__main__":
190
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
191
+ # Check for SPACE_HOST and SPACE_ID at startup for information
192
+ # space_host_startup = os.getenv("SPACE_HOST")
193
+ # space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
194
+ space_host_startup = "artyomboyko-final-assignment-template.hf.space"
195
+ space_id_startup = "artyomboyko/Final_Assignment_Template"
196
+
197
+ if space_host_startup:
198
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
199
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
200
+ else:
201
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
202
+
203
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
204
+ print(f"✅ SPACE_ID found: {space_id_startup}")
205
+ print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
206
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
207
+ else:
208
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
209
+
210
+ print("-"*(60 + len(" App Starting ")) + "\n")
211
+
212
+ print("Launching Gradio Interface for Basic Agent Evaluation...")
213
+ demo.launch(debug=True, share=False)
gaia_dataset.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from smolagents import FinalAnswerTool
2
+ from datasets import load_dataset, Dataset
3
+ import json
4
+
5
+ gaia_dataset = load_dataset("gaia-benchmark/GAIA", "2023_level1", trust_remote_code=True, split="validation")
6
+
7
+ def get_example_by_feature_value(dataset: Dataset, feature_name: str, feature_value: str):
8
+
9
+ for example in dataset:
10
+ if example[feature_name] == feature_value:
11
+ return example
12
+
13
+ return None
14
+
15
+
16
+ def get_question(task_id: str) -> str:
17
+
18
+ question_data = get_example_by_feature_value(gaia_dataset, "task_id", task_id)
19
+
20
+ question_text = "Question: " + question_data["Question"] + "\n\n"
21
+
22
+ if question_data["file_name"]:
23
+ question_text = question_text + "File path: " + question_data["file_path"] + "\n\n"
24
+
25
+ question_text = question_text + "Tools required:\n" + question_data["Annotator Metadata"]['Tools'] + "\n\n"
26
+ question_text = question_text + "Approximately, the problem can be solved as follows::\n" + question_data["Annotator Metadata"]["Steps"] + "\n\n"
27
+
28
+ return question_text
29
+
30
+
31
+
32
+ if __name__ == "__main__":
33
+ id = "a1e91b78-d3d8-4675-bb8d-62741b4b68a6" # Question without file
34
+ # id = "cca530fc-4052-43b2-b130-b30968d8aa44" # Question with file
35
+
36
+ print(get_question(id))
packages.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ build-essential
2
+ cmake
3
+ curl
4
+ ffmpeg
5
+ g++
6
+ git
7
+ git-lfs
8
+ htop
9
+ iotop
10
+ libxml2
11
+ libopenblas-dev
12
+ libssl-dev
13
+ python3-pip
14
+ python3-wheel
15
+ python3-setuptools
16
+ python3-packaging
17
+ python-is-python3
18
+ wget
19
+ zlib1g
requirements.txt ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ accelerate
2
+ av
3
+ beautifulsoup4
4
+ bitsandbytes
5
+ datasets
6
+ duckduckgo-search
7
+ evaluate
8
+ ffmpeg
9
+ gradio
10
+ gradio[oauth]
11
+ gradio_client
12
+ hf_xet
13
+ huggingface_hub
14
+ ipykernel
15
+ ipython
16
+ ipywidgets
17
+ librosa
18
+ openai
19
+ opencv-python
20
+ openpyxl
21
+ pyproject-toml
22
+ requests
23
+ selenium
24
+ smolagents[all]==1.9.2
25
+ tavily-python
26
+ tqdm
27
+ transformers
28
+ torchao
29
+ uuid
30
+ wikipedia
31
+ yt_dlp
32
+ qwen_vl_utils
tools.py ADDED
@@ -0,0 +1,397 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from smolagents import DuckDuckGoSearchTool, VisitWebpageTool, SpeechToTextTool, FinalAnswerTool, PythonInterpreterTool, tool
2
+
3
+ from transformers import Qwen2_5_VLForConditionalGeneration, AutoTokenizer, AutoProcessor, pipeline
4
+ from qwen_vl_utils import process_vision_info
5
+ import torch
6
+
7
+ from typing import List, Any, Optional
8
+ from markdownify import markdownify
9
+ from tavily import TavilyClient
10
+
11
+ import os
12
+ import uuid
13
+ import json
14
+ import traceback
15
+ import requests
16
+ import datetime
17
+ import yt_dlp
18
+ import pandas as pd
19
+ import wikipedia as wiki
20
+ from bs4 import BeautifulSoup
21
+
22
+ import requests
23
+ from bs4 import BeautifulSoup
24
+ from markdownify import markdownify as md
25
+
26
+
27
+ @tool
28
+ def video_analyzer(file_path: str, query: str) -> str:
29
+ """
30
+
31
+ An artificial intelligence tool that takes as input a text string containing
32
+ the absolute path to a video file in MP4 format and a string with
33
+ a detailed text query to analyze the video.
34
+
35
+ Args:
36
+ file_path: Absolute path to an Excel file.
37
+ query: detailed text query to analyze the video.
38
+
39
+ Returns:
40
+ str: Row of text with the results of video file analysis
41
+
42
+ Examples:
43
+ >>> video_analyzer("/test/1.mp4", "Identify separate bird species. What is the highest number of bird species to be on camera simultaneously?")
44
+ The video shows a group of Emperor penguins and a single Albatross. Therefore, the highest number of bird species to be on camera simultaneously is 2.
45
+
46
+ """
47
+
48
+ model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
49
+ "Qwen/Qwen2.5-VL-3B-Instruct", torch_dtype="auto", device_map="auto"
50
+ )
51
+
52
+ processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-3B-Instruct")
53
+
54
+ text = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant. " + query
55
+
56
+ messages = [
57
+ {
58
+ "role": "user",
59
+ "content": [
60
+ {"type": "video", "video": f"file://{file_path}", "fps": 1.0,},
61
+ {"type": "text", "text": text},
62
+ ],
63
+ }
64
+ ]
65
+
66
+ # Preparation for inference
67
+ text = processor.apply_chat_template(
68
+ messages, tokenize=False, add_generation_prompt=True
69
+ )
70
+ image_inputs, video_inputs = process_vision_info(messages)
71
+ inputs = processor(
72
+ text=[text],
73
+ images=image_inputs,
74
+ videos=video_inputs,
75
+ padding=True,
76
+ return_tensors="pt",
77
+ )
78
+ inputs = inputs.to("cuda")
79
+
80
+ # Inference: Generation of the output
81
+ generated_ids = model.generate(**inputs, max_new_tokens=128)
82
+ generated_ids_trimmed = [
83
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
84
+ ]
85
+ output_text = processor.batch_decode(
86
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
87
+ )
88
+
89
+ return output_text[0]
90
+
91
+
92
+ # https://wikipedia.readthedocs.io/en/latest/code.html
93
+ @tool
94
+ def wikipedia_available_titles(query: str) -> List[str]:
95
+ """This insturment returns the titles of the articles available on wikipedia."
96
+
97
+ Args:
98
+ query: str
99
+ The query that will be used to search for articles on wikipedia.
100
+
101
+ Returns:
102
+ list : list of strings with available article titles
103
+
104
+ """
105
+ try:
106
+ wiki.set_rate_limiting(rate_limit=True, min_wait=datetime.timedelta(milliseconds=100))
107
+ titles = wiki.search(query)
108
+ except Exception as e:
109
+ print("Exception occurred: ", e, "with query: ", query)
110
+
111
+ return titles
112
+
113
+
114
+ @tool
115
+ def wikipedia_summary(title: str) -> str:
116
+ """This instrument returns the summary of a wikipedia article.
117
+
118
+ Args:
119
+ title: str
120
+ The title of the wikipedia article to summarize.
121
+
122
+ Returns:
123
+ str : The summary of the article.
124
+ """
125
+ try:
126
+ wiki.set_rate_limiting(rate_limit=True, min_wait=datetime.timedelta(milliseconds=100))
127
+ summary = wiki.summary(title, )
128
+ except Exception as e:
129
+ print("Exception occurred: ", e, "with title: ", title)
130
+ summary = ""
131
+
132
+ return summary
133
+
134
+
135
+ @tool
136
+ def reverse_text(text: str) -> str:
137
+ """This tool returns a reversed string of text.
138
+
139
+ Args:
140
+ text: str
141
+ The line of text to be reversed
142
+
143
+ Returns:
144
+ str : Reversed line of text.
145
+
146
+ Examples:
147
+ >>> reverse_text("ecnetnes siht dnatsrednu uoy fI")
148
+ If you understand this sentence
149
+
150
+ """
151
+ return text[::-1]
152
+
153
+
154
+ tavily_access_token = os.getenv("TAVILY_ACCESS_TOKEN")
155
+
156
+
157
+ @tool
158
+ def tavily_search(request: str) -> str:
159
+ """
160
+ This is an ultimatum tool for finding information on the internet.
161
+ Don't use it to search YouTube! It's useless!
162
+
163
+ Args:
164
+ request: A string containing a query to search in the Internet.
165
+
166
+ Returns:
167
+ str: JSON string with execution results containing the following fields:
168
+ - query: The search query to execute with Tavily.
169
+ - answer: A short answer to the user's query, generated by an LLM. Included in the response only if include_answer is requested
170
+ - images: List of query-related images. If include_image_descriptions is true, each item will have url and description.
171
+ - results: A list of sorted search results, ranked by relevancy. Contains the following fields:
172
+ - title: The title of the search result.
173
+ - url: The URL of the search result.
174
+ - content: A short description of the search result.
175
+ - score: The relevance score of the search result.
176
+ - raw_content: The cleaned and parsed HTML content of the search result. Only if include_raw_content is true.
177
+ """
178
+
179
+ client = TavilyClient(tavily_access_token)
180
+ response = client.search(query=request, include_raw_content=False, max_results=3, search_depth='advanced')
181
+
182
+ return response
183
+
184
+ @tool
185
+ def tavily_extract_web_page(url: str) -> str:
186
+ """
187
+ This is an ultimatum tool that allows you to retrieve the contents of a web page.
188
+ In other words, to view the website. Don't use YouTube to extract pages! It's useless!
189
+
190
+ Args:
191
+ url: The URL of the web page from which you want to retrieve information.
192
+
193
+ Returns:
194
+ str: The parsed and cleaned HTML content of the web page. The raw content extracted.
195
+ """
196
+
197
+ client = TavilyClient(tavily_access_token)
198
+ response = client.extract([url], extract_depth="advanced")
199
+
200
+ return response["results"][0]['raw_content']
201
+
202
+
203
+ @tool
204
+ def download_youtube_video_audio(url: str) -> tuple[bool, str, str]:
205
+ """
206
+ Downloads a YouTube video to a specified directory. Video and audio are downloaded separately.
207
+ The video is downloaded in mp4 format and the audio in mp3 format.
208
+
209
+ Args:
210
+ url: The URL of the YouTube video.
211
+
212
+ Returns:
213
+ Returns three strings:
214
+ bool: Execution result. True - success, False - error in file upload process.
215
+ str: The absolute path to the downloaded video file.
216
+ str: The absolute path to the downloaded audio file.
217
+ """
218
+ try:
219
+ # Генерация имен файлов
220
+ guid = str(uuid.uuid4())
221
+ output_dir="./downloads"
222
+
223
+ abs_output_dir = os.path.abspath(output_dir)
224
+
225
+ video_path = os.path.join(abs_output_dir, f"{guid}.mp4")
226
+ audio_path = os.path.join(abs_output_dir, f"{guid}.mp3") # Расширение будет добавлено позже автоматически
227
+
228
+ format_priority = (
229
+ 'bestvideo[height=360][ext=mp4]/' # 1. Точное 720p в MP4
230
+ 'bestvideo[height<360][ext=mp4]/' # 2. Наилучшее качество ниже 720p в MP4
231
+ 'worstvideo[height>=360]' # 3. Если нет 720p, берёт лучшее (макс. 1080p)
232
+ )
233
+
234
+ video_options = {
235
+ 'format': format_priority,
236
+ 'outtmpl': video_path,
237
+ 'quiet': True,
238
+ 'no_warnings': True,
239
+ }
240
+
241
+ # Настройки для аудио
242
+ audio_options = {
243
+ 'format': 'bestaudio/best[ext=mp3]',
244
+ 'outtmpl': audio_path,
245
+ 'quiet': True,
246
+ 'no_warnings': True,
247
+ }
248
+
249
+ # Создание папки, если она не существует.
250
+ os.makedirs(output_dir, exist_ok=True)
251
+
252
+ # Загрузка
253
+ with yt_dlp.YoutubeDL(video_options) as ydl:
254
+ ydl.download([url])
255
+
256
+ with yt_dlp.YoutubeDL(audio_options) as ydl:
257
+ ydl.download([url])
258
+
259
+ return True, video_path, audio_path
260
+
261
+ except Exception as e:
262
+
263
+ # Удаляем файлы если что-то пошло не так
264
+ for path in [video_path, audio_path]:
265
+ try:
266
+ os.remove(path)
267
+ except:
268
+ pass
269
+
270
+ return False, None, None
271
+
272
+
273
+ @tool
274
+ def transcribe_audio_file(path: str) -> str:
275
+ """
276
+ The tool takes as input the absolute path to the mp3 file to be transcribed and returns the English text.
277
+
278
+ Args:
279
+ path: Absolute path to an audio file in mp3 format.
280
+
281
+ Returns:
282
+ str: A string of transcripts of an audio file in English.
283
+ """
284
+
285
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
286
+
287
+ transcribe = pipeline(
288
+ "automatic-speech-recognition",
289
+ model="openai/whisper-base",
290
+ chunk_length_s=30,
291
+ batch_size=2,
292
+ device=device,
293
+ )
294
+ try:
295
+ transcription = transcribe(path, batch_size=8, generate_kwargs={"language": "english", "task": "transcribe"})["text"]
296
+ except Exception as e:
297
+ print("ERROR: {e}, {path}")
298
+ traceback.print_exc()
299
+ return None
300
+
301
+ return transcription
302
+
303
+ @tool
304
+ def get_excel_data(file_path: str) -> pd.DataFrame:
305
+ """
306
+ The tool takes as input an absolute path to the Excel file whose contents are to be output and returns a string of text with the contents of the file.
307
+
308
+ Args:
309
+ file_path: Absolute path to an Excel file.
310
+
311
+ Returns:
312
+ str: A row with the contents of an Excel file
313
+ """
314
+ return str(pd.read_excel(file_path))
315
+
316
+
317
+ @tool
318
+ def multiply(a: int, b: int) -> int:
319
+ """Multiply two numbers.
320
+ Args:
321
+ a: first int
322
+ b: second int
323
+ """
324
+ return a * b
325
+
326
+ @tool
327
+ def add(a: int, b: int) -> int:
328
+ """Add two numbers.
329
+
330
+ Args:
331
+ a: first int
332
+ b: second int
333
+ """
334
+ return a + b
335
+
336
+ @tool
337
+ def subtract(a: int, b: int) -> int:
338
+ """Subtract two numbers.
339
+
340
+ Args:
341
+ a: first int
342
+ b: second int
343
+ """
344
+ return a - b
345
+
346
+ @tool
347
+ def divide(a: int, b: int) -> int:
348
+ """Divide two numbers.
349
+
350
+ Args:
351
+ a: first int
352
+ b: second int
353
+ """
354
+ if b == 0:
355
+ raise ValueError("Cannot divide by zero.")
356
+ return a / b
357
+
358
+ @tool
359
+ def modulus(a: int, b: int) -> int:
360
+ """Get the modulus of two numbers.
361
+
362
+ Args:
363
+ a: first int
364
+ b: second int
365
+ """
366
+ return a % b
367
+
368
+
369
+ available_tools = [
370
+ reverse_text,
371
+ multiply,
372
+ add,
373
+ subtract,
374
+ divide,
375
+ modulus,
376
+ download_youtube_video_audio,
377
+ transcribe_audio_file,
378
+ get_excel_data,
379
+ wikipedia_available_titles,
380
+ wikipedia_summary,
381
+ video_analyzer,
382
+ FinalAnswerTool(),
383
+ DuckDuckGoSearchTool(),
384
+ tavily_search,
385
+ tavily_extract_web_page,
386
+ # VisitWebpageTool(),
387
+ PythonInterpreterTool(),
388
+ # SpeechToTextTool(),
389
+
390
+ ]
391
+
392
+
393
+ if __name__ == "__main__":
394
+ file = "/workspaces/Final_Assignment_Template/downloads/60cc887f-cb60-4fc6-88c8-a8bbc6a4659a.mp4"
395
+ text = "Identify separate bird species. What is the highest number of bird species to be on camera simultaneously?"
396
+
397
+ print(video_analyzer(file, text))