Spaces:
Sleeping
Sleeping
sandesh-bharadwaj
commited on
Commit
·
d715645
1
Parent(s):
3278a88
Added comments and cleanup of code
Browse files
app.py
CHANGED
@@ -4,6 +4,7 @@ from moviepy.editor import VideoFileClip, AudioFileClip, CompositeAudioClip
|
|
4 |
from moviepy.audio.fx.volumex import volumex
|
5 |
import shutil, tempfile, os
|
6 |
|
|
|
7 |
video_model_map = {
|
8 |
"Fast": "flash",
|
9 |
"Quality": "pro",
|
@@ -31,21 +32,25 @@ genre_map = {
|
|
31 |
}
|
32 |
|
33 |
# Methods for Gradio state updates
|
|
|
34 |
def on_advanced_change(state):
|
35 |
return gr.Accordion(open=state, visible=state)
|
36 |
|
|
|
37 |
def upload_file(file):
|
38 |
return gr.Video(file.name, label=file.name, height=640, show_download_button=False, show_label=False, visible=True)
|
39 |
|
|
|
40 |
def on_vdc_change(content):
|
41 |
return gr.Textbox(content, label="Video Description", visible=True)
|
42 |
|
|
|
43 |
def on_mp_change(content):
|
44 |
return gr.Textbox(content, label="Music Prompt", visible=True)
|
45 |
|
46 |
# Global state variables for Gradio
|
47 |
video_duration = 0
|
48 |
-
audio_paths = None
|
49 |
|
50 |
|
51 |
# Function to generate unique directory for each session
|
@@ -56,7 +61,8 @@ def create_session_dir():
|
|
56 |
def cleanup_session_dir():
|
57 |
if os.path.exists(session_dir):
|
58 |
shutil.rmtree(session_dir, ignore_errors=True)
|
59 |
-
|
|
|
60 |
def on_select_dropdown(value, evt: gr.EventData):
|
61 |
if value > 0:
|
62 |
orig_clip_vol = gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=True, interactive=True, step=1)
|
@@ -67,18 +73,22 @@ def on_select_dropdown(value, evt: gr.EventData):
|
|
67 |
else:
|
68 |
return gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=False, interactive=False, step=1), gr.Slider(minimum=0, maximum=200, value=100, label="Generated Music Volume (%)", visible=False, interactive=False, step=1), gr.Button(visible=False, interactive=False)
|
69 |
|
70 |
-
#
|
71 |
def generate_video_description(video_descriptor, google_api_key, toggle_advanced, video_file, genre, bpm, user_keywords):
|
72 |
global video_duration
|
73 |
try:
|
|
|
74 |
if google_api_key == "":
|
75 |
raise gr.Error("Please enter your Google API Key before continuing!")
|
76 |
if video_file is None:
|
77 |
raise gr.Error("Please upload a video before generating music.")
|
|
|
|
|
78 |
video_descriptor = DescribeVideo(
|
79 |
model=video_model_map[video_descriptor], google_api_key=google_api_key
|
80 |
)
|
81 |
-
|
|
|
82 |
if not toggle_advanced:
|
83 |
video_description = video_descriptor.describe_video(
|
84 |
video_file, genre=None,
|
@@ -91,30 +101,36 @@ def generate_video_description(video_descriptor, google_api_key, toggle_advanced
|
|
91 |
bpm=bpm,
|
92 |
user_keywords=user_keywords
|
93 |
)
|
|
|
94 |
video_duration = VideoFileClip(video_file).duration
|
95 |
|
|
|
96 |
gr.Info("Video Description generated successfully.")
|
97 |
gr.Info("Music Prompt generated successfully.")
|
98 |
|
99 |
-
# Return the
|
100 |
return video_description["Content Description"], video_description["Music Prompt"]
|
101 |
|
102 |
except Exception as e:
|
103 |
raise gr.Error("Exception raised: ", e)
|
104 |
-
|
|
|
105 |
def generate_music(music_generator, music_prompt, num_samples):
|
106 |
global video_duration, audio_paths, session_dir
|
107 |
try:
|
|
|
108 |
audio_generator = GenerateAudio(model=music_model_map[music_generator])
|
109 |
if audio_generator.device == "cpu":
|
110 |
gr.Warning("The music generator model is running on CPU. For faster results, consider using a GPU.")
|
111 |
|
|
|
112 |
music_prompt = [music_prompt] * num_samples
|
113 |
audio_generator.generate_audio(music_prompt, duration=video_duration)
|
114 |
audio_paths = audio_generator.save_audio(audio_dir=session_dir)
|
115 |
|
116 |
gr.Info("Music generated successfully.")
|
117 |
-
|
|
|
118 |
show_players = [gr.Audio(visible=True, value=audio_path, show_label=False, scale=0.5) for audio_path in audio_paths]
|
119 |
hide_players = [gr.Audio(visible=False) for _ in range(5-len(audio_paths))]
|
120 |
|
@@ -125,7 +141,7 @@ def generate_music(music_generator, music_prompt, num_samples):
|
|
125 |
except Exception as e:
|
126 |
raise gr.Error("Exception raised: ",e)
|
127 |
|
128 |
-
|
129 |
def mix_music_with_video(video_file, dropdown_index, orig_clip_vol, generated_audio_vol):
|
130 |
global session_dir, audio_paths
|
131 |
orig_clip = VideoFileClip(video_file)
|
@@ -134,6 +150,7 @@ def mix_music_with_video(video_file, dropdown_index, orig_clip_vol, generated_au
|
|
134 |
orig_clip_audio = orig_clip.audio
|
135 |
generated_audio = AudioFileClip(audio_paths[dropdown_index-1])
|
136 |
|
|
|
137 |
if orig_clip_audio:
|
138 |
orig_clip_audio = volumex(
|
139 |
orig_clip_audio, float(orig_clip_vol / 100)
|
@@ -144,20 +161,23 @@ def mix_music_with_video(video_file, dropdown_index, orig_clip_vol, generated_au
|
|
144 |
generated_audio, float(generated_audio_vol / 100)
|
145 |
)
|
146 |
|
|
|
147 |
if orig_clip_audio is not None:
|
148 |
orig_clip.audio = CompositeAudioClip([orig_clip_audio, generated_audio])
|
149 |
else:
|
150 |
orig_clip.audio = CompositeAudioClip([generated_audio])
|
151 |
-
|
|
|
152 |
final_video_path = f"{session_dir}/final_video.mp4"
|
153 |
orig_clip.write_videofile(final_video_path)
|
154 |
|
|
|
155 |
orig_clip.close()
|
156 |
generated_audio.close()
|
157 |
|
158 |
return gr.Video(final_video_path, height=640, show_download_button=False, show_label=False, visible=True), gr.DownloadButton("Download final video", value=final_video_path, visible=True, interactive=True)
|
159 |
|
160 |
-
|
161 |
with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
162 |
# Create session-specific temp dir
|
163 |
session_dir = create_session_dir()
|
@@ -165,6 +185,7 @@ with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
|
165 |
toggle_advanced = gr.State(False)
|
166 |
with gr.Row():
|
167 |
with gr.Column(scale=1) as sideBar:
|
|
|
168 |
google_api_key = gr.Textbox(label="Enter your Google API Key to get started:", info="https://ai.google.dev/gemini-api/docs/api-key", type="password")
|
169 |
video_descriptor = gr.Dropdown(["Fast", "Quality"], label="Select Video Descriptor", value="Fast", interactive=True)
|
170 |
music_generator = gr.Dropdown(["Fast", "Balanced", "Quality"], label="Select Music Generator", value="Fast", interactive=True)
|
@@ -192,12 +213,14 @@ with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
|
192 |
|
193 |
generate_music_btn = gr.Button("Generate Music")
|
194 |
|
|
|
195 |
toggle_advanced.change(on_advanced_change, inputs=toggle_advanced, outputs=[advanced_settings])
|
196 |
|
197 |
advanced_settings_btn.click(lambda x: not x, toggle_advanced, toggle_advanced)
|
198 |
|
199 |
|
200 |
with gr.Column(scale=3.5) as MainWindow:
|
|
|
201 |
gr.Image("assets/VidTune-Logo-Without-BG.png", width=200, interactive=False, show_download_button=False, show_label=False)
|
202 |
gr.Markdown(
|
203 |
"""
|
@@ -205,28 +228,30 @@ with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
|
205 |
<p>VidTune is a web application to effortlessly tailor perfect soundtracks for your videos with AI.</p>
|
206 |
""",
|
207 |
)
|
|
|
208 |
uploaded_file = gr.UploadButton(label="Upload Video (Limit 200MB)", file_count="single", type="filepath", file_types=["video"])
|
209 |
-
|
210 |
video_file = gr.Video(height=640, show_download_button=False, show_label=False, visible=False)
|
|
|
211 |
|
|
|
212 |
video_description_box = gr.Textbox(label="Video Description", visible=True)
|
213 |
music_prompt_box = gr.Textbox(label="Music Prompt", visible=True)
|
214 |
|
|
|
215 |
audio_players = [gr.Audio(visible=False) for _ in range(5)]
|
216 |
audio_players_selections = gr.Dropdown(choices=["None"], visible=False, interactive=False, label="")
|
217 |
|
|
|
218 |
orig_clip_vol= gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=False, interactive=False, step=1)
|
219 |
-
|
220 |
generated_audio_vol = gr.Slider(minimum=0, maximum=200, value=100, label="Generated Music Volume (%)", visible=False, interactive=False, step=1)
|
221 |
-
|
222 |
mix_music_button = gr.Button(visible=False)
|
223 |
|
224 |
-
|
225 |
-
|
226 |
download_video_btn = gr.DownloadButton(visible=False, interactive=False)
|
227 |
|
228 |
-
uploaded_file.upload(upload_file, uploaded_file, video_file)
|
229 |
|
|
|
230 |
generate_music_btn.click(
|
231 |
generate_video_description,
|
232 |
inputs=[video_descriptor, google_api_key, toggle_advanced, video_file, genre, bpm, user_keywords],
|
@@ -235,8 +260,10 @@ with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
|
235 |
inputs=[music_generator, music_prompt_box, num_samples],
|
236 |
outputs=[*audio_players, audio_players_selections])
|
237 |
|
|
|
238 |
audio_players_selections.select(on_select_dropdown, audio_players_selections, outputs=[orig_clip_vol, generated_audio_vol,mix_music_button])
|
239 |
|
|
|
240 |
mix_music_button.click(
|
241 |
mix_music_with_video,
|
242 |
inputs = [video_file, audio_players_selections, orig_clip_vol, generated_audio_vol],
|
@@ -244,6 +271,7 @@ with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
|
244 |
|
245 |
)
|
246 |
|
|
|
247 |
demo.unload(cleanup_session_dir)
|
248 |
|
249 |
|
|
|
4 |
from moviepy.audio.fx.volumex import volumex
|
5 |
import shutil, tempfile, os
|
6 |
|
7 |
+
# Maps for model selection based on user input
|
8 |
video_model_map = {
|
9 |
"Fast": "flash",
|
10 |
"Quality": "pro",
|
|
|
32 |
}
|
33 |
|
34 |
# Methods for Gradio state updates
|
35 |
+
# Function to toggle visibility of advanced settings accordion
|
36 |
def on_advanced_change(state):
|
37 |
return gr.Accordion(open=state, visible=state)
|
38 |
|
39 |
+
# Function to display the uploaded video
|
40 |
def upload_file(file):
|
41 |
return gr.Video(file.name, label=file.name, height=640, show_download_button=False, show_label=False, visible=True)
|
42 |
|
43 |
+
# Function to update video description textbox content
|
44 |
def on_vdc_change(content):
|
45 |
return gr.Textbox(content, label="Video Description", visible=True)
|
46 |
|
47 |
+
# Function to update music prompt textbox content
|
48 |
def on_mp_change(content):
|
49 |
return gr.Textbox(content, label="Music Prompt", visible=True)
|
50 |
|
51 |
# Global state variables for Gradio
|
52 |
video_duration = 0
|
53 |
+
audio_paths = None # Paths to the generated audio files
|
54 |
|
55 |
|
56 |
# Function to generate unique directory for each session
|
|
|
61 |
def cleanup_session_dir():
|
62 |
if os.path.exists(session_dir):
|
63 |
shutil.rmtree(session_dir, ignore_errors=True)
|
64 |
+
|
65 |
+
# Event handler for dropdown selection to display sliders and buttons
|
66 |
def on_select_dropdown(value, evt: gr.EventData):
|
67 |
if value > 0:
|
68 |
orig_clip_vol = gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=True, interactive=True, step=1)
|
|
|
73 |
else:
|
74 |
return gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=False, interactive=False, step=1), gr.Slider(minimum=0, maximum=200, value=100, label="Generated Music Volume (%)", visible=False, interactive=False, step=1), gr.Button(visible=False, interactive=False)
|
75 |
|
76 |
+
# Function to generate video description
|
77 |
def generate_video_description(video_descriptor, google_api_key, toggle_advanced, video_file, genre, bpm, user_keywords):
|
78 |
global video_duration
|
79 |
try:
|
80 |
+
# Check for Google API key and uploaded video
|
81 |
if google_api_key == "":
|
82 |
raise gr.Error("Please enter your Google API Key before continuing!")
|
83 |
if video_file is None:
|
84 |
raise gr.Error("Please upload a video before generating music.")
|
85 |
+
|
86 |
+
# Initialize video descriptor model
|
87 |
video_descriptor = DescribeVideo(
|
88 |
model=video_model_map[video_descriptor], google_api_key=google_api_key
|
89 |
)
|
90 |
+
|
91 |
+
# Generate video description based on advanced settings
|
92 |
if not toggle_advanced:
|
93 |
video_description = video_descriptor.describe_video(
|
94 |
video_file, genre=None,
|
|
|
101 |
bpm=bpm,
|
102 |
user_keywords=user_keywords
|
103 |
)
|
104 |
+
# Get the duration of the uploaded video
|
105 |
video_duration = VideoFileClip(video_file).duration
|
106 |
|
107 |
+
# Provide success messages
|
108 |
gr.Info("Video Description generated successfully.")
|
109 |
gr.Info("Music Prompt generated successfully.")
|
110 |
|
111 |
+
# Return the generated content to update the UI
|
112 |
return video_description["Content Description"], video_description["Music Prompt"]
|
113 |
|
114 |
except Exception as e:
|
115 |
raise gr.Error("Exception raised: ", e)
|
116 |
+
|
117 |
+
# Function to generate music based on the video description
|
118 |
def generate_music(music_generator, music_prompt, num_samples):
|
119 |
global video_duration, audio_paths, session_dir
|
120 |
try:
|
121 |
+
# Initialize audio generator model
|
122 |
audio_generator = GenerateAudio(model=music_model_map[music_generator])
|
123 |
if audio_generator.device == "cpu":
|
124 |
gr.Warning("The music generator model is running on CPU. For faster results, consider using a GPU.")
|
125 |
|
126 |
+
# Generate multiple samples of music
|
127 |
music_prompt = [music_prompt] * num_samples
|
128 |
audio_generator.generate_audio(music_prompt, duration=video_duration)
|
129 |
audio_paths = audio_generator.save_audio(audio_dir=session_dir)
|
130 |
|
131 |
gr.Info("Music generated successfully.")
|
132 |
+
|
133 |
+
# Show audio players for the generated music and provide selection dropdown
|
134 |
show_players = [gr.Audio(visible=True, value=audio_path, show_label=False, scale=0.5) for audio_path in audio_paths]
|
135 |
hide_players = [gr.Audio(visible=False) for _ in range(5-len(audio_paths))]
|
136 |
|
|
|
141 |
except Exception as e:
|
142 |
raise gr.Error("Exception raised: ",e)
|
143 |
|
144 |
+
# Function to mix selected generated music with the original video
|
145 |
def mix_music_with_video(video_file, dropdown_index, orig_clip_vol, generated_audio_vol):
|
146 |
global session_dir, audio_paths
|
147 |
orig_clip = VideoFileClip(video_file)
|
|
|
150 |
orig_clip_audio = orig_clip.audio
|
151 |
generated_audio = AudioFileClip(audio_paths[dropdown_index-1])
|
152 |
|
153 |
+
# Adjust volume of original and generated audio
|
154 |
if orig_clip_audio:
|
155 |
orig_clip_audio = volumex(
|
156 |
orig_clip_audio, float(orig_clip_vol / 100)
|
|
|
161 |
generated_audio, float(generated_audio_vol / 100)
|
162 |
)
|
163 |
|
164 |
+
# Combine the original and generated audio
|
165 |
if orig_clip_audio is not None:
|
166 |
orig_clip.audio = CompositeAudioClip([orig_clip_audio, generated_audio])
|
167 |
else:
|
168 |
orig_clip.audio = CompositeAudioClip([generated_audio])
|
169 |
+
|
170 |
+
# Save the final video with mixed audio
|
171 |
final_video_path = f"{session_dir}/final_video.mp4"
|
172 |
orig_clip.write_videofile(final_video_path)
|
173 |
|
174 |
+
# Close clips to release resources
|
175 |
orig_clip.close()
|
176 |
generated_audio.close()
|
177 |
|
178 |
return gr.Video(final_video_path, height=640, show_download_button=False, show_label=False, visible=True), gr.DownloadButton("Download final video", value=final_video_path, visible=True, interactive=True)
|
179 |
|
180 |
+
# Gradio Blocks interface
|
181 |
with gr.Blocks(delete_cache=(1800, 3600)) as demo:
|
182 |
# Create session-specific temp dir
|
183 |
session_dir = create_session_dir()
|
|
|
185 |
toggle_advanced = gr.State(False)
|
186 |
with gr.Row():
|
187 |
with gr.Column(scale=1) as sideBar:
|
188 |
+
# Sidebar inputs for selecting models and settings
|
189 |
google_api_key = gr.Textbox(label="Enter your Google API Key to get started:", info="https://ai.google.dev/gemini-api/docs/api-key", type="password")
|
190 |
video_descriptor = gr.Dropdown(["Fast", "Quality"], label="Select Video Descriptor", value="Fast", interactive=True)
|
191 |
music_generator = gr.Dropdown(["Fast", "Balanced", "Quality"], label="Select Music Generator", value="Fast", interactive=True)
|
|
|
213 |
|
214 |
generate_music_btn = gr.Button("Generate Music")
|
215 |
|
216 |
+
# Toggle advanced settings visibility
|
217 |
toggle_advanced.change(on_advanced_change, inputs=toggle_advanced, outputs=[advanced_settings])
|
218 |
|
219 |
advanced_settings_btn.click(lambda x: not x, toggle_advanced, toggle_advanced)
|
220 |
|
221 |
|
222 |
with gr.Column(scale=3.5) as MainWindow:
|
223 |
+
# Main window with UI elements
|
224 |
gr.Image("assets/VidTune-Logo-Without-BG.png", width=200, interactive=False, show_download_button=False, show_label=False)
|
225 |
gr.Markdown(
|
226 |
"""
|
|
|
228 |
<p>VidTune is a web application to effortlessly tailor perfect soundtracks for your videos with AI.</p>
|
229 |
""",
|
230 |
)
|
231 |
+
# Upload video button and video player
|
232 |
uploaded_file = gr.UploadButton(label="Upload Video (Limit 200MB)", file_count="single", type="filepath", file_types=["video"])
|
|
|
233 |
video_file = gr.Video(height=640, show_download_button=False, show_label=False, visible=False)
|
234 |
+
uploaded_file.upload(upload_file, uploaded_file, video_file)
|
235 |
|
236 |
+
# Display generated video description and music prompt
|
237 |
video_description_box = gr.Textbox(label="Video Description", visible=True)
|
238 |
music_prompt_box = gr.Textbox(label="Music Prompt", visible=True)
|
239 |
|
240 |
+
# Audio players and dropdown selection
|
241 |
audio_players = [gr.Audio(visible=False) for _ in range(5)]
|
242 |
audio_players_selections = gr.Dropdown(choices=["None"], visible=False, interactive=False, label="")
|
243 |
|
244 |
+
# Mixing options
|
245 |
orig_clip_vol= gr.Slider(minimum=0, maximum=200, value=100, label="Original Audio Volume (%)", visible=False, interactive=False, step=1)
|
|
|
246 |
generated_audio_vol = gr.Slider(minimum=0, maximum=200, value=100, label="Generated Music Volume (%)", visible=False, interactive=False, step=1)
|
|
|
247 |
mix_music_button = gr.Button(visible=False)
|
248 |
|
249 |
+
# Generate output video and download
|
250 |
+
output_video = gr.Video(height=640, show_download_button=False, show_label=False, visible=False)
|
251 |
download_video_btn = gr.DownloadButton(visible=False, interactive=False)
|
252 |
|
|
|
253 |
|
254 |
+
# Generate video description and music (sequential call) on button click
|
255 |
generate_music_btn.click(
|
256 |
generate_video_description,
|
257 |
inputs=[video_descriptor, google_api_key, toggle_advanced, video_file, genre, bpm, user_keywords],
|
|
|
260 |
inputs=[music_generator, music_prompt_box, num_samples],
|
261 |
outputs=[*audio_players, audio_players_selections])
|
262 |
|
263 |
+
# Automatically enable mixing options when a selection is made in the dropdown
|
264 |
audio_players_selections.select(on_select_dropdown, audio_players_selections, outputs=[orig_clip_vol, generated_audio_vol,mix_music_button])
|
265 |
|
266 |
+
# Mix music and video on selection from dropdown
|
267 |
mix_music_button.click(
|
268 |
mix_music_with_video,
|
269 |
inputs = [video_file, audio_players_selections, orig_clip_vol, generated_audio_vol],
|
|
|
271 |
|
272 |
)
|
273 |
|
274 |
+
# Cleanup function on unload event
|
275 |
demo.unload(cleanup_session_dir)
|
276 |
|
277 |
|