Upload 4 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ffmpeg filter=lfs diff=lfs merge=lfs -text
+ffmpeg.exe filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,37 +1,67 @@
-import gradio as gr
-from transformers import (
-    AutoProcessor,
-    VisionEncoderDecoderModel,
-    ViTImageProcessor,
-    AutoTokenizer,
-    MusicgenForConditionalGeneration,
-)
-import torch
-from PIL import Image
-from openai import OpenAI
-from scipy.io import wavfile
-import numpy as np
-import pic2song 
-import uuid
-
-def greet(image,image_ins):
-    file = pic2song.pic2song(image,image_ins)
-    return file
-
-
-with gr.Blocks() as demo:
-    with gr.Column():
-        with gr.Row():
-            inp = gr.Image(type="pil",image_mode="RGB", height="500px")
-            with gr.Column():
-                image_ins=gr.Slider(minimum=1, maximum=60,step=1, value=10, label="Generating length (seconds) 生成长度(秒)")
-                with gr.Row():
-                    out1 = gr.Audio()
-                    out2 = gr.Audio()
-                with gr.Row():    
-                    out3 = gr.Audio()
-                    out4 = gr.Audio()
-    btn = gr.Button("Run")
-    btn.click(fn=greet, inputs=[inp,image_ins], outputs=[out1,out2,out3,out4])
-
-demo.launch()
+import gradio as gr
+from transformers import (
+    AutoProcessor,
+    VisionEncoderDecoderModel,
+    ViTImageProcessor,
+    AutoTokenizer,
+    MusicgenForConditionalGeneration,
+)
+import torch
+from PIL import Image
+from openai import OpenAI
+from scipy.io import wavfile
+import numpy as np
+import pic2song
+import uuid
+
+
+def greet(image, image_ins):
+    file = pic2song.pic2song(image, image_ins)
+    return file
+
+
+with gr.Blocks() as demo:
+    with gr.Column():
+        with gr.Row():
+            inp = gr.Image(type="pil", image_mode="RGB", height="500px")
+            with gr.Column():
+                image_ins = gr.Slider(
+                    minimum=1,
+                    maximum=60,
+                    step=1,
+                    value=5,
+                    label="Generating length (seconds) 生成长度(秒)",
+                )
+                with gr.Row():
+                    out1 = gr.Audio()
+                    out2 = gr.Audio()
+                with gr.Row():
+                    out3 = gr.Audio()
+                    out4 = gr.Audio()
+    btn = gr.Button("Run")
+    with gr.Column():
+        with gr.Row():
+            video_out1 = gr.Video(interactive=False,height="300px",show_download_button=True)
+   
+            video_out2 = gr.Video(interactive=False,height="300px",show_download_button=True)
+
+            video_out3 = gr.Video(interactive=False,height="300px",show_download_button=True)
+
+            video_out4 = gr.Video(interactive=False,height="300px",show_download_button=True)
+    
+    btn.click(
+        fn=greet,
+        inputs=[inp, image_ins],
+        outputs=[
+            out1,
+            video_out1,
+            out2,
+            video_out2,
+            out3,
+            video_out3,
+            out4,
+            video_out4,
+        ],
+    )
+
+demo.launch()

ffmpeg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f63b646b3ba2f936002476792ee6809db934581e0c19a1611c995804b7cf63b3
+size 124581480

ffmpeg.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:445335a651bdc74e9ed3a998fda1cbb0d1ca114732241f023421d8c4994a85b2
+size 122695680

pic2song.py

@@ -12,6 +12,7 @@ from scipy.io import wavfile
 import numpy as np
 import uuid
 import os
+import subprocess
 
 # 从预训练模型中加载VisionEncoderDecoderModel模型
 model = VisionEncoderDecoderModel.from_pretrained(
@@ -29,11 +30,11 @@ feature_extractor = ViTImageProcessor.from_pretrained(
 tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
 
 # 检测并设置设备为cuda或cpu
-#device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-device = torch.device("cpu")
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# device = torch.device("cpu")
 # 将模型移至所选设备
-model.to(device)
-Musicgenmodel.to(device)
+model.to(torch.device("cpu"))
+Musicgenmodel.to(torch.device("cuda"))
 sampling_rate = Musicgenmodel.config.audio_encoder.sampling_rate
 
 # 设定生成文本的最大长度和beam搜索的数量
@@ -49,9 +50,11 @@ def predict_step(image_paths):
         image_paths = image_paths.convert(mode="RGB")
 
     # 使用feature_extractor处理器将图片转换为像素值张量
-    pixel_values = feature_extractor(images=image_paths, return_tensors="pt").pixel_values
+    pixel_values = feature_extractor(
+        images=image_paths, return_tensors="pt"
+    ).pixel_values
     # 将像素值张量移至所选设备
-    pixel_values = pixel_values.to(device)
+    pixel_values = pixel_values.to(torch.device("cpu"))
 
     # 使用模型生成图片描述文本的索引序列
     output_ids = model.generate(pixel_values, **gen_kwargs)
@@ -68,8 +71,8 @@ def pic2song(image_paths, image_ins):
     in_max_new_tokens = int(image_ins * 50)
     # 调用predict_step函数并输出预测的图片描述文本列表
     print(image_paths)
-    user_messagge = predict_step(image_paths)
-    print(user_messagge)
+    user_message = predict_step(image_paths)
+    print(user_message)
 
     client = OpenAI(
         api_key=os.environ.get("deepseekapi"),
@@ -82,7 +85,7 @@ def pic2song(image_paths, image_ins):
             {"role": "system", "content": "You are a helpful assistant"},
             {
                 "role": "user",
-                "content": user_messagge
+                "content": user_message
                 + " 根据这个提示,编一个类似'a catchy beat for a podcast intro.'这样的描述,英文一句话,回复结果必须符合'a catchy beat for a podcast intro.'格式",
             },
         ],
@@ -90,7 +93,6 @@ def pic2song(image_paths, image_ins):
     )
 
     music_message = response.choices[0].message.content
-    # music_message = 'a catchy beat for tiktok.'
     print(music_message)
 
     inputs = processor(
@@ -99,63 +101,41 @@ def pic2song(image_paths, image_ins):
         return_tensors="pt",
     )
 
-    
-    audio_values = Musicgenmodel.generate(
-        **inputs.to(device),
-        do_sample=True,
-        guidance_scale=3,
-        max_new_tokens=in_max_new_tokens
-    )
-    
-    
-    out_directory = "out"
-
-    # 检查目录是否存在
-    if not os.path.exists(out_directory):
-        # 如果不存在，创建目录
-        os.makedirs(out_directory)
-        print("已创建 out 目录")
-    
-    
-    output_path1 = "out/" + str(uuid.uuid4()) + ".wav"
-    wavfile.write(
-        output_path1, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy()
-    )
-
-    audio_values = Musicgenmodel.generate(
-        **inputs.to(device),
-        do_sample=True,
-        guidance_scale=3,
-        max_new_tokens=in_max_new_tokens
-    )
-
-    output_path2 = "out/" + str(uuid.uuid4()) + ".wav"
-    wavfile.write(
-        output_path2, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy()
-    )
-
-    audio_values = Musicgenmodel.generate(
-        **inputs.to(device),
-        do_sample=True,
-        guidance_scale=3,
-        max_new_tokens=in_max_new_tokens
-    )
-
-    output_path3 = "out/" + str(uuid.uuid4()) + ".wav"
-    wavfile.write(
-        output_path3, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy()
-    )
-
-    audio_values = Musicgenmodel.generate(
-        **inputs.to(device),
-        do_sample=True,
-        guidance_scale=3,
-        max_new_tokens=in_max_new_tokens
-    )
-
-    output_path4 = "out/" + str(uuid.uuid4()) + ".wav"
-    wavfile.write(
-        output_path4, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy()
-    )
-
-    return output_path1, output_path2, output_path3, output_path4
+    # 生成音频
+    output_paths = []
+    for i in range(4):
+        audio_values = Musicgenmodel.generate(
+            **inputs.to(device),
+            do_sample=True,
+            guidance_scale=3,
+            max_new_tokens=in_max_new_tokens,
+        )
+
+        output_path = f"out/{uuid.uuid4()}.wav"
+        wavfile.write(
+            output_path, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy()
+        )
+        output_paths.append(output_path)
+        voutput_path = f"out/{uuid.uuid4()}.mp4"
+        temp_image_path = "temp_image.jpg"
+        image_paths.save(temp_image_path)
+        ffmpeg_cmd = [
+            'ffmpeg',
+            '-loop', '1',
+            '-i', temp_image_path,
+            '-i', output_path,
+            '-c:v', 'libx264',
+            '-preset', 'slow',
+            '-tune', 'stillimage',
+            '-c:a', 'aac',
+            '-b:a', '320k',
+            '-pix_fmt', 'yuv420p',
+            '-shortest',
+            voutput_path
+        ]
+        subprocess.run(ffmpeg_cmd)
+
+        # 删除临时图像文件
+        os.remove(temp_image_path)
+        output_paths.append(voutput_path)
+    return output_paths