app.py

import gradio as gr
from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
import torch
import librosa

# Load the model and processor
processor = Wav2Vec2Processor.from_pretrained("SpeechResearch/whisper-ft-normal")
model = Wav2Vec2ForCTC.from_pretrained("SpeechResearch/whisper-ft-normal")

def transcribe_speech(audio_path):
    speech, _ = librosa.load(audio_path, sr=16000)
    input_values = processor(speech, return_tensors="pt", padding="longest").input_values
    with torch.no_grad():
        logits = model(input_values).logits
    predicted_ids = torch.argmax(logits, dim=-1)
    transcription = processor.batch_decode(predicted_ids)
    return transcription[0]

def pipe(text, voice, image_in):
    # Assuming voice is a file path to the audio file
    transcription = transcribe_speech(voice)
    # Now use this transcription with your get_dreamtalk function
    video = get_dreamtalk(image_in, transcription)
    return video

with gr.Blocks() as demo:
    with gr.Column():
        gr.HTML("""
        <h1 style="text-align: center;">
        Talking Image 
        </h1>
        <h3 style="text-align: center;">
        Clone your voice and make your photos speak. 
        </h3>
        """)
        with gr.Row():
            with gr.Column():
                image_in = gr.Image(label="Portrait IN", type="filepath", value="./creatus.jpg")
            with gr.Column():
                voice = gr.Audio(type="filepath", label="Upload or Record Speaker audio (Optional voice cloning)")
                text = gr.Textbox(label="text")
                submit_btn = gr.Button('Submit')
            with gr.Column():
                video_o = gr.Video(label="Video result")
    submit_btn.click(
        fn=pipe,
        inputs=[text, voice, image_in],
        outputs=[video_o],
        concurrency_limit=3
    )
demo.queue(max_size=10).launch(show_error=True, show_api=False)