Update app.py

app.py

@@ -8,9 +8,9 @@ import os
 from mutagen.mp3 import MP3
 import cv2
 from dotenv import load_dotenv
-from transformers import pipeline
-
-
+from transformers import pipeline, AutoProcessor, AutoModel
+import torch
+import soundfile as sf
 
 # Load environment variables
 load_dotenv()
@@ -24,6 +24,22 @@ def resize(img_list):
         resize_img_list.append(np.array(imResize))
     return resize_img_list
 
+def text2speech(text):
+    # Using Microsoft's SpeechT5 model instead of FastSpeech2
+    processor = AutoProcessor.from_pretrained("microsoft/speecht5_tts")
+    model = AutoModel.from_pretrained("microsoft/speecht5_tts")
+    
+    # Preprocessing text input
+    inputs = processor(text=text, return_tensors="pt")
+    
+    # Generate speech with default speaker embedding
+    speaker_embeddings = torch.zeros((1, model.config.speaker_embedding_size))
+    speech = model.generate_speech(inputs["input_ids"], speaker_embeddings)
+    
+    # Save as flac file
+    sf.write("speech_output.flac", speech.numpy(), samplerate=16000)
+    return "speech_output.flac"
+
 def merge_audio_video(entities_num, resize_img_list, text_input):
     speech = text2speech(text_input)
     wav_audio = AudioSegment.from_file(speech, "flac")
@@ -41,18 +57,6 @@ def merge_audio_video(entities_num, resize_img_list, text_input):
     
     return mergedclip
 
-def text2speech(text):
-    # Generate speech from text using FastSpeech2
-    speech_output = fastspeech(text)
-    # Save the output as a .flac file (assuming the output is in numpy format)
-    with open("speech_output.flac", "wb") as f:
-        f.write(speech_output["audio"])
-    return "speech_output.flac"
-    
-# Load FastSpeech2 model from Hugging Face directly
-fastspeech = pipeline("text-to-speech", model="facebook/fastspeech2-en-ljspeech", use_auth_token=HF_TOKEN)
-
-
 def engine(text_input):
     ner = gr.Interface.load("huggingface/flair/ner-english-ontonotes-large", api_key=HF_TOKEN)
     entities = ner(text_input)
@@ -68,15 +72,10 @@ def engine(text_input):
     
     resize_img_list = resize(img_list)
     mergedclip = merge_audio_video(entities_num, resize_img_list, text_input)
-    mergedclip.to_videofile('mergedvideo.mp4')
+    mergedclip.write_videofile('mergedvideo.mp4')
     
     return 'mergedvideo.mp4'
 
-
-
-
-
-
 app = gr.Interface(
     fn=engine, 
     inputs=gr.Textbox(lines=5, label="Input Text"), 
@@ -87,4 +86,4 @@ app = gr.Interface(
     ],
     title="AI Pipeline Multi Model 🎭🎞️🍿 Movie Maker 🎬 🧠 🎨",
     article="<br><div></div>"
-).launch(debug=True)
+).launch(debug=True)