Create app.py

app.py

@@ -0,0 +1,179 @@
+#pip install openai-whisper
+#!apt-get install ffmpeg
+#!pip install playsound
+#!pip install pydub
+#!pip install librosa
+#!pip install gradio
+#!pip install ollama
+import gradio as gr
+import os
+import librosa
+import soundfile as sf
+import numpy as np
+from pydub import AudioSegment
+import whisper  # For speech-to-text
+import ollama  # For AI text evaluation
+
+# Define the directories where your .wav, background music, and response files are located
+wav_directory = "./files"
+bg_directory = "./files/bg"
+response_directory = "./files/response"
+
+# Load Whisper model for speech-to-text
+whisper_model = whisper.load_model("base")
+
+# Function to list all .wav files in the directory
+def list_wav_files():
+    return [f for f in os.listdir(wav_directory) if f.endswith('.mp3')]
+
+# Function to list all background music files in the directory
+def list_bg_files():
+    bg_files = [f for f in os.listdir(bg_directory) if f.endswith('.mp3')]
+    bg_files.insert(0, "None")  # Add "None" as the first option
+    return bg_files
+
+# Function to adjust the speed and pitch of the selected .wav file and add background music
+def adjust_audio(selected_file, speed, pitch, bg_music):
+    if selected_file:
+        # Load the selected .wav file using librosa
+        file_path = os.path.join(wav_directory, selected_file)
+        y, sr = librosa.load(file_path, sr=None)
+        
+        # Adjust the speed using librosa's time_stretch function
+        if speed != 1.0:
+            y = librosa.effects.time_stretch(y=y, rate=speed)
+        
+        # Adjust the pitch using librosa's pitch_shift function
+        if pitch != 0:
+            y = librosa.effects.pitch_shift(y=y, sr=sr, n_steps=pitch)
+        
+        # Save the adjusted audio to a temporary file
+        temp_file = os.path.join(wav_directory, "temp_adjusted.wav")
+        sf.write(temp_file, y, sr)
+        
+        # If background music is selected and not "None", overlay it
+        if bg_music and bg_music != "None":
+            bg_path = os.path.join(bg_directory, bg_music)
+            modified_voice = AudioSegment.from_file(temp_file)
+            bg = AudioSegment.from_file(bg_path)
+            bg = bg - 20  # Reduce background music volume by 20 dB
+            
+            # Ensure the background music is at least as long as the modified voice
+            if len(bg) < len(modified_voice):
+                bg = bg * (len(modified_voice) // len(bg) + 1)
+            bg = bg[:len(modified_voice)]
+            
+            # Overlay the background music
+            final_audio = modified_voice.overlay(bg)
+            
+            # Save the final audio with background music
+            final_file = os.path.join(wav_directory, "temp_final.wav")
+            final_audio.export(final_file, format="wav")
+            return final_file
+        
+        return temp_file
+    return None
+
+# Function to evaluate the original audio and return the appropriate response file and message
+def evaluate_audio(selected_file):
+    if selected_file:
+        # Transcribe the selected .mp3 file using Whisper
+        file_path = os.path.join(wav_directory, selected_file)
+        result = whisper_model.transcribe(file_path)
+        english_text = result["text"]
+
+        # Create a prompt to evaluate if the text is meaningful English
+        prompt = f'''
+        Text: {english_text}
+        
+        Instructions:
+        1. Read the text above.
+        2. If the text is not in meaningful English, write only 'no'.
+        3. If the text is in meaningful English, write only 'yes'.
+        4. Do not write anything else except 'no' or 'yes'.
+        '''
+        
+        # Use Ollama to evaluate the text
+        response = ollama.chat(model='phi3', messages=[{'role': 'user', 'content': prompt}]).message.content
+
+        print(response)
+        
+        # Determine which response file to play and the appropriate message
+        if "yes" in response.lower():
+            message = "Your inquiry is in the English language."
+            response_file = os.path.join(response_directory, "res.mp3")
+        else:
+            message = "Your inquiry is in Piglatin language."
+            response_file = os.path.join(response_directory, "nres.mp3")
+        
+        return response_file, message
+    return None, ""
+
+# Create the Gradio interface
+with gr.Blocks() as demo:
+    # Dropdown to list available .wav files
+    available_musics = gr.Dropdown(choices=list_wav_files(), label="Available Voices")
+    
+    # Dropdown to select playback speed
+    speed_options = [1, 1.25, 1.5, 1.75, 2]
+    speed_selector = gr.Dropdown(choices=speed_options, label="Select Playback Speed", value=1)
+    
+    # Dropdown to select pitch shift
+    pitch_options = [0, 1, 2, 3, 4, 5]
+    pitch_selector = gr.Dropdown(choices=pitch_options, label="Select Pitch Shift", value=0)
+    
+    # Dropdown to select background music
+    bg_music_selector = gr.Dropdown(choices=list_bg_files(), label="Select Background Sound")
+    
+    # Audio component to play the selected .wav file
+    audio_player = gr.Audio(label="")
+    
+    # Link the dropdowns to the audio player
+    available_musics.change(fn=adjust_audio, inputs=[available_musics, speed_selector, pitch_selector, bg_music_selector], outputs=audio_player)
+    speed_selector.change(fn=adjust_audio, inputs=[available_musics, speed_selector, pitch_selector, bg_music_selector], outputs=audio_player)
+    pitch_selector.change(fn=adjust_audio, inputs=[available_musics, speed_selector, pitch_selector, bg_music_selector], outputs=audio_player)
+    bg_music_selector.change(fn=adjust_audio, inputs=[available_musics, speed_selector, pitch_selector, bg_music_selector], outputs=audio_player)
+    
+    # New section for AI-generated response
+    with gr.Group():
+        gr.Markdown("### AI Generated Response By Voice Agent")
+        ai_response_audio = gr.Audio(label="AI Response")
+        ai_response_message = gr.Markdown("")  # Placeholder for the message
+        
+        available_musics.change(fn=evaluate_audio, inputs=available_musics, outputs=[ai_response_audio, ai_response_message])
+
+    with gr.Group():
+            gr.Markdown("### Project Explanation")
+            gr.Markdown("""
+            As per the assignment requirements, I have developed a web interface that allows users to customize voice settings, including:
+            
+            - **Voice Selection (choose from available voices)
+            - **Speed Modification (adjust speaking speed)
+            - **Pitch Adjustment (alter voice pitch)
+            - **Background Sound Addition (enhance the audio with background effects)
+            - **Handling Pig Latin Language Input
+            The assignment did not explicitly specify how to handle user speech in Pig Latin. Based on my assumption:
+            
+            - **If the user's speech is in Pig Latin, the voice assistant will respond in Pig Latin with:
+            "Thank you, I received your query. I will get back to you soon."
+            - **If the user's speech is in English, the voice assistant will reply in Pig Latin:
+            "Sorry, please speak in Pig Latin. I cannot understand otherwise."
+            Technology Stack
+            The project utilizes the following technologies:
+            
+            Audio Processing: librosa, pydub, soundfile
+            Speech-to-Text Model: Whisper (for transcribing speech)
+            LLM Model: PH3 (for generating AI responses)
+            Optimized for Efficiency: Due to computational constraints, I used lightweight deep learning and NLP models. However, these can be upgraded for improved performance.
+            How to Use the Voice Assistant
+            - **Select a voice from the dropdown menu.
+            - **Adjust the speed of the voice.
+            - **Modify the pitch as desired.
+            - **Add background sound if needed.
+            - **Listen to the modified voice output.
+            - **Receive an AI-generated response from the voice assistant.
+    
+            """)
+
+# Launch the Gradio interface
+demo.launch(share=True)