from huggingface_hub import InferenceClient
from transformers import pipeline
import gradio as gr
import edge_tts
import tempfile
import os
from streaming_stt_nemo import Model
import torch
import random
# Initialize the inference client with your Hugging Face token
client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1")
# Initialize the ASR pipeline
asr = pipeline("automatic-speech-recognition", "facebook/wav2vec2-base-960h")
def speech_to_text(speech):
"""Converts speech to text using the ASR pipeline."""
# breakpoint()
return asr(speech)["text"]
def classify_mood(input_string):
"""Classifies the mood based on keywords in the input string."""
input_string = input_string.lower()
mood_words = {"happy", "sad", "instrumental", "party"}
for word in mood_words:
if word in input_string:
return word, True
return None, False
def generate(
prompt, history, temperature=0.1, max_new_tokens=2048, top_p=0.8, repetition_penalty=1.0,
):
temperature = float(temperature)
if temperature < 1e-2:
temperature = 1e-2
top_p = float(top_p)
generate_kwargs = dict(
temperature=temperature,
max_new_tokens=max_new_tokens,
top_p=top_p,
repetition_penalty=repetition_penalty,
do_sample=True,
seed=42,
)
formatted_prompt = format_prompt(prompt, history)
stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
output = ""
for response in stream:
output += response.token.text
mood, is_classified = classify_mood(output)
# Print the chatbot's response
if is_classified:
print("Chatbot:", mood.capitalize())
playlist_message = f"Playing {mood.capitalize()} playlist for you!"
output=playlist_message
return output
# yield output
return output
def format_prompt(message, history):
"""Formats the prompt including fixed instructions and conversation history."""
fixed_prompt = """
You are a smart mood analyser, who determines user mood. Based on the user input, classify the mood of the user into one of the four moods {Happy, Sad, Instrumental, Party}. If you are finding it difficult to classify into one of these four moods, keep the conversation going on until we classify the user’s mood. Return a single-word reply from one of the options if you have classified. Suppose you classify a sentence as happy, then just respond with "happy".
Note: Do not write anything else other than the classified mood if classified.
Note: If any question or any user text cannot be classified, follow up with a question to know the user's mood until you classify the mood.
Note: Mood should be classified only from any of these 4 classes {Happy, Sad, Instrumental, Party}, if not any of these 4 then continue with a follow-up question until you classify the mood.
Note: if user asks something like i need a coffee then do not classify the mood directly and ask more follow-up questions as asked in examples.
Examples
User: What is C programming?
LLM Response: C programming is a programming language. How are you feeling now after knowing the answer?
User: Can I get a coffee?
LLM Response: It sounds like you're in need of a little pick-me-up. How are you feeling right now? Are you looking for something upbeat, something to relax to, or maybe some instrumental music while you enjoy your coffee?
User: I feel like rocking
LLM Response: Party
User: I'm feeling so energetic today!
LLM Response: Happy
User: I'm feeling down today.
LLM Response: Sad
User: I'm ready to have some fun tonight!
LLM Response: Party
User: I need some background music while I am stuck in traffic.
LLM Response: Instrumental
User: Hi
LLM Response: Hi, how are you doing?
User: Feeling okay only.
LLM Response: Are you having a good day?
User: I don't know
LLM Response: Do you want to listen to some relaxing music?
User: No
LLM Response: How about listening to some rock and roll music?
User: Yes
LLM Response: Party
User: Where do I find an encyclopedia?
LLM Response: You can find it in any of the libraries or on the Internet. Does this answer make you happy?
User: I need a coffee
LLM Response: It sounds like you're in need of a little pick-me-up. How are you feeling right now? Are you looking for something upbeat, something to relax to, or maybe some instrumental music while you enjoy your coffee?
User: I just got promoted at work!
LLM Response: Happy
User: Today is my birthday!
LLM Response: Happy
User: I won a prize in the lottery.
LLM Response: Happy
User: I am so excited about my vacation next week!
LLM Response: Happy
User: I aced my exams!
LLM Response: Happy
User: I had a wonderful time with my family today.
LLM Response: Happy
User: I just finished a great workout!
LLM Response: Happy
User: I am feeling really good about myself today.
LLM Response: Happy
User: I finally finished my project and it was a success!
LLM Response: Happy
User: I just heard my favorite song on the radio.
LLM Response: Happy
User: My pet passed away yesterday.
LLM Response: Sad
User: I lost my job today.
LLM Response: Sad
User: I'm feeling really lonely.
LLM Response: Sad
User: I didn't get the results I wanted.
LLM Response: Sad
User: I had a fight with my best friend.
LLM Response: Sad
User: I'm feeling really overwhelmed with everything.
LLM Response: Sad
User: I just got some bad news.
LLM Response: Sad
User: I'm missing my family.
LLM Response: Sad
User: I am feeling really down today.
LLM Response: Sad
User: Nothing seems to be going right.
LLM Response: Sad
User: I need some music while I study.
LLM Response: Instrumental
User: I want to listen to something soothing while I work.
LLM Response: Instrumental
User: Do you have any recommendations for background music?
LLM Response: Instrumental
User: I'm looking for some relaxing tunes.
LLM Response: Instrumental
User: I need some music to focus on my tasks.
LLM Response: Instrumental
User: Can you suggest some ambient music for meditation?
LLM Response: Instrumental
User: What's good for background music during reading?
LLM Response: Instrumental
User: I need some calm music to help me sleep.
LLM Response: Instrumental
User: I prefer instrumental music while cooking.
LLM Response: Instrumental
User: What's the best music to play while doing yoga?
LLM Response: Instrumental
User: Let's have a blast tonight!
LLM Response: Party
User: I'm in the mood to dance!
LLM Response: Party
User: I want to celebrate all night long!
LLM Response: Party
User: Time to hit the club!
LLM Response: Party
User: I feel like partying till dawn.
LLM Response: Party
User: Let's get this party started!
LLM Response: Party
User: I'm ready to party hard tonight.
LLM Response: Party
User: I'm in the mood for some loud music and dancing!
LLM Response: Party
User: Tonight's going to be epic!
LLM Response: Party
User: Lets turn up the music and have some fun!
LLM Response: Party
""" # Include your fixed prompt and instructions here
prompt = f"{fixed_prompt}"
for user_prompt, bot_response in history:
prompt += f"\nUser: {user_prompt}\nLLM Response: {bot_response}"
prompt += f"\nUser: {message}\nLLM Response:"
return prompt
async def process_speech(speech_file):
"""Processes speech input to text and then calls generate."""
text = speech_to_text(speech_file)
reply = generate(text, history="")
communicate = edge_tts.Communicate(reply)
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
tmp_path = tmp_file.name
await communicate.save(tmp_path)
yield tmp_path
DESCRIPTION = """ # Mood-Based Music Recommender⚡
### Hi! I'm a music recommender app.
### What kind of music do you want to listen to, or how are you feeling today?
"""
# Gradio interface setup
with gr.Blocks(css="style.css") as demo:
gr.Markdown(DESCRIPTION)
with gr.Row():
input = gr.Audio(label="User", sources="microphone", type="filepath", waveform_options=False)
output = gr.Audio(label="AI", type="filepath",
interactive=False,
autoplay=True,
elem_classes="audio")
gr.Interface(
batch=True,
max_batch_size=10,
fn=process_speech,
inputs=[input],
outputs=[output], live=True)
if __name__ == "__main__":
demo.queue(max_size=200).launch()