from elevenlabs import VoiceSettings from elevenlabs.client import ElevenLabs from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer from ai71 import AI71 from datetime import datetime import os import time from pydub import AudioSegment from base64 import b64encode import gradio as gr import concurrent.futures AI71_API_KEY = os.getenv('AI71_API_KEY') XI_API_KEY = os.getenv('ELEVEN_LABS_API_KEY') client = ElevenLabs(api_key=XI_API_KEY) translator = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_1.2B") tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_1.2B") transcriber = gr.load("models/openai/whisper-large-v3-turbo") # transcriber = whisper.load_model("turbo") language_codes = {"English":"en", "Hindi":"hi", "Portuguese":"pt", "Chinese":"zh", "Spanish":"es", "French":"fr", "German":"de", "Japanese":"ja", "Arabic":"ar", "Russian":"ru", "Korean":"ko", "Indonesian":"id", "Italian":"it", "Dutch":"nl","Turkish":"tr", "Polish":"pl", "Swedish":"sv", "Filipino":"fil", "Malay":"ms", "Romanian":"ro", "Ukrainian":"uk", "Greek":"el", "Czech":"cs", "Danish":"da", "Finnish":"fi", "Bulgarian":"bg", "Croatian":"hr", "Slovak":"sk"} # meeting_texts = [] n_participants = 4 # This can be adjusted based on the number of people in the call language_choices = ["English", "Polish", "Hindi", "Arabic"] def clear_all(): global meeting_texts # meeting_texts = [] # Reset meeting texts return [None] * (n_participants * 4 + 1)+[gr.State([])] # Reset outputs of transcripts, translated texts, and dubbed videos def wait_for_dubbing_completion(dubbing_id: str) -> bool: """ Waits for the dubbing process to complete by periodically checking the status. Args: dubbing_id (str): The dubbing project id. Returns: bool: True if the dubbing is successful, False otherwise. """ MAX_ATTEMPTS = 120 CHECK_INTERVAL = 10 # In seconds for _ in range(MAX_ATTEMPTS): metadata = client.dubbing.get_dubbing_project_metadata(dubbing_id) if metadata.status == "dubbed": return True elif metadata.status == "dubbing": print( "Dubbing in progress... Will check status again in", CHECK_INTERVAL, "seconds.", ) time.sleep(CHECK_INTERVAL) else: print("Dubbing failed:", metadata.error_message) return False print("Dubbing timed out") return False def download_dubbed_file(dubbing_id: str, language_code: str) -> str: """ Downloads the dubbed file for a given dubbing ID and language code. Args: dubbing_id: The ID of the dubbing project. language_code: The language code for the dubbing. Returns: The file path to the downloaded dubbed file. """ dir_path = f"data/{dubbing_id}" os.makedirs(dir_path, exist_ok=True) file_path = f"{dir_path}/{language_code}.mp4" with open(file_path, "wb") as file: for chunk in client.dubbing.get_dubbed_file(dubbing_id, language_code): file.write(chunk) return file_path def create_dub_from_file( input_file_path: str, file_format: str, source_language: str, target_language: str, ): # ) -> Optional[str]: """ Dubs an audio or video file from one language to another and saves the output. Args: input_file_path (str): The file path of the audio or video to dub. file_format (str): The file format of the input file. source_language (str): The language of the input file. target_language (str): The target language to dub into. Returns: Optional[str]: The file path of the dubbed file or None if operation failed. """ if not os.path.isfile(input_file_path): raise FileNotFoundError(f"The input file does not exist: {input_file_path}") with open(input_file_path, "rb") as audio_file: response = client.dubbing.dub_a_video_or_an_audio_file( file=(os.path.basename(input_file_path), audio_file, file_format), # Optional file target_lang=target_language, # The target language to dub the content into. Can be none if dubbing studio editor is enabled and running manual mode # mode="automatic", # automatic or manual. source_lang=source_language, # Source language num_speakers=1, # Number of speakers to use for the dubbing. watermark=True, # Whether to apply watermark to the output video. ) # rest of the code dubbing_id = response.dubbing_id if wait_for_dubbing_completion(dubbing_id): output_file_path = download_dubbed_file(dubbing_id, target_language) return output_file_path else: return None def summarize(meeting_texts): meeting_texts = ', '.join([f"{k}: {v}" for i in meeting_texts for k, v in i.items()]) meeting_date_time = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # meeting_texts = meeting_date_time + '\n' + meeting_texts # meeting_conversation_processed ='\n'.join(mt) # print("M:", session_conversation_processed) minutes_of_meeting = "" for chunk in AI71(AI71_API_KEY.strip()).chat.completions.create( model="tiiuae/falcon-180b-chat", messages=[ {"role": "system", "content": f"""You are an expereiced Secretary who can summarize meeting discussions into minutes of meeting. Summarize the meeting discussions provided in json format as Speakerwise conversation. Strictly consider ONLY the context given in user content for summarization. Do not generalize the summary with irrelevant content. Ensure to mention the title as 'Minutes of Meeting held on {meeting_date_time} and present the summary with better viewing format and title in bold letters"""}, {"role": "user", "content": meeting_texts}, ], stream=True, ): if chunk.choices[0].delta.content: summary = chunk.choices[0].delta.content minutes_of_meeting += summary minutes_of_meeting = minutes_of_meeting.replace('User:', '').strip() print("\n") print("minutes_of_meeting:", minutes_of_meeting) return minutes_of_meeting # Placeholder function for speech to text conversion def speech_to_text(video): print(video, type(video)) print('Started transcribing') audio = AudioSegment.from_file(video) audio.export('temp.wav', format="wav") # transcript = transcriber.transcribe(video).text # transcript = transcriber.transcribe(video).text transcript = transcriber("temp.wav").split("'")[1].strip() print('transcript:', transcript) return transcript # Placeholder function for translating text def translate_text(text, source_language,target_language): tokenizer.src_lang = source_language encoded_ln = tokenizer(text, return_tensors="pt") generated_tokens = translator.generate(**encoded_ln, forced_bos_token_id=tokenizer.get_lang_id(target_language)) translated_text = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0] print('translated_text:', translated_text) return translated_text # Placeholder function for dubbing (text-to-speech in another language) def synthesize_speech(video, source_language,target_language): print('Started dubbing') dub_video = create_dub_from_file(input_file_path = video, file_format = 'audio/mpeg', source_language = source_language, target_language = target_language) return dub_video # # This function handles the processing when any participant speaks # def process_speaker(video, speaker_idx, n_participants, *language_list): # transcript = speech_to_text(video) # # Create outputs for each participant # outputs = [] # global meeting_texts # def process_translation_dubbing(i): # if i != speaker_idx: # participant_language = language_codes[language_list[i]] # speaker_language = language_codes[language_list[speaker_idx]] # translated_text = translate_text(transcript, speaker_language, participant_language) # dubbed_video = synthesize_speech(video, speaker_language, participant_language) # return translated_text, dubbed_video # return None, None # with concurrent.futures.ThreadPoolExecutor() as executor: # futures = [executor.submit(process_translation_dubbing, i) for i in range(n_participants)] # results = [f.result() for f in futures] # for i, (translated_text, dubbed_video) in enumerate(results): # if i == speaker_idx: # outputs.insert(0, transcript) # else: # outputs.append(translated_text) # outputs.append(dubbed_video) # if speaker_idx == 0: # meeting_texts.append({f"Speaker_{speaker_idx+1}":outputs[0]}) # else: # meeting_texts.append({f"Speaker_{speaker_idx+1}":outputs[1]}) # print(len(outputs)) # print(outputs) # print('meeting_texts: ',meeting_texts) # return outputs # def create_participant_row(i, language_choices): # """Creates the UI for a single participant.""" # with gr.Row(): # video_input = gr.Video(label=f"Participant {i+1} Video", interactive=True) # language_dropdown = gr.Dropdown(choices=language_choices, label=f"Participant {i+1} Language", value=language_choices[i]) # transcript_output = gr.Textbox(label=f"Participant {i+1} Transcript") # translated_text = gr.Textbox(label="Speaker's Translated Text") # dubbed_video = gr.Video(label="Speaker's Dubbed Video") # return video_input, language_dropdown, transcript_output, translated_text, dubbed_video # # Main dynamic Gradio interface # def create_gradio_interface(n_participants, language_choices): # with gr.Blocks() as demo: # gr.Markdown("""# LinguaPolis: Bridging Languages, Uniting Teams Globally - Multilingual Conference Call Simulation # ## Record your video or upload your video and press the corresponding Submit button at the bottom""") # video_inputs = [] # language_dropdowns = [] # transcript_outputs = [] # translated_texts = [] # dubbed_videos = [] # clear_button = gr.Button("Clear All") # # Create a row for each participant # for i in range(n_participants): # video_input, language_dropdown, transcript_output, translated_text, dubbed_video = create_participant_row(i, language_choices) # video_inputs.append(video_input) # language_dropdowns.append(language_dropdown) # transcript_outputs.append(transcript_output) # translated_texts.append(translated_text) # dubbed_videos.append(dubbed_video) # # Create dynamic processing buttons for each participant # for i in range(n_participants): # gr.Button(f"Submit Speaker {i+1}'s Speech").click( # process_speaker, # [video_inputs[i], gr.State(i), gr.State(n_participants)] + [language_dropdowns[j] for j in range(n_participants)], # [transcript_outputs[i]] + [k for j in zip(translated_texts[:i]+translated_texts[i+1:], dubbed_videos[:i]+dubbed_videos[i+1:]) for k in j] # ) # minutes = gr.Textbox(label="Minutes of Meeting") # gr.Button(f"Generate Minutes of meeting").click(summarize, None, minutes) # # Clear button to reset inputs and outputs # clear_button.click(clear_all, None, [*video_inputs, *transcript_outputs, *translated_texts, *dubbed_videos, minutes]) # # Launch with .queue() to keep it running properly in Jupyter # demo.queue().launch(debug=True, share=True) # create_gradio_interface(n_participants, language_choices) # def create_dub_from_file( # input_file_path: str, # file_format: str, # source_language: str, # target_language: str, # ): # # ) -> Optional[str]: # """ # Dubs an audio or video file from one language to another and saves the output. # Args: # input_file_path (str): The file path of the audio or video to dub. # file_format (str): The file format of the input file. # source_language (str): The language of the input file. # target_language (str): The target language to dub into. # Returns: # Optional[str]: The file path of the dubbed file or None if operation failed. # """ # if not os.path.isfile(input_file_path): # raise FileNotFoundError(f"The input file does not exist: {input_file_path}") # with open(input_file_path, "rb") as audio_file: # response = client.dubbing.dub_a_video_or_an_audio_file( # file=(os.path.basename(input_file_path), audio_file, file_format), # Optional file # target_lang=target_language, # The target language to dub the content into. Can be none if dubbing studio editor is enabled and running manual mode # # mode="automatic", # automatic or manual. # source_lang=source_language, # Source language # num_speakers=1, # Number of speakers to use for the dubbing. # watermark=True, # Whether to apply watermark to the output video. # ) # # rest of the code # dubbing_id = response.dubbing_id # if wait_for_dubbing_completion(dubbing_id): # output_file_path = download_dubbed_file(dubbing_id, target_language) # return output_file_path # else: # return None # # Modify the summarize function to accept and return meeting_texts # def summarize(meeting_texts): # meeting_texts = ', '.join([f"{k}: {v}" for i in meeting_texts for k, v in i.items()]) # meeting_date_time = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # # meeting_texts_str = meeting_date_time + '\n' + mt # minutes_of_meeting = "" # for chunk in AI71(AI71_API_KEY.strip()).chat.completions.create( # model="tiiuae/falcon-180b-chat", # messages=[ # {"role": "system", "content": f"""You are an experienced Secretary who can summarize meeting discussions into minutes of meeting. # Summarize the meetings discussions provided as Speakerwise conversation. # Strictly consider only the context given in user content for summarization. # Ensure to mention the title as 'Minutes of Meeting held on {meeting_date_time}' and present the summary with better viewing format and title in bold letters."""}, # {"role": "user", "content": meeting_texts}, # ], # stream=True, # ): # if chunk.choices[0].delta.content: # summary = chunk.choices[0].delta.content # minutes_of_meeting += summary # minutes_of_meeting = minutes_of_meeting.replace('User:', '').strip() # print("minutes_of_meeting:", minutes_of_meeting) # return minutes_of_meeting # # Placeholder function for speech to text conversion # def speech_to_text(video): # print(video, type(video)) # print('Started transcribing') # audio = AudioSegment.from_file(video) # audio.export('temp.wav', format="wav") # # transcript = transcriber.transcribe(video).text # # transcript = transcriber.transcribe(video).text # transcript = transcriber("temp.wav").split("'")[1].strip() # print('transcript:', transcript) # return transcript # # Placeholder function for translating text # def translate_text(text, source_language,target_language): # tokenizer.src_lang = source_language # encoded_ln = tokenizer(text, return_tensors="pt") # generated_tokens = translator.generate(**encoded_ln, forced_bos_token_id=tokenizer.get_lang_id(target_language)) # translated_text = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0] # print('translated_text:', translated_text) # return translated_text # # Placeholder function for dubbing (text-to-speech in another language) # def synthesize_speech(video, source_language,target_language): # print('Started dubbing') # dub_video = create_dub_from_file(input_file_path = video, # file_format = 'audio/mpeg', # source_language = source_language, # target_language = target_language) # return dub_video # Update process_speaker function to accept and return meeting_texts def process_speaker(video, speaker_idx, n_participants, meeting_texts, *language_list): transcript = speech_to_text(video) # Create outputs for each participant outputs = [] def process_translation_dubbing(i): if i != speaker_idx: participant_language = language_codes[language_list[i]] speaker_language = language_codes[language_list[speaker_idx]] translated_text = translate_text(transcript, speaker_language, participant_language) dubbed_video = synthesize_speech(video, speaker_language, participant_language) return translated_text, dubbed_video return None, None with concurrent.futures.ThreadPoolExecutor() as executor: futures = [executor.submit(process_translation_dubbing, i) for i in range(n_participants)] results = [f.result() for f in futures] for i, (translated_text, dubbed_video) in enumerate(results): if i == speaker_idx: outputs.insert(0, transcript) else: outputs.append(translated_text) outputs.append(dubbed_video) if speaker_idx == 0: meeting_texts.append({f"Speaker_{speaker_idx+1}": outputs[0]}) else: meeting_texts.append({f"Speaker_{speaker_idx+1}": outputs[1]}) print(len(outputs)) print(outputs) print("meeting_texts:", meeting_texts) print('outputs:', outputs) outputs.append(meeting_texts) print(len(outputs)) return outputs def create_participant_row(i, language_choices): """Creates the UI for a single participant.""" with gr.Row(): video_input = gr.Video(label=f"Participant {i+1} Video", interactive=True) language_dropdown = gr.Dropdown(choices=language_codes.keys(), label=f"Participant {i+1} Language", value=language_choices[i]) transcript_output = gr.Textbox(label=f"Participant {i+1} Transcript") translated_text = gr.Textbox(label="Speaker's Translated Text") dubbed_video = gr.Video(label="Speaker's Dubbed Video") return video_input, language_dropdown, transcript_output, translated_text, dubbed_video # Modify the Gradio interface to manage the meeting_texts between function calls def create_gradio_interface(n_participants, language_choices): with gr.Blocks() as demo: gr.Markdown("""# LinguaPolis: Bridging Languages, Uniting Teams Globally - Multilingual Conference Call Simulation ## Assume it is a virtual conference call where the speakers speak one by one ### Select a language, record your video or upload your video and press the corresponding 'Submit button' at the bottom #### After the output generation, repeat the above for as many speakers as you want, one by one. When you finish, press 'Generate Minutes of Meeting button' to get the meeting summary""") video_inputs = [] language_dropdowns = [] transcript_outputs = [] translated_texts = [] dubbed_videos = [] clear_button = gr.Button("Clear All") meeting_texts = gr.State([]) # Initialize meeting_texts as a Gradio State # Create a row for each participant for i in range(n_participants): video_input, language_dropdown, transcript_output, translated_text, dubbed_video = create_participant_row(i, language_choices) video_inputs.append(video_input) language_dropdowns.append(language_dropdown) transcript_outputs.append(transcript_output) translated_texts.append(translated_text) dubbed_videos.append(dubbed_video) # Create dynamic processing buttons for each participant for i in range(n_participants): gr.Button(f"Submit Speaker {i+1}'s Speech").click( process_speaker, [video_inputs[i], gr.State(i), gr.State(n_participants), meeting_texts] + [language_dropdowns[j] for j in range(n_participants)], [transcript_outputs[i]] + [k for j in zip(translated_texts[:i]+translated_texts[i+1:], dubbed_videos[:i]+dubbed_videos[i+1:]) for k in j] + [meeting_texts] ) minutes = gr.Textbox(label="Minutes of Meeting") gr.Button(f"Generate Minutes of meeting").click(summarize, [meeting_texts], minutes) # Clear button to reset inputs and outputs clear_button.click(clear_all, None, [*video_inputs, *transcript_outputs, *translated_texts, *dubbed_videos, minutes, meeting_texts]) demo.launch(debug=True, share=True) create_gradio_interface(4, language_choices)