Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import langcodes | |
| import torch | |
| import uuid | |
| import json | |
| import librosa | |
| import os | |
| import tempfile | |
| import soundfile as sf | |
| import scipy.io.wavfile as wav | |
| from transformers import pipeline, VitsModel, AutoTokenizer, set_seed | |
| from huggingface_hub import InferenceClient | |
| from langdetect import detect, DetectorFactory | |
| from nemo.collections.asr.models import EncDecMultiTaskModel | |
| # Constants | |
| SAMPLE_RATE = 16000 # Hz | |
| # load ASR model | |
| canary_model = EncDecMultiTaskModel.from_pretrained('nvidia/canary-1b') | |
| decode_cfg = canary_model.cfg.decoding | |
| decode_cfg.beam.beam_size = 1 | |
| canary_model.change_decoding_strategy(decode_cfg) | |
| client = InferenceClient("HuggingFaceH4/zephyr-7b-beta") | |
| image_pipe = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base") | |
| summary_pipe = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6") | |
| ner_pipe = pipeline("ner", model="dslim/bert-base-NER") | |
| tts_model = VitsModel.from_pretrained("facebook/mms-tts-eng") | |
| tts_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-eng") | |
| # Function to convert audio to text using ASR | |
| def gen_text(audio_filepath, action, source_lang, target_lang): | |
| if audio_filepath is None: | |
| raise gr.Error("Please provide some input audio.") | |
| utt_id = uuid.uuid4() | |
| with tempfile.TemporaryDirectory() as tmpdir: | |
| # Convert to 16 kHz | |
| data, sr = librosa.load(audio_filepath, sr=None, mono=True) | |
| if sr != SAMPLE_RATE: | |
| data = librosa.resample(data, orig_sr=sr, target_sr=SAMPLE_RATE) | |
| converted_audio_filepath = os.path.join(tmpdir, f"{utt_id}.wav") | |
| sf.write(converted_audio_filepath, data, SAMPLE_RATE) | |
| # Transcribe audio | |
| duration = len(data) / SAMPLE_RATE | |
| manifest_data = { | |
| "audio_filepath": converted_audio_filepath, | |
| "taskname": action, | |
| "source_lang": source_lang, | |
| "target_lang": source_lang if action=="asr" else target_lang, | |
| "pnc": "no", | |
| "answer": "predict", | |
| "duration": str(duration), | |
| } | |
| manifest_filepath = os.path.join(tmpdir, f"{utt_id}.json") | |
| with open(manifest_filepath, 'w') as fout: | |
| fout.write(json.dumps(manifest_data)) | |
| predicted_text = canary_model.transcribe(manifest_filepath)[0] | |
| # if duration < 40: | |
| # predicted_text = canary_model.transcribe(manifest_filepath)[0] | |
| # else: | |
| # predicted_text = get_buffered_pred_feat_multitaskAED( | |
| # frame_asr, | |
| # canary_model.cfg.preprocessor, | |
| # model_stride_in_secs, | |
| # canary_model.device, | |
| # manifest=manifest_filepath, | |
| # )[0].text | |
| return predicted_text | |
| # Function to convert text to speech using TTS | |
| def gen_translated_speech(text, lang): | |
| set_seed(555) # Make it deterministic | |
| match lang: | |
| case "en": | |
| model = "facebook/mms-tts-eng" | |
| case "fr": | |
| model = "facebook/mms-tts-fra" | |
| case "de": | |
| model = "facebook/mms-tts-deu" | |
| case "es": | |
| model = "facebook/mms-tts-spa" | |
| case _: | |
| model = "facebook/mms-tts" | |
| # load TTS model | |
| tts_model = VitsModel.from_pretrained(model) | |
| tts_tokenizer = AutoTokenizer.from_pretrained(model) | |
| input_text = tts_tokenizer(text, return_tensors="pt") | |
| with torch.no_grad(): | |
| outputs = tts_model(**input_text) | |
| waveform_np = outputs.waveform[0].cpu().numpy() | |
| output_file = f"{str(uuid.uuid4())}.wav" | |
| wav.write(output_file, rate=tts_model.config.sampling_rate, data=waveform_np) | |
| return output_file | |
| # Root function for Gradio interface | |
| def start_process(audio_filepath, source_lang, target_lang): | |
| transcription = gen_text(audio_filepath, "asr", source_lang, target_lang) | |
| print("Done transcribing") | |
| translation = gen_text(audio_filepath, "s2t_translation", source_lang, target_lang) | |
| print("Done translation") | |
| audio_output_filepath = gen_translated_speech(translation, target_lang) | |
| print("Done speaking") | |
| return transcription, translation, audio_output_filepath | |
| def gen_speech(text): | |
| set_seed(555) # Make it deterministic | |
| input_text = tts_tokenizer(text, return_tensors="pt") | |
| with torch.no_grad(): | |
| outputs = tts_model(**input_text) | |
| waveform_np = outputs.waveform[0].cpu().numpy() | |
| output_file = f"{str(uuid.uuid4())}.wav" | |
| wav.write(output_file, rate=tts_model.config.sampling_rate, data=waveform_np) | |
| return output_file | |
| def detect_language(text): | |
| DetectorFactory.seed = 0 # Ensure consistent results | |
| return detect(text) | |
| def language_name_to_code(language_name): | |
| try: | |
| language = langcodes.find(language_name) | |
| return language.language | |
| except langcodes.LanguageTagError: | |
| return None | |
| def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p,): | |
| messages = [{"role": "system", "content": system_message}] | |
| for val in history: | |
| if val[0]: | |
| messages.append({"role": "user", "content": val[0]}) | |
| if val[1]: | |
| messages.append({"role": "assistant", "content": val[1]}) | |
| messages.append({"role": "user", "content": message}) | |
| response = "" | |
| for message in client.chat_completion( | |
| messages, | |
| max_tokens=max_tokens, | |
| stream=True, | |
| temperature=temperature, | |
| top_p=top_p, | |
| ): | |
| token = message.choices[0].delta.content | |
| response += token | |
| yield response | |
| def launch_image_pipe(input): | |
| out = image_pipe(input) | |
| text = out[0]['generated_text'] | |
| audio_output_filepath = gen_speech(text) | |
| return text, audio_output_filepath | |
| def translate(input_text, source, target): | |
| try: | |
| model = f"Helsinki-NLP/opus-mt-{source}-{target}" | |
| pipe = pipeline("translation", model=model) | |
| translation = pipe(input_text) | |
| return translation[0]['translation_text'], "" | |
| except KeyError: | |
| return "", f"Error: Translation direction {source} to {target} is not supported by Helsinki Translation Models" | |
| def summarize_translate(input_text, target_lang): | |
| output = summary_pipe(input_text) | |
| input_text_summary = output[0]['summary_text'] | |
| # source = 'en' | |
| source = detect_language(input_text_summary) | |
| target = language_name_to_code(target_lang) | |
| print(f"source_detect:{source}, target_lang:{target_lang}, target_code:{target}") | |
| summary_translated = translate(input_text_summary, source, target) | |
| return input_text_summary, summary_translated[0] | |
| def merge_tokens(tokens): | |
| merged_tokens = [] | |
| for token in tokens: | |
| if merged_tokens and token['entity'].startswith('I-') and merged_tokens[-1]['entity'].endswith(token['entity'][2:]): | |
| # If current token continues the entity of the last one, merge them | |
| last_token = merged_tokens[-1] | |
| last_token['word'] += token['word'].replace('##', '') | |
| last_token['end'] = token['end'] | |
| last_token['score'] = (last_token['score'] + token['score']) / 2 | |
| else: | |
| # Otherwise, add the token to the list | |
| merged_tokens.append(token) | |
| return merged_tokens | |
| def ner(input): | |
| output = ner_pipe(input) | |
| merged_tokens = merge_tokens(output) | |
| return {"text": input, "entities": merged_tokens} | |
| def create_playground_header(): | |
| gr.Markdown(""" | |
| # 🤗 Hugging Face Labs | |
| **Explore different LLM on Hugging Face platform. Just play and enjoy** | |
| """) | |
| def create_playground_footer(): | |
| gr.Markdown(""" | |
| **To Learn More about 🤗 Hugging Face, [Click Here](https://huggingface.co/docs)** | |
| """) | |
| # Create Gradio interface | |
| playground = gr.Blocks() | |
| with playground: | |
| create_playground_header() | |
| with gr.Tabs(): | |
| ## ================================================================================================================================ | |
| ## Speech Translator | |
| ## ================================================================================================================================ | |
| with gr.TabItem("Speech Translator"): | |
| with gr.Row(): | |
| gr.Markdown(""" | |
| ## Your AI Translate Assistant | |
| ### Gets input audio from user, transcribe and translate it. Convert back to speech. | |
| - category: [Automatic Speech Recognition](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition), model: [nvidia/canary-1b](https://huggingface.co/nvidia/canary-1b) | |
| - category: [Text-to-Speech](https://huggingface.co/models?pipeline_tag=text-to-speech), model: [facebook/mms-tts](https://huggingface.co/facebook/mms-tts) | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| source_lang = gr.Dropdown( | |
| choices=["en", "de", "es", "fr"], value="en", label="Source Language" | |
| ) | |
| with gr.Column(): | |
| target_lang = gr.Dropdown( | |
| choices=["en", "de", "es", "fr"], value="fr", label="Target Language" | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| input_audio = gr.Audio(sources=["microphone"], type="filepath", label="Input Audio") | |
| with gr.Column(): | |
| translated_speech = gr.Audio(type="filepath", label="Generated Speech") | |
| with gr.Row(): | |
| with gr.Column(): | |
| transcipted_text = gr.Textbox(label="Transcription") | |
| with gr.Column(): | |
| translated_text = gr.Textbox(label="Translation") | |
| with gr.Row(): | |
| with gr.Column(): | |
| submit_button = gr.Button(value="Start Process", variant="primary") | |
| with gr.Column(): | |
| clear_button = gr.ClearButton(components=[input_audio, source_lang, target_lang, transcipted_text, translated_text, translated_speech], value="Clear") | |
| with gr.Row(): | |
| gr.Examples( | |
| examples=[ | |
| ["audio/sample_en.wav","en","fr"], | |
| ["audio/sample_fr.wav","fr","de"], | |
| ["audio/sample_de.wav","de","es"], | |
| ["audio/sample_es.wav","es","en"] | |
| ], | |
| inputs=[input_audio, source_lang, target_lang], | |
| outputs=[transcipted_text, translated_text, translated_speech], | |
| run_on_click=True, cache_examples=True, fn=start_process | |
| ) | |
| submit_button.click(start_process, inputs=[input_audio, source_lang, target_lang], outputs=[transcipted_text, translated_text, translated_speech]) | |
| ## ================================================================================================================================ | |
| ## Image Captioning | |
| ## ================================================================================================================================ | |
| with gr.TabItem("Image"): | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| gr.Markdown(""" | |
| ## Image Captioning | |
| ### Upload a image, check what AI understand and have vision on it. | |
| - category: Image-to-Text, model: [Salesforce/blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base) | |
| - category: Text-to-Speech, model: [facebook/mms-tts-eng](https://huggingface.co/facebook/mms-tts-eng) | |
| """) | |
| with gr.Column(scale=1): | |
| ITT_button = gr.Button(value="Start Process", variant="primary") | |
| with gr.Row(): | |
| with gr.Column(): | |
| img = gr.Image(type='pil') | |
| with gr.Column(): | |
| generated_textbox = gr.Textbox(lines=2, placeholder="", label="Generated Text") | |
| audio_output = gr.Audio(type="filepath", label="Generated Speech") | |
| ITT_Clear_button = gr.ClearButton(components=[img, generated_textbox, audio_output], value="Clear") | |
| gr.Examples( | |
| examples=[ | |
| ["image/lion-dog-costume.jpg"], | |
| ["image/dog-halloween.jpeg"] | |
| ], | |
| inputs=[img], | |
| outputs=[generated_textbox, audio_output], | |
| run_on_click=True, cache_examples=True, fn=launch_image_pipe) | |
| ITT_button.click(launch_image_pipe, inputs=[img], outputs=[generated_textbox, audio_output]) | |
| # generate_audio_button.click(generate_audio, inputs=[generated_textbox], outputs=[audio_output]) | |
| ## ================================================================================================================================ | |
| ## Text Summarization and Translation | |
| ## ================================================================================================================================ | |
| with gr.TabItem("Text"): | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| gr.Markdown(""" | |
| ## Text Summarization and Translation | |
| ### Summarize the paragraph and translate it into other language. | |
| - pipeline: summarization, model: [sshleifer/distilbart-cnn-12-6](https://huggingface.co/sshleifer/distilbart-cnn-12-6) | |
| - pipeline: translation, model: [Helsinki-NLP/opus-mt-en-{target](https://huggingface.co/Helsinki-NLP/opus-mt-en-fr) | |
| """) | |
| with gr.Column(scale=1): | |
| text_pipeline_button = gr.Button(value="Start Process", variant="primary") | |
| with gr.Row(): | |
| with gr.Column(): | |
| source_text = gr.Textbox(label="Text to summarize", lines=18) | |
| with gr.Column(): | |
| summary_textoutput = gr.Textbox(lines=3, placeholder="", label="Text Summarization") | |
| target_language_dropdown = gr.Dropdown( choices=["Chinese", "French", "Spanish"], | |
| value="Chinese", | |
| label="Translate to Language") | |
| translated_textbox = gr.Textbox(lines=3, placeholder="", label="Translated Result") | |
| Text_Clear_button = gr.ClearButton(components=[source_text, summary_textoutput, translated_textbox], value="Clear") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Examples( | |
| examples=[ | |
| ["The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.","French"], | |
| ["Tower Bridge is a Grade I listed combined bascule, suspension, and, until 1960, cantilever bridge in London, built between 1886 and 1894, designed by Horace Jones and engineered by John Wolfe Barry with the help of Henry Marc Brunel. It crosses the River Thames close to the Tower of London and is one of five London bridges owned and maintained by the City Bridge Foundation, a charitable trust founded in 1282. The bridge was constructed to connect the 39 per cent of London's population that lived east of London Bridge, while allowing shipping to access the Pool of London between the Tower of London and London Bridge. The bridge was opened by Edward, Prince of Wales and Alexandra, Princess of Wales on 30 June 1894.","Chinese"] | |
| ], | |
| inputs=[source_text, target_language_dropdown], | |
| outputs=[summary_textoutput, translated_textbox], | |
| run_on_click=True, cache_examples=True, fn=summarize_translate) | |
| text_pipeline_button.click(summarize_translate, inputs=[source_text, target_language_dropdown], outputs=[summary_textoutput, translated_textbox]) | |
| ## ================================================================================================================================ | |
| ## Find entities | |
| ## ================================================================================================================================ | |
| with gr.TabItem("Name Entity"): | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| gr.Markdown(""" | |
| ## Find entities | |
| ### Entities involved Name, Organization, and Location. | |
| > pipeline: ner, model: [dslim/bert-base-NER](https://huggingface.co/dslim/bert-base-NER) | |
| """) | |
| with gr.Column(scale=1): | |
| ner_pipeline_button = gr.Button(value="Start Process", variant="primary") | |
| with gr.Row(): | |
| with gr.Column(): | |
| ner_text_input = gr.Textbox(label="Text to find entities", lines=5) | |
| with gr.Column(): | |
| ner_text_output = gr.HighlightedText(label="Text with entities") | |
| Ner_Clear_button = gr.ClearButton(components=[ner_text_input, ner_text_output], value="Clear") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Examples(examples=[ | |
| "My name is Ray, I'm learning through Hugging Face and DeepLearning.AI and I live in Caversham, Reading", | |
| "My name is Raymond, I work at A&O IT Group" | |
| ], inputs=[ner_text_input], outputs=[ner_text_output], run_on_click=True, cache_examples=True, fn=ner) | |
| ner_pipeline_button.click(ner, inputs=[ner_text_input], outputs=[ner_text_output]) | |
| ## ================================================================================================================================ | |
| ## Chatbot | |
| ## ================================================================================================================================ | |
| with gr.TabItem("Chatbot"): | |
| gr.ChatInterface( | |
| respond, | |
| additional_inputs=[ | |
| gr.Textbox(value="You are a friendly Chatbot.", label="System message"), | |
| gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), | |
| gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), | |
| gr.Slider( | |
| minimum=0.1, | |
| maximum=1.0, | |
| value=0.95, | |
| step=0.05, | |
| label="Top-p (nucleus sampling)", | |
| ), | |
| ], | |
| ) | |
| create_playground_footer() | |
| playground.launch() |