Spaces:
Running
on
Zero
Running
on
Zero
| # coding=utf-8 | |
| import io | |
| import numpy as np | |
| import torchaudio | |
| import torch | |
| import soundfile as sf | |
| import gradio as gr | |
| import spaces | |
| from inspiremusic.cli.inference import InspireMusicUnified, set_env_variables | |
| import os | |
| import sys | |
| def get_args(): | |
| parser = argparse.ArgumentParser( | |
| description='Run inference with your model') | |
| parser.add_argument('-m', '--model_name', default="InspireMusic-1.5B-Long", | |
| help='Model name') | |
| parser.add_argument('-d', '--model_dir', | |
| help='Model folder path') | |
| parser.add_argument('-t', '--text', | |
| default="Experience soothing and sensual instrumental jazz with a touch of Bossa Nova, perfect for a relaxing restaurant or spa ambiance.", | |
| help='Prompt text') | |
| parser.add_argument('-a', '--audio_prompt', default=None, | |
| help='Prompt audio') | |
| parser.add_argument('-c', '--chorus', default="intro", | |
| help='Chorus tag generation mode (e.g., random, verse, chorus, intro, outro)') | |
| parser.add_argument('--fast', type=bool, default=False, | |
| help='Enable fast inference mode (without flow matching)') | |
| parser.add_argument('-g', '--gpu', type=int, default=0, | |
| help='GPU ID for this rank, -1 for CPU') | |
| parser.add_argument('--task', default='text-to-music', | |
| choices=['text-to-music', 'continuation', 'reconstruct', 'super_resolution'], | |
| help='Inference task type: text-to-music, continuation, reconstruct, super_resolution') | |
| parser.add_argument('-r', '--result_dir', default="exp/inspiremusic", | |
| help='Directory to save generated audio') | |
| parser.add_argument('-o', '--output_fn', default="output_audio", | |
| help='Output file name') | |
| parser.add_argument('-f', '--format', type=str, default="wav", | |
| choices=["wav", "mp3", "m4a", "flac"], | |
| help='Format of output audio') | |
| parser.add_argument('--sample_rate', type=int, default=24000, | |
| help='Sampling rate of input audio') | |
| parser.add_argument('--output_sample_rate', type=int, default=48000, | |
| choices=[24000, 48000], | |
| help='Sampling rate of generated output audio') | |
| parser.add_argument('-s', '--time_start', type=float, default=0.0, | |
| help='Start time in seconds') | |
| parser.add_argument('-e', '--time_end', type=float, default=30.0, | |
| help='End time in seconds') | |
| parser.add_argument('--max_audio_prompt_length', type=float, default=5.0, | |
| help='Maximum audio prompt length in seconds') | |
| parser.add_argument('--min_generate_audio_seconds', type=float, | |
| default=10.0, | |
| help='Minimum generated audio length in seconds') | |
| parser.add_argument('--max_generate_audio_seconds', type=float, | |
| default=30.0, | |
| help='Maximum generated audio length in seconds') | |
| parser.add_argument('--fp16', type=bool, default=True, | |
| help='Inference with fp16 model') | |
| parser.add_argument('--fade_out', type=bool, default=True, | |
| help='Apply fade out effect to generated audio') | |
| parser.add_argument('--fade_out_duration', type=float, default=1.0, | |
| help='Fade out duration in seconds') | |
| parser.add_argument('--trim', type=bool, default=False, | |
| help='Trim the silence ending of generated audio') | |
| args = parser.parse_args() | |
| if not args.model_dir: | |
| args.model_dir = os.path.join("./pretrained_models", args.model_name) | |
| print(args) | |
| return args | |
| def InspireMusic(args): | |
| set_env_variables() | |
| model = InspireMusicUnified(model_name=args.model_name, | |
| model_dir=args.model_dir, | |
| min_generate_audio_seconds=args.min_generate_audio_seconds, | |
| max_generate_audio_seconds=args.max_generate_audio_seconds, | |
| sample_rate=args.sample_rate, | |
| output_sample_rate=args.output_sample_rate, | |
| load_jit=True, | |
| load_onnx=False, | |
| fast=args.fast, | |
| fp16=args.fp16, | |
| gpu=args.gpu, | |
| result_dir=args.result_dir) | |
| model.inference(task=args.task, | |
| text=args.text, | |
| audio_prompt=args.audio_prompt, | |
| chorus=args.chorus, | |
| time_start=args.time_start, | |
| time_end=args.time_end, | |
| output_fn=args.output_fn, | |
| max_audio_prompt_length=args.max_audio_prompt_length, | |
| fade_out_duration=args.fade_out_duration, | |
| output_format=args.format, | |
| fade_out_mode=args.fade_out, | |
| trim=args.trim) | |
| return os.path.join(args.result_dir, f"{args.output_fn}.{args.format}") | |
| audio_examples = [ | |
| ["example/inspiremusic/inspiremusic_01.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_noflow_01.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_w_cfm_intro.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_w_cfm_verse.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_w_cfm_chorus.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_w_cfm_outro.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_w_cfm_verse_ras.wav", "text-to-music"], | |
| ["example/inspiremusic/inspiremusic_wo_cfm_verse_topk.wav", "text-to-music"], | |
| ["example/ras/chorus/chorus_01.wav", "music-continuation"], | |
| ["example/ras/chorus/chorus_02.wav", "music-continuation"], | |
| ["example/ras/chorus/chorus_03.wav", "music-continuation"], | |
| ["example/ras/chorus/chorus_04.wav", "music-continuation"], | |
| ["example/ras/chorus/chorus_05.wav", "music-continuation"], | |
| ] | |
| description = """ | |
| # InspireMusic is a music generation model with text to music generation capability, including text to music, music continuation. | |
| ## Usage | |
| ### Input text descriptions of the music, click submit, then generate music. | |
| *Example Texts* | |
| - `Experience soothing and sensual instrumental jazz with a touch of Bossa Nova, perfect for a relaxing restaurant or spa ambiance.` | |
| - `The instrumental rock piece features a prominent bass guitar, delivering a pure and energetic sound.` | |
| - `A serene blend of instrumental and light pop, featuring soothing melodies and a gentle, soulful keyboard performance.` | |
| Recommended audio prompt duration is 5 seconds, generate audio length is below 30 seconds. To generate audio longer than 30 seconds, local deployment is recommended, github repo. | |
| """ | |
| html_content = """ | |
| <div> | |
| <h2 style="font-size: 22px;margin-left: 0px;">Music Generation Model: InspireMusic</h2> | |
| <p style="font-size: 18px;margin-left: 20px;">InspireMusic is a unified music, song and audio generation framework through the audio tokenization and detokenization process integrated with an autoregressive transformer. The toolkit provides both inference and training code for music generation. Featuring a unified framework, InspireMusic incorporates autoregressive Transformer and conditional flow-matching modeling (CFM), allowing for the controllable generation of music, songs, and audio with both textual and structural music conditioning, as well as neural audio tokenizers. Currently, the toolkit supports text-to-music generation and plans to expand its capabilities to include text-to-song and text-to-audio generation in the future.</p> | |
| <h2 style="font-size: 22px;margin-left: 0px;">Usage</h2> <p style="font-size: 18px;margin-left: 20px;">Input a text description of music or input through a microphone, then select the chorus and duration. The music is generated based on the input text. The chorus labels are placed in the front of the text.</p> | |
| <p style="font-size: 18px;margin-left: 20px;">Recommended select audio duration is below 30 seconds. For audio longer than 30 seconds, local deployment is recommended.</p> | |
| <h2 style="font-size: 22px;margin-left: 0px;">Repo & Demo</h2> | |
| <p style="font-size: 18px;margin-left: 20px;"><a href="https://github.com/FunAudioLLM/InspireMusic" target="_blank">Code</a> </p> | |
| <p style="font-size: 18px;margin-left: 20px;"><a href="https://iris2c.github.io/InspireMusic" target="_blank">Demo</a></p> | |
| <h2 style="font-size: 22px;margin-left: 0px;">Models</h2> | |
| <p style="font-size: 18px;margin-left: 20px;"><a href="https://modelscope.cn/models/iic/InspireMusic-1.5B-Long/summary" target="_blank">Modelscope Model</a>: </p> | |
| <p style="font-size: 18px;margin-left: 20px;"><a href="https://huggingface.co/FunAudioLLM/InspireMusic-1.5B-Long" target="_blank">Huggingface Model</a></p> | |
| </div> | |
| """ | |
| def music_generation(task, text=None, audio=None): | |
| args = get_args() | |
| args.task = task | |
| args.text = text if text | |
| args.audio_prompt = audio if audio | |
| generate_audio_path = InspireMusic(args) | |
| return generate_audio_path | |
| demo = gr.Blocks() | |
| t2m_demo = gr.Interface( | |
| fn=music_generation, | |
| inputs = [ | |
| gr.Dropdown(["Text-To-Music"], value="text-to-music", multiselect=False, info="Choose a task."), | |
| gr.Text(label="Input Text"), | |
| ], | |
| outputs = [ | |
| gr.Audio(label="Generated Music", type="generated audio filepath"), | |
| ], | |
| title = "<a href='https://github.com/FunAudioLLM/InspireMusic' target='_blank'>InspireMusic<a/>: A Unified Framework for Music, Song, Audio Generation.", | |
| description = ("InspireMusic ([Github Repo](https://github.com/FunAudioLLM/InspireMusic)) is a fundamental AIGC toolkit and models designed for music, song, and audio generation using PyTorch." | |
| "To try it, simply type text to generation music, or click one of the examples. "), | |
| article = ("<p style='text-align: center'><a href='' target='_blank'>InspireMusic</a> </p>" | |
| "<p style='text-align: center'><a href='https://openreview.net/forum?id=yBlVlS2Fd9' target='_blank'>WavTokenizer: an Efficient Acoustic Discrete Codec Tokenizer for Audio Language Modeling</a> </p>"), | |
| examples = [ | |
| ["example/inspiremusic/inspiremusic_01.wav", "24000 Hz"], | |
| ["example/ras/chorus/chorus_01.wav", "48000 Hz"], | |
| ], | |
| cache_examples = True, | |
| ) | |
| con_demo = gr.Interface( | |
| fn=music_generation, | |
| inputs = [ | |
| gr.Dropdown(["Music Continuation"], value="continuation", multiselect=False, info="Choose a task."), | |
| gr.Text(label="Input Text"), | |
| gr.Audio(label="Input Audio Prompt", type="audio prompt filepath"), | |
| ], | |
| outputs = [ | |
| gr.Audio(label="Generated Music", type="generated audio filepath"), | |
| ], | |
| title = "<a href='https://github.com/FunAudioLLM/InspireMusic' target='_blank'>InspireMusic<a/>: A Unified Framework for Music, Song, Audio Generation.", | |
| description = ("InspireMusic ([Github Repo](https://github.com/FunAudioLLM/InspireMusic)) is a fundamental AIGC toolkit and models designed for music, song, and audio generation using PyTorch." | |
| "To try it, simply type text to generation music, or click one of the examples. "), | |
| article = ("<p style='text-align: center'><a href='' target='_blank'>InspireMusic</a> </p>" | |
| "<p style='text-align: center'><a href='https://openreview.net/forum?id=yBlVlS2Fd9' target='_blank'>WavTokenizer: an Efficient Acoustic Discrete Codec Tokenizer for Audio Language Modeling</a> </p>"), | |
| examples = [ | |
| ["example/inspiremusic/inspiremusic_01.wav", "24000 Hz"], | |
| ["example/ras/chorus/chorus_01.wav", "48000 Hz"], | |
| ], | |
| cache_examples = True, | |
| ) | |
| con_demo = gr.Interface( | |
| fn=music_generation, | |
| inputs = [ | |
| gr.Dropdown(["Music Continuation"], value="continuation", multiselect=False, info="Choose a task."), | |
| gr.Text(label="Input Text"), | |
| gr.Audio(label="Input Audio Prompt", type="audio prompt filepath"), | |
| ], | |
| outputs = [ | |
| gr.Audio(label="Generated Music", type="generated audio filepath"), | |
| ], | |
| title = "<a href='https://github.com/FunAudioLLM/InspireMusic' target='_blank'>InspireMusic<a/>: A Unified Framework for Music, Song, Audio Generation.", | |
| description = ("InspireMusic ([Github Repo](https://github.com/FunAudioLLM/InspireMusic)) is a fundamental AIGC toolkit and models designed for music, song, and audio generation using PyTorch." | |
| "To try it, simply type text to generation music, or click one of the examples. "), | |
| article = ("<p style='text-align: center'><a href='' target='_blank'>InspireMusic</a> </p>" | |
| "<p style='text-align: center'><a href='https://openreview.net/forum?id=yBlVlS2Fd9' target='_blank'>WavTokenizer: an Efficient Acoustic Discrete Codec Tokenizer for Audio Language Modeling</a> </p>"), | |
| examples = [ | |
| ["example/inspiremusic/inspiremusic_01.wav", "24000 Hz"], | |
| ["example/ras/chorus/chorus_01.wav", "48000 Hz"], | |
| ], | |
| cache_examples = True, | |
| ) | |
| with demo: | |
| gr.TabbedInterface([t2m_demo, con_demo,], | |
| ["Task 1: Text-to-Music", | |
| "Task 2: Music Continuation"]) | |
| # gr.TabbedInterface([t2m_demo, con_demo, fast_demo], ["Task 1: Text-to-Music", "Task 2: Music Continuation", "Task 3: Without Flow Matching"]) | |
| demo.launch() | |