Create app.py

app.py

@@ -0,0 +1,137 @@
+import os
+import torch
+import shutil
+import librosa
+import binascii
+import warnings
+import midi2audio
+import pytube as pt    # to download the youtube videos as audios
+import gradio as gr
+from transformers import Pop2PianoForConditionalGeneration, Pop2PianoProcessor
+
+
+yt_video_dir = "./yt_dir"
+outputs_dir = "./midi_wav_outputs"
+os.makedirs(outputs_dir, exist_ok=True)
+os.makedirs(yt_video_dir, exist_ok=True)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = Pop2PianoForConditionalGeneration.from_pretrained("sweetcocoa/pop2piano").to(device)
+processor = Pop2PianoProcessor.from_pretrained("sweetcocoa/pop2piano")
+composers = model.generation_config.composer_to_feature_token.keys()
+
+def get_audio_from_yt_video(yt_link):
+    try:
+        yt = pt.YouTube(yt_link)
+        t = yt.streams.filter(only_audio=True)
+        filename = os.path.join(yt_video_dir, binascii.hexlify(os.urandom(8)).decode() + ".mp4")
+        t[0].download(filename=filename)
+    except:
+        warnings.warn(f"Video Not Found at {yt_link}")
+        filename = None
+    
+    return filename, filename
+
+def prepare_output_file(tokenizer_output):
+    # Add some random values so that no two file names are same
+    output_file_name = "output_" + binascii.hexlify(os.urandom(8)).decode()
+    midi_output = os.path.join(outputs_dir, output_file_name + ".mid")
+        
+    # write the .mid file
+    tokenizer_output[0].write(midi_output)
+    
+    # convert .mid file to .wav using `midi2audio`
+    wav_output = midi_output.replace(".mid", ".wav")
+    midi2audio.FluidSynth().midi_to_audio(midi_output, wav_output)
+
+    from IPython.display import Audio
+    return wav_output, wav_output, midi_output
+    
+def inference(file_uploaded, composer):
+    # to save the native sampling rate of the file, sr=None is used, but this can cause some silent errors where the
+    # generated output will not be upto the desired quality. If that happens please consider switching sr to 44100 Hz.
+    waveform, sr = librosa.load(file_uploaded, sr=None) 
+    
+    inputs = processor(audio=waveform, sampling_rate=sr, return_tensors="pt").to(device)
+    model_output = model.generate(input_features=inputs["input_features"], composer=composer)
+    tokenizer_output = processor.batch_decode(token_ids=model_output.to("cpu"), feature_extractor_output=inputs.to("cpu"))["pretty_midi_objects"]
+
+    return prepare_output_file(tokenizer_output)    
+
+
+block = gr.Blocks()
+
+
+with block:
+    gr.HTML(
+        """
+            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+              <div
+                style="
+                  display: inline-flex;
+                  align-items: center;
+                  gap: 0.8rem;
+                  font-size: 1.75rem;
+                "
+              >
+                <h1 style="font-weight: 900; margin-bottom: 7px;">
+                  Pop2piano
+                </h1>
+              </div>
+              <p style="margin-bottom: 10px; font-size: 94%">
+                A demo for Pop2Piano:Pop Audio-based Piano Cover Generation.<br>
+                Please select the composer(Arranger) and upload the pop audio or enter the YouTube link and then click Generate.
+              </p>
+            </div>
+        """
+    )
+    with gr.Group():
+        with gr.Box():
+            with gr.Row().style(mobile_collapse=False, equal_height=True):
+                file_uploaded = gr.Audio(label="Upload an audio", type="filepath")
+
+                with gr.Column():
+                    with gr.Row():
+                        yt_link = gr.Textbox(label="Enter YouTube link of the Video")
+                        yt_btn = gr.Button("Get Audio from the YT link(Press this before pressing Generate)")
+                    
+                    
+                    yt_audio_path = gr.Audio(label="Audio Extracted from the YouTube Video", interactive=False)
+                    
+                    yt_btn.click(get_audio_from_yt_video, inputs=[yt_link], outputs=[yt_audio_path, file_uploaded])
+
+        with gr.Box():
+            with gr.Row().style(mobile_collapse=False, equal_height=True):
+                composer = gr.Dropdown(label="Arranger", choices=composers, value="composer1")
+                
+            with gr.Row().style(mobile_collapse=False, equal_height=True):
+                btn = gr.Button("Generate")
+        
+        with gr.Box():
+            with gr.Row().style(mobile_collapse=False, equal_height=True):
+                wav_output2 = gr.File(label="Download the Generated MIDI (.wav)")
+                wav_output1 = gr.Audio(label="Listen to the Generated MIDI")
+                midi_output = gr.File(label="Download the Generated MIDI (.mid)")
+                btn.click(inference, inputs=[file_uploaded, composer], outputs=[wav_output1, wav_output2, midi_output])
+                
+        gr.Examples([
+            ["./examples/custom_song.mp3", "composer1"],
+            ["./examples/BornThisWay.mp3", "composer1"],
+            ["./examples/Sk8erBoi.mp3", "composer2"],
+        ],
+            fn=inference,
+            inputs=[file_uploaded, composer],
+            outputs=[wav_output1, wav_output2, midi_output],
+            cache_examples=True
+        )
+        gr.HTML(
+            """
+        <div class="footer">
+                    <p><a href="http://sweetcocoa.github.io/pop2piano_samples" style="text-decoration: underline;" target="_blank">Project Page</a>
+                    </p>
+        </div>
+        """
+        )
+
+block.launch(debug=False)
+shutil.rmtree("./midi_wav_outputs")