Upload 2 files

app.py

@@ -1,42 +1,57 @@
 import gradio as gr
-from musc.model import PretrainedModel
-from json import load as json_load
-from mido import MidiFile,MidiTrack
-from os import remove as os_remove
-Model = PretrainedModel(json_load(open("violin.json")),"violin_model.pt").to("cpu")
-def merge_violin_tracks(input_midi, output_midi):
-    mid = MidiFile(input_midi)
-    new_mid = MidiFile(ticks_per_beat=mid.ticks_per_beat)
-    new_track = MidiTrack()
-    new_mid.tracks.append(new_track)
-    events = []
-    for track in mid.tracks:
-        current_time = 0
-        for msg in track:
-            current_time += msg.time  
-            events.append((current_time, msg))
-    events.sort(key=lambda x: x[0])
-    last_time = 0
-    for event_time, msg in events:
-        delta_time = event_time - last_time
-        new_track.append(msg.copy(time=delta_time))
-        last_time = event_time  
-    for track in mid.tracks:
-        for msg in track:
-            if msg.type == 'set_tempo':
-                new_track.insert(0, msg)  
-    new_mid.save(output_midi)
+import numpy as np
+from librosa import load as librosa_load
+from pydub import AudioSegment
+from pretty_midi import PrettyMIDI
+from transformers import Pop2PianoForConditionalGeneration, Pop2PianoProcessor
+from os import listdir as os_listdir, remove as os_remove
+from io import BytesIO
+from base64 import b64encode
+model = Pop2PianoForConditionalGeneration.from_pretrained("sweetcocoa/pop2piano").to("cpu")
+processor = Pop2PianoProcessor.from_pretrained("sweetcocoa/pop2piano")
+soundfonts = [i.removesuffix(".sf2") for i in os_listdir("soundfonts")]
 
-def transcribe_and_generate_midi(music_file_path, model=Model, batch_size=32):
-    model.transcribe(music_file_path, batch_size=batch_size).write("output.mid")
-    merge_violin_tracks("output.mid","output.mid")
-    os_remove(music_file_path)
-    return "output.mid"
+def librosa_to_audiosegment(y, sr):
+    epsilon = 1e-8
+    if np.max(np.abs(y)) > 0:
+        y = y / (np.max(np.abs(y)) + epsilon) * 32767
+    return AudioSegment(y.astype(np.int16).tobytes(), frame_rate=sr, sample_width=2, channels=1)
 
+def inference(file_upload, composer, sf2_files):
+    sf2_files = ["soundfonts/" + i + ".sf2" for i in sf2_files]
+    audio_data, audio_sr = librosa_load(file_upload, sr=None)
+    inputs = processor(audio=audio_data, sampling_rate=audio_sr, return_tensors="pt").to("cpu")
+    midi = processor.batch_decode(
+        token_ids=model.generate(input_features=inputs["input_features"], composer="composer" + str(composer)),
+        feature_extractor_output=inputs
+    )["pretty_midi_objects"][0]
+    with open("output.mid", "wb") as midi_file:
+        midi.write(midi_file)
+    midi = PrettyMIDI("output.mid")
+    final_mix = librosa_to_audiosegment(audio_data, audio_sr).apply_gain(-16)
+    for sf2_file in sf2_files:
+        sf_audio_data = midi.fluidsynth(fs=44100, sf2_path=sf2_file)
+        epsilon = 1e-8
+        sf_audio_data = np.int16(sf_audio_data / (np.max(np.abs(sf_audio_data)) + epsilon) * 32767)
+        sf_audio_segment = librosa_to_audiosegment(sf_audio_data, 44100)
+        if len(sf_audio_segment) < len(final_mix):
+            sf_audio_segment = sf_audio_segment.append(AudioSegment.silent(duration=len(final_mix) - len(sf_audio_segment)))
+        elif len(sf_audio_segment) > len(final_mix):
+            sf_audio_segment = sf_audio_segment[:len(final_mix)]
+        final_mix = final_mix.overlay(sf_audio_segment)
+    final_mix.export("output.mp3", format="mp3")
+    os_remove(file_upload)
+    return "output.mid", "output.mp3", f'<div style="display: flex; justify-content: center; align-items: center;"><iframe style="width: 100%; height: 500px; overflow:hidden" srcdoc=\'{open("midi_viz.html").read().replace("{midi_data}", b64encode(open("output.mid","rb").read()).decode("utf-8"))}\'></iframe></div>'
 gr.Interface(
-    fn=transcribe_and_generate_midi, 
-    inputs=gr.Audio(label="Upload your Audio file",type="filepath"),
-    outputs=gr.File(label="Download MIDI file"),
-    title="Audio2Violin",
-    description="Upload a Audio file, and it will be transcribed into Violin MIDI format."
-).launch()
+    inference,
+    [
+        gr.Audio(sources="upload", type="filepath", label="Audio"),
+        gr.Number(1, minimum=1, maximum=21, label="Composer"),
+        gr.Dropdown(soundfonts, multiselect=True, label="Instrument")
+    ],
+    [
+        gr.File(label="MIDI"),
+        gr.Audio(label="Instrument Audio"),
+        gr.HTML()
+    ]
+).launch()

midi_viz.html

@@ -0,0 +1,92 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <style>
+        #midi-section midi-player {
+            display: block;
+            width: inherit;
+            margin: 4px;
+            margin-bottom: 0;
+        }
+
+        #midi-section midi-player::part(control-panel) {
+            background: #ff5;
+            border: 2px solid #000;
+            border-radius: 10px 10px 0 0;
+        }
+
+        #midi-section midi-player::part(play-button) {
+            color: #353;
+            border: 2px solid currentColor;
+            background-color: #4d4;
+            border-radius: 20px;
+            transition: all 0.2s;
+            content: "hello";
+        }
+
+        #midi-section midi-player::part(play-button):hover {
+            color: #0a0;
+            background-color: #5f5;
+            border-radius: 10px;
+        }
+
+        #midi-section midi-player::part(time) {
+            font-family: monospace;
+        }
+
+        #midi-section midi-visualizer .piano-roll-visualizer {
+            background: #ffd;
+            border: 2px solid black;
+            border-top: none;
+            border-radius: 0 0 10px 10px;
+            margin: 4px;
+            margin-top: 0;
+            overflow: auto;
+        }
+
+        #midi-section midi-visualizer svg rect.note {
+            opacity: 0.6;
+            stroke-width: 2;
+        }
+
+        #midi-section midi-visualizer svg rect.note[data-instrument="0"] {
+            fill: #e22;
+            stroke: #500;
+        }
+
+        #midi-section midi-visualizer svg rect.note[data-instrument="2"] {
+            fill: #2ee;
+            stroke: #055;
+        }
+
+        #midi-section midi-visualizer svg rect.note[data-is-drum="true"] {
+            fill: #888;
+            stroke: #888;
+        }
+
+        #midi-section midi-visualizer svg rect.note.active {
+            opacity: 0.9;
+            stroke: #000;
+        }
+    </style>
+</head>
+
+<body>
+    <section id="midi-section"><midi-player sound-font="" visualizer="#midi-section midi-visualizer"></midi-player><midi-visualizer></midi-visualizer></section>
+    <script src="https://cdn.jsdelivr.net/combine/npm/[email protected],npm/@magenta/[email protected]/es6/core.js,npm/focus-visible@5,npm/[email protected]"></script>
+</body>
+<script>
+    function base64ToBlobUrl(base64, mimeType = "audio/midi") {
+        const byteCharacters = atob(base64.split(",")[1] || base64);
+        const byteNumbers = new Array(byteCharacters.length);
+        for (let i = 0; i < byteCharacters.length; i++) {
+            byteNumbers[i] = byteCharacters.charCodeAt(i);
+        }
+        return URL.createObjectURL(new Blob([new Uint8Array(byteNumbers)], { type: mimeType }));
+    }
+    let midi_data = "{midi_data}";
+    document.querySelector("midi-player").src = base64ToBlobUrl(midi_data);
+</script>
+
+</html>