Hugging Face's logo

Spaces:
shethjenil
/
Audio2Midi
Running

App Files Community
shethjenil commited on
Commit
2395189
·
verified ·
1 Parent(s): 8b2762e

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. app.py +53 -0
  2. midi_viz.html +92 -0
  3. packages.txt +2 -0
  4. requirements.txt +9 -0
  5. soundfonts.zip +3 -0
app.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from numpy import max as np_max,abs as np_abs,int16 as np_int16
3
+ from librosa import load as librosa_load
4
+ from pydub import AudioSegment
5
+ from transformers import Pop2PianoForConditionalGeneration, Pop2PianoProcessor
6
+ from os import listdir as os_listdir
7
+ from base64 import b64encode
8
+ from shutil import unpack_archive
9
+ model = Pop2PianoForConditionalGeneration.from_pretrained("sweetcocoa/pop2piano").to("cpu")
10
+ processor = Pop2PianoProcessor.from_pretrained("sweetcocoa/pop2piano")
11
+ unpack_archive("soundfonts.zip","soundfonts")
12
+ soundfonts = [i.removesuffix(".sf2") for i in os_listdir("soundfonts")]
13
+ def librosa_to_audiosegment(y, sr):
14
+ epsilon = 1e-8
15
+ if np_max(np_abs(y)) > 0:
16
+ y = y / (np_max(np_abs(y)) + epsilon) * 32767
17
+ return AudioSegment(y.astype(np_int16).tobytes(), frame_rate=sr, sample_width=2, channels=1)
18
+ def inference(file_upload, composer, sf2_files,volume=-16):
19
+ sf2_files = ["soundfonts/" + i + ".sf2" for i in sf2_files]
20
+ audio_data, audio_sr = librosa_load(file_upload, sr=None)
21
+ inputs = processor(audio=audio_data, sampling_rate=audio_sr, return_tensors="pt").to("cpu")
22
+ midi = processor.batch_decode(
23
+ token_ids=model.generate(input_features=inputs["input_features"], composer="composer" + str(composer)),
24
+ feature_extractor_output=inputs
25
+ )["pretty_midi_objects"][0]
26
+ midi.write(open("output.mid", "wb"))
27
+ final_mix = librosa_to_audiosegment(audio_data, audio_sr).apply_gain(volume)
28
+ for sf2_file in sf2_files:
29
+ sf_audio_data = midi.fluidsynth(fs=44100, sf2_path=sf2_file)
30
+ epsilon = 1e-8
31
+ sf_audio_data = np_int16(sf_audio_data / (np_max(np_abs(sf_audio_data)) + epsilon) * 32767)
32
+ sf_audio_segment = librosa_to_audiosegment(sf_audio_data, 44100)
33
+ if len(sf_audio_segment) < len(final_mix):
34
+ sf_audio_segment = sf_audio_segment.append(AudioSegment.silent(duration=len(final_mix) - len(sf_audio_segment)))
35
+ elif len(sf_audio_segment) > len(final_mix):
36
+ sf_audio_segment = sf_audio_segment[:len(final_mix)]
37
+ final_mix = final_mix.overlay(sf_audio_segment)
38
+ final_mix.export("output.mp3", format="mp3")
39
+ return "output.mid", "output.mp3", f'<div style="display: flex; justify-content: center; align-items: center;"><iframe style="width: 100%; height: 500px; overflow:hidden" srcdoc=\'{open("midi_viz.html").read().replace("{midi_data}", b64encode(open("output.mid","rb").read()).decode("utf-8"))}\'></iframe></div>'
40
+
41
+ gr.Interface(
42
+ inference,
43
+ [
44
+ gr.Audio(sources="upload", type="filepath", label="Audio"),
45
+ gr.Number(1, minimum=1, maximum=21, label="Composer"),
46
+ gr.Dropdown(soundfonts, multiselect=True, label="Instrument")
47
+ ],
48
+ [
49
+ gr.File(label="MIDI"),
50
+ gr.Audio(label="Instrument Audio"),
51
+ gr.HTML()
52
+ ]
53
+ ).launch()
midi_viz.html ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+
4
+ <head>
5
+ <style>
6
+ #midi-section midi-player {
7
+ display: block;
8
+ width: inherit;
9
+ margin: 4px;
10
+ margin-bottom: 0;
11
+ }
12
+
13
+ #midi-section midi-player::part(control-panel) {
14
+ background: #ff5;
15
+ border: 2px solid #000;
16
+ border-radius: 10px 10px 0 0;
17
+ }
18
+
19
+ #midi-section midi-player::part(play-button) {
20
+ color: #353;
21
+ border: 2px solid currentColor;
22
+ background-color: #4d4;
23
+ border-radius: 20px;
24
+ transition: all 0.2s;
25
+ content: "hello";
26
+ }
27
+
28
+ #midi-section midi-player::part(play-button):hover {
29
+ color: #0a0;
30
+ background-color: #5f5;
31
+ border-radius: 10px;
32
+ }
33
+
34
+ #midi-section midi-player::part(time) {
35
+ font-family: monospace;
36
+ }
37
+
38
+ #midi-section midi-visualizer .piano-roll-visualizer {
39
+ background: #ffd;
40
+ border: 2px solid black;
41
+ border-top: none;
42
+ border-radius: 0 0 10px 10px;
43
+ margin: 4px;
44
+ margin-top: 0;
45
+ overflow: auto;
46
+ }
47
+
48
+ #midi-section midi-visualizer svg rect.note {
49
+ opacity: 0.6;
50
+ stroke-width: 2;
51
+ }
52
+
53
+ #midi-section midi-visualizer svg rect.note[data-instrument="0"] {
54
+ fill: #e22;
55
+ stroke: #500;
56
+ }
57
+
58
+ #midi-section midi-visualizer svg rect.note[data-instrument="2"] {
59
+ fill: #2ee;
60
+ stroke: #055;
61
+ }
62
+
63
+ #midi-section midi-visualizer svg rect.note[data-is-drum="true"] {
64
+ fill: #888;
65
+ stroke: #888;
66
+ }
67
+
68
+ #midi-section midi-visualizer svg rect.note.active {
69
+ opacity: 0.9;
70
+ stroke: #000;
71
+ }
72
+ </style>
73
+ </head>
74
+
75
+ <body>
76
+ <section id="midi-section"><midi-player sound-font="" visualizer="#midi-section midi-visualizer"></midi-player><midi-visualizer></midi-visualizer></section>
77
+ <script src="https://cdn.jsdelivr.net/combine/npm/tone@14.7.58,npm/@magenta/music@1.23.1/es6/core.js,npm/focus-visible@5,npm/html-midi-player@1.5.0"></script>
78
+ </body>
79
+ <script>
80
+ function base64ToBlobUrl(base64, mimeType = "audio/midi") {
81
+ const byteCharacters = atob(base64.split(",")[1] || base64);
82
+ const byteNumbers = new Array(byteCharacters.length);
83
+ for (let i = 0; i < byteCharacters.length; i++) {
84
+ byteNumbers[i] = byteCharacters.charCodeAt(i);
85
+ }
86
+ return URL.createObjectURL(new Blob([new Uint8Array(byteNumbers)], { type: mimeType }));
87
+ }
88
+ let midi_data = "{midi_data}";
89
+ document.querySelector("midi-player").src = base64ToBlobUrl(midi_data);
90
+ </script>
91
+
92
+ </html>
packages.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ fluidsynth
2
+ ffmpeg
requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ torch
2
+ librosa
3
+ transformers
4
+ pretty-midi
5
+ essentia
6
+ resampy
7
+ pyfluidsynth
8
+ pydub
9
+ numpy<1.24
soundfonts.zip ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6d75d166a3f47a65eb1d33a6a0edb75f295a839822adcc0cd9e6c8d0f9973ea
3
+ size 30200010