signature options and MidiSynthesizer optimise

app.py

@@ -11,7 +11,7 @@ import tqdm
 from huggingface_hub import hf_hub_download
 
 import MIDI
-from midi_synthesizer import synthesis
+from midi_synthesizer import MidiSynthesizer
 from midi_tokenizer import MIDITokenizer
 
 MAX_SEED = np.iinfo(np.int32).max
@@ -121,12 +121,28 @@ def send_msgs(msgs):
     return json.dumps(msgs)
 
 
-def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, mid, midi_events,
+def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, time_sig, key_sig, mid, midi_events,
         reduce_cc_st, remap_track_channel, add_default_instr, remove_empty_channels, seed, seed_rand,
         gen_events, temp, top_p, top_k, allow_cc):
     model = models[model_name]
     tokenizer = model[2]
     bpm = int(bpm)
+    if time_sig == "auto":
+        time_sig = None
+        time_sig_nn = 4
+        time_sig_dd = 2
+    else:
+        time_sig_nn, time_sig_dd = time_sig.split('/')
+        time_sig_nn = int(time_sig_nn)
+        time_sig_dd = {2: 1, 4: 2, 8: 3}[int(time_sig_dd)]
+    if key_sig == 0:
+        key_sig = None
+        key_sig_sf = 0
+        key_sig_mi = 0
+    else:
+        key_sig = (key_sig - 1)
+        key_sig_sf = key_sig // 2 - 7
+        key_sig_mi = key_sig % 2
     gen_events = int(gen_events)
     max_len = gen_events
     if seed_rand:
@@ -137,6 +153,11 @@ def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, mid, midi_events,
     if tab == 0:
         i = 0
         mid = [[tokenizer.bos_id] + [tokenizer.pad_id] * (tokenizer.max_token_seq - 1)]
+        if tokenizer.version == "v2":
+            if time_sig is not None:
+                mid.append(tokenizer.event2tokens(["time_signature", 0, 0, 0, time_sig_nn - 1, time_sig_dd - 1]))
+            if key_sig is not None:
+                mid.append(tokenizer.event2tokens(["key_signature", 0, 0, 0, key_sig_sf + 7, key_sig_mi]))
         if bpm != 0:
             mid.append(tokenizer.event2tokens(["set_tempo", 0, 0, 0, bpm]))
         patches = {}
@@ -148,7 +169,7 @@ def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, mid, midi_events,
         if drum_kit != "None":
             patches[9] = drum_kits2number[drum_kit]
         for i, (c, p) in enumerate(patches.items()):
-            mid.append(tokenizer.event2tokens(["patch_change", 0, 0, i, c, p]))
+            mid.append(tokenizer.event2tokens(["patch_change", 0, 0, i + 1, c, p]))
         mid_seq = mid
         mid = np.asarray(mid, dtype=np.int64)
         if len(instruments) > 0:
@@ -181,11 +202,11 @@ def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, mid, midi_events,
         init_msgs = [create_msg("visualizer_clear", tokenizer.version),
                      create_msg("visualizer_append", events)]
     yield mid_seq, None, None, seed, send_msgs(init_msgs)
-    t = time.time() + 1
     midi_generator = generate(model, mid, max_len=max_len, temp=temp, top_p=top_p, top_k=top_k,
                               disable_patch_change=disable_patch_change, disable_control_change=not allow_cc,
                               disable_channels=disable_channels, generator=generator)
     events = []
+    t = time.time() + 1
     for i, token_seq in enumerate(midi_generator):
         token_seq = token_seq.tolist()
         mid_seq.append(token_seq)
@@ -200,7 +221,7 @@ def run(model_name, tab, mid_seq, instruments, drum_kit, bpm, mid, midi_events,
     mid = tokenizer.detokenize(mid_seq)
     with open(f"output.mid", 'wb') as f:
         f.write(MIDI.score2midi(mid))
-    audio = synthesis(MIDI.score2opus(mid), soundfont_path)
+    audio = synthesizer.synthesis(MIDI.score2opus(mid))
     events = [tokenizer.tokens2event(tokens) for tokens in mid_seq]
     yield mid_seq, "output.mid", (44100, audio), seed, send_msgs([create_msg("visualizer_end", events)])
 
@@ -212,7 +233,7 @@ def cancel_run(model_name, mid_seq):
     mid = tokenizer.detokenize(mid_seq)
     with open(f"output.mid", 'wb') as f:
         f.write(MIDI.score2midi(mid))
-    audio = synthesis(MIDI.score2opus(mid), soundfont_path)
+    audio = synthesizer.synthesis(MIDI.score2opus(mid))
     events = [tokenizer.tokens2event(tokens) for tokens in mid_seq]
     return "output.mid", (44100, audio), send_msgs([create_msg("visualizer_end", events)])
 
@@ -268,6 +289,8 @@ number2drum_kits = {-1: "None", 0: "Standard", 8: "Room", 16: "Power", 24: "Elec
                     40: "Blush", 48: "Orchestra"}
 patch2number = {v: k for k, v in MIDI.Number2patch.items()}
 drum_kits2number = {v: k for k, v in number2drum_kits.items()}
+key_signatures = ['C♭', 'A♭m', 'G♭', 'E♭m', 'D♭', 'B♭m', 'A♭', 'Fm', 'E♭', 'Cm', 'B♭', 'Gm', 'F', 'Dm',
+                  'C', 'Am', 'G', 'Em', 'D', 'Bm', 'A', 'F♯m', 'E', 'C♯m', 'B', 'G♯m', 'F♯', 'D♯m', 'C♯', 'A♯m']
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -276,6 +299,7 @@ if __name__ == "__main__":
     parser.add_argument("--max-gen", type=int, default=1024, help="max")
     opt = parser.parse_args()
     soundfont_path = hf_hub_download_retry(repo_id="skytnt/midi-model", filename="soundfont.sf2")
+    synthesizer = MidiSynthesizer(soundfont_path)
     models_info = {"generic pretrain model (tv2o-large) by asigalov61": ["asigalov61/Music-Llama", "", "tv2o-large"],
                    "generic pretrain model (tv2o-medium) by asigalov61": ["asigalov61/Music-Llama-Medium", "", "tv2o-medium"],
                    "generic pretrain model (tv1-medium) by skytnt": ["skytnt/midi-model", "", "tv1-medium"],
@@ -324,6 +348,16 @@ if __name__ == "__main__":
                 input_bpm = gr.Slider(label="BPM (beats per minute, auto if 0)", minimum=0, maximum=255,
                                       step=1,
                                       value=0)
+                input_time_sig = gr.Radio(label="time signature (only for tv2 models)",
+                                          value="auto",
+                                          choices=["auto", "4/4", "2/4", "3/4", "6/4", "7/4",
+                                                   "2/2", "3/2", "4/2", "3/8", "5/8", "6/8", "7/8", "9/8", "12/8"]
+                                          )
+                input_key_sig = gr.Radio(label="key signature (only for tv2 models)",
+                                         value="auto",
+                                         choices=["auto"] + key_signatures,
+                                         type="index"
+                                         )
                 example1 = gr.Examples([
                     [[], "None"],
                     [["Acoustic Grand"], "None"],
@@ -375,8 +409,9 @@ if __name__ == "__main__":
         output_audio = gr.Audio(label="output audio", format="mp3", elem_id="midi_audio")
         output_midi = gr.File(label="output midi", file_types=[".mid"])
         run_event = run_btn.click(run, [input_model, tab_select, output_midi_seq, input_instruments,
-                                        input_drum_kit, input_bpm, input_midi, input_midi_events, input_reduce_cc_st,
-                                        input_remap_track_channel, input_add_default_instr, input_remove_empty_channels,
+                                        input_drum_kit, input_bpm, input_time_sig, input_key_sig, input_midi,
+                                        input_midi_events, input_reduce_cc_st, input_remap_track_channel,
+                                        input_add_default_instr, input_remove_empty_channels,
                                         input_seed, input_seed_rand, input_gen_events, input_temp, input_top_p,
                                         input_top_k, input_allow_cc],
                                   [output_midi_seq, output_midi, output_audio, input_seed, js_msg],

midi_synthesizer.py

@@ -1,53 +1,76 @@
 import fluidsynth
 import numpy as np
 
+class MidiSynthesizer:
+    def __init__(self, soundfont_path, sample_rate=44100):
+        self.soundfont_path = soundfont_path
+        self.sample_rate = sample_rate
+        fl = fluidsynth.Synth(samplerate=float(sample_rate))
+        sfid = fl.sfload(soundfont_path)
+        self.devices = [[fl, sfid, False]]
 
-def synthesis(midi_opus, soundfont_path, sample_rate=44100):
-    ticks_per_beat = midi_opus[0]
-    event_list = []
-    for track_idx, track in enumerate(midi_opus[1:]):
-        abs_t = 0
-        for event in track:
-            abs_t += event[1]
-            event_new = [*event]
-            event_new[1] = abs_t
-            event_list.append(event_new)
-    event_list = sorted(event_list, key=lambda e: e[1])
+    def get_fluidsynth(self):
+        for device in self.devices:
+            if not device[2]:
+                device[2] = True
+                return device
+        fl = fluidsynth.Synth(samplerate=float(self.sample_rate))
+        sfid = fl.sfload(self.soundfont_path)
+        device = [fl, sfid, True]
+        self.devices.append(device)
+        return device
 
-    tempo = int((60 / 120) * 10 ** 6)  # default 120 bpm
-    ss = np.empty((0, 2), dtype=np.int16)
-    fl = fluidsynth.Synth(samplerate=float(sample_rate))
-    sfid = fl.sfload(soundfont_path)
-    last_t = 0
-    for c in range(16):
-        fl.program_select(c, sfid, 128 if c == 9 else 0, 0)
-    for event in event_list:
-        name = event[0]
-        sample_len = int(((event[1] / ticks_per_beat) * tempo / (10 ** 6)) * sample_rate)
-        sample_len -= int(((last_t / ticks_per_beat) * tempo / (10 ** 6)) * sample_rate)
-        last_t = event[1]
-        if sample_len > 0:
-            sample = fl.get_samples(sample_len).reshape(sample_len, 2)
-            ss = np.concatenate([ss, sample])
-        if name == "set_tempo":
-            tempo = event[2]
-        elif name == "patch_change":
-            c, p = event[2:4]
-            fl.program_select(c, sfid, 128 if c == 9 else 0, p)
-        elif name == "control_change":
-            c, cc, v = event[2:5]
-            fl.cc(c, cc, v)
-        elif name == "note_on" and event[3] > 0:
-            c, p, v = event[2:5]
-            fl.noteon(c, p, v)
-        elif name == "note_off" or (name == "note_on" and event[3] == 0):
-            c, p = event[2:4]
-            fl.noteoff(c, p)
+    def release_fluidsynth(self, device):
+        device[0].system_reset()
+        device[0].get_samples(self.sample_rate*5) # wait for silence
+        device[2] = False
 
-    fl.delete()
-    if ss.shape[0] > 0:
-        max_val = np.abs(ss).max()
-        if max_val != 0:
-            ss = (ss / max_val) * np.iinfo(np.int16).max
-    ss = ss.astype(np.int16)
-    return ss
+    def synthesis(self, midi_opus):
+        ticks_per_beat = midi_opus[0]
+        event_list = []
+        for track_idx, track in enumerate(midi_opus[1:]):
+            abs_t = 0
+            for event in track:
+                abs_t += event[1]
+                event_new = [*event]
+                event_new[1] = abs_t
+                event_list.append(event_new)
+        event_list = sorted(event_list, key=lambda e: e[1])
+
+        tempo = int((60 / 120) * 10 ** 6)  # default 120 bpm
+        ss = np.empty((0, 2), dtype=np.int16)
+        device = self.get_fluidsynth()
+        fl, sfid = device[:-1]
+        last_t = 0
+        for c in range(16):
+            fl.program_select(c, sfid, 128 if c == 9 else 0, 0)
+        for event in event_list:
+            name = event[0]
+            sample_len = int(((event[1] / ticks_per_beat) * tempo / (10 ** 6)) * self.sample_rate)
+            sample_len -= int(((last_t / ticks_per_beat) * tempo / (10 ** 6)) * self.sample_rate)
+            last_t = event[1]
+            if sample_len > 0:
+                sample = fl.get_samples(sample_len).reshape(sample_len, 2)
+                ss = np.concatenate([ss, sample])
+            if name == "set_tempo":
+                tempo = event[2]
+            elif name == "patch_change":
+                c, p = event[2:4]
+                fl.program_select(c, sfid, 128 if c == 9 else 0, p)
+            elif name == "control_change":
+                c, cc, v = event[2:5]
+                fl.cc(c, cc, v)
+            elif name == "note_on" and event[3] > 0:
+                c, p, v = event[2:5]
+                fl.noteon(c, p, v)
+            elif name == "note_off" or (name == "note_on" and event[3] == 0):
+                c, p = event[2:4]
+                fl.noteoff(c, p)
+
+        self.release_fluidsynth(device)
+        if ss.shape[0] > 0:
+            max_val = np.abs(ss).max()
+            if max_val != 0:
+                ss = (ss / max_val) * np.iinfo(np.int16).max
+        ss = ss.astype(np.int16)
+        return ss