diff --git a/ALL_DATA/08 Hi-Hat 150BPM.mid b/ALL_DATA/08 Hi-Hat 150BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..84f78b9c01212fe7a5157345a3c5fa37dc5491cb Binary files /dev/null and b/ALL_DATA/08 Hi-Hat 150BPM.mid differ diff --git a/ALL_DATA/09 Hi-Hat 140BPM.mid b/ALL_DATA/09 Hi-Hat 140BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..abd471d662b0e55e07d665021c5969571160b51c Binary files /dev/null and b/ALL_DATA/09 Hi-Hat 140BPM.mid differ diff --git a/ALL_DATA/10 Hi-Hat 140BPM.mid b/ALL_DATA/10 Hi-Hat 140BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..b66769885324155cddf416989173412e967201e3 Binary files /dev/null and b/ALL_DATA/10 Hi-Hat 140BPM.mid differ diff --git a/ALL_DATA/140 - HiHat MIDI 16.mid b/ALL_DATA/140 - HiHat MIDI 16.mid new file mode 100644 index 0000000000000000000000000000000000000000..327e4756e10c3313fbf65f47a411279783be70d4 Binary files /dev/null and b/ALL_DATA/140 - HiHat MIDI 16.mid differ diff --git a/ALL_DATA/6 God Pt 1_ 130 BPM.mid b/ALL_DATA/6 God Pt 1_ 130 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..44a0c71f59cfb25c32930becaeb4af6e086f611e Binary files /dev/null and b/ALL_DATA/6 God Pt 1_ 130 BPM.mid differ diff --git a/ALL_DATA/Bandz.mid b/ALL_DATA/Bandz.mid new file mode 100644 index 0000000000000000000000000000000000000000..17eed74e6c76cc4f12942b258e3d2fe715de8ca2 Binary files /dev/null and b/ALL_DATA/Bandz.mid differ diff --git a/ALL_DATA/City Lights MIDI 130 BPM.mid b/ALL_DATA/City Lights MIDI 130 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..2b4d75f3507a475124a8a28be7837269d0827be5 Binary files /dev/null and b/ALL_DATA/City Lights MIDI 130 BPM.mid differ diff --git a/ALL_DATA/Class.mid b/ALL_DATA/Class.mid new file mode 100644 index 0000000000000000000000000000000000000000..246271edc324bbbf10493fd9434aaa12fc3cd135 Binary files /dev/null and b/ALL_DATA/Class.mid differ diff --git a/ALL_DATA/Cymatics - Bankhead Trap Hihat MIDI - 138 BPM.mid b/ALL_DATA/Cymatics - Bankhead Trap Hihat MIDI - 138 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..fff6a1bbd684f89bd4ce5f743fbab8c353256bec Binary files /dev/null and b/ALL_DATA/Cymatics - Bankhead Trap Hihat MIDI - 138 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Deja Vu Trap Hihat MIDI - 150 BPM.mid b/ALL_DATA/Cymatics - Deja Vu Trap Hihat MIDI - 150 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..8281f6821da6401be7a7b8ea88425dd68451bc3f Binary files /dev/null and b/ALL_DATA/Cymatics - Deja Vu Trap Hihat MIDI - 150 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Fire Away Dancehall Hihat MIDI - 96 BPM.mid b/ALL_DATA/Cymatics - Fire Away Dancehall Hihat MIDI - 96 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..5326107cdb8ccc377e4b1bca729ec33851dc2408 Binary files /dev/null and b/ALL_DATA/Cymatics - Fire Away Dancehall Hihat MIDI - 96 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Gucci Trap Hihat MIDI - 129 BPM.mid b/ALL_DATA/Cymatics - Gucci Trap Hihat MIDI - 129 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..1fbdc3c7275ff67bd973c993310df686a58e6d7a Binary files /dev/null and b/ALL_DATA/Cymatics - Gucci Trap Hihat MIDI - 129 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Hot Trap Hihat MIDI - 124 BPM.mid b/ALL_DATA/Cymatics - Hot Trap Hihat MIDI - 124 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..f9ea8de0cdef5a55206d544525a9510bdb19a840 Binary files /dev/null and b/ALL_DATA/Cymatics - Hot Trap Hihat MIDI - 124 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Tesla Trap Hihat MIDI - 145 BPM.mid b/ALL_DATA/Cymatics - Tesla Trap Hihat MIDI - 145 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..d74ee2d5739b114ca6bdcf9c85a2d82b6b891b5b Binary files /dev/null and b/ALL_DATA/Cymatics - Tesla Trap Hihat MIDI - 145 BPM.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 18.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 18.mid new file mode 100644 index 0000000000000000000000000000000000000000..d2518ce062d1bbbe6014cc4351c375cf5d65a403 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 18.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 24.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 24.mid new file mode 100644 index 0000000000000000000000000000000000000000..a97cc9a35b08064807c43a63358ba9dbcda4ad71 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 24.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 26.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 26.mid new file mode 100644 index 0000000000000000000000000000000000000000..80c51fb131c17305f4e5c5f322ba4b9ce40a1825 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 26.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 27.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 27.mid new file mode 100644 index 0000000000000000000000000000000000000000..6db51a878374c16aad05cf803145da6f1fd4d8f9 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 27.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 30.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 30.mid new file mode 100644 index 0000000000000000000000000000000000000000..ccf70906325958c9efc41adbd03513c32ecfbf77 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 30.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 32.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 32.mid new file mode 100644 index 0000000000000000000000000000000000000000..b27f8249be74b28e09a02772f2b9512e1928070a Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 32.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 4.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 4.mid new file mode 100644 index 0000000000000000000000000000000000000000..b3fc60b48981c63f9a9d5ff49c7b8e7fd896004a Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 4.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 5.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 5.mid new file mode 100644 index 0000000000000000000000000000000000000000..3614b02fa30a047731f795da499457ab3ba0561f Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 5.mid differ diff --git a/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 6.mid b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 6.mid new file mode 100644 index 0000000000000000000000000000000000000000..79028250706add21dc74ff3a14c23cc66f475407 Binary files /dev/null and b/ALL_DATA/Cymatics - VYPR Modern Hihat MIDI 6.mid differ diff --git a/ALL_DATA/Cymatics - Whip Trap Hihat MIDI - 160 BPM.mid b/ALL_DATA/Cymatics - Whip Trap Hihat MIDI - 160 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..ac31784912e2214cc05cc48e0bc2fddc36d237a8 Binary files /dev/null and b/ALL_DATA/Cymatics - Whip Trap Hihat MIDI - 160 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Wraith Trap Hihat MIDI - 157 BPM.mid b/ALL_DATA/Cymatics - Wraith Trap Hihat MIDI - 157 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..7c5c0ada74c1f35eeef2c411dbae8f2ef04dc98f Binary files /dev/null and b/ALL_DATA/Cymatics - Wraith Trap Hihat MIDI - 157 BPM.mid differ diff --git a/ALL_DATA/Cymatics - Zone Drill Hihat MIDI - 136 BPM.mid b/ALL_DATA/Cymatics - Zone Drill Hihat MIDI - 136 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..54f2ab0520192c635341e09425da32001f82d787 Binary files /dev/null and b/ALL_DATA/Cymatics - Zone Drill Hihat MIDI - 136 BPM.mid differ diff --git a/ALL_DATA/Fly Shit 71 BPM.mid b/ALL_DATA/Fly Shit 71 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..da3a21a00c088291aa5efdfa7ee66198f115cc65 Binary files /dev/null and b/ALL_DATA/Fly Shit 71 BPM.mid differ diff --git a/ALL_DATA/Gucci.mid b/ALL_DATA/Gucci.mid new file mode 100644 index 0000000000000000000000000000000000000000..e89b8fa275e565d8cf328e7b65e37eb4b4ce9838 Binary files /dev/null and b/ALL_DATA/Gucci.mid differ diff --git a/ALL_DATA/HH - 61.mid b/ALL_DATA/HH - 61.mid new file mode 100644 index 0000000000000000000000000000000000000000..aabac991665d96f2b926299c4d5860aadbe84a87 Binary files /dev/null and b/ALL_DATA/HH - 61.mid differ diff --git a/ALL_DATA/HH - 62.mid b/ALL_DATA/HH - 62.mid new file mode 100644 index 0000000000000000000000000000000000000000..57406886a837fe02940a4a908671aa5b589a254f Binary files /dev/null and b/ALL_DATA/HH - 62.mid differ diff --git a/ALL_DATA/HH - 63.mid b/ALL_DATA/HH - 63.mid new file mode 100644 index 0000000000000000000000000000000000000000..ff61aee14b96cc9094063ea1198f276af64bfd79 Binary files /dev/null and b/ALL_DATA/HH - 63.mid differ diff --git a/ALL_DATA/HH - CarbonFibes.mid b/ALL_DATA/HH - CarbonFibes.mid new file mode 100644 index 0000000000000000000000000000000000000000..4dbd0fc93fb1018781dc1be144e3a3b8424c7a6e Binary files /dev/null and b/ALL_DATA/HH - CarbonFibes.mid differ diff --git a/ALL_DATA/HH - Jinx.mid b/ALL_DATA/HH - Jinx.mid new file mode 100644 index 0000000000000000000000000000000000000000..fba59072ec64410589cb3e5ab4bb039d9728018e Binary files /dev/null and b/ALL_DATA/HH - Jinx.mid differ diff --git a/ALL_DATA/HH - Manual.mid b/ALL_DATA/HH - Manual.mid new file mode 100644 index 0000000000000000000000000000000000000000..3317f3023117c963560ed387168be1f9001396af Binary files /dev/null and b/ALL_DATA/HH - Manual.mid differ diff --git a/ALL_DATA/HH - SLO OnMyOwn.mid b/ALL_DATA/HH - SLO OnMyOwn.mid new file mode 100644 index 0000000000000000000000000000000000000000..f391ff401e60451cfe1657b6659e718a38170618 Binary files /dev/null and b/ALL_DATA/HH - SLO OnMyOwn.mid differ diff --git a/ALL_DATA/HH - SLO Screwed.mid b/ALL_DATA/HH - SLO Screwed.mid new file mode 100644 index 0000000000000000000000000000000000000000..c8a434bf9df41cec79a7dc16abf0353b8994f649 Binary files /dev/null and b/ALL_DATA/HH - SLO Screwed.mid differ diff --git a/ALL_DATA/HH - SLO Tillertime.mid b/ALL_DATA/HH - SLO Tillertime.mid new file mode 100644 index 0000000000000000000000000000000000000000..f141e93433ed3d8eb0d7773fd33b33291a9d17ce Binary files /dev/null and b/ALL_DATA/HH - SLO Tillertime.mid differ diff --git a/ALL_DATA/HH Alpengeist.mid b/ALL_DATA/HH Alpengeist.mid new file mode 100644 index 0000000000000000000000000000000000000000..2df1338597d3dd1afc8f175f56126cf0aac25b0b Binary files /dev/null and b/ALL_DATA/HH Alpengeist.mid differ diff --git a/ALL_DATA/HH Boyarmira.mid b/ALL_DATA/HH Boyarmira.mid new file mode 100644 index 0000000000000000000000000000000000000000..58306353c2f6f4bcc61ccd7c392f3382fd434563 Binary files /dev/null and b/ALL_DATA/HH Boyarmira.mid differ diff --git a/ALL_DATA/HH GetEmCarti.mid b/ALL_DATA/HH GetEmCarti.mid new file mode 100644 index 0000000000000000000000000000000000000000..dc0e3d93dcb33fdbcaf9efe6c0db91ed0a9f86df Binary files /dev/null and b/ALL_DATA/HH GetEmCarti.mid differ diff --git a/ALL_DATA/HH Juke.mid b/ALL_DATA/HH Juke.mid new file mode 100644 index 0000000000000000000000000000000000000000..6f75793ac5cb0158c72e570e3b4b9016b3fc5e94 Binary files /dev/null and b/ALL_DATA/HH Juke.mid differ diff --git a/ALL_DATA/HH Maria.mid b/ALL_DATA/HH Maria.mid new file mode 100644 index 0000000000000000000000000000000000000000..0a88c5b9b1421bbf4e95ecb6e12856b074d91edc Binary files /dev/null and b/ALL_DATA/HH Maria.mid differ diff --git a/ALL_DATA/HH Missed Call.mid b/ALL_DATA/HH Missed Call.mid new file mode 100644 index 0000000000000000000000000000000000000000..23e9c7a4a57fc6ecdd030bb828d3f7ec7ba88f0a Binary files /dev/null and b/ALL_DATA/HH Missed Call.mid differ diff --git a/ALL_DATA/HH Walka.mid b/ALL_DATA/HH Walka.mid new file mode 100644 index 0000000000000000000000000000000000000000..8a01e59eecb49ac1bf002a051f366fe77c9d87ab Binary files /dev/null and b/ALL_DATA/HH Walka.mid differ diff --git a/ALL_DATA/HH_21_114.mid b/ALL_DATA/HH_21_114.mid new file mode 100644 index 0000000000000000000000000000000000000000..2e702bb96091459c9459ec4cccd0bdec557a4540 Binary files /dev/null and b/ALL_DATA/HH_21_114.mid differ diff --git a/ALL_DATA/HH_29_138.mid b/ALL_DATA/HH_29_138.mid new file mode 100644 index 0000000000000000000000000000000000000000..f03cc7182caad11cb6c446e55a0df5e3269de654 Binary files /dev/null and b/ALL_DATA/HH_29_138.mid differ diff --git a/ALL_DATA/HH_35_134.mid b/ALL_DATA/HH_35_134.mid new file mode 100644 index 0000000000000000000000000000000000000000..65e88e5da30ea7ec2488e0593d23da7e244e964c Binary files /dev/null and b/ALL_DATA/HH_35_134.mid differ diff --git a/ALL_DATA/HH_47_122.mid b/ALL_DATA/HH_47_122.mid new file mode 100644 index 0000000000000000000000000000000000000000..bd53a019324d21502dbc8e14679993f09c69d39c Binary files /dev/null and b/ALL_DATA/HH_47_122.mid differ diff --git a/ALL_DATA/HH_5_120BPM.mid b/ALL_DATA/HH_5_120BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..c6b57eb130af673f4df20b6c3b91bd6233ccf391 Binary files /dev/null and b/ALL_DATA/HH_5_120BPM.mid differ diff --git a/ALL_DATA/Hats - 117bpm - Gmin.mid b/ALL_DATA/Hats - 117bpm - Gmin.mid new file mode 100644 index 0000000000000000000000000000000000000000..042a6ff9508bb18a941a2225394e479bf8b36d37 Binary files /dev/null and b/ALL_DATA/Hats - 117bpm - Gmin.mid differ diff --git a/ALL_DATA/Hats - 122bpm - Dmin.mid b/ALL_DATA/Hats - 122bpm - Dmin.mid new file mode 100644 index 0000000000000000000000000000000000000000..9da8aeab14d5cae1e3008000caf372f46a9f8df8 Binary files /dev/null and b/ALL_DATA/Hats - 122bpm - Dmin.mid differ diff --git a/ALL_DATA/Hats - 130bpm - Amin.mid b/ALL_DATA/Hats - 130bpm - Amin.mid new file mode 100644 index 0000000000000000000000000000000000000000..68dc6bb2c4cd3a1dbabd7f9feac60b9e8996f4b0 Binary files /dev/null and b/ALL_DATA/Hats - 130bpm - Amin.mid differ diff --git a/ALL_DATA/Hats 2 - 130bpm - C#min.mid b/ALL_DATA/Hats 2 - 130bpm - C#min.mid new file mode 100644 index 0000000000000000000000000000000000000000..18f402ffb1bb2e630f797afe0ebf7af7b3b09091 Binary files /dev/null and b/ALL_DATA/Hats 2 - 130bpm - C#min.mid differ diff --git a/ALL_DATA/Hats 4 - 128bpm - D#min.mid b/ALL_DATA/Hats 4 - 128bpm - D#min.mid new file mode 100644 index 0000000000000000000000000000000000000000..98dc826e6db4e5b7674b92dc86ba1bfc25a924b2 Binary files /dev/null and b/ALL_DATA/Hats 4 - 128bpm - D#min.mid differ diff --git a/ALL_DATA/Hats 4 - 130bpm - Amin 2.mid b/ALL_DATA/Hats 4 - 130bpm - Amin 2.mid new file mode 100644 index 0000000000000000000000000000000000000000..80d22ea638bab206dc455228862605f2e7518fc9 Binary files /dev/null and b/ALL_DATA/Hats 4 - 130bpm - Amin 2.mid differ diff --git a/ALL_DATA/Hats 5 - 100bpm - C#min.mid b/ALL_DATA/Hats 5 - 100bpm - C#min.mid new file mode 100644 index 0000000000000000000000000000000000000000..4a263fbd755991d29f88b17d114ffe7f0b92754d Binary files /dev/null and b/ALL_DATA/Hats 5 - 100bpm - C#min.mid differ diff --git a/ALL_DATA/Hats Full - 155bpm - C#min.mid b/ALL_DATA/Hats Full - 155bpm - C#min.mid new file mode 100644 index 0000000000000000000000000000000000000000..f5db8768943bd2ad77f1c24d9ec9317cac6a6278 Binary files /dev/null and b/ALL_DATA/Hats Full - 155bpm - C#min.mid differ diff --git a/ALL_DATA/HiHats - Closed HiHat Pattern - 148Bpm.mid b/ALL_DATA/HiHats - Closed HiHat Pattern - 148Bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..b41b24716b427c4578f2190f623e349161ef2ddf Binary files /dev/null and b/ALL_DATA/HiHats - Closed HiHat Pattern - 148Bpm.mid differ diff --git a/ALL_DATA/HiHats - Closed HiHat Pattern - 150Bpm (2).mid b/ALL_DATA/HiHats - Closed HiHat Pattern - 150Bpm (2).mid new file mode 100644 index 0000000000000000000000000000000000000000..9df3bff06c68079f3c6b0446f5fce573729d8479 Binary files /dev/null and b/ALL_DATA/HiHats - Closed HiHat Pattern - 150Bpm (2).mid differ diff --git a/ALL_DATA/HiHats - Closed HiHat Pattern - 156Bpm.mid b/ALL_DATA/HiHats - Closed HiHat Pattern - 156Bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..7aaa21c136561f37c44e8cef6cc9c1c20f56be74 Binary files /dev/null and b/ALL_DATA/HiHats - Closed HiHat Pattern - 156Bpm.mid differ diff --git a/ALL_DATA/HiHats - Closed HiHat Pattern - 160Bpm.mid b/ALL_DATA/HiHats - Closed HiHat Pattern - 160Bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..6d4bc7ffd3537caa192868f2d2a4840418fe27a8 Binary files /dev/null and b/ALL_DATA/HiHats - Closed HiHat Pattern - 160Bpm.mid differ diff --git a/ALL_DATA/Jump Out The House (Hi-Hat) @rteekbeats.mid b/ALL_DATA/Jump Out The House (Hi-Hat) @rteekbeats.mid new file mode 100644 index 0000000000000000000000000000000000000000..06f282e09ae56a4fdb7042a9471a078d69feef60 Binary files /dev/null and b/ALL_DATA/Jump Out The House (Hi-Hat) @rteekbeats.mid differ diff --git a/ALL_DATA/NM - DownTheHatch 150 BPM_A#4.mid b/ALL_DATA/NM - DownTheHatch 150 BPM_A#4.mid new file mode 100644 index 0000000000000000000000000000000000000000..24e28be6ba2b7e61e4ff460e96555145494c14b3 Binary files /dev/null and b/ALL_DATA/NM - DownTheHatch 150 BPM_A#4.mid differ diff --git a/ALL_DATA/NM - HH 1648.mid b/ALL_DATA/NM - HH 1648.mid new file mode 100644 index 0000000000000000000000000000000000000000..05b8d0d7712c1b77a9728882aa23518339ea5564 Binary files /dev/null and b/ALL_DATA/NM - HH 1648.mid differ diff --git a/ALL_DATA/NM - HH Dro.mid b/ALL_DATA/NM - HH Dro.mid new file mode 100644 index 0000000000000000000000000000000000000000..c6cf5991153a8ba69257bfd928f2ff00cd6330d9 Binary files /dev/null and b/ALL_DATA/NM - HH Dro.mid differ diff --git a/ALL_DATA/NM - HH EverydayWeLit.mid b/ALL_DATA/NM - HH EverydayWeLit.mid new file mode 100644 index 0000000000000000000000000000000000000000..a70923fed9f252bbf3e54f3f30fe27babd7892ae Binary files /dev/null and b/ALL_DATA/NM - HH EverydayWeLit.mid differ diff --git a/ALL_DATA/NM - HH Want.mid b/ALL_DATA/NM - HH Want.mid new file mode 100644 index 0000000000000000000000000000000000000000..57a41136d4b26c420060ca16ffefed09203b8487 Binary files /dev/null and b/ALL_DATA/NM - HH Want.mid differ diff --git a/ALL_DATA/NM - HH Wavs.mid b/ALL_DATA/NM - HH Wavs.mid new file mode 100644 index 0000000000000000000000000000000000000000..8513efbcccb00fa014117b6c00088d46f46bbb23 Binary files /dev/null and b/ALL_DATA/NM - HH Wavs.mid differ diff --git a/ALL_DATA/NM - Hats 8 2.mid b/ALL_DATA/NM - Hats 8 2.mid new file mode 100644 index 0000000000000000000000000000000000000000..9be0c308da5c92961d1940ebc342ed9dc0ce019d Binary files /dev/null and b/ALL_DATA/NM - Hats 8 2.mid differ diff --git a/ALL_DATA/NM - Hats 8 3.mid b/ALL_DATA/NM - Hats 8 3.mid new file mode 100644 index 0000000000000000000000000000000000000000..9be0c308da5c92961d1940ebc342ed9dc0ce019d Binary files /dev/null and b/ALL_DATA/NM - Hats 8 3.mid differ diff --git a/ALL_DATA/NM - Hats 8.mid b/ALL_DATA/NM - Hats 8.mid new file mode 100644 index 0000000000000000000000000000000000000000..9be0c308da5c92961d1940ebc342ed9dc0ce019d Binary files /dev/null and b/ALL_DATA/NM - Hats 8.mid differ diff --git a/ALL_DATA/NM - Hats 9.mid b/ALL_DATA/NM - Hats 9.mid new file mode 100644 index 0000000000000000000000000000000000000000..bae200e6f5d2de3631d9bfcda579a1cb66ee8a4b Binary files /dev/null and b/ALL_DATA/NM - Hats 9.mid differ diff --git a/ALL_DATA/New Horizons - 15. Hihat MIDI (fast 2 step) (prod. Avid Beats).mid b/ALL_DATA/New Horizons - 15. Hihat MIDI (fast 2 step) (prod. Avid Beats).mid new file mode 100644 index 0000000000000000000000000000000000000000..046d4e20880f77d716759ddbd6c0865f29a605ec Binary files /dev/null and b/ALL_DATA/New Horizons - 15. Hihat MIDI (fast 2 step) (prod. Avid Beats).mid differ diff --git a/ALL_DATA/New Horizons - 7. Hihat MIDI 120 bpm (prod. Avid Beats).mid b/ALL_DATA/New Horizons - 7. Hihat MIDI 120 bpm (prod. Avid Beats).mid new file mode 100644 index 0000000000000000000000000000000000000000..e078cb0a7cd9093e6391fece39ff931508976fff Binary files /dev/null and b/ALL_DATA/New Horizons - 7. Hihat MIDI 120 bpm (prod. Avid Beats).mid differ diff --git a/ALL_DATA/Plug.mid b/ALL_DATA/Plug.mid new file mode 100644 index 0000000000000000000000000000000000000000..c4cde22c07c022d80e7ff5038c5b10b38cd2475e Binary files /dev/null and b/ALL_DATA/Plug.mid differ diff --git a/ALL_DATA/Rockstar Made (Hi-Hat) @rteekbeats.mid b/ALL_DATA/Rockstar Made (Hi-Hat) @rteekbeats.mid new file mode 100644 index 0000000000000000000000000000000000000000..5389712ac1fc7523a3a4bb3ebc199bcd286d222d Binary files /dev/null and b/ALL_DATA/Rockstar Made (Hi-Hat) @rteekbeats.mid differ diff --git a/ALL_DATA/Sicko Main - Open Hi-Hat.mid b/ALL_DATA/Sicko Main - Open Hi-Hat.mid new file mode 100644 index 0000000000000000000000000000000000000000..764c7b7bfc071ca3d34d199644831c87436a2b9a Binary files /dev/null and b/ALL_DATA/Sicko Main - Open Hi-Hat.mid differ diff --git a/ALL_DATA/Speakers .mid b/ALL_DATA/Speakers .mid new file mode 100644 index 0000000000000000000000000000000000000000..27640ab7f9dc2613c67ce1876d2360d25a8ef382 Binary files /dev/null and b/ALL_DATA/Speakers .mid differ diff --git a/ALL_DATA/Trippy 73 BPM.mid b/ALL_DATA/Trippy 73 BPM.mid new file mode 100644 index 0000000000000000000000000000000000000000..53582df1bd4e7c29c0ff103b0e8c67b7b4a8ea08 Binary files /dev/null and b/ALL_DATA/Trippy 73 BPM.mid differ diff --git a/ALL_DATA/Vamp Anthem (Hi-Hat) @rteekbeats.mid b/ALL_DATA/Vamp Anthem (Hi-Hat) @rteekbeats.mid new file mode 100644 index 0000000000000000000000000000000000000000..bfdd783a291f9c3cc4ab340f40f707d1e81e5e57 Binary files /dev/null and b/ALL_DATA/Vamp Anthem (Hi-Hat) @rteekbeats.mid differ diff --git a/ALL_DATA/World.mid b/ALL_DATA/World.mid new file mode 100644 index 0000000000000000000000000000000000000000..6c720d93235e36bea9d2dbffef8d74637dbcf59d Binary files /dev/null and b/ALL_DATA/World.mid differ diff --git a/ALL_DATA/[@prodcrooked] HiTek 120 bpm.mid b/ALL_DATA/[@prodcrooked] HiTek 120 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..58eb359d15d118fcf536e0cf0e72074177e23dba Binary files /dev/null and b/ALL_DATA/[@prodcrooked] HiTek 120 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] cobalt 146b bpm.mid b/ALL_DATA/[@prodcrooked] cobalt 146b bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..e5d5481240f2a7eea750209fb10b7df70dc7928f Binary files /dev/null and b/ALL_DATA/[@prodcrooked] cobalt 146b bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] droptop 154 bpm.mid b/ALL_DATA/[@prodcrooked] droptop 154 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..f164a5f6363faa03b3f79d5889e9d4a145b1c03b Binary files /dev/null and b/ALL_DATA/[@prodcrooked] droptop 154 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] gotit 114 bpm.mid b/ALL_DATA/[@prodcrooked] gotit 114 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..c681569db9b6838b964dbe523d2d59310e79bdf9 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] gotit 114 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] lost 121 bpm.mid b/ALL_DATA/[@prodcrooked] lost 121 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..3a902247f8af0495f1d28af5bcc7222814c42f47 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] lost 121 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] lowkey 120 bpm.mid b/ALL_DATA/[@prodcrooked] lowkey 120 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..160feddb15f68d690780390afbfcc917c093fcbc Binary files /dev/null and b/ALL_DATA/[@prodcrooked] lowkey 120 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] reasonable 152 bpm.mid b/ALL_DATA/[@prodcrooked] reasonable 152 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..4714861214fbc8ec5c9cae95603ef2b6e48b6537 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] reasonable 152 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] rolls 165 bpm.mid b/ALL_DATA/[@prodcrooked] rolls 165 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..725cec8e5425bf8b8644a899695232bab8eefe7e Binary files /dev/null and b/ALL_DATA/[@prodcrooked] rolls 165 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] tyson 127 bpm.mid b/ALL_DATA/[@prodcrooked] tyson 127 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..32a1fc4557ae475f9a78fd0a6ea313e75d2461e6 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] tyson 127 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] uncharted 130 bpm.mid b/ALL_DATA/[@prodcrooked] uncharted 130 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..85f65a6ff9ece3314b7691de0fa5d5a63d899cd7 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] uncharted 130 bpm.mid differ diff --git a/ALL_DATA/[@prodcrooked] wire 160 bpm.mid b/ALL_DATA/[@prodcrooked] wire 160 bpm.mid new file mode 100644 index 0000000000000000000000000000000000000000..bb1e7057893c3327bb2bdf30b1274bb2af174636 Binary files /dev/null and b/ALL_DATA/[@prodcrooked] wire 160 bpm.mid differ diff --git a/ALL_DATA/kookup - backpedal [HH MIDI].mid b/ALL_DATA/kookup - backpedal [HH MIDI].mid new file mode 100644 index 0000000000000000000000000000000000000000..5218eb1f83761305f41b13cac765ad7b4d7a70ad Binary files /dev/null and b/ALL_DATA/kookup - backpedal [HH MIDI].mid differ diff --git a/ALL_DATA/kookup - inhaler [HH MIDI].mid b/ALL_DATA/kookup - inhaler [HH MIDI].mid new file mode 100644 index 0000000000000000000000000000000000000000..696f6480d968c1a470dbd1637781db0e8e816cc7 Binary files /dev/null and b/ALL_DATA/kookup - inhaler [HH MIDI].mid differ diff --git a/ALL_DATA/kookup - pacekeeper [HH MIDI].mid b/ALL_DATA/kookup - pacekeeper [HH MIDI].mid new file mode 100644 index 0000000000000000000000000000000000000000..83be70d1a9631d4cbe757c3061d5624fcff6e9ad Binary files /dev/null and b/ALL_DATA/kookup - pacekeeper [HH MIDI].mid differ diff --git a/ALL_DATA/kookup - pegasus [HH MIDI].mid b/ALL_DATA/kookup - pegasus [HH MIDI].mid new file mode 100644 index 0000000000000000000000000000000000000000..599b6d009b1382f968bbeef0f747396c7bd405b4 Binary files /dev/null and b/ALL_DATA/kookup - pegasus [HH MIDI].mid differ diff --git a/ALL_DATA/kookup - weeknd [HH MIDI].mid b/ALL_DATA/kookup - weeknd [HH MIDI].mid new file mode 100644 index 0000000000000000000000000000000000000000..e20f0e756f6ffe916e440d1ee3093632c5d9cf2f Binary files /dev/null and b/ALL_DATA/kookup - weeknd [HH MIDI].mid differ diff --git a/ALL_DATA/slim Hi Hat Midi (12).mid b/ALL_DATA/slim Hi Hat Midi (12).mid new file mode 100644 index 0000000000000000000000000000000000000000..e30a7407b26a0e2396e9e35a0e4e002c2c8773f0 Binary files /dev/null and b/ALL_DATA/slim Hi Hat Midi (12).mid differ diff --git a/ALL_DATA/slim Hi Hat Midi (2).mid b/ALL_DATA/slim Hi Hat Midi (2).mid new file mode 100644 index 0000000000000000000000000000000000000000..3d4a614730ad850ac8ffced8571a908023e740ac Binary files /dev/null and b/ALL_DATA/slim Hi Hat Midi (2).mid differ diff --git a/ALL_DATA/slim Hi Hat Midi (28).mid b/ALL_DATA/slim Hi Hat Midi (28).mid new file mode 100644 index 0000000000000000000000000000000000000000..1ef7699fc2883f96bcdc6c0b2da4daed4f385bae Binary files /dev/null and b/ALL_DATA/slim Hi Hat Midi (28).mid differ diff --git a/train.py b/train.py new file mode 100644 index 0000000000000000000000000000000000000000..042d06a0e00afd97614700ce73c189d8b030cde1 --- /dev/null +++ b/train.py @@ -0,0 +1,471 @@ +import argparse +import os +import random +from pathlib import Path +from typing import Union +#CHANGED VERSION +import lightning as pl +import numpy as np +import torch +import torch.nn.functional as F +from lightning import Trainer +from lightning.fabric.utilities import rank_zero_only +from lightning.pytorch.callbacks import ModelCheckpoint +from peft import LoraConfig, TaskType +from safetensors.torch import save_file as safe_save_file +from torch import optim +from torch.optim.lr_scheduler import LambdaLR +from torch.utils.data import Dataset, DataLoader + +import MIDI +from midi_model import MIDIModel, MIDIModelConfig, config_name_list +from midi_tokenizer import MIDITokenizerV1, MIDITokenizerV2 + +EXTENSION = [".mid", ".midi"] + + +def file_ext(fname): + return os.path.splitext(fname)[1].lower() + + +class MidiDataset(Dataset): + def __init__(self, midi_list, tokenizer: Union[MIDITokenizerV1, MIDITokenizerV2], max_len=2048, min_file_size=1, + max_file_size=384000, + aug=True, check_quality=False, rand_start=True): + + self.tokenizer = tokenizer + self.midi_list = midi_list + self.max_len = max_len + self.min_file_size = min_file_size + self.max_file_size = max_file_size + self.aug = aug + self.check_quality = check_quality + self.rand_start = rand_start + + def __len__(self): + return len(self.midi_list) + + def load_midi(self, index): + path = self.midi_list[index] + try: + with open(path, 'rb') as f: + datas = f.read() + if len(datas) > self.max_file_size: # large midi file will spend too much time to load + raise ValueError("file too large") + elif len(datas) < self.min_file_size: + raise ValueError("file too small") + mid = MIDI.midi2score(datas) + if max([0] + [len(track) for track in mid[1:]]) == 0: + raise ValueError("empty track") + mid = self.tokenizer.tokenize(mid) + if self.check_quality and not self.tokenizer.check_quality(mid)[0]: + raise ValueError("bad quality") + if self.aug: + mid = self.tokenizer.augment(mid) + except Exception: + mid = self.load_midi(random.randint(0, self.__len__() - 1)) + return mid + + def __getitem__(self, index): + mid = self.load_midi(index) + mid = np.asarray(mid, dtype=np.int16) + # if mid.shape[0] < self.max_len: + # mid = np.pad(mid, ((0, self.max_len - mid.shape[0]), (0, 0)), + # mode="constant", constant_values=self.tokenizer.pad_id) + if self.rand_start: + start_idx = random.randrange(0, max(1, mid.shape[0] - self.max_len)) + start_idx = random.choice([0, start_idx]) + else: + max_start = max(1, mid.shape[0] - self.max_len) + start_idx = (index * (max_start // 8)) % max_start + mid = mid[start_idx: start_idx + self.max_len] + mid = mid.astype(np.int64) + mid = torch.from_numpy(mid) + return mid + + def collate_fn(self, batch): + max_len = max([len(mid) for mid in batch]) + batch = [F.pad(mid, (0, 0, 0, max_len - mid.shape[0]), mode="constant", value=self.tokenizer.pad_id) for mid in batch] + batch = torch.stack(batch) + return batch + + +def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): + """ Create a schedule with a learning rate that decreases linearly after + linearly increasing during a warmup period. + """ + + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +class TrainMIDIModel(MIDIModel, pl.LightningModule): + def __init__(self, config: MIDIModelConfig, + lr=2e-4, weight_decay=0.01, warmup=1e3, max_step=1e6, sample_seq=False, + gen_example_interval=1, example_batch=8): + super(TrainMIDIModel, self).__init__(config) + self.lr = lr + self.weight_decay = weight_decay + self.warmup = warmup + self.max_step = max_step + self.sample_seq = sample_seq + self.gen_example_interval = gen_example_interval + self.example_batch = example_batch + self.last_save_step = 0 + self.gen_example_count = 0 + + def configure_optimizers(self): + param_optimizer = list(self.named_parameters()) + no_decay = ['bias', 'norm'] # no decay for bias and Norm + optimizer_grouped_parameters = [ + { + 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], + 'weight_decay': self.weight_decay}, + { + 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], + 'weight_decay': 0.0 + } + ] + optimizer = optim.AdamW( + optimizer_grouped_parameters, + lr=self.lr, + betas=(0.9, 0.99), + eps=1e-08, + ) + lr_scheduler = get_linear_schedule_with_warmup( + optimizer=optimizer, + num_warmup_steps=self.warmup, + num_training_steps=self.max_step, + ) + return { + "optimizer": optimizer, + "lr_scheduler": { + "scheduler": lr_scheduler, + "interval": "step", + "frequency": 1 + } + } + + def compute_accuracy(self, logits, labels): + out = torch.argmax(logits, dim=-1) + out = out.flatten() + labels = labels.flatten() + + mask = (labels != self.tokenizer.pad_id) + out = out[mask] + labels = labels[mask] + + num_right = (out == labels) + num_right = torch.sum(num_right).type(torch.float32) + acc = num_right / len(labels) + + return acc + + def training_step(self, batch, batch_idx): + x = batch[:, :-1].contiguous() # (batch_size, midi_sequence_length, token_sequence_length) + y = batch[:, 1:].contiguous() + hidden = self.forward(x) + if self.sample_seq: # to reduce vram + rand_idx = [-1] + random.sample(list(range(y.shape[1] - 2)), min(127, (y.shape[1] - 2) // 2)) + hidden = hidden[:, rand_idx] + y = y[:, rand_idx] + hidden = hidden.reshape(-1, hidden.shape[-1]) + y = y.reshape(-1, y.shape[-1]) # (batch_size*midi_sequence_length, token_sequence_length) + x = y[:, :-1] + logits = self.forward_token(hidden, x) + loss = F.cross_entropy( + logits.view(-1, self.tokenizer.vocab_size), + y.view(-1), + reduction="mean", + ignore_index=self.tokenizer.pad_id + ) + self.log("train/loss", loss) + self.log("train/lr", self.lr_schedulers().get_last_lr()[0]) + return loss + + def validation_step(self, batch, batch_idx): + x = batch[:, :-1].contiguous() # (batch_size, midi_sequence_length, token_sequence_length) + y = batch[:, 1:].contiguous() + hidden = self.forward(x) + hidden = hidden.reshape(-1, hidden.shape[-1]) + y = y.reshape(-1, y.shape[-1]) # (batch_size*midi_sequence_length, token_sequence_length) + x = y[:, :-1] + logits = self.forward_token(hidden, x) + loss = F.cross_entropy( + logits.view(-1, self.tokenizer.vocab_size), + y.view(-1), + reduction="mean", + ignore_index=self.tokenizer.pad_id + ) + acc = self.compute_accuracy(logits, y) + self.log_dict({"val/loss": loss, "val/acc": acc}, sync_dist=True) + return loss + + @rank_zero_only + def gen_example(self, save_dir): + base_dir = f"{save_dir}/sample/{self.global_step}" + if not os.path.exists(base_dir): + Path(base_dir).mkdir(parents=True) + midis = self.generate(batch_size=self.example_batch) + midis = [self.tokenizer.detokenize(midi) for midi in midis] + imgs = [self.tokenizer.midi2img(midi) for midi in midis] + for i, (img, midi) in enumerate(zip(imgs, midis)): + img.save(f"{base_dir}/0_{i}.png") + with open(f"{base_dir}/0_{i}.mid", 'wb') as f: + f.write(MIDI.score2midi(midi)) + prompt = val_dataset.load_midi(random.randint(0, len(val_dataset) - 1)) + prompt = np.asarray(prompt, dtype=np.int16) + ori = prompt[:512] + img = self.tokenizer.midi2img(self.tokenizer.detokenize(ori)) + img.save(f"{base_dir}/1_ori.png") + prompt = prompt[:256].astype(np.int64) + midis = self.generate(prompt, batch_size=self.example_batch) + midis = [self.tokenizer.detokenize(midi) for midi in midis] + imgs = [self.tokenizer.midi2img(midi) for midi in midis] + for i, (img, midi) in enumerate(zip(imgs, midis)): + img.save(f"{base_dir}/1_{i}.png") + with open(f"{base_dir}/1_{i}.mid", 'wb') as f: + f.write(MIDI.score2midi(midi)) + + @rank_zero_only + def save_peft(self, save_dir): + adapter_name = self.active_adapters()[0] + adapter_config = self.peft_config[adapter_name] + if not os.path.exists(save_dir): + os.makedirs(save_dir, exist_ok=True) + adapter_config.save_pretrained(save_dir) + adapter_state_dict = self.get_adapter_state_dict(adapter_name) + safe_save_file(adapter_state_dict, + os.path.join(save_dir, "adapter_model.safetensors"), + metadata={"format": "pt"}) + + def on_save_checkpoint(self, checkpoint): + if self.global_step == self.last_save_step: + return + self.last_save_step = self.global_step + trainer = self.trainer + if len(trainer.loggers) > 0: + if trainer.loggers[0].save_dir is not None: + save_dir = trainer.loggers[0].save_dir + else: + save_dir = trainer.default_root_dir + name = trainer.loggers[0].name + version = trainer.loggers[0].version + version = version if isinstance(version, str) else f"version_{version}" + save_dir = os.path.join(save_dir, str(name), version) + else: + save_dir = trainer.default_root_dir + self.config.save_pretrained(os.path.join(save_dir, "checkpoints")) + if self._hf_peft_config_loaded: + self.save_peft(os.path.join(save_dir, "lora")) + self.gen_example_count += 1 + if self.gen_example_interval>0 and self.gen_example_count % self.gen_example_interval == 0: + try: + self.gen_example(save_dir) + except Exception as e: + print(e) + + +def get_midi_list(path): + all_files = { + os.path.join(root, fname) + for root, _dirs, files in os.walk(path) + for fname in files + } + print(f"All files found: {all_files}") # Debug: Print all files found + all_midis = sorted( + fname for fname in all_files if file_ext(fname) in EXTENSION + ) + print(f"MIDI files after filtering: {all_midis}") # Debug: Print MIDI files + return all_midis + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + # model args + parser.add_argument( + "--resume", type=str, default="", help="resume training from ckpt" + ) + parser.add_argument( + "--ckpt", type=str, default="", help="load ckpt" + ) + parser.add_argument( + "--config", type=str, default="tv2o-medium", help="model config name or file" + ) + parser.add_argument( + "--task", type=str, default="train", choices=["train", "lora"], help="Full train or lora" + ) + + # dataset args + parser.add_argument( + "--data", type=str, default="data", help="dataset path" + ) + parser.add_argument( + "--data-val-split", + type=int, + default=128, + help="the number of midi files divided into the validation set", + ) + parser.add_argument("--max-len", type=int, default=512, help="max seq length for training") + + parser.add_argument( + "--quality", action="store_true", default=False, help="check dataset quality" + ) + + # training args + parser.add_argument("--seed", type=int, default=0, help="seed") + parser.add_argument("--lr", type=float, default=1e-4, help="learning rate") + parser.add_argument("--weight-decay", type=float, default=0.01, help="weight decay") + parser.add_argument("--warmup-step", type=int, default=1e2, help="warmup step") + parser.add_argument("--max-step", type=int, default=1e4, help="max training step") + + parser.add_argument("--grad-clip", type=float, default=1.0, help="gradient clip val") + parser.add_argument( + "--sample-seq", action="store_true", default=False, help="sample midi seq to reduce vram" + ) + parser.add_argument( + "--gen-example-interval", type=int, default=1, help="generate example interval. set 0 to disable" + ) + parser.add_argument("--batch-size-train", type=int, default=1, help="batch size for training") + + parser.add_argument("--batch-size-val", type=int, default=1, help="batch size for validation") + + parser.add_argument( + "--batch-size-gen-example", type=int, default=8, help="batch size for generate example" + ) + parser.add_argument("--workers-train", type=int, default=1, help="workers num for training dataloader") + + parser.add_argument("--workers-val", type=int, default=1, help="workers num for validation dataloader") + + parser.add_argument("--acc-grad", type=int, default=4, help="gradient accumulation") + + parser.add_argument( + "--accelerator", + type=str, + default="mps", + choices=["cpu", "gpu", "tpu", "ipu", "hpu", "auto", "mps"], + help="accelerator", + ) + parser.add_argument("--precision", type=str, default="16-mixed", help="precision") + + parser.add_argument("--devices", type=int, default=1, help="devices num") + parser.add_argument("--nodes", type=int, default=1, help="nodes num") + parser.add_argument( + "--disable-benchmark", action="store_true", default=False, help="disable cudnn benchmark" + ) + parser.add_argument( + "--log-step", type=int, default=1, help="log training loss every n steps" + ) + parser.add_argument("--val-step", type=int, default=10000, help="validate and save every n steps") + + + opt = parser.parse_args() + print(opt) + opt.data = "/Users/ethanlum/Desktop/midi-composer/data" + print(f"Dataset directory: {opt.data}") + + if not os.path.exists("lightning_logs"): + os.mkdir("lightning_logs") + if not os.path.exists("sample"): + os.mkdir("sample") + pl.seed_everything(opt.seed) + print("---load dataset---") + if opt.config in config_name_list: + config = MIDIModelConfig.from_name(opt.config) + else: + config = MIDIModelConfig.from_name("tv2o-small") + tokenizer = config.tokenizer + midi_list = get_midi_list(opt.data) + print(f"Number of MIDI files found: {len(midi_list)}") + import os + print(f"Files in dataset directory: {os.listdir(opt.data)}") + random.shuffle(midi_list) + full_dataset_len = len(midi_list) + train_dataset_len = full_dataset_len - opt.data_val_split + train_midi_list = midi_list[:train_dataset_len] + val_midi_list = midi_list[train_dataset_len:] + train_dataset = MidiDataset(train_midi_list, tokenizer, max_len=opt.max_len, aug=False, check_quality=opt.quality, + rand_start=True) + val_dataset = MidiDataset(val_midi_list, tokenizer, max_len=opt.max_len, aug=False, check_quality=opt.quality, + rand_start=False) + train_dataloader = DataLoader( + train_dataset, + batch_size=opt.batch_size_train, + #batch_size = 8, + shuffle=True, + persistent_workers=True, + num_workers=opt.workers_train, + pin_memory=True, + collate_fn=train_dataset.collate_fn + ) + val_dataloader = DataLoader( + val_dataset, + batch_size=opt.batch_size_val, + shuffle=False, + persistent_workers=True, + num_workers=opt.workers_val, + pin_memory=True, + collate_fn=val_dataset.collate_fn + ) + print(f"train: {len(train_dataset)} val: {len(val_dataset)}") + torch.backends.cuda.enable_mem_efficient_sdp(True) + torch.backends.cuda.enable_flash_sdp(True) + model = TrainMIDIModel(config, lr=opt.lr, weight_decay=opt.weight_decay, + warmup=opt.warmup_step, max_step=opt.max_step, + sample_seq=opt.sample_seq, gen_example_interval=opt.gen_example_interval, + example_batch=opt.batch_size_gen_example) + if opt.ckpt: + ckpt = torch.load(opt.ckpt, map_location="cpu") + state_dict = ckpt.get("state_dict", ckpt) + model.load_state_dict(state_dict, strict=False) + elif opt.task == "lora": + raise ValueError("--ckpt must be set to train lora") + if opt.task == "lora": + model.requires_grad_(False) + lora_config = LoraConfig( + r=64, + target_modules=["q_proj", "o_proj", "k_proj", "v_proj", "gate_proj", "up_proj", "down_proj"], + task_type=TaskType.CAUSAL_LM, + bias="none", + lora_alpha=128, + lora_dropout=0 + ) + model.add_adapter(lora_config) + print("---start train---") + checkpoint_callback = ModelCheckpoint( + monitor="val/loss", + mode="min", + save_top_k=1, + save_last=True, + auto_insert_metric_name=False, + filename="epoch={epoch},loss={val/loss:.4f}", + ) + callbacks = [checkpoint_callback] + + trainer = Trainer( + val_check_interval=300, # Validate less frequently + check_val_every_n_epoch=2, # Validate every 2 epochs + max_epochs=10, + precision=16, # Use 16-bit precision to reduce memory + accumulate_grad_batches=1, # Minimal gradient accumulation + gradient_clip_val=opt.grad_clip, # Retain gradient clipping + accelerator="mps", # Ensure MPS accelerator is used + devices=1, # Use only one device to avoid memory overrun + enable_checkpointing=True, # Keep checkpoints enabled + num_sanity_val_steps=0, # Skip sanity validation for speed + num_nodes=opt.nodes, + max_steps=opt.max_step // 2, # Halve total steps for faster training + benchmark=not opt.disable_benchmark, + log_every_n_steps=10, + strategy="auto", + callbacks=callbacks, +) + ckpt_path = opt.resume + if ckpt_path == "": + ckpt_path = None + print("---start train---") + trainer.fit(model, train_dataloader, val_dataloader, ckpt_path=ckpt_path) diff --git a/trained_model.pth b/trained_model.pth new file mode 100644 index 0000000000000000000000000000000000000000..2ec362758719960a9d6a733168c2707d98002ec9 --- /dev/null +++ b/trained_model.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8765b2470b0146696f18b637428ddf2b7e9a18247a31d874111f0359bb23c91 +size 467741754