diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,2712 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 96 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > Model has 532793243 parameters + + > EPOCH: 0/5 + --> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > TRAINING (2024-12-22 17:44:01)  + + --> TIME: 2024-12-22 17:44:03 -- STEP: 0/2902 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.10459499806165695 (0.10459499806165695) + | > loss_mel_ce: 3.311910390853882 (3.311910390853882) + | > loss: 0.4270631670951843 (0.4270631670951843) + | > current_lr: 5e-06 + | > step_time: 0.8415 (0.8415403366088867) + | > loader_time: 1.0344 (1.0344254970550537) + + + --> TIME: 2024-12-22 17:44:48 -- STEP: 50/2902 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.08981729298830032 (0.09741745188832283) + | > loss_mel_ce: 3.367861747741699 (3.394744510650635) + | > loss: 0.43220987915992737 (0.4365202432870865) + | > current_lr: 5e-06 + | > step_time: 0.5525 (0.591734619140625) + | > loader_time: 0.0148 (0.015042276382446288) + + + --> TIME: 2024-12-22 17:45:34 -- STEP: 100/2902 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.08311788737773895 (0.0911357863992453) + | > loss_mel_ce: 3.351332426071167 (3.394733407497406) + | > loss: 0.429306298494339 (0.4357336476445197) + | > current_lr: 5e-06 + | > step_time: 0.6436 (0.5889876151084902) + | > loader_time: 0.015 (0.014310407638549804) + + + --> TIME: 2024-12-22 17:46:21 -- STEP: 150/2902 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.07448040693998337 (0.08718039671579998) + | > loss_mel_ce: 3.401618719100952 (3.3927350441614785) + | > loss: 0.4345123767852783 (0.43498942931493106) + | > current_lr: 5e-06 + | > step_time: 0.6071 (0.5917791159947713) + | > loader_time: 0.0119 (0.013932906786600748) + + + --> TIME: 2024-12-22 17:47:08 -- STEP: 200/2902 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.0731874480843544 (0.08409653138369327) + | > loss_mel_ce: 3.291797637939453 (3.390307182073593) + | > loss: 0.42062312364578247 (0.43430046319961535) + | > current_lr: 5e-06 + | > step_time: 0.5773 (0.5956235480308533) + | > loader_time: 0.0131 (0.01358080267906189) + + + --> TIME: 2024-12-22 17:47:56 -- STEP: 250/2902 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.06732229888439178 (0.08167070341110234) + | > loss_mel_ce: 3.3784215450286865 (3.3851796493530273) + | > loss: 0.43071797490119934 (0.4333562933206557) + | > current_lr: 5e-06 + | > step_time: 0.6582 (0.5984164400100709) + | > loader_time: 0.012 (0.013197577476501465) + + + --> TIME: 2024-12-22 17:48:43 -- STEP: 300/2902 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.07035832107067108 (0.0798381366580725) + | > loss_mel_ce: 3.4118311405181885 (3.3834915582338967) + | > loss: 0.435273677110672 (0.43291621098915717) + | > current_lr: 5e-06 + | > step_time: 0.6434 (0.5990857418378196) + | > loader_time: 0.0125 (0.01297310829162598) + + + --> TIME: 2024-12-22 17:49:30 -- STEP: 350/2902 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.06829076260328293 (0.0782227069245918) + | > loss_mel_ce: 3.2637181282043457 (3.381631039210728) + | > loss: 0.41650110483169556 (0.4324817172970089) + | > current_lr: 5e-06 + | > step_time: 0.6559 (0.6008020734786992) + | > loader_time: 0.01 (0.01282343115125384) + + + --> TIME: 2024-12-22 17:50:17 -- STEP: 400/2902 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.07181409001350403 (0.07684497562237082) + | > loss_mel_ce: 3.338163137435913 (3.378450380563736) + | > loss: 0.42624714970588684 (0.4319119187444447) + | > current_lr: 5e-06 + | > step_time: 0.5614 (0.6016418385505681) + | > loader_time: 0.0103 (0.012634595632553106) + + + --> TIME: 2024-12-22 17:51:04 -- STEP: 450/2902 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.0650668740272522 (0.07570000054107771) + | > loss_mel_ce: 3.341148853302002 (3.374886935022142) + | > loss: 0.42577695846557617 (0.43132336596647886) + | > current_lr: 5e-06 + | > step_time: 0.6586 (0.6017436684502502) + | > loader_time: 0.0104 (0.012495169109768342) + + + --> TIME: 2024-12-22 17:51:51 -- STEP: 500/2902 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.06929623335599899 (0.07468750177323819) + | > loss_mel_ce: 3.3347978591918945 (3.3740318789482115) + | > loss: 0.42551174759864807 (0.43108992183208455) + | > current_lr: 5e-06 + | > step_time: 0.6249 (0.6018227620124823) + | > loader_time: 0.0123 (0.01235325384140015) + + + --> TIME: 2024-12-22 17:52:39 -- STEP: 550/2902 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.06491728872060776 (0.07386755391278053) + | > loss_mel_ce: 3.290853977203369 (3.372003945870833) + | > loss: 0.41947141289711 (0.430733936862512) + | > current_lr: 5e-06 + | > step_time: 0.5923 (0.6032628406177872) + | > loader_time: 0.01 (0.012248729792508213) + + + --> TIME: 2024-12-22 17:53:26 -- STEP: 600/2902 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.0667271837592125 (0.07305075627441214) + | > loss_mel_ce: 3.284752607345581 (3.3701720937093085) + | > loss: 0.41893497109413147 (0.4304028554260729) + | > current_lr: 5e-06 + | > step_time: 0.4854 (0.6030249472459164) + | > loader_time: 0.01 (0.012161032358805342) + + + --> TIME: 2024-12-22 17:54:14 -- STEP: 650/2902 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.06883151084184647 (0.07234381749652904) + | > loss_mel_ce: 3.3112661838531494 (3.367854071763844) + | > loss: 0.4225122034549713 (0.4300247354232346) + | > current_lr: 5e-06 + | > step_time: 0.5225 (0.6042447258875925) + | > loader_time: 0.0096 (0.012069188264700085) + + + --> TIME: 2024-12-22 17:55:01 -- STEP: 700/2902 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.06538775563240051 (0.07166379387357402) + | > loss_mel_ce: 3.388728380203247 (3.3648004106112874) + | > loss: 0.43176451325416565 (0.4295580248321804) + | > current_lr: 5e-06 + | > step_time: 0.656 (0.60397415297372) + | > loader_time: 0.0132 (0.012077730042593824) + + + --> TIME: 2024-12-22 17:55:47 -- STEP: 750/2902 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.058807797729969025 (0.0710751066207886) + | > loss_mel_ce: 3.3790392875671387 (3.364291217168171) + | > loss: 0.4297308921813965 (0.42942078975836423) + | > current_lr: 5e-06 + | > step_time: 0.6502 (0.6036211016972853) + | > loader_time: 0.0099 (0.011993129412333174) + + + --> TIME: 2024-12-22 17:56:34 -- STEP: 800/2902 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.062027063220739365 (0.07054349425248804) + | > loss_mel_ce: 3.3717057704925537 (3.363725082874297) + | > loss: 0.429216593503952 (0.4292835713922976) + | > current_lr: 5e-06 + | > step_time: 0.6413 (0.6034053999185554) + | > loader_time: 0.0159 (0.011959144175052647) + + + --> TIME: 2024-12-22 17:57:21 -- STEP: 850/2902 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.059973880648612976 (0.07002651334685442) + | > loss_mel_ce: 3.4005930423736572 (3.362588267606846) + | > loss: 0.4325708746910095 (0.42907684687305886) + | > current_lr: 5e-06 + | > step_time: 0.6503 (0.6033904190624451) + | > loader_time: 0.0131 (0.011911653350381295) + + + --> TIME: 2024-12-22 17:58:09 -- STEP: 900/2902 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.059261124581098557 (0.06958188860780673) + | > loss_mel_ce: 3.326876640319824 (3.3624700564808307) + | > loss: 0.4232672154903412 (0.42900649242930927) + | > current_lr: 5e-06 + | > step_time: 0.5363 (0.6036524536874549) + | > loader_time: 0.0099 (0.011872834629482697) + + + --> TIME: 2024-12-22 17:58:56 -- STEP: 950/2902 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.05842612311244011 (0.06918677367662138) + | > loss_mel_ce: 3.364814043045044 (3.361136853820398) + | > loss: 0.42790502309799194 (0.4287904527626538) + | > current_lr: 5e-06 + | > step_time: 0.5597 (0.603434893959446) + | > loader_time: 0.0105 (0.01180140896847374) + + + --> TIME: 2024-12-22 17:59:43 -- STEP: 1000/2902 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.061046820133924484 (0.06879791953414686) + | > loss_mel_ce: 3.3686323165893555 (3.360496532917022) + | > loss: 0.42870989441871643 (0.428661805897951) + | > current_lr: 5e-06 + | > step_time: 0.5852 (0.6037456159591665) + | > loader_time: 0.0103 (0.01176662635803223) + + + --> TIME: 2024-12-22 18:00:31 -- STEP: 1050/2902 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.06030765920877457 (0.06841222434881197) + | > loss_mel_ce: 3.262373685836792 (3.3596309841246823) + | > loss: 0.41533517837524414 (0.42850540047600144) + | > current_lr: 5e-06 + | > step_time: 0.6931 (0.6041640540531695) + | > loader_time: 0.0133 (0.011732036953880679) + + + --> TIME: 2024-12-22 18:01:17 -- STEP: 1100/2902 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.06249762699007988 (0.06807924053547065) + | > loss_mel_ce: 3.444939374923706 (3.3585651447556235) + | > loss: 0.4384296238422394 (0.42833054761994965) + | > current_lr: 5e-06 + | > step_time: 0.6096 (0.6037762509692791) + | > loader_time: 0.01 (0.011697546135295524) + + + --> TIME: 2024-12-22 18:02:05 -- STEP: 1150/2902 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.05816451832652092 (0.06773634191764444) + | > loss_mel_ce: 3.32035756111145 (3.3562073423551486) + | > loss: 0.4223152697086334 (0.4279929601109548) + | > current_lr: 5e-06 + | > step_time: 0.6184 (0.6039227991518759) + | > loader_time: 0.0113 (0.01168110660884692) + + + --> TIME: 2024-12-22 18:02:51 -- STEP: 1200/2902 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.061584342271089554 (0.06741090319119396) + | > loss_mel_ce: 3.287616491317749 (3.35481663107872) + | > loss: 0.4186500906944275 (0.4277784414092702) + | > current_lr: 5e-06 + | > step_time: 0.5726 (0.6035150116682043) + | > loader_time: 0.0105 (0.011648295323054) + + + --> TIME: 2024-12-22 18:03:38 -- STEP: 1250/2902 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.062285855412483215 (0.06711042538881305) + | > loss_mel_ce: 3.24466609954834 (3.352927883148192) + | > loss: 0.41336899995803833 (0.42750478818416615) + | > current_lr: 5e-06 + | > step_time: 0.6217 (0.6035623750686641) + | > loader_time: 0.0109 (0.011627090263366705) + + + --> TIME: 2024-12-22 18:04:25 -- STEP: 1300/2902 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.060854967683553696 (0.0668398949761804) + | > loss_mel_ce: 3.3223676681518555 (3.351386453921977) + | > loss: 0.42290282249450684 (0.4272782932107267) + | > current_lr: 5e-06 + | > step_time: 0.6664 (0.6035717503841104) + | > loader_time: 0.0106 (0.01160200100678665) + + + --> TIME: 2024-12-22 18:05:13 -- STEP: 1350/2902 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.058691028505563736 (0.0665982692633515) + | > loss_mel_ce: 3.4332950115203857 (3.351136484146117) + | > loss: 0.4364982545375824 (0.4272168437639874) + | > current_lr: 5e-06 + | > step_time: 0.6591 (0.6039325431541159) + | > loader_time: 0.0106 (0.01157924245904994) + + + --> TIME: 2024-12-22 18:05:59 -- STEP: 1400/2902 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.0591551810503006 (0.06633740439478851) + | > loss_mel_ce: 3.4388298988342285 (3.349815227644783) + | > loss: 0.4372481405735016 (0.42701907856123805) + | > current_lr: 5e-06 + | > step_time: 0.5407 (0.6033196180207391) + | > loader_time: 0.0105 (0.011555085863385888) + + + --> TIME: 2024-12-22 18:06:46 -- STEP: 1450/2902 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.06207405775785446 (0.06609510640132035) + | > loss_mel_ce: 3.350438356399536 (3.348956268409202) + | > loss: 0.4265640377998352 (0.42688142141391505) + | > current_lr: 5e-06 + | > step_time: 0.6589 (0.6032376984892222) + | > loader_time: 0.0106 (0.011531794975543852) + + + --> TIME: 2024-12-22 18:07:33 -- STEP: 1500/2902 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.05779704079031944 (0.0658860725015404) + | > loss_mel_ce: 3.186833381652832 (3.3477389181454966) + | > loss: 0.4055787920951843 (0.4267031234502794) + | > current_lr: 5e-06 + | > step_time: 0.6797 (0.6032957612673445) + | > loader_time: 0.0106 (0.01151966810226441) + + + --> TIME: 2024-12-22 18:08:20 -- STEP: 1550/2902 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.06286037713289261 (0.0656808182981708) + | > loss_mel_ce: 3.3885998725891113 (3.3471834834929424) + | > loss: 0.4314325451850891 (0.42660803737178943) + | > current_lr: 5e-06 + | > step_time: 0.4494 (0.6032871904680812) + | > loader_time: 0.0097 (0.011501272724520782) + + + --> TIME: 2024-12-22 18:09:06 -- STEP: 1600/2902 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.056488897651433945 (0.06548411942552791) + | > loss_mel_ce: 3.230045795440674 (3.3461627419292914) + | > loss: 0.41081684827804565 (0.42645585726946605) + | > current_lr: 5e-06 + | > step_time: 0.4419 (0.6029248094558723) + | > loader_time: 0.0102 (0.011486575603485115) + + + --> TIME: 2024-12-22 18:09:52 -- STEP: 1650/2902 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.05772990733385086 (0.0652738300598031) + | > loss_mel_ce: 3.189204454421997 (3.344762894890524) + | > loss: 0.405866801738739 (0.4262545901898183) + | > current_lr: 5e-06 + | > step_time: 0.5127 (0.602524100650441) + | > loader_time: 0.0102 (0.011477450457486245) + + + --> TIME: 2024-12-22 18:10:40 -- STEP: 1700/2902 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.05655720457434654 (0.065094419630135) + | > loss_mel_ce: 3.2922372817993164 (3.3436829961047443) + | > loss: 0.41859930753707886 (0.4260971765483128) + | > current_lr: 5e-06 + | > step_time: 0.6106 (0.6027139033990753) + | > loader_time: 0.0104 (0.011463808171889368) + + + --> TIME: 2024-12-22 18:11:27 -- STEP: 1750/2902 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.05985289812088013 (0.06492117699129257) + | > loss_mel_ce: 3.4752132892608643 (3.342860545022146) + | > loss: 0.44188326597213745 (0.4259727147987912) + | > current_lr: 5e-06 + | > step_time: 0.6503 (0.6031666985920503) + | > loader_time: 0.0117 (0.011454036848885678) + + + --> TIME: 2024-12-22 18:12:14 -- STEP: 1800/2902 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.060121070593595505 (0.06474052529368152) + | > loss_mel_ce: 3.321464776992798 (3.3419743609428396) + | > loss: 0.4226982295513153 (0.42583936037288783) + | > current_lr: 5e-06 + | > step_time: 0.5076 (0.6028450124793587) + | > loader_time: 0.0107 (0.011442316770553594) + + + --> TIME: 2024-12-22 18:13:01 -- STEP: 1850/2902 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.05790895223617554 (0.06457843535856637) + | > loss_mel_ce: 3.2426202297210693 (3.340656888936016) + | > loss: 0.4125661551952362 (0.4256544151016185) + | > current_lr: 5e-06 + | > step_time: 0.6492 (0.6027948614068939) + | > loader_time: 0.0105 (0.01143438712970631) + + + --> TIME: 2024-12-22 18:13:48 -- STEP: 1900/2902 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.05779070779681206 (0.06441327211103959) + | > loss_mel_ce: 3.4901840686798096 (3.3398845655039726) + | > loss: 0.44349685311317444 (0.4255372292744487) + | > current_lr: 5e-06 + | > step_time: 0.6231 (0.6030444751287767) + | > loader_time: 0.0109 (0.011421437263488775) + + + --> TIME: 2024-12-22 18:14:36 -- STEP: 1950/2902 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.057499103248119354 (0.06425976193104062) + | > loss_mel_ce: 3.2813806533813477 (3.339388028902884) + | > loss: 0.41735997796058655 (0.42545597343872765) + | > current_lr: 5e-06 + | > step_time: 0.6133 (0.6034406099563995) + | > loader_time: 0.0109 (0.01141151843926846) + + + --> TIME: 2024-12-22 18:15:24 -- STEP: 2000/2902 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.06024341285228729 (0.06411207320168634) + | > loss_mel_ce: 3.2993335723876953 (3.3388525167703618) + | > loss: 0.4199471175670624 (0.42537057338655004) + | > current_lr: 5e-06 + | > step_time: 0.5848 (0.6038204357624058) + | > loader_time: 0.011 (0.011401470899581915) + + + --> TIME: 2024-12-22 18:16:12 -- STEP: 2050/2902 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.05515234172344208 (0.0639634985585767) + | > loss_mel_ce: 3.3303256034851074 (3.338357712699145) + | > loss: 0.42318475246429443 (0.4252901510785267) + | > current_lr: 5e-06 + | > step_time: 0.6663 (0.6038993442349322) + | > loader_time: 0.0111 (0.011391908250203952) + + + --> TIME: 2024-12-22 18:17:00 -- STEP: 2100/2902 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.05871572345495224 (0.06382640003980644) + | > loss_mel_ce: 3.335447311401367 (3.337795670827229) + | > loss: 0.4242703914642334 (0.4252027585534823) + | > current_lr: 5e-06 + | > step_time: 0.6248 (0.6043373265720553) + | > loader_time: 0.0109 (0.011387435368129188) + + + --> TIME: 2024-12-22 18:17:48 -- STEP: 2150/2902 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.058568090200424194 (0.06369089260177577) + | > loss_mel_ce: 3.3574507236480713 (3.3373832586199734) + | > loss: 0.42700234055519104 (0.42513426865256115) + | > current_lr: 5e-06 + | > step_time: 0.6186 (0.6046538991706321) + | > loader_time: 0.0107 (0.011380267919496052) + + + --> TIME: 2024-12-22 18:18:34 -- STEP: 2200/2902 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.05417061224579811 (0.06356014532629754) + | > loss_mel_ce: 3.2483224868774414 (3.336935182593085) + | > loss: 0.41281163692474365 (0.42506191578778346) + | > current_lr: 5e-06 + | > step_time: 0.3652 (0.604359569874677) + | > loader_time: 0.0147 (0.011376231258565735) + + + --> TIME: 2024-12-22 18:19:21 -- STEP: 2250/2902 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.05976838991045952 (0.06343722468614592) + | > loss_mel_ce: 3.3819050788879395 (3.3364632027943935) + | > loss: 0.4302091896533966 (0.4249875532388684) + | > current_lr: 5e-06 + | > step_time: 0.6488 (0.6042012519836432) + | > loader_time: 0.0112 (0.011366784837510855) + + + --> TIME: 2024-12-22 18:20:08 -- STEP: 2300/2902 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.05424568057060242 (0.06331661111958663) + | > loss_mel_ce: 3.2424044609069824 (3.335641909993212) + | > loss: 0.4120812714099884 (0.42486981498158466) + | > current_lr: 5e-06 + | > step_time: 0.6175 (0.6043975578183717) + | > loader_time: 0.0108 (0.011360070187112565) + + + --> TIME: 2024-12-22 18:20:56 -- STEP: 2350/2902 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.05863575264811516 (0.06319462328356651) + | > loss_mel_ce: 3.1610910892486572 (3.334841926757325) + | > loss: 0.4024658501148224 (0.4247545686173943) + | > current_lr: 5e-06 + | > step_time: 0.5305 (0.6044632607317989) + | > loader_time: 0.0111 (0.011355122505350318) + + + --> TIME: 2024-12-22 18:21:43 -- STEP: 2400/2902 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.05514562875032425 (0.0630755390987423) + | > loss_mel_ce: 3.3872921466827393 (3.33422359963258) + | > loss: 0.43030473589897156 (0.4246623922015226) + | > current_lr: 5e-06 + | > step_time: 0.608 (0.6043834758798285) + | > loader_time: 0.0113 (0.011349765062332156) + + + --> TIME: 2024-12-22 18:22:31 -- STEP: 2450/2902 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.05645584315061569 (0.06295087482552163) + | > loss_mel_ce: 3.2656829357147217 (3.33336971107794) + | > loss: 0.41526734828948975 (0.4245400731417593) + | > current_lr: 5e-06 + | > step_time: 0.6592 (0.6047812546515954) + | > loader_time: 0.0114 (0.011347482253094114) + + + --> TIME: 2024-12-22 18:23:18 -- STEP: 2500/2902 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.05541292577981949 (0.06283749222308417) + | > loss_mel_ce: 3.3705124855041504 (3.3327593852043123) + | > loss: 0.42824068665504456 (0.4244496095895763) + | > current_lr: 5e-06 + | > step_time: 0.6591 (0.6047453528404236) + | > loader_time: 0.0115 (0.011346870326995856) + + + --> TIME: 2024-12-22 18:24:05 -- STEP: 2550/2902 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.05708501115441322 (0.06273307577652111) + | > loss_mel_ce: 3.210287570953369 (3.3319304627063198) + | > loss: 0.4084215760231018 (0.42433294222635337) + | > current_lr: 5e-06 + | > step_time: 0.6514 (0.6046757781271843) + | > loader_time: 0.0109 (0.011345180156184185) + + + --> TIME: 2024-12-22 18:24:51 -- STEP: 2600/2902 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.055396050214767456 (0.06263674674269117) + | > loss_mel_ce: 3.3255748748779297 (3.331115982624198) + | > loss: 0.42262136936187744 (0.42421909111050415) + | > current_lr: 5e-06 + | > step_time: 0.5778 (0.6045672522141382) + | > loader_time: 0.0115 (0.011345681593968331) + + + --> TIME: 2024-12-22 18:25:38 -- STEP: 2650/2902 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.054033081978559494 (0.06253566949575595) + | > loss_mel_ce: 3.202986717224121 (3.3304516717622836) + | > loss: 0.4071274697780609 (0.42412341760014544) + | > current_lr: 5e-06 + | > step_time: 0.5688 (0.6044498934835748) + | > loader_time: 0.011 (0.011339990417912336) + + + --> TIME: 2024-12-22 18:26:26 -- STEP: 2700/2902 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.056942086666822433 (0.06243673637647341) + | > loss_mel_ce: 3.1709163188934326 (3.3299163836903016) + | > loss: 0.4034822881221771 (0.42404413991504203) + | > current_lr: 5e-06 + | > step_time: 0.6471 (0.6046004780133557) + | > loader_time: 0.011 (0.011335281795925578) + + + --> TIME: 2024-12-22 18:27:12 -- STEP: 2750/2902 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.056967683136463165 (0.06234114655310478) + | > loss_mel_ce: 3.262450933456421 (3.32917199074138) + | > loss: 0.41492733359336853 (0.4239391420754515) + | > current_lr: 5e-06 + | > step_time: 0.6701 (0.6045005709908213) + | > loader_time: 0.0113 (0.01133127125826751) + + + --> TIME: 2024-12-22 18:28:00 -- STEP: 2800/2902 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.05768461897969246 (0.06225454275762417) + | > loss_mel_ce: 3.3250513076782227 (3.3285182338952994) + | > loss: 0.42284199595451355 (0.4238465970115998) + | > current_lr: 5e-06 + | > step_time: 0.4433 (0.6045015747206539) + | > loader_time: 0.0102 (0.011324909329414391) + + + --> TIME: 2024-12-22 18:28:47 -- STEP: 2850/2902 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.05957743152976036 (0.06216866457279334) + | > loss_mel_ce: 3.287506580352783 (3.32793902003974) + | > loss: 0.41838550567626953 (0.4237634605064723) + | > current_lr: 5e-06 + | > step_time: 0.6157 (0.6046459482427213) + | > loader_time: 0.0111 (0.011319453674450277) + + + --> TIME: 2024-12-22 18:29:34 -- STEP: 2900/2902 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.057000596076250076 (0.06207586748718207) + | > loss_mel_ce: 3.371854782104492 (3.327414204334388) + | > loss: 0.42860692739486694 (0.42368625892647344) + | > current_lr: 5e-06 + | > step_time: 0.6715 (0.6046704241325099) + | > loader_time: 0.0111 (0.011315281719997034) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11512538640186042 (+0) + | > avg_loss_text_ce: 0.05481784877301467 (+0) + | > avg_loss_mel_ce: 3.244625385361489 (+0) + | > avg_loss: 3.299443240117545 (+0) + + + > EPOCH: 1/5 + --> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > TRAINING (2024-12-22 18:31:16)  + + --> TIME: 2024-12-22 18:32:02 -- STEP: 48/2902 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.05890386551618576 (0.05689182776647309) + | > loss_mel_ce: 3.2426769733428955 (3.2889040211836496) + | > loss: 0.41269761323928833 (0.41822448062400025) + | > current_lr: 5e-06 + | > step_time: 0.4931 (0.6041691402594247) + | > loader_time: 0.0098 (0.010133753220240274) + + + --> TIME: 2024-12-22 18:32:49 -- STEP: 98/2902 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.05669895187020302 (0.05675384869837031) + | > loss_mel_ce: 3.316187858581543 (3.2876880923096015) + | > loss: 0.42161086201667786 (0.41805524302988634) + | > current_lr: 5e-06 + | > step_time: 0.6463 (0.6044869422912595) + | > loader_time: 0.0098 (0.010064643256518305) + + + --> TIME: 2024-12-22 18:33:36 -- STEP: 148/2902 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.05863890051841736 (0.056781802452295214) + | > loss_mel_ce: 3.3207154273986816 (3.284461807560276) + | > loss: 0.4224192798137665 (0.41765545167633006) + | > current_lr: 5e-06 + | > step_time: 0.5437 (0.6032516183079898) + | > loader_time: 0.0098 (0.010022640228271484) + + + --> TIME: 2024-12-22 18:34:23 -- STEP: 198/2902 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.05307390168309212 (0.05668377612877374) + | > loss_mel_ce: 3.1541969776153564 (3.281082079868124) + | > loss: 0.40090885758399963 (0.4172207321783509) + | > current_lr: 5e-06 + | > step_time: 0.6491 (0.6029641315190476) + | > loader_time: 0.0103 (0.010148859987355242) + + + --> TIME: 2024-12-22 18:35:11 -- STEP: 248/2902 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.056664470583200455 (0.05664264377687247) + | > loss_mel_ce: 3.267181873321533 (3.284639359481873) + | > loss: 0.4154807925224304 (0.4176602506589505) + | > current_lr: 5e-06 + | > step_time: 0.5497 (0.6068478284343594) + | > loader_time: 0.0098 (0.010098123742688087) + + + --> TIME: 2024-12-22 18:35:57 -- STEP: 298/2902 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.05854443460702896 (0.056619707457471205) + | > loss_mel_ce: 3.436552047729492 (3.283927426242189) + | > loss: 0.43688705563545227 (0.41756839210155033) + | > current_lr: 5e-06 + | > step_time: 0.54 (0.6054019759965423) + | > loader_time: 0.0097 (0.0100771468757783) + + + --> TIME: 2024-12-22 18:36:45 -- STEP: 348/2902 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.05434746667742729 (0.05662827220499172) + | > loss_mel_ce: 3.4035985469818115 (3.2863656522213733) + | > loss: 0.4322432577610016 (0.41787424092662756) + | > current_lr: 5e-06 + | > step_time: 0.6651 (0.6075942810924571) + | > loader_time: 0.0103 (0.010104454111778875) + + + --> TIME: 2024-12-22 18:37:33 -- STEP: 398/2902 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.055893950164318085 (0.05664088315887367) + | > loss_mel_ce: 3.214393138885498 (3.2846952245463084) + | > loss: 0.408785879611969 (0.4176670137661786) + | > current_lr: 5e-06 + | > step_time: 0.5718 (0.6087509783069092) + | > loader_time: 0.0097 (0.010112432978261063) + + + --> TIME: 2024-12-22 18:38:20 -- STEP: 448/2902 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.055915918201208115 (0.05663545990994732) + | > loss_mel_ce: 3.395284652709961 (3.2834518684872562) + | > loss: 0.4314000606536865 (0.4175109162128398) + | > current_lr: 5e-06 + | > step_time: 0.5086 (0.6075441422207012) + | > loader_time: 0.0099 (0.010116907102721085) + + + --> TIME: 2024-12-22 18:39:06 -- STEP: 498/2902 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.05525524914264679 (0.05658867202669262) + | > loss_mel_ce: 3.2936763763427734 (3.2832315762837734) + | > loss: 0.4186164438724518 (0.41747753112670416) + | > current_lr: 5e-06 + | > step_time: 0.6469 (0.60625598756185) + | > loader_time: 0.0103 (0.010127688986230572) + + + --> TIME: 2024-12-22 18:39:54 -- STEP: 548/2902 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.05756908282637596 (0.05664774154170152) + | > loss_mel_ce: 3.1639130115509033 (3.28317973996601) + | > loss: 0.4026852548122406 (0.41747843528533507) + | > current_lr: 5e-06 + | > step_time: 0.6413 (0.6067891307990916) + | > loader_time: 0.0103 (0.010137229505246578) + + + --> TIME: 2024-12-22 18:40:40 -- STEP: 598/2902 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.054091762751340866 (0.05661526037049931) + | > loss_mel_ce: 3.264618158340454 (3.2823297997382173) + | > loss: 0.4148387312889099 (0.4173681325836726) + | > current_lr: 5e-06 + | > step_time: 0.6618 (0.6058910582775254) + | > loader_time: 0.0104 (0.010126844297683365) + + + --> TIME: 2024-12-22 18:41:28 -- STEP: 648/2902 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.05606652796268463 (0.056563418749489905) + | > loss_mel_ce: 3.352484703063965 (3.2804698237666376) + | > loss: 0.42606890201568604 (0.417129155294395) + | > current_lr: 5e-06 + | > step_time: 0.4843 (0.6063799847055359) + | > loader_time: 0.0097 (0.010125751112714238) + + + --> TIME: 2024-12-22 18:42:15 -- STEP: 698/2902 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.05525405332446098 (0.05655210185529168) + | > loss_mel_ce: 3.2965216636657715 (3.2807158672365553) + | > loss: 0.4189719557762146 (0.41715849598704247) + | > current_lr: 5e-06 + | > step_time: 0.6065 (0.6062579441890324) + | > loader_time: 0.01 (0.010130226782878017) + + + --> TIME: 2024-12-22 18:43:03 -- STEP: 748/2902 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.058175548911094666 (0.05650098778725786) + | > loss_mel_ce: 3.269888162612915 (3.280304757987752) + | > loss: 0.41600796580314636 (0.4171007181552644) + | > current_lr: 5e-06 + | > step_time: 0.6982 (0.6072237536231458) + | > loader_time: 0.0106 (0.010144633724090253) + + + --> TIME: 2024-12-22 18:43:49 -- STEP: 798/2902 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.056960977613925934 (0.056491326578054814) + | > loss_mel_ce: 3.241190195083618 (3.2806498221586224) + | > loss: 0.41226890683174133 (0.4171426434415326) + | > current_lr: 5e-06 + | > step_time: 0.6506 (0.6060144674210314) + | > loader_time: 0.0101 (0.010138096964747698) + + + --> TIME: 2024-12-22 18:44:36 -- STEP: 848/2902 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.05707242339849472 (0.05649265555998767) + | > loss_mel_ce: 3.3955237865448 (3.28043776075795) + | > loss: 0.4315745234489441 (0.4171163019074023) + | > current_lr: 5e-06 + | > step_time: 0.5557 (0.6060532727893788) + | > loader_time: 0.0099 (0.010141247004832874) + + + --> TIME: 2024-12-22 18:45:23 -- STEP: 898/2902 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.05551436170935631 (0.05646666096635412) + | > loss_mel_ce: 3.2455344200134277 (3.279986635613813) + | > loss: 0.412631094455719 (0.4170566619371785) + | > current_lr: 5e-06 + | > step_time: 0.5661 (0.605561435355375) + | > loader_time: 0.0097 (0.010150369397781486) + + + --> TIME: 2024-12-22 18:46:10 -- STEP: 948/2902 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.055860064923763275 (0.05645387268424788) + | > loss_mel_ce: 3.196481227874756 (3.2784469580348534) + | > loss: 0.40654265880584717 (0.4168626036384942) + | > current_lr: 5e-06 + | > step_time: 0.6979 (0.6058803857127317) + | > loader_time: 0.0103 (0.010132792881269507) + + + --> TIME: 2024-12-22 18:46:57 -- STEP: 998/2902 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.05521142855286598 (0.05644223373196646) + | > loss_mel_ce: 3.2823712825775146 (3.2776020464295135) + | > loss: 0.4171978533267975 (0.41675553484287925) + | > current_lr: 5e-06 + | > step_time: 0.5983 (0.6054262338515987) + | > loader_time: 0.0113 (0.01014956730401111) + + + --> TIME: 2024-12-22 18:47:43 -- STEP: 1048/2902 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.0541345551609993 (0.05641971721548502) + | > loss_mel_ce: 3.2423479557037354 (3.2767769084631944) + | > loss: 0.41206032037734985 (0.4166495781249673) + | > current_lr: 5e-06 + | > step_time: 0.5016 (0.6050974510553224) + | > loader_time: 0.0105 (0.010205373281740973) + + + --> TIME: 2024-12-22 18:48:30 -- STEP: 1098/2902 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.05483408272266388 (0.056407355969747996) + | > loss_mel_ce: 3.241896867752075 (3.2765870491663613) + | > loss: 0.41209137439727783 (0.4166243005482878) + | > current_lr: 5e-06 + | > step_time: 0.6277 (0.6046719931079604) + | > loader_time: 0.01 (0.010229592983405675) + + + --> TIME: 2024-12-22 18:49:18 -- STEP: 1148/2902 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.05799783021211624 (0.05640103503451633) + | > loss_mel_ce: 3.125845432281494 (3.2763871357416035) + | > loss: 0.3979804217815399 (0.41659852130280145) + | > current_lr: 5e-06 + | > step_time: 0.6496 (0.6050292589938597) + | > loader_time: 0.0108 (0.010255601763309925) + + + --> TIME: 2024-12-22 18:50:04 -- STEP: 1198/2902 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.055139560252428055 (0.05637540748165624) + | > loss_mel_ce: 3.3101248741149902 (3.275573732259876) + | > loss: 0.42065805196762085 (0.4164936424346519) + | > current_lr: 5e-06 + | > step_time: 0.5471 (0.6046004488392543) + | > loader_time: 0.0106 (0.01028222253605202) + + + --> TIME: 2024-12-22 18:50:51 -- STEP: 1248/2902 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.05775659531354904 (0.05636790161952375) + | > loss_mel_ce: 3.269582986831665 (3.275295549669328) + | > loss: 0.41591745615005493 (0.41645793134394365) + | > current_lr: 5e-06 + | > step_time: 0.4634 (0.6042926706946812) + | > loader_time: 0.0102 (0.010291477999626075) + + + --> TIME: 2024-12-22 18:51:38 -- STEP: 1298/2902 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.05557722970843315 (0.056327807755770586) + | > loss_mel_ce: 3.2988786697387695 (3.274222171361716) + | > loss: 0.41930699348449707 (0.4163187472680684) + | > current_lr: 5e-06 + | > step_time: 0.6939 (0.6040190265432156) + | > loader_time: 0.0109 (0.010298490891654985) + + + --> TIME: 2024-12-22 18:52:26 -- STEP: 1348/2902 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.05759159103035927 (0.05631046397583862) + | > loss_mel_ce: 3.2881672382354736 (3.273693052703621) + | > loss: 0.4182198643684387 (0.41625043948786195) + | > current_lr: 5e-06 + | > step_time: 0.7071 (0.6047909778020497) + | > loader_time: 0.0107 (0.010317974699003411) + + + --> TIME: 2024-12-22 18:53:13 -- STEP: 1398/2902 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.05378188565373421 (0.056270858414140715) + | > loss_mel_ce: 3.1348822116851807 (3.2728953762286395) + | > loss: 0.3985830247402191 (0.4161457792244244) + | > current_lr: 5e-06 + | > step_time: 0.5446 (0.6049843866255487) + | > loader_time: 0.0102 (0.010335659605580175) + + + --> TIME: 2024-12-22 18:54:00 -- STEP: 1448/2902 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.05448736995458603 (0.056238353370755736) + | > loss_mel_ce: 3.215205430984497 (3.272332094486248) + | > loss: 0.40871161222457886 (0.41607130596733227) + | > current_lr: 5e-06 + | > step_time: 0.4742 (0.6047209682385556) + | > loader_time: 0.0101 (0.010342140882713364) + + + --> TIME: 2024-12-22 18:54:48 -- STEP: 1498/2902 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.05498555675148964 (0.056216870995365244) + | > loss_mel_ce: 3.271206855773926 (3.272168317847005) + | > loss: 0.41577404737472534 (0.4160481485847797) + | > current_lr: 5e-06 + | > step_time: 0.6284 (0.6049662217278979) + | > loader_time: 0.0101 (0.01034619254327426) + + + --> TIME: 2024-12-22 18:55:35 -- STEP: 1548/2902 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.057937052100896835 (0.056200356920579446) + | > loss_mel_ce: 3.3727030754089355 (3.2725960398213205) + | > loss: 0.42883002758026123 (0.4160995496104854) + | > current_lr: 5e-06 + | > step_time: 0.6974 (0.6050552325347286) + | > loader_time: 0.0115 (0.010363647796078863) + + + --> TIME: 2024-12-22 18:56:22 -- STEP: 1598/2902 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.05774551257491112 (0.056181561288065986) + | > loss_mel_ce: 3.3820343017578125 (3.271953099361798) + | > loss: 0.42997246980667114 (0.4160168326654482) + | > current_lr: 5e-06 + | > step_time: 0.6635 (0.6048786176161103) + | > loader_time: 0.0117 (0.010390232889464262) + + + --> TIME: 2024-12-22 18:57:09 -- STEP: 1648/2902 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.053543396294116974 (0.05615658306706588) + | > loss_mel_ce: 3.2377357482910156 (3.271739111073967) + | > loss: 0.4114098846912384 (0.41598696182075057) + | > current_lr: 5e-06 + | > step_time: 0.5607 (0.6048709005117405) + | > loader_time: 0.0114 (0.010413509549446476) + + + --> TIME: 2024-12-22 18:57:56 -- STEP: 1698/2902 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.05661720037460327 (0.05613417721609063) + | > loss_mel_ce: 3.2408382892608643 (3.27091189141268) + | > loss: 0.41218194365501404 (0.41588075864441404) + | > current_lr: 5e-06 + | > step_time: 0.6238 (0.6048471832163056) + | > loader_time: 0.0106 (0.010442374030608591) + + + --> TIME: 2024-12-22 18:58:44 -- STEP: 1748/2902 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.05818689614534378 (0.05609983744342689) + | > loss_mel_ce: 3.2191317081451416 (3.2704604229883425) + | > loss: 0.4096648395061493 (0.4158200326139103) + | > current_lr: 5e-06 + | > step_time: 0.6999 (0.6053034963269524) + | > loader_time: 0.0111 (0.010458539225144026) + + + --> TIME: 2024-12-22 18:59:31 -- STEP: 1798/2902 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.05465058237314224 (0.056072617156868246) + | > loss_mel_ce: 3.252976655960083 (3.27008914642525) + | > loss: 0.4134534001350403 (0.4157702205443674) + | > current_lr: 5e-06 + | > step_time: 0.6463 (0.6055046300071227) + | > loader_time: 0.0109 (0.01047546341103627) + + + --> TIME: 2024-12-22 19:00:19 -- STEP: 1848/2902 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.052592262625694275 (0.05605650214817035) + | > loss_mel_ce: 3.2938497066497803 (3.2699186003311382) + | > loss: 0.41830524802207947 (0.41574688796034626) + | > current_lr: 5e-06 + | > step_time: 0.6226 (0.6058445509636036) + | > loader_time: 0.0114 (0.01049227864195258) + + + --> TIME: 2024-12-22 19:01:07 -- STEP: 1898/2902 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.05527101084589958 (0.05603910592611333) + | > loss_mel_ce: 3.3503198623657227 (3.269640341573823) + | > loss: 0.425698846578598 (0.41570993103832793) + | > current_lr: 5e-06 + | > step_time: 0.6871 (0.6060072334347831) + | > loader_time: 0.0114 (0.010505890193050855) + + + --> TIME: 2024-12-22 19:01:54 -- STEP: 1948/2902 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.05501137673854828 (0.056014667678188364) + | > loss_mel_ce: 3.2952277660369873 (3.269260302216611) + | > loss: 0.4187798798084259 (0.41565937137334497) + | > current_lr: 5e-06 + | > step_time: 0.7268 (0.6060542482615002) + | > loader_time: 0.0113 (0.010517563908007111) + + + --> TIME: 2024-12-22 19:02:41 -- STEP: 1998/2902 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.05698436126112938 (0.05600299724788815) + | > loss_mel_ce: 3.2746849060058594 (3.268788534122426) + | > loss: 0.4164586663246155 (0.41559894159212485) + | > current_lr: 5e-06 + | > step_time: 0.6491 (0.6059473711210442) + | > loader_time: 0.0108 (0.010526264155352558) + + + --> TIME: 2024-12-22 19:03:29 -- STEP: 2048/2902 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.05508394539356232 (0.05598831629686173) + | > loss_mel_ce: 3.2314789295196533 (3.2680820173118272) + | > loss: 0.4108203649520874 (0.41550879187707324) + | > current_lr: 5e-06 + | > step_time: 0.4947 (0.6060070462990548) + | > loader_time: 0.0106 (0.010532886837609112) + + + --> TIME: 2024-12-22 19:04:17 -- STEP: 2098/2902 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.05668158456683159 (0.05596930274581829) + | > loss_mel_ce: 3.2369794845581055 (3.2679047324296064) + | > loss: 0.41170763969421387 (0.4154842545920263) + | > current_lr: 5e-06 + | > step_time: 0.5536 (0.6062503820152025) + | > loader_time: 0.0104 (0.010541449624998894) + + + --> TIME: 2024-12-22 19:05:04 -- STEP: 2148/2902 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.05353817716240883 (0.0559521284911719) + | > loss_mel_ce: 3.2572579383850098 (3.267594929513985) + | > loss: 0.41384950280189514 (0.4154433824219073) + | > current_lr: 5e-06 + | > step_time: 0.6125 (0.6061867110334069) + | > loader_time: 0.0107 (0.010544300412332551) + + + --> TIME: 2024-12-22 19:05:52 -- STEP: 2198/2902 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.05389314889907837 (0.055934098555119174) + | > loss_mel_ce: 3.159824848175049 (3.2674999072185087) + | > loss: 0.4017147421836853 (0.41542925089733296) + | > current_lr: 5e-06 + | > step_time: 0.6574 (0.6065022010386691) + | > loader_time: 0.011 (0.010550862990041777) + + + --> TIME: 2024-12-22 19:06:38 -- STEP: 2248/2902 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.05729653686285019 (0.05592138946281738) + | > loss_mel_ce: 3.3485758304595947 (3.2666910624801035) + | > loss: 0.4257340431213379 (0.41532655668799556) + | > current_lr: 5e-06 + | > step_time: 0.5645 (0.6061298031611797) + | > loader_time: 0.0105 (0.010557977436276102) + + + --> TIME: 2024-12-22 19:07:25 -- STEP: 2298/2902 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.05610555410385132 (0.055886846526912524) + | > loss_mel_ce: 3.221963405609131 (3.2663745807501616) + | > loss: 0.40975862741470337 (0.41528267858856327) + | > current_lr: 5e-06 + | > step_time: 0.5627 (0.6061801232909823) + | > loader_time: 0.0105 (0.010561991713169462) + + + --> TIME: 2024-12-22 19:08:12 -- STEP: 2348/2902 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.054220084100961685 (0.05587315915098484) + | > loss_mel_ce: 3.2626988887786865 (3.266057619853564) + | > loss: 0.41461488604545593 (0.41524134751479097) + | > current_lr: 5e-06 + | > step_time: 0.6679 (0.6060712704057382) + | > loader_time: 0.011 (0.01056742150292081) + + + --> TIME: 2024-12-22 19:08:59 -- STEP: 2398/2902 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.05538155138492584 (0.05585878610523964) + | > loss_mel_ce: 3.1319658756256104 (3.2655375949336256) + | > loss: 0.3984184265136719 (0.41517454782210345) + | > current_lr: 5e-06 + | > step_time: 0.534 (0.6057613339396293) + | > loader_time: 0.0107 (0.01057357794051374) + + + --> TIME: 2024-12-22 19:09:46 -- STEP: 2448/2902 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.054799508303403854 (0.05584115826439573) + | > loss_mel_ce: 3.246894359588623 (3.265253891547521) + | > loss: 0.4127117395401001 (0.4151368814404883) + | > current_lr: 5e-06 + | > step_time: 0.5644 (0.6058000731312372) + | > loader_time: 0.0108 (0.010580366538241025) + + + --> TIME: 2024-12-22 19:10:32 -- STEP: 2498/2902 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.05230073258280754 (0.0558200412056296) + | > loss_mel_ce: 3.3292624950408936 (3.264872017147257) + | > loss: 0.4226953983306885 (0.4150865075347708) + | > current_lr: 5e-06 + | > step_time: 0.5618 (0.6056058732293148) + | > loader_time: 0.0108 (0.010589490612570631) + + + --> TIME: 2024-12-22 19:11:20 -- STEP: 2548/2902 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.05251738056540489 (0.05579752388324007) + | > loss_mel_ce: 3.153108596801758 (3.2645589915137836) + | > loss: 0.4007032513618469 (0.4150445646777445) + | > current_lr: 5e-06 + | > step_time: 0.5492 (0.6057547577883424) + | > loader_time: 0.0105 (0.010596270651046307) + + + --> TIME: 2024-12-22 19:12:06 -- STEP: 2598/2902 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.052732329815626144 (0.05577686714658008) + | > loss_mel_ce: 3.3292698860168457 (3.264208226042403) + | > loss: 0.4227502644062042 (0.41499813687948195) + | > current_lr: 5e-06 + | > step_time: 0.5166 (0.6055064683147716) + | > loader_time: 0.0106 (0.010603652264724981) + + + --> TIME: 2024-12-22 19:12:53 -- STEP: 2648/2902 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.05599590763449669 (0.05576192359566642) + | > loss_mel_ce: 3.354013204574585 (3.263891820370971) + | > loss: 0.4262511432170868 (0.41495671823305663) + | > current_lr: 5e-06 + | > step_time: 0.594 (0.6053740501043655) + | > loader_time: 0.0115 (0.010613989884039431) + + + --> TIME: 2024-12-22 19:13:39 -- STEP: 2698/2902 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.05634847655892372 (0.05574999848246749) + | > loss_mel_ce: 3.174736976623535 (3.2637718163922242) + | > loss: 0.40388569235801697 (0.41494022710752804) + | > current_lr: 5e-06 + | > step_time: 0.6399 (0.6048974610859585) + | > loader_time: 0.0112 (0.010621621486785422) + + + --> TIME: 2024-12-22 19:14:26 -- STEP: 2748/2902 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.052691757678985596 (0.05573155193994961) + | > loss_mel_ce: 3.3672266006469727 (3.2633937294479716) + | > loss: 0.4274897873401642 (0.4148906604258081) + | > current_lr: 5e-06 + | > step_time: 0.6579 (0.6049603481480119) + | > loader_time: 0.0111 (0.010630740001906142) + + + --> TIME: 2024-12-22 19:15:14 -- STEP: 2798/2902 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.05469852685928345 (0.05571502604900596) + | > loss_mel_ce: 3.3810243606567383 (3.263473064836389) + | > loss: 0.4294653534889221 (0.4148985116267903) + | > current_lr: 5e-06 + | > step_time: 0.6643 (0.6051521071201562) + | > loader_time: 0.0122 (0.010641849236968946) + + + --> TIME: 2024-12-22 19:16:01 -- STEP: 2848/2902 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.05531623214483261 (0.05570412047071318) + | > loss_mel_ce: 3.2582409381866455 (3.2633204451772606) + | > loss: 0.41419464349746704 (0.4148780709823196) + | > current_lr: 5e-06 + | > step_time: 0.3849 (0.6051703050230323) + | > loader_time: 0.0101 (0.010653467241967678) + + + --> TIME: 2024-12-22 19:16:48 -- STEP: 2898/2902 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.05267462506890297 (0.05568916210486938) + | > loss_mel_ce: 3.2610042095184326 (3.2627478573880087) + | > loss: 0.41420984268188477 (0.41480462774480104) + | > current_lr: 5e-06 + | > step_time: 0.6969 (0.6050742793856527) + | > loader_time: 0.0117 (0.010668510317884718) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11507114858338323 (-5.423781847718745e-05) + | > avg_loss_text_ce: 0.05247822260916836 (-0.002339626163846309) + | > avg_loss_mel_ce: 3.202105414987815 (-0.04251997037367383) + | > avg_loss: 3.254583633307255 (-0.04485960681029022) + + + > EPOCH: 2/5 + --> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > TRAINING (2024-12-22 19:18:12)  + + --> TIME: 2024-12-22 19:18:56 -- STEP: 46/2902 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.05449153855443001 (0.054341975761496505) + | > loss_mel_ce: 3.34686017036438 (3.262047643246858) + | > loss: 0.4251689612865448 (0.4145487028619517) + | > current_lr: 5e-06 + | > step_time: 0.607 (0.5991597175598146) + | > loader_time: 0.0104 (0.010427303936170496) + + + --> TIME: 2024-12-22 19:19:43 -- STEP: 96/2902 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.055128224194049835 (0.05449626039868841) + | > loss_mel_ce: 3.243983268737793 (3.2608101293444633) + | > loss: 0.41238895058631897 (0.41441329941153526) + | > current_lr: 5e-06 + | > step_time: 0.5481 (0.597640375296275) + | > loader_time: 0.0102 (0.010360459486643466) + + + --> TIME: 2024-12-22 19:20:31 -- STEP: 146/2902 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.05616119131445885 (0.05466639781243181) + | > loss_mel_ce: 3.2564191818237305 (3.253628141259494) + | > loss: 0.41407254338264465 (0.4135368174477799) + | > current_lr: 5e-06 + | > step_time: 0.6546 (0.6053444686001294) + | > loader_time: 0.0103 (0.010333493964312823) + + + --> TIME: 2024-12-22 19:21:17 -- STEP: 196/2902 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.05680551752448082 (0.05473522758301424) + | > loss_mel_ce: 3.142683982849121 (3.2478924880222397) + | > loss: 0.3999361991882324 (0.41282846444115345) + | > current_lr: 5e-06 + | > step_time: 0.6581 (0.6036743071614478) + | > loader_time: 0.0103 (0.010324716567993155) + + + --> TIME: 2024-12-22 19:22:05 -- STEP: 246/2902 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.053551316261291504 (0.05457976252991494) + | > loss_mel_ce: 3.1189703941345215 (3.245878595646804) + | > loss: 0.39656519889831543 (0.41255729506170846) + | > current_lr: 5e-06 + | > step_time: 0.6213 (0.6064855470889952) + | > loader_time: 0.0099 (0.01028650562937666) + + + --> TIME: 2024-12-22 19:22:53 -- STEP: 296/2902 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.05468261241912842 (0.05454644439993679) + | > loss_mel_ce: 3.2226357460021973 (3.244215094559901) + | > loss: 0.4096648097038269 (0.4123451929237391) + | > current_lr: 5e-06 + | > step_time: 0.5772 (0.6066698633335732) + | > loader_time: 0.0104 (0.010362294880119523) + + + --> TIME: 2024-12-22 19:23:39 -- STEP: 346/2902 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.056551966816186905 (0.05455692140439343) + | > loss_mel_ce: 3.2596166133880615 (3.2438248712892466) + | > loss: 0.41452106833457947 (0.41229772490228533) + | > current_lr: 5e-06 + | > step_time: 0.5327 (0.6049295891227059) + | > loader_time: 0.0097 (0.010348508812788594) + + + --> TIME: 2024-12-22 19:24:26 -- STEP: 396/2902 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.057674016803503036 (0.05454431032065792) + | > loss_mel_ce: 3.2081027030944824 (3.243972488726027) + | > loss: 0.4082220792770386 (0.41231460033944156) + | > current_lr: 5e-06 + | > step_time: 0.6618 (0.6051323636613708) + | > loader_time: 0.0102 (0.010334551936448218) + + + --> TIME: 2024-12-22 19:25:13 -- STEP: 446/2902 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.053182203322649 (0.05448943492643235) + | > loss_mel_ce: 3.3238401412963867 (3.2430883688777006) + | > loss: 0.42212778329849243 (0.4121972257647279) + | > current_lr: 5e-06 + | > step_time: 0.5257 (0.6037687779542039) + | > loader_time: 0.0097 (0.010317636712249615) + + + --> TIME: 2024-12-22 19:26:00 -- STEP: 496/2902 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.05793963000178337 (0.0545288963588856) + | > loss_mel_ce: 3.1546380519866943 (3.2435592682130867) + | > loss: 0.40157219767570496 (0.41226102081277677) + | > current_lr: 5e-06 + | > step_time: 0.5773 (0.6035887161570208) + | > loader_time: 0.0103 (0.010317006899464512) + + + --> TIME: 2024-12-22 19:26:46 -- STEP: 546/2902 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.05591798573732376 (0.05447477271486988) + | > loss_mel_ce: 3.255568742752075 (3.2395439558413432) + | > loss: 0.4139358401298523 (0.4117523413765561) + | > current_lr: 5e-06 + | > step_time: 0.6578 (0.6032865440452491) + | > loader_time: 0.0105 (0.010307735575861107) + + + --> TIME: 2024-12-22 19:27:34 -- STEP: 596/2902 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.056878942996263504 (0.05450224877653906) + | > loss_mel_ce: 3.32279109954834 (3.2391228695843832) + | > loss: 0.4224587678909302 (0.41170314004357217) + | > current_lr: 5e-06 + | > step_time: 0.6144 (0.6037989190760873) + | > loader_time: 0.0104 (0.01030485742044128) + + + --> TIME: 2024-12-22 19:28:21 -- STEP: 646/2902 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.05395687371492386 (0.05448623928284682) + | > loss_mel_ce: 3.2676842212677 (3.238311486716608) + | > loss: 0.4152051508426666 (0.41159971593518735) + | > current_lr: 5e-06 + | > step_time: 0.6494 (0.6037543484301021) + | > loader_time: 0.0105 (0.010310063052103604) + + + --> TIME: 2024-12-22 19:29:08 -- STEP: 696/2902 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.05541800707578659 (0.05446035207795172) + | > loss_mel_ce: 3.185835123062134 (3.2377473331730924) + | > loss: 0.4051566421985626 (0.41152596079755094) + | > current_lr: 5e-06 + | > step_time: 0.4914 (0.6042240799843576) + | > loader_time: 0.0098 (0.010309737989272186) + + + --> TIME: 2024-12-22 19:29:55 -- STEP: 746/2902 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.05779316648840904 (0.05444739832375548) + | > loss_mel_ce: 3.315509557723999 (3.2381504930695626) + | > loss: 0.421662837266922 (0.41157473659707133) + | > current_lr: 5e-06 + | > step_time: 0.6138 (0.6041072274660613) + | > loader_time: 0.016 (0.010313577690329045) + + + --> TIME: 2024-12-22 19:30:42 -- STEP: 796/2902 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.051639851182699203 (0.05440891647814357) + | > loss_mel_ce: 3.1917884349823 (3.237836667640723) + | > loss: 0.40542852878570557 (0.41153069817690385) + | > current_lr: 5e-06 + | > step_time: 0.7215 (0.6042170922959863) + | > loader_time: 0.0113 (0.010306383197631065) + + + --> TIME: 2024-12-22 19:31:30 -- STEP: 846/2902 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.0565703846514225 (0.05434837889357803) + | > loss_mel_ce: 3.3043394088745117 (3.2371244619360473) + | > loss: 0.4201137125492096 (0.4114341052429613) + | > current_lr: 5e-06 + | > step_time: 0.6267 (0.6048971769657534) + | > loader_time: 0.0102 (0.010311559018795649) + + + --> TIME: 2024-12-22 19:32:17 -- STEP: 896/2902 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.05269205942749977 (0.05433550530246326) + | > loss_mel_ce: 3.2086503505706787 (3.236831199909958) + | > loss: 0.4076678156852722 (0.41139583825133735) + | > current_lr: 5e-06 + | > step_time: 0.5897 (0.6048591102340385) + | > loader_time: 0.0105 (0.010325204581022259) + + + --> TIME: 2024-12-22 19:33:05 -- STEP: 946/2902 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.05327130854129791 (0.05433601268171382) + | > loss_mel_ce: 3.1741154193878174 (3.236998047435509) + | > loss: 0.40342333912849426 (0.4114167576864456) + | > current_lr: 5e-06 + | > step_time: 0.5356 (0.6051080012674306) + | > loader_time: 0.0099 (0.010325364532228752) + + + --> TIME: 2024-12-22 19:33:52 -- STEP: 996/2902 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.05429665371775627 (0.05431595303671786) + | > loss_mel_ce: 3.098682165145874 (3.2360697728563013) + | > loss: 0.3941223621368408 (0.41129821604753586) + | > current_lr: 5e-06 + | > step_time: 0.6609 (0.6053404793681862) + | > loader_time: 0.0107 (0.010332633931952783) + + + --> TIME: 2024-12-22 19:34:38 -- STEP: 1046/2902 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.0514579601585865 (0.05428189642186033) + | > loss_mel_ce: 3.1680314540863037 (3.2349629500172092) + | > loss: 0.40243616700172424 (0.4111556061481198) + | > current_lr: 5e-06 + | > step_time: 0.5487 (0.604390150502352) + | > loader_time: 0.0102 (0.010338225966429842) + + + --> TIME: 2024-12-22 19:35:25 -- STEP: 1096/2902 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.05452772602438927 (0.05426657630171437) + | > loss_mel_ce: 3.296816825866699 (3.234400193839176) + | > loss: 0.41891807317733765 (0.4110833465726704) + | > current_lr: 5e-06 + | > step_time: 0.6502 (0.6042889208689218) + | > loader_time: 0.012 (0.010351204524075022) + + + --> TIME: 2024-12-22 19:36:12 -- STEP: 1146/2902 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.054743699729442596 (0.054242047489365564) + | > loss_mel_ce: 3.1826770305633545 (3.2341083564891435) + | > loss: 0.4046775996685028 (0.4110438008426994) + | > current_lr: 5e-06 + | > step_time: 0.6152 (0.6043572690058332) + | > loader_time: 0.0101 (0.010360107579988538) + + + --> TIME: 2024-12-22 19:36:59 -- STEP: 1196/2902 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.04921133816242218 (0.05422345994594804) + | > loss_mel_ce: 3.323059320449829 (3.2337135487176893) + | > loss: 0.42153382301330566 (0.41099212650072603) + | > current_lr: 5e-06 + | > step_time: 0.6726 (0.6041809440057805) + | > loader_time: 0.011 (0.010368075857194368) + + + --> TIME: 2024-12-22 19:37:47 -- STEP: 1246/2902 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.05187971889972687 (0.05421067672642622) + | > loss_mel_ce: 3.2639362812042236 (3.2332898234861807) + | > loss: 0.41447699069976807 (0.4109375628958158) + | > current_lr: 5e-06 + | > step_time: 0.639 (0.6047971242886284) + | > loader_time: 0.0106 (0.01038035209833331) + + + --> TIME: 2024-12-22 19:38:34 -- STEP: 1296/2902 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.055725544691085815 (0.05422138063708482) + | > loss_mel_ce: 3.212235927581787 (3.2331566674473833) + | > loss: 0.4084951877593994 (0.41092225642483876) + | > current_lr: 5e-06 + | > step_time: 0.4821 (0.6050901449756845) + | > loader_time: 0.0104 (0.010385573278238736) + + + --> TIME: 2024-12-22 19:39:21 -- STEP: 1346/2902 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.054803065955638885 (0.05419990306866825) + | > loss_mel_ce: 3.3109726905822754 (3.2331086253694745) + | > loss: 0.42072197794914246 (0.41091356646784316) + | > current_lr: 5e-06 + | > step_time: 0.653 (0.6051710766102181) + | > loader_time: 0.0105 (0.010394215760932447) + + + --> TIME: 2024-12-22 19:40:07 -- STEP: 1396/2902 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.05577743798494339 (0.05419548767963078) + | > loss_mel_ce: 3.1629726886749268 (3.2319712032561303) + | > loss: 0.4023437798023224 (0.4107708367799275) + | > current_lr: 5e-06 + | > step_time: 0.6159 (0.6044754990533964) + | > loader_time: 0.0104 (0.010397913804368499) + + + --> TIME: 2024-12-22 19:40:54 -- STEP: 1446/2902 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.05135389044880867 (0.05418163613721544) + | > loss_mel_ce: 3.1880338191986084 (3.231893382296686) + | > loss: 0.4049234688282013 (0.41075937767328863) + | > current_lr: 5e-06 + | > step_time: 0.6488 (0.604256208177093) + | > loader_time: 0.0108 (0.010400912738604823) + + + --> TIME: 2024-12-22 19:41:40 -- STEP: 1496/2902 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.055541761219501495 (0.05417019395994392) + | > loss_mel_ce: 3.259507417678833 (3.2317391237154354) + | > loss: 0.41438114643096924 (0.4107386650925967) + | > current_lr: 5e-06 + | > step_time: 0.5431 (0.6038286035711105) + | > loader_time: 0.0104 (0.010405490742647704) + + + --> TIME: 2024-12-22 19:42:27 -- STEP: 1546/2902 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.05562349036335945 (0.05414642839852774) + | > loss_mel_ce: 3.2322540283203125 (3.230981718368085) + | > loss: 0.41098469495773315 (0.41064101872052416) + | > current_lr: 5e-06 + | > step_time: 0.6215 (0.603749968564525) + | > loader_time: 0.0106 (0.010414998867533335) + + + --> TIME: 2024-12-22 19:43:15 -- STEP: 1596/2902 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.05427920073270798 (0.05413814238914497) + | > loss_mel_ce: 3.1687004566192627 (3.2307760164253683) + | > loss: 0.4028724431991577 (0.4106142702407404) + | > current_lr: 5e-06 + | > step_time: 0.6616 (0.6040683033173532) + | > loader_time: 0.0114 (0.0104308143295441) + + + --> TIME: 2024-12-22 19:44:02 -- STEP: 1646/2902 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.05014851316809654 (0.054124010054627726) + | > loss_mel_ce: 3.308337926864624 (3.2305475654462987) + | > loss: 0.41981080174446106 (0.41058394731104997) + | > current_lr: 5e-06 + | > step_time: 0.6745 (0.6039762705528275) + | > loader_time: 0.0109 (0.010435233718714472) + + + --> TIME: 2024-12-22 19:44:50 -- STEP: 1696/2902 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.05018627643585205 (0.05409667860576005) + | > loss_mel_ce: 3.0870144367218018 (3.229944757836044) + | > loss: 0.3921501040458679 (0.4105051799308295) + | > current_lr: 5e-06 + | > step_time: 0.6483 (0.604342738271883) + | > loader_time: 0.0112 (0.010440538233181211) + + + --> TIME: 2024-12-22 19:45:36 -- STEP: 1746/2902 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.054639771580696106 (0.05408420456618174) + | > loss_mel_ce: 3.236097574234009 (3.2295827566553217) + | > loss: 0.41134217381477356 (0.4104583704969998) + | > current_lr: 5e-06 + | > step_time: 0.6497 (0.6042213617458091) + | > loader_time: 0.0105 (0.010453733381946467) + + + --> TIME: 2024-12-22 19:46:24 -- STEP: 1796/2902 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.0547931008040905 (0.05406024138378872) + | > loss_mel_ce: 3.2425832748413086 (3.2288573825014186) + | > loss: 0.4121720492839813 (0.4103647032897826) + | > current_lr: 5e-06 + | > step_time: 0.654 (0.6042706026531794) + | > loader_time: 0.0114 (0.01046769297733604) + + + --> TIME: 2024-12-22 19:47:11 -- STEP: 1846/2902 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.05211947485804558 (0.0540442640871972) + | > loss_mel_ce: 3.1296088695526123 (3.2281167809392355) + | > loss: 0.39771604537963867 (0.4102701309055313) + | > current_lr: 5e-06 + | > step_time: 0.5564 (0.6042208054212929) + | > loader_time: 0.0106 (0.010471764778393445) + + + --> TIME: 2024-12-22 19:47:58 -- STEP: 1896/2902 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.056086037307977676 (0.05404073305483044) + | > loss_mel_ce: 3.290001392364502 (3.2276765414934108) + | > loss: 0.41826093196868896 (0.4102146595810785) + | > current_lr: 5e-06 + | > step_time: 0.578 (0.6043190175219416) + | > loader_time: 0.0106 (0.010481623280400462) + + + --> TIME: 2024-12-22 19:48:45 -- STEP: 1946/2902 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.05454717576503754 (0.054029209030081904) + | > loss_mel_ce: 3.259446620941162 (3.2277184737059454) + | > loss: 0.4142492115497589 (0.41021846057148387) + | > current_lr: 5e-06 + | > step_time: 0.6551 (0.604525358311808) + | > loader_time: 0.0114 (0.01049390493414507) + + + --> TIME: 2024-12-22 19:49:33 -- STEP: 1996/2902 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.05354653671383858 (0.0540268157099286) + | > loss_mel_ce: 3.304413080215454 (3.22762779996008) + | > loss: 0.41974493861198425 (0.41020682715402074) + | > current_lr: 5e-06 + | > step_time: 0.6331 (0.6046111153696238) + | > loader_time: 0.0113 (0.010508355850686048) + + + --> TIME: 2024-12-22 19:50:20 -- STEP: 2046/2902 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.055073026567697525 (0.05401429145770805) + | > loss_mel_ce: 3.216965675354004 (3.2274121992282727) + | > loss: 0.4090048372745514 (0.4101783115405838) + | > current_lr: 5e-06 + | > step_time: 0.6854 (0.604751432746036) + | > loader_time: 0.0113 (0.010519989657844845) + + + --> TIME: 2024-12-22 19:51:08 -- STEP: 2096/2902 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.05682572349905968 (0.05401011059089131) + | > loss_mel_ce: 3.143216848373413 (3.2272642189992284) + | > loss: 0.4000053107738495 (0.4101592913724992) + | > current_lr: 5e-06 + | > step_time: 0.6136 (0.6048859395371127) + | > loader_time: 0.0113 (0.01052777946905325) + + + --> TIME: 2024-12-22 19:51:55 -- STEP: 2146/2902 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.052709128707647324 (0.053994231788790474) + | > loss_mel_ce: 3.2841806411743164 (3.2269571499633423) + | > loss: 0.41711121797561646 (0.41011892291061897) + | > current_lr: 5e-06 + | > step_time: 0.5301 (0.6049023500026538) + | > loader_time: 0.0102 (0.010533556689309487) + + + --> TIME: 2024-12-22 19:52:42 -- STEP: 2196/2902 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.05199521407485008 (0.053986151086909545) + | > loss_mel_ce: 3.114041566848755 (3.2264509198878404) + | > loss: 0.3957546055316925 (0.4100546340283364) + | > current_lr: 5e-06 + | > step_time: 0.6118 (0.6049591221444579) + | > loader_time: 0.011 (0.010538012820731954) + + + --> TIME: 2024-12-22 19:53:29 -- STEP: 2246/2902 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.05536806583404541 (0.05397705866300891) + | > loss_mel_ce: 3.229199171066284 (3.226278807155169) + | > loss: 0.4105709195137024 (0.41003198335187707) + | > current_lr: 5e-06 + | > step_time: 0.6545 (0.6048499229011621) + | > loader_time: 0.0109 (0.010542242100274884) + + + --> TIME: 2024-12-22 19:54:16 -- STEP: 2296/2902 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.05085474252700806 (0.05395412064102261) + | > loss_mel_ce: 3.0547423362731934 (3.2258496954258287) + | > loss: 0.3881996273994446 (0.4099754771081411) + | > current_lr: 5e-06 + | > step_time: 0.5028 (0.6047402797883378) + | > loader_time: 0.0106 (0.010546214281474247) + + + --> TIME: 2024-12-22 19:55:04 -- STEP: 2346/2902 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.055554307997226715 (0.053945425656470924) + | > loss_mel_ce: 3.1857361793518066 (3.225720517267451) + | > loss: 0.4051613211631775 (0.4099582429508417) + | > current_lr: 5e-06 + | > step_time: 0.6962 (0.6051811104845186) + | > loader_time: 0.0113 (0.01055516936488366) + + + --> TIME: 2024-12-22 19:55:51 -- STEP: 2396/2902 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.05265699699521065 (0.053927266866105536) + | > loss_mel_ce: 3.222463846206665 (3.225377675388409) + | > loss: 0.40939009189605713 (0.4099131178900672) + | > current_lr: 5e-06 + | > step_time: 0.6853 (0.6051519718114415) + | > loader_time: 0.0114 (0.010562321379507113) + + + --> TIME: 2024-12-22 19:56:37 -- STEP: 2446/2902 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.05529283732175827 (0.05392556344802422) + | > loss_mel_ce: 3.248408555984497 (3.2250945345846596) + | > loss: 0.4129626750946045 (0.40987751231462516) + | > current_lr: 5e-06 + | > step_time: 0.6224 (0.6047231179227202) + | > loader_time: 0.0109 (0.010566833754201832) + + + --> TIME: 2024-12-22 19:57:24 -- STEP: 2496/2902 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.05385390669107437 (0.05390973614815337) + | > loss_mel_ce: 3.188469171524048 (3.224661102279637) + | > loss: 0.4052903950214386 (0.4098213548676515) + | > current_lr: 5e-06 + | > step_time: 0.5605 (0.6046044805493085) + | > loader_time: 0.0112 (0.010573913558171335) + + + --> TIME: 2024-12-22 19:58:11 -- STEP: 2546/2902 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.05382354557514191 (0.05389312902891253) + | > loss_mel_ce: 3.26172137260437 (3.2242961726784207) + | > loss: 0.41444310545921326 (0.40977366277999216) + | > current_lr: 5e-06 + | > step_time: 0.6391 (0.6046588743004547) + | > loader_time: 0.0113 (0.010582529459052066) + + + --> TIME: 2024-12-22 19:58:58 -- STEP: 2596/2902 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.05468406900763512 (0.053886171581025585) + | > loss_mel_ce: 3.3646421432495117 (3.2239668191122135) + | > loss: 0.42741578817367554 (0.4097316239620392) + | > current_lr: 5e-06 + | > step_time: 0.6687 (0.6046174602075035) + | > loader_time: 0.0109 (0.010590319181627414) + + + --> TIME: 2024-12-22 19:59:46 -- STEP: 2646/2902 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.051759347319602966 (0.05386880528849135) + | > loss_mel_ce: 3.094008207321167 (3.223380103853732) + | > loss: 0.3932209312915802 (0.40965611376684885) + | > current_lr: 5e-06 + | > step_time: 0.6131 (0.6047673099255417) + | > loader_time: 0.0108 (0.010597522837234462) + + + --> TIME: 2024-12-22 20:00:34 -- STEP: 2696/2902 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.054853543639183044 (0.05384999631492295) + | > loss_mel_ce: 3.108388662338257 (3.2230357261545906) + | > loss: 0.39540526270866394 (0.40961071541560495) + | > current_lr: 5e-06 + | > step_time: 0.4407 (0.6050107980517307) + | > loader_time: 0.01 (0.010605064980707816) + + + --> TIME: 2024-12-22 20:01:20 -- STEP: 2746/2902 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.05507945269346237 (0.053841534410505) + | > loss_mel_ce: 3.3420369625091553 (3.2224513486798254) + | > loss: 0.4246395528316498 (0.40953661047752443) + | > current_lr: 5e-06 + | > step_time: 0.6173 (0.6048634139100554) + | > loader_time: 0.0111 (0.010611069176375911) + + + --> TIME: 2024-12-22 20:02:08 -- STEP: 2796/2902 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.051820311695337296 (0.053828303700843266) + | > loss_mel_ce: 3.2003228664398193 (3.2219526886257808) + | > loss: 0.406517893075943 (0.40947262414108854) + | > current_lr: 5e-06 + | > step_time: 0.6264 (0.6049439221492642) + | > loader_time: 0.0108 (0.010617568598625836) + + + --> TIME: 2024-12-22 20:02:55 -- STEP: 2846/2902 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.05038287490606308 (0.0538107970950684) + | > loss_mel_ce: 3.0824766159057617 (3.22168692776793) + | > loss: 0.3916074335575104 (0.4094372156926299) + | > current_lr: 5e-06 + | > step_time: 0.5422 (0.6049977293001112) + | > loader_time: 0.0104 (0.010621224585671881) + + + --> TIME: 2024-12-22 20:03:42 -- STEP: 2896/2902 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.05103882774710655 (0.053802881721736494) + | > loss_mel_ce: 3.1896164417266846 (3.221427488063578) + | > loss: 0.4050818979740143 (0.4094037962820153) + | > current_lr: 5e-06 + | > step_time: 0.5014 (0.60504338013533) + | > loader_time: 0.0104 (0.010624508749055609) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11469822218923863 (-0.0003729263941446015) + | > avg_loss_text_ce: 0.05099275098605589 (-0.0014854716231124665) + | > avg_loss_mel_ce: 3.170435007172402 (-0.03167040781541308) + | > avg_loss: 3.2214277597388823 (-0.033155873568372574) + + + > EPOCH: 3/5 + --> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > TRAINING (2024-12-22 20:05:08)  + + --> TIME: 2024-12-22 20:05:50 -- STEP: 44/2902 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.05118555575609207 (0.053070816888727924) + | > loss_mel_ce: 3.2175662517547607 (3.2166340080174534) + | > loss: 0.4085939824581146 (0.40871310301802377) + | > current_lr: 5e-06 + | > step_time: 0.6559 (0.6008733131668783) + | > loader_time: 0.0103 (0.010414784604852848) + + + --> TIME: 2024-12-22 20:06:38 -- STEP: 94/2902 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.05431003123521805 (0.05314608088674697) + | > loss_mel_ce: 3.2067341804504395 (3.222269390491729) + | > loss: 0.4076305329799652 (0.409426933590402) + | > current_lr: 5e-06 + | > step_time: 0.5549 (0.6045856957739972) + | > loader_time: 0.0099 (0.010405466911640572) + + + --> TIME: 2024-12-22 20:07:25 -- STEP: 144/2902 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.057936280965805054 (0.05344269391045803) + | > loss_mel_ce: 3.121586322784424 (3.2203697181410256) + | > loss: 0.3974403142929077 (0.409226551031073) + | > current_lr: 5e-06 + | > step_time: 0.5058 (0.6065922164254718) + | > loader_time: 0.0099 (0.010360252526071334) + + + --> TIME: 2024-12-22 20:08:11 -- STEP: 194/2902 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.051768817007541656 (0.05326584741934059) + | > loss_mel_ce: 3.128735303878784 (3.2146200224296333) + | > loss: 0.39756301045417786 (0.4084857332030523) + | > current_lr: 5e-06 + | > step_time: 0.4981 (0.6034614535951123) + | > loader_time: 0.0094 (0.010401970332430815) + + + --> TIME: 2024-12-22 20:08:59 -- STEP: 244/2902 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.0531751923263073 (0.05316113559987213) + | > loss_mel_ce: 3.196899890899658 (3.213496774923606) + | > loss: 0.4062593877315521 (0.40833223892039944) + | > current_lr: 5e-06 + | > step_time: 0.6879 (0.6043678590508755) + | > loader_time: 0.0105 (0.010358643336374256) + + + --> TIME: 2024-12-22 20:09:46 -- STEP: 294/2902 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.053236380219459534 (0.05315211574946131) + | > loss_mel_ce: 3.101793050765991 (3.210114073591167) + | > loss: 0.3943786919116974 (0.4079082736960887) + | > current_lr: 5e-06 + | > step_time: 0.6839 (0.6065582020753093) + | > loader_time: 0.0105 (0.010334769884745283) + + + --> TIME: 2024-12-22 20:10:33 -- STEP: 344/2902 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.05260626599192619 (0.05303334600703661) + | > loss_mel_ce: 3.1087124347686768 (3.20666435984678) + | > loss: 0.3951648473739624 (0.4074622132750446) + | > current_lr: 5e-06 + | > step_time: 0.4613 (0.6061097487460737) + | > loader_time: 0.0096 (0.010324067154596021) + + + --> TIME: 2024-12-22 20:11:20 -- STEP: 394/2902 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.052920322865247726 (0.053000804972905796) + | > loss_mel_ce: 3.29287052154541 (3.2060848898088867) + | > loss: 0.41822385787963867 (0.40738571181817723) + | > current_lr: 5e-06 + | > step_time: 0.6541 (0.6054662000104255) + | > loader_time: 0.0105 (0.010301449577215363) + + + --> TIME: 2024-12-22 20:12:07 -- STEP: 444/2902 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.049667783081531525 (0.052979113911656106) + | > loss_mel_ce: 3.1026852130889893 (3.20582131061468) + | > loss: 0.3940441310405731 (0.40735005298713317) + | > current_lr: 5e-06 + | > step_time: 0.4883 (0.6050507517548295) + | > loader_time: 0.0094 (0.01028660664687286) + + + --> TIME: 2024-12-22 20:12:54 -- STEP: 494/2902 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.04981204494833946 (0.052981702980362935) + | > loss_mel_ce: 3.082789182662964 (3.206828746718433) + | > loss: 0.3915751576423645 (0.40747630620292336) + | > current_lr: 5e-06 + | > step_time: 0.6535 (0.6049649213489733) + | > loader_time: 0.0106 (0.010281438286970505) + + + --> TIME: 2024-12-22 20:13:41 -- STEP: 544/2902 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.054514117538928986 (0.05301835546133055) + | > loss_mel_ce: 3.2055437564849854 (3.2056176938554812) + | > loss: 0.4075072407722473 (0.40732950602165047) + | > current_lr: 5e-06 + | > step_time: 0.3799 (0.6048707545680159) + | > loader_time: 0.0091 (0.01027688208748313) + + + --> TIME: 2024-12-22 20:14:27 -- STEP: 594/2902 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.05195319652557373 (0.05302652499014486) + | > loss_mel_ce: 3.207526922225952 (3.204179445099749) + | > loss: 0.40743499994277954 (0.4071507459437406) + | > current_lr: 5e-06 + | > step_time: 0.5395 (0.6040624107174604) + | > loader_time: 0.01 (0.01028768141261656) + + + --> TIME: 2024-12-22 20:15:15 -- STEP: 644/2902 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.05081777274608612 (0.05300852938653526) + | > loss_mel_ce: 3.2010488510131836 (3.204077336358725) + | > loss: 0.40648332238197327 (0.4071357330272656) + | > current_lr: 5e-06 + | > step_time: 0.6665 (0.604685469813969) + | > loader_time: 0.0108 (0.010288058600810744) + + + --> TIME: 2024-12-22 20:16:01 -- STEP: 694/2902 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.053493183106184006 (0.0530088954156947) + | > loss_mel_ce: 3.2406392097473145 (3.2033663084596284) + | > loss: 0.41176655888557434 (0.4070469001442279) + | > current_lr: 5e-06 + | > step_time: 0.5094 (0.6040704161014617) + | > loader_time: 0.0095 (0.010290941175191477) + + + --> TIME: 2024-12-22 20:16:48 -- STEP: 744/2902 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.05268382281064987 (0.05297802821783135) + | > loss_mel_ce: 3.216268301010132 (3.202414394386351) + | > loss: 0.4086190164089203 (0.4069240525845558) + | > current_lr: 5e-06 + | > step_time: 0.6145 (0.6038787175891229) + | > loader_time: 0.0111 (0.0103129840666248) + + + --> TIME: 2024-12-22 20:17:35 -- STEP: 794/2902 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.05258984863758087 (0.05294387003111304) + | > loss_mel_ce: 3.1116855144500732 (3.201499309587837) + | > loss: 0.3955344259738922 (0.4068053971309805) + | > current_lr: 5e-06 + | > step_time: 0.6243 (0.6037636656004484) + | > loader_time: 0.0104 (0.010329933550856275) + + + --> TIME: 2024-12-22 20:18:22 -- STEP: 844/2902 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.04872863367199898 (0.052906146908611534) + | > loss_mel_ce: 3.1472809314727783 (3.2004845972874696) + | > loss: 0.3995012044906616 (0.4066738426261603) + | > current_lr: 5e-06 + | > step_time: 0.6961 (0.6039798511148075) + | > loader_time: 0.0112 (0.010345070565481321) + + + --> TIME: 2024-12-22 20:19:10 -- STEP: 894/2902 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.05376376956701279 (0.052866009224447945) + | > loss_mel_ce: 3.118699550628662 (3.2001385982137913) + | > loss: 0.39655792713165283 (0.4066255754339081) + | > current_lr: 5e-06 + | > step_time: 0.6157 (0.6042345548902859) + | > loader_time: 0.0104 (0.010358344788519328) + + + --> TIME: 2024-12-22 20:19:56 -- STEP: 944/2902 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.05543249472975731 (0.05286841211698449) + | > loss_mel_ce: 3.1901323795318604 (3.199281110349346) + | > loss: 0.4056956171989441 (0.4065186898460832) + | > current_lr: 5e-06 + | > step_time: 0.3571 (0.6036462536302674) + | > loader_time: 0.0096 (0.010371917637728028) + + + --> TIME: 2024-12-22 20:20:43 -- STEP: 994/2902 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.05254547297954559 (0.05283306972301706) + | > loss_mel_ce: 3.26566481590271 (3.198850807769436) + | > loss: 0.4147762954235077 (0.40646048432864407) + | > current_lr: 5e-06 + | > step_time: 0.6149 (0.6033839968368571) + | > loader_time: 0.0107 (0.010391604252744248) + + + --> TIME: 2024-12-22 20:21:29 -- STEP: 1044/2902 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.051356155425310135 (0.05281972371298695) + | > loss_mel_ce: 3.2884762287139893 (3.1981828160212853) + | > loss: 0.4174790382385254 (0.4063753171099556) + | > current_lr: 5e-06 + | > step_time: 0.6524 (0.603219235993894) + | > loader_time: 0.0111 (0.010409164702755281) + + + --> TIME: 2024-12-22 20:22:17 -- STEP: 1094/2902 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.05341365188360214 (0.052800002707069756) + | > loss_mel_ce: 3.23124623298645 (3.198679782354851) + | > loss: 0.4105824828147888 (0.40643497279945634) + | > current_lr: 5e-06 + | > step_time: 0.638 (0.6037313306135502) + | > loader_time: 0.0117 (0.010436894034990664) + + + --> TIME: 2024-12-22 20:23:04 -- STEP: 1144/2902 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.05161362886428833 (0.05279735522995987) + | > loss_mel_ce: 3.176171064376831 (3.1983392276130336) + | > loss: 0.4034730792045593 (0.4063920725423555) + | > current_lr: 5e-06 + | > step_time: 0.5035 (0.603374870715442) + | > loader_time: 0.0107 (0.010455987878612703) + + + --> TIME: 2024-12-22 20:23:50 -- STEP: 1194/2902 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.053719062358140945 (0.05276699765106444) + | > loss_mel_ce: 3.240354299545288 (3.1977713854868) + | > loss: 0.4117591679096222 (0.40631729763756236) + | > current_lr: 5e-06 + | > step_time: 0.6053 (0.6026648664394623) + | > loader_time: 0.0107 (0.010469512124756475) + + + --> TIME: 2024-12-22 20:24:37 -- STEP: 1244/2902 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.05204193294048309 (0.0527393467712345) + | > loss_mel_ce: 3.062506914138794 (3.197535026878406) + | > loss: 0.3893186151981354 (0.4062842964598985) + | > current_lr: 5e-06 + | > step_time: 0.6637 (0.6031234285455812) + | > loader_time: 0.0112 (0.010495160170306727) + + + --> TIME: 2024-12-22 20:25:25 -- STEP: 1294/2902 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.053739190101623535 (0.052719865308087074) + | > loss_mel_ce: 3.235835313796997 (3.1971923310016006) + | > loss: 0.41119682788848877 (0.4062390243389874) + | > current_lr: 5e-06 + | > step_time: 0.6554 (0.6032189197112826) + | > loader_time: 0.0112 (0.010511806243354049) + + + --> TIME: 2024-12-22 20:26:12 -- STEP: 1344/2902 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.05264301598072052 (0.05269939124506589) + | > loss_mel_ce: 3.1769728660583496 (3.1967695257848225) + | > loss: 0.4037019908428192 (0.40618361446208173) + | > current_lr: 5e-06 + | > step_time: 0.5749 (0.6032951499025029) + | > loader_time: 0.0109 (0.010520114075569876) + + + --> TIME: 2024-12-22 20:26:59 -- STEP: 1394/2902 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.05526312068104744 (0.05267769771185805) + | > loss_mel_ce: 3.0526609420776367 (3.1962102015357114) + | > loss: 0.3884904980659485 (0.40611098721687544) + | > current_lr: 5e-06 + | > step_time: 0.5445 (0.6036030539823234) + | > loader_time: 0.0101 (0.010525740715147253) + + + --> TIME: 2024-12-22 20:27:46 -- STEP: 1444/2902 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.05535884574055672 (0.052661094868715626) + | > loss_mel_ce: 3.212167739868164 (3.195650814973087) + | > loss: 0.40844082832336426 (0.4060389885983307) + | > current_lr: 5e-06 + | > step_time: 0.5163 (0.6032200695074831) + | > loader_time: 0.0104 (0.010530813083754342) + + + --> TIME: 2024-12-22 20:28:32 -- STEP: 1494/2902 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.0518801212310791 (0.05264195443007602) + | > loss_mel_ce: 3.2703099250793457 (3.195997038519527) + | > loss: 0.4152737557888031 (0.4060798740011939) + | > current_lr: 5e-06 + | > step_time: 0.6595 (0.6027859074364076) + | > loader_time: 0.0111 (0.01054092423823304) + + + --> TIME: 2024-12-22 20:29:18 -- STEP: 1544/2902 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.049537673592567444 (0.052636940536509066) + | > loss_mel_ce: 3.1364898681640625 (3.195794070477313) + | > loss: 0.3982534408569336 (0.4060538762630266) + | > current_lr: 5e-06 + | > step_time: 0.4407 (0.6026072866558418) + | > loader_time: 0.0101 (0.010545256224320955) + + + --> TIME: 2024-12-22 20:30:05 -- STEP: 1594/2902 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.05353932827711105 (0.05263680126868572) + | > loss_mel_ce: 3.1715071201324463 (3.1952996852257325) + | > loss: 0.40313079953193665 (0.4059920606753756) + | > current_lr: 5e-06 + | > step_time: 0.5432 (0.6026972758126834) + | > loader_time: 0.0105 (0.010556090132355535) + + + --> TIME: 2024-12-22 20:30:52 -- STEP: 1644/2902 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.0515349842607975 (0.05263029766062824) + | > loss_mel_ce: 3.0010695457458496 (3.1949205655251114) + | > loss: 0.3815755546092987 (0.40594385780049635) + | > current_lr: 5e-06 + | > step_time: 0.6357 (0.6025581902251055) + | > loader_time: 0.0106 (0.010555189311359335) + + + --> TIME: 2024-12-22 20:31:39 -- STEP: 1694/2902 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.050352662801742554 (0.05262529455705897) + | > loss_mel_ce: 3.185516834259033 (3.1946503501011607) + | > loss: 0.4044836759567261 (0.4059094554769949) + | > current_lr: 5e-06 + | > step_time: 0.6068 (0.6024062957223231) + | > loader_time: 0.0111 (0.010554100856308105) + + + --> TIME: 2024-12-22 20:32:26 -- STEP: 1744/2902 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.052729882299900055 (0.0525995482361679) + | > loss_mel_ce: 3.1776161193847656 (3.194501636908689) + | > loss: 0.40379324555397034 (0.4058876480531254) + | > current_lr: 5e-06 + | > step_time: 0.5961 (0.6025670185821876) + | > loader_time: 0.0107 (0.01056057829922492) + + + --> TIME: 2024-12-22 20:33:13 -- STEP: 1794/2902 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.05042431131005287 (0.05259022799630961) + | > loss_mel_ce: 3.1463003158569336 (3.1944169817428) + | > loss: 0.3995905816555023 (0.40587590109876687) + | > current_lr: 5e-06 + | > step_time: 0.5049 (0.6027706765809595) + | > loader_time: 0.01 (0.010568745160182578) + + + --> TIME: 2024-12-22 20:34:01 -- STEP: 1844/2902 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.05469150468707085 (0.05258134340184566) + | > loss_mel_ce: 3.293409824371338 (3.194005653553051) + | > loss: 0.4185126721858978 (0.40582337445395616) + | > current_lr: 5e-06 + | > step_time: 0.5551 (0.6029681765076406) + | > loader_time: 0.0106 (0.010575436847587466) + + + --> TIME: 2024-12-22 20:34:48 -- STEP: 1894/2902 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.05276293680071831 (0.052574189625672584) + | > loss_mel_ce: 3.239795446395874 (3.19401058891381) + | > loss: 0.4115698039531708 (0.40582309715221637) + | > current_lr: 5e-06 + | > step_time: 0.6411 (0.6032363728963344) + | > loader_time: 0.011 (0.010578809217515436) + + + --> TIME: 2024-12-22 20:35:36 -- STEP: 1944/2902 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.049916088581085205 (0.05257261406683935) + | > loss_mel_ce: 3.2820544242858887 (3.193959850955893) + | > loss: 0.41649630665779114 (0.40581655793597166) + | > current_lr: 5e-06 + | > step_time: 0.4468 (0.6033671674905009) + | > loader_time: 0.0091 (0.010581779504509125) + + + --> TIME: 2024-12-22 20:36:23 -- STEP: 1994/2902 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.053316082805395126 (0.05255969537005442) + | > loss_mel_ce: 3.1072206497192383 (3.193483378368252) + | > loss: 0.39506709575653076 (0.4057553840377026) + | > current_lr: 5e-06 + | > step_time: 0.6845 (0.60350839544084) + | > loader_time: 0.0105 (0.0105668879081397) + + + --> TIME: 2024-12-22 20:37:10 -- STEP: 2044/2902 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.04926644265651703 (0.05255933851550412) + | > loss_mel_ce: 3.3206586837768555 (3.193047124462352) + | > loss: 0.4212406277656555 (0.4057008076291494) + | > current_lr: 5e-06 + | > step_time: 0.5429 (0.6033082252145989) + | > loader_time: 0.0109 (0.010563473062272637) + + + --> TIME: 2024-12-22 20:37:57 -- STEP: 2094/2902 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.0534113310277462 (0.05254988536174627) + | > loss_mel_ce: 3.1378417015075684 (3.192547170116203) + | > loss: 0.3989066183567047 (0.4056371316925729) + | > current_lr: 5e-06 + | > step_time: 0.4968 (0.6034088593613234) + | > loader_time: 0.0105 (0.010568689594068405) + + + --> TIME: 2024-12-22 20:38:44 -- STEP: 2144/2902 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.05344215780496597 (0.05255435538867406) + | > loss_mel_ce: 3.1524195671081543 (3.1925554472341475) + | > loss: 0.40073272585868835 (0.40563872507981835) + | > current_lr: 5e-06 + | > step_time: 0.5771 (0.6033291484215376) + | > loader_time: 0.0115 (0.010582063958716036) + + + --> TIME: 2024-12-22 20:39:30 -- STEP: 2194/2902 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.051915958523750305 (0.052540452967490436) + | > loss_mel_ce: 3.1636173725128174 (3.1927434840632665) + | > loss: 0.4019416570663452 (0.40566049191554) + | > current_lr: 5e-06 + | > step_time: 0.5515 (0.6030407171414578) + | > loader_time: 0.0109 (0.0105995729823708) + + + --> TIME: 2024-12-22 20:40:18 -- STEP: 2244/2902 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.052167460322380066 (0.0525299172994553) + | > loss_mel_ce: 3.199925184249878 (3.1927137905051146) + | > loss: 0.4065115749835968 (0.4056554632709626) + | > current_lr: 5e-06 + | > step_time: 0.6492 (0.6033633877249335) + | > loader_time: 0.0117 (0.01061397854657097) + + + --> TIME: 2024-12-22 20:41:04 -- STEP: 2294/2902 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.04624050855636597 (0.05252224427129046) + | > loss_mel_ce: 3.2247681617736816 (3.1923680543483797) + | > loss: 0.40887609124183655 (0.40561128713664757) + | > current_lr: 5e-06 + | > step_time: 0.5472 (0.6029953439025756) + | > loader_time: 0.0107 (0.01062412925045076) + + + --> TIME: 2024-12-22 20:41:50 -- STEP: 2344/2902 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.05163954943418503 (0.052516960615209565) + | > loss_mel_ce: 3.2304461002349854 (3.1922061061899796) + | > loss: 0.41026070713996887 (0.4055903831388763) + | > current_lr: 5e-06 + | > step_time: 0.5826 (0.602789519996774) + | > loader_time: 0.0112 (0.010636577528898213) + + + --> TIME: 2024-12-22 20:42:37 -- STEP: 2394/2902 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.053269702941179276 (0.05250816199066448) + | > loss_mel_ce: 3.2108967304229736 (3.1919024122089956) + | > loss: 0.4080207943916321 (0.40555132153687523) + | > current_lr: 5e-06 + | > step_time: 0.5341 (0.6027643641732401) + | > loader_time: 0.0111 (0.010651086705270757) + + + --> TIME: 2024-12-22 20:43:24 -- STEP: 2444/2902 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.052502501755952835 (0.05249957667294825) + | > loss_mel_ce: 3.2452847957611084 (3.1916058965868737) + | > loss: 0.4122233986854553 (0.4055131839042599) + | > current_lr: 5e-06 + | > step_time: 0.6571 (0.6027423118023176) + | > loader_time: 0.0114 (0.010663800953648083) + + + --> TIME: 2024-12-22 20:44:11 -- STEP: 2494/2902 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.05313148349523544 (0.05248425879128094) + | > loss_mel_ce: 3.137269973754883 (3.191296629553905) + | > loss: 0.39880019426345825 (0.4054726108418718) + | > current_lr: 5e-06 + | > step_time: 0.5558 (0.6027711930041723) + | > loader_time: 0.0112 (0.010675928931094774) + + + --> TIME: 2024-12-22 20:44:59 -- STEP: 2544/2902 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.05024084821343422 (0.05247156317902251) + | > loss_mel_ce: 3.179081439971924 (3.1910943921257107) + | > loss: 0.4036652743816376 (0.40544574424249563) + | > current_lr: 5e-06 + | > step_time: 0.5751 (0.6028636446538974) + | > loader_time: 0.0115 (0.010689315181108381) + + + --> TIME: 2024-12-22 20:45:46 -- STEP: 2594/2902 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.05368289723992348 (0.05246220678579332) + | > loss_mel_ce: 3.191403388977051 (3.1909259006254644) + | > loss: 0.4056357741355896 (0.40542351324096854) + | > current_lr: 5e-06 + | > step_time: 0.5432 (0.602958301350809) + | > loader_time: 0.0108 (0.010702524078563248) + + + --> TIME: 2024-12-22 20:46:33 -- STEP: 2644/2902 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.053973011672496796 (0.052453115493701664) + | > loss_mel_ce: 3.285907506942749 (3.190714084615143) + | > loss: 0.4174850583076477 (0.4053958998302648) + | > current_lr: 5e-06 + | > step_time: 0.658 (0.6029926955970453) + | > loader_time: 0.0119 (0.01071605108869959) + + + --> TIME: 2024-12-22 20:47:21 -- STEP: 2694/2902 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.05342477560043335 (0.05244395500650519) + | > loss_mel_ce: 3.285519599914551 (3.190686175446201) + | > loss: 0.4173680543899536 (0.4053912661366756) + | > current_lr: 5e-06 + | > step_time: 0.6651 (0.6032455267513263) + | > loader_time: 0.0117 (0.010733391146528872) + + + --> TIME: 2024-12-22 20:48:08 -- STEP: 2744/2902 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.047638945281505585 (0.05242956084343159) + | > loss_mel_ce: 3.2169618606567383 (3.1906095680620497) + | > loss: 0.40807509422302246 (0.4053798909394108) + | > current_lr: 5e-06 + | > step_time: 0.5872 (0.6032850741471233) + | > loader_time: 0.0117 (0.01075017017803804) + + + --> TIME: 2024-12-22 20:48:56 -- STEP: 2794/2902 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.05027558282017708 (0.05241230539524224) + | > loss_mel_ce: 3.273003339767456 (3.190310692053311) + | > loss: 0.4154098629951477 (0.4053403744974047) + | > current_lr: 5e-06 + | > step_time: 0.6852 (0.6034290926236311) + | > loader_time: 0.0123 (0.010766002904882409) + + + --> TIME: 2024-12-22 20:49:44 -- STEP: 2844/2902 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.05199229717254639 (0.05240542293197455) + | > loss_mel_ce: 3.114150047302246 (3.19000683672485) + | > loss: 0.39576780796051025 (0.4053015322666798) + | > current_lr: 5e-06 + | > step_time: 0.709 (0.6036963121632833) + | > loader_time: 0.0122 (0.0107818767323608) + + + --> TIME: 2024-12-22 20:50:31 -- STEP: 2894/2902 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.054400309920310974 (0.05239259631970916) + | > loss_mel_ce: 3.325909376144409 (3.189384226808896) + | > loss: 0.422538697719574 (0.405222102701046) + | > current_lr: 5e-06 + | > step_time: 0.6677 (0.6037184401224148) + | > loader_time: 0.0114 (0.01079611524517321) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11509281095832281 (+0.0003945887690841865) + | > avg_loss_text_ce: 0.04980169842490041 (-0.0011910525611554812) + | > avg_loss_mel_ce: 3.1442968448003126 (-0.026138162372089546) + | > avg_loss: 3.1940985436391354 (-0.027329216099746922) + + > BEST MODEL : /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000/best_model_11608.pth + + > EPOCH: 4/5 + --> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 + + > TRAINING (2024-12-22 20:52:03)  + + --> TIME: 2024-12-22 20:52:44 -- STEP: 42/2902 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.05242399126291275 (0.05177849477955273) + | > loss_mel_ce: 3.0292856693267822 (3.157657742500305) + | > loss: 0.385213702917099 (0.40117952866213663) + | > current_lr: 5e-06 + | > step_time: 0.6336 (0.6046405576524281) + | > loader_time: 0.0103 (0.01020863510313488) + + + --> TIME: 2024-12-22 20:53:31 -- STEP: 92/2902 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.052793268114328384 (0.05141946446636449) + | > loss_mel_ce: 3.267806053161621 (3.165517534898675) + | > loss: 0.4150749146938324 (0.4021171247181685) + | > current_lr: 5e-06 + | > step_time: 0.6089 (0.6012791395187376) + | > loader_time: 0.0104 (0.010277999484020733) + + + --> TIME: 2024-12-22 20:54:18 -- STEP: 142/2902 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.050963904708623886 (0.05146324702761542) + | > loss_mel_ce: 3.2492852210998535 (3.1661211789493824) + | > loss: 0.41253113746643066 (0.40219805274211184) + | > current_lr: 5e-06 + | > step_time: 0.6675 (0.6018522910668817) + | > loader_time: 0.0105 (0.010207364256952853) + + + --> TIME: 2024-12-22 20:55:05 -- STEP: 192/2902 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.04937686026096344 (0.05135606706608087) + | > loss_mel_ce: 3.152395248413086 (3.1661600942413006) + | > loss: 0.4002215266227722 (0.4021895203428964) + | > current_lr: 5e-06 + | > step_time: 0.5679 (0.6053141777714087) + | > loader_time: 0.0099 (0.010187438378731413) + + + --> TIME: 2024-12-22 20:55:52 -- STEP: 242/2902 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.04982029274106026 (0.051418416833212556) + | > loss_mel_ce: 3.183880567550659 (3.1688949736681846) + | > loss: 0.4042125940322876 (0.4025391741725039) + | > current_lr: 5e-06 + | > step_time: 0.6461 (0.6039103515877204) + | > loader_time: 0.011 (0.010216754330091244) + + + --> TIME: 2024-12-22 20:56:39 -- STEP: 292/2902 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.05215315520763397 (0.0514818048487379) + | > loss_mel_ce: 3.3319084644317627 (3.1706454353789764) + | > loss: 0.42300769686698914 (0.4027659054080101) + | > current_lr: 5e-06 + | > step_time: 0.5685 (0.6028565535806625) + | > loader_time: 0.0114 (0.010344660445435411) + + + --> TIME: 2024-12-22 20:57:26 -- STEP: 342/2902 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.05081651732325554 (0.05145429278457024) + | > loss_mel_ce: 3.119997262954712 (3.1713683577308878) + | > loss: 0.39635172486305237 (0.4028528318767659) + | > current_lr: 5e-06 + | > step_time: 0.62 (0.6045072148417866) + | > loader_time: 0.0111 (0.010437014507271393) + + + --> TIME: 2024-12-22 20:58:14 -- STEP: 392/2902 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.050988949835300446 (0.05142504952809943) + | > loss_mel_ce: 3.1205058097839355 (3.16914310382337) + | > loss: 0.3964368402957916 (0.40257101977358056) + | > current_lr: 5e-06 + | > step_time: 0.5909 (0.6051843859711471) + | > loader_time: 0.0115 (0.010557485478264945) + + + --> TIME: 2024-12-22 20:59:01 -- STEP: 442/2902 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.05060780048370361 (0.051467611888482555) + | > loss_mel_ce: 3.14922833442688 (3.1712528445602004) + | > loss: 0.39997953176498413 (0.40284005764922404) + | > current_lr: 5e-06 + | > step_time: 0.5165 (0.6043184364543239) + | > loader_time: 0.0101 (0.010559999565193556) + + + --> TIME: 2024-12-22 20:59:48 -- STEP: 492/2902 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.052262645214796066 (0.05145950533631371) + | > loss_mel_ce: 3.0933821201324463 (3.1706015079971253) + | > loss: 0.39320558309555054 (0.40275762751092753) + | > current_lr: 5e-06 + | > step_time: 0.6487 (0.6053038733761483) + | > loader_time: 0.0103 (0.010541675536613154) + + + --> TIME: 2024-12-22 21:00:36 -- STEP: 542/2902 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.05357429012656212 (0.05147910170339569) + | > loss_mel_ce: 3.2249159812927246 (3.1703968549566506) + | > loss: 0.4098112881183624 (0.4027344952870119) + | > current_lr: 5e-06 + | > step_time: 0.543 (0.6057071637403478) + | > loader_time: 0.01 (0.01051323088332736) + + + --> TIME: 2024-12-22 21:01:23 -- STEP: 592/2902 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.04849829524755478 (0.051450166826111234) + | > loss_mel_ce: 3.157019853591919 (3.1691637586902925) + | > loss: 0.4006897807121277 (0.4025767413848961) + | > current_lr: 5e-06 + | > step_time: 0.6251 (0.606176301031499) + | > loader_time: 0.0106 (0.010490711073617685) + + + --> TIME: 2024-12-22 21:02:11 -- STEP: 642/2902 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.05318274348974228 (0.05145047221483657) + | > loss_mel_ce: 3.197951555252075 (3.1695703424025923) + | > loss: 0.40639179944992065 (0.4026276024516871) + | > current_lr: 5e-06 + | > step_time: 0.6083 (0.6072793760775032) + | > loader_time: 0.01 (0.010462552216194134) + + + --> TIME: 2024-12-22 21:02:58 -- STEP: 692/2902 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.04917445406317711 (0.05141469234912899) + | > loss_mel_ce: 3.0380802154541016 (3.16752684770981) + | > loss: 0.385906845331192 (0.40236769310828596) + | > current_lr: 5e-06 + | > step_time: 0.5506 (0.6069841088587149) + | > loader_time: 0.0104 (0.010431977365747355) + + + --> TIME: 2024-12-22 21:03:46 -- STEP: 742/2902 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.0537167526781559 (0.051418948411499715) + | > loss_mel_ce: 3.1005213260650635 (3.1670297853387592) + | > loss: 0.39427974820137024 (0.40230609211478285) + | > current_lr: 5e-06 + | > step_time: 0.5324 (0.607316173954473) + | > loader_time: 0.0098 (0.010424615237912084) + + + --> TIME: 2024-12-22 21:04:33 -- STEP: 792/2902 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.05441666394472122 (0.051431453706828334) + | > loss_mel_ce: 3.1533031463623047 (3.1675818153101987) + | > loss: 0.40096497535705566 (0.4023766589887216) + | > current_lr: 5e-06 + | > step_time: 0.3769 (0.60757312028095) + | > loader_time: 0.0092 (0.01040240610488738) + + + --> TIME: 2024-12-22 21:05:21 -- STEP: 842/2902 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.051836397498846054 (0.051435001375874825) + | > loss_mel_ce: 3.162421464920044 (3.167984573121875) + | > loss: 0.40178224444389343 (0.4024274471678248) + | > current_lr: 5e-06 + | > step_time: 0.608 (0.6077996503712163) + | > loader_time: 0.0101 (0.010387159016806273) + + + --> TIME: 2024-12-22 21:06:08 -- STEP: 892/2902 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.050409939140081406 (0.05143925132054518) + | > loss_mel_ce: 3.1199047565460205 (3.1676170697661257) + | > loss: 0.3962893486022949 (0.40238204050491755) + | > current_lr: 5e-06 + | > step_time: 0.6262 (0.6074925163936188) + | > loader_time: 0.0109 (0.01040899967398879) + + + --> TIME: 2024-12-22 21:06:54 -- STEP: 942/2902 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.054377250373363495 (0.05144039717465323) + | > loss_mel_ce: 3.2185378074645996 (3.1676341611615157) + | > loss: 0.40911439061164856 (0.4023843201355824) + | > current_lr: 5e-06 + | > step_time: 0.5024 (0.6071113489742238) + | > loader_time: 0.0099 (0.010440987133422985) + + + --> TIME: 2024-12-22 21:07:42 -- STEP: 992/2902 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.04666450247168541 (0.05143451606929903) + | > loss_mel_ce: 3.063586473464966 (3.167725384956406) + | > loss: 0.3887813687324524 (0.4023949879732345) + | > current_lr: 5e-06 + | > step_time: 0.5517 (0.607087874124127) + | > loader_time: 0.0109 (0.010459479064710682) + + + --> TIME: 2024-12-22 21:08:29 -- STEP: 1042/2902 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.050158604979515076 (0.05141120305331335) + | > loss_mel_ce: 3.0921571254730225 (3.167083098426204) + | > loss: 0.39278945326805115 (0.4023117879530749) + | > current_lr: 5e-06 + | > step_time: 0.6452 (0.6070227060116642) + | > loader_time: 0.0119 (0.01050893679232607) + + + --> TIME: 2024-12-22 21:09:16 -- STEP: 1092/2902 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.053189586848020554 (0.05139344183885025) + | > loss_mel_ce: 3.1477901935577393 (3.1663511314671555) + | > loss: 0.4001224637031555 (0.40221807196900083) + | > current_lr: 5e-06 + | > step_time: 0.6439 (0.6070524135352056) + | > loader_time: 0.0118 (0.010553024190686133) + + + --> TIME: 2024-12-22 21:10:04 -- STEP: 1142/2902 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.05265435203909874 (0.051385245982381374) + | > loss_mel_ce: 3.2024030685424805 (3.165868102147158) + | > loss: 0.4068821668624878 (0.4021566687396445) + | > current_lr: 5e-06 + | > step_time: 0.6971 (0.6074808042646498) + | > loader_time: 0.011 (0.010573815130310598) + + + --> TIME: 2024-12-22 21:10:52 -- STEP: 1192/2902 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.05063112452626228 (0.051397483155206335) + | > loss_mel_ce: 3.160621166229248 (3.1657690583859512) + | > loss: 0.40140652656555176 (0.402145817875862) + | > current_lr: 5e-06 + | > step_time: 0.5908 (0.6079006377082538) + | > loader_time: 0.0106 (0.01057791729901461) + + + --> TIME: 2024-12-22 21:11:39 -- STEP: 1242/2902 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.049502015113830566 (0.05138907669750962) + | > loss_mel_ce: 3.150031805038452 (3.165537585766804) + | > loss: 0.39994174242019653 (0.40211583297225795) + | > current_lr: 5e-06 + | > step_time: 0.5612 (0.6078281895933898) + | > loader_time: 0.01 (0.010567935000679155) + + + --> TIME: 2024-12-22 21:12:25 -- STEP: 1292/2902 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.05091576278209686 (0.05138804009270739) + | > loss_mel_ce: 3.257941484451294 (3.165761600337899) + | > loss: 0.4136071503162384 (0.4021437052095272) + | > current_lr: 5e-06 + | > step_time: 0.6488 (0.6072963262120997) + | > loader_time: 0.0104 (0.010560713869868423) + + + --> TIME: 2024-12-22 21:13:12 -- STEP: 1342/2902 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.05174795538187027 (0.05137953288220256) + | > loss_mel_ce: 3.145700693130493 (3.1660235461108552) + | > loss: 0.39968109130859375 (0.4021753851107801) + | > current_lr: 5e-06 + | > step_time: 0.6661 (0.6069889692187127) + | > loader_time: 0.0101 (0.010547872094212637) + + + --> TIME: 2024-12-22 21:14:00 -- STEP: 1392/2902 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.051908206194639206 (0.05137049605074369) + | > loss_mel_ce: 3.218794107437134 (3.165184286819107) + | > loss: 0.40883779525756836 (0.4020693480497465) + | > current_lr: 5e-06 + | > step_time: 0.6109 (0.6072547183982253) + | > loader_time: 0.0106 (0.010541640650266897) + + + --> TIME: 2024-12-22 21:14:46 -- STEP: 1442/2902 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.04997578635811806 (0.05138166409228693) + | > loss_mel_ce: 3.3202295303344727 (3.16502617648835) + | > loss: 0.42127567529678345 (0.40205098029313935) + | > current_lr: 5e-06 + | > step_time: 0.6776 (0.6069498730102618) + | > loader_time: 0.0105 (0.010534495156614863) + + + --> TIME: 2024-12-22 21:15:34 -- STEP: 1492/2902 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.050916023552417755 (0.0513936553010592) + | > loss_mel_ce: 3.1688055992126465 (3.1647604515341587) + | > loss: 0.40246519446372986 (0.4020192635922269) + | > current_lr: 5e-06 + | > step_time: 0.6073 (0.6068949724969521) + | > loader_time: 0.0098 (0.010515906855503937) + + + --> TIME: 2024-12-22 21:16:21 -- STEP: 1542/2902 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.05477416515350342 (0.05138281299629949) + | > loss_mel_ce: 3.264493942260742 (3.164743644718066) + | > loss: 0.4149085283279419 (0.4020158074090765) + | > current_lr: 5e-06 + | > step_time: 0.6556 (0.6069266871874115) + | > loader_time: 0.0101 (0.0104985054674161) + + + --> TIME: 2024-12-22 21:17:09 -- STEP: 1592/2902 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.05062420293688774 (0.05138683669390764) + | > loss_mel_ce: 3.109247922897339 (3.1645478925213744) + | > loss: 0.39498400688171387 (0.4019918413505186) + | > current_lr: 5e-06 + | > step_time: 0.586 (0.6073737449981449) + | > loader_time: 0.0101 (0.010484467499220199) + + + --> TIME: 2024-12-22 21:17:56 -- STEP: 1642/2902 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.04851849004626274 (0.051371706259435754) + | > loss_mel_ce: 3.08614444732666 (3.164616929918491) + | > loss: 0.3918328583240509 (0.40199857971990616) + | > current_lr: 5e-06 + | > step_time: 0.6683 (0.6072920318352599) + | > loader_time: 0.0103 (0.010469061593626077) + + + --> TIME: 2024-12-22 21:18:44 -- STEP: 1692/2902 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.051790542900562286 (0.05136777905760503) + | > loss_mel_ce: 3.2155044078826904 (3.164545286349935) + | > loss: 0.4084118604660034 (0.40198913338125203) + | > current_lr: 5e-06 + | > step_time: 0.624 (0.6075981038118357) + | > loader_time: 0.0102 (0.010463569603913226) + + + --> TIME: 2024-12-22 21:19:30 -- STEP: 1742/2902 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.05148990824818611 (0.05135159586886655) + | > loss_mel_ce: 3.149303674697876 (3.1640812554124027) + | > loss: 0.4000991880893707 (0.4019291066274854) + | > current_lr: 5e-06 + | > step_time: 0.6116 (0.6072361833329305) + | > loader_time: 0.011 (0.010460974702057696) + + + --> TIME: 2024-12-22 21:20:18 -- STEP: 1792/2902 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.05156005173921585 (0.051346307745136835) + | > loss_mel_ce: 3.0656654834747314 (3.1633247026641453) + | > loss: 0.38965320587158203 (0.4018338766181844) + | > current_lr: 5e-06 + | > step_time: 0.6007 (0.6074373084785673) + | > loader_time: 0.0105 (0.010466490872204309) + + + --> TIME: 2024-12-22 21:21:05 -- STEP: 1842/2902 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.05219785124063492 (0.051339257172795236) + | > loss_mel_ce: 3.2246766090393066 (3.1631171858660174) + | > loss: 0.409609317779541 (0.4018070556538892) + | > current_lr: 5e-06 + | > step_time: 0.6783 (0.6072862473165811) + | > loader_time: 0.0112 (0.010471460744172307) + + + --> TIME: 2024-12-22 21:21:53 -- STEP: 1892/2902 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.04937213659286499 (0.051324877614277474) + | > loss_mel_ce: 3.2181973457336426 (3.162961993736646) + | > loss: 0.40844619274139404 (0.40178585920412013) + | > current_lr: 5e-06 + | > step_time: 0.6131 (0.6075245814898039) + | > loader_time: 0.0101 (0.010493385993401046) + + + --> TIME: 2024-12-22 21:22:39 -- STEP: 1942/2902 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.04959951713681221 (0.05130738955486016) + | > loss_mel_ce: 3.0643162727355957 (3.1629433719063393) + | > loss: 0.38923946022987366 (0.4017813454787839) + | > current_lr: 5e-06 + | > step_time: 0.682 (0.6071882382971364) + | > loader_time: 0.0106 (0.01049361550599245) + + + --> TIME: 2024-12-22 21:23:26 -- STEP: 1992/2902 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.05019519850611687 (0.05130434854060829) + | > loss_mel_ce: 2.967393398284912 (3.162763780140015) + | > loss: 0.37719857692718506 (0.40175851636442816) + | > current_lr: 5e-06 + | > step_time: 0.5847 (0.6066965500274327) + | > loader_time: 0.0108 (0.010494193877561031) + + + --> TIME: 2024-12-22 21:24:13 -- STEP: 2042/2902 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.04995539039373398 (0.05128760228296567) + | > loss_mel_ce: 3.0459234714508057 (3.1625825939636156) + | > loss: 0.38698485493659973 (0.4017337748049291) + | > current_lr: 5e-06 + | > step_time: 0.5468 (0.6069906450275334) + | > loader_time: 0.0106 (0.010499159572874069) + + + --> TIME: 2024-12-22 21:25:00 -- STEP: 2092/2902 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.04973485693335533 (0.051272201517429436) + | > loss_mel_ce: 3.0299553871154785 (3.1628303902782857) + | > loss: 0.3849612772464752 (0.4017628242420195) + | > current_lr: 5e-06 + | > step_time: 0.5548 (0.6068838057508879) + | > loader_time: 0.0104 (0.010503764020325355) + + + --> TIME: 2024-12-22 21:25:48 -- STEP: 2142/2902 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.05021126568317413 (0.051260509819529745) + | > loss_mel_ce: 2.955406904220581 (3.1625539609809317) + | > loss: 0.37570226192474365 (0.40172680913288955) + | > current_lr: 5e-06 + | > step_time: 0.5523 (0.6070832329081319) + | > loader_time: 0.0112 (0.010521110366372504) + + + --> TIME: 2024-12-22 21:26:35 -- STEP: 2192/2902 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.04956125468015671 (0.051253811561631654) + | > loss_mel_ce: 3.1815543174743652 (3.1624464504692664) + | > loss: 0.4038894474506378 (0.40171253303215454) + | > current_lr: 5e-06 + | > step_time: 0.5867 (0.6070984596536095) + | > loader_time: 0.011 (0.010532227310821093) + + + --> TIME: 2024-12-22 21:27:23 -- STEP: 2242/2902 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.050890352576971054 (0.05124137310990786) + | > loss_mel_ce: 3.058953285217285 (3.1622287470990247) + | > loss: 0.3887304663658142 (0.4016837653148187) + | > current_lr: 5e-06 + | > step_time: 0.5351 (0.6072680375918427) + | > loader_time: 0.0102 (0.010540215273609555) + + + --> TIME: 2024-12-22 21:28:10 -- STEP: 2292/2902 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.05306136608123779 (0.05123496174578263) + | > loss_mel_ce: 3.2562832832336426 (3.1622462749065092) + | > loss: 0.41366809606552124 (0.4016851548749116) + | > current_lr: 5e-06 + | > step_time: 0.5083 (0.6071446213422633) + | > loader_time: 0.0101 (0.010546645152839275) + + + --> TIME: 2024-12-22 21:28:57 -- STEP: 2342/2902 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.04956350103020668 (0.05122417288837929) + | > loss_mel_ce: 3.125633955001831 (3.1618559527458223) + | > loss: 0.39689967036247253 (0.4016350159703107) + | > current_lr: 5e-06 + | > step_time: 0.6593 (0.6069975350271614) + | > loader_time: 0.0107 (0.010552020586015414) + + + --> TIME: 2024-12-22 21:29:44 -- STEP: 2392/2902 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.048557110130786896 (0.0512131784301263) + | > loss_mel_ce: 3.2124428749084473 (3.161652133217623) + | > loss: 0.4076249897480011 (0.40160816420729306) + | > current_lr: 5e-06 + | > step_time: 0.5845 (0.6070272774002609) + | > loader_time: 0.0109 (0.010562337860215857) + + + --> TIME: 2024-12-22 21:30:31 -- STEP: 2442/2902 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.05165933817625046 (0.051207817576819456) + | > loss_mel_ce: 3.187281847000122 (3.161279260379971) + | > loss: 0.40486764907836914 (0.4015608849728525) + | > current_lr: 5e-06 + | > step_time: 0.6548 (0.6070269353071448) + | > loader_time: 0.0117 (0.010570833177277526) + + + --> TIME: 2024-12-22 21:31:19 -- STEP: 2492/2902 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.049455221742391586 (0.05119555806785959) + | > loss_mel_ce: 3.148606300354004 (3.1611325667145564) + | > loss: 0.3997576832771301 (0.40154101581194596) + | > current_lr: 5e-06 + | > step_time: 0.6502 (0.6071065388751442) + | > loader_time: 0.0112 (0.010577304692367864) + + + --> TIME: 2024-12-22 21:32:05 -- STEP: 2542/2902 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.04723748564720154 (0.05118038502824431) + | > loss_mel_ce: 3.230195999145508 (3.1610278290711853) + | > loss: 0.4096791744232178 (0.40152602700808215) + | > current_lr: 5e-06 + | > step_time: 0.691 (0.6068531577742812) + | > loader_time: 0.0111 (0.01058123244909111) + + + --> TIME: 2024-12-22 21:32:52 -- STEP: 2592/2902 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.04967782273888588 (0.05117524785915222) + | > loss_mel_ce: 3.1262459754943848 (3.160997681779626) + | > loss: 0.39699047803878784 (0.4015216164773813) + | > current_lr: 5e-06 + | > step_time: 0.5813 (0.6068024520336834) + | > loader_time: 0.0114 (0.010590111307891791) + + + --> TIME: 2024-12-22 21:33:40 -- STEP: 2642/2902 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.05493231117725372 (0.051161204823585306) + | > loss_mel_ce: 3.163623809814453 (3.1608336640523294) + | > loss: 0.4023195207118988 (0.401499358870873) + | > current_lr: 5e-06 + | > step_time: 0.4954 (0.6067724356048478) + | > loader_time: 0.0103 (0.010597871344347599) + + + --> TIME: 2024-12-22 21:34:26 -- STEP: 2692/2902 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.04973088577389717 (0.051152218153968636) + | > loss_mel_ce: 3.069262981414795 (3.1608455970241267) + | > loss: 0.3898742198944092 (0.4014997271624398) + | > current_lr: 5e-06 + | > step_time: 0.6973 (0.606688487760024) + | > loader_time: 0.0115 (0.010608202103874123) + + + --> TIME: 2024-12-22 21:35:14 -- STEP: 2742/2902 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.05162692070007324 (0.05113960035895226) + | > loss_mel_ce: 3.1889121532440186 (3.1607107881383247) + | > loss: 0.4050673842430115 (0.40148129882080374) + | > current_lr: 5e-06 + | > step_time: 0.6184 (0.606666887093424) + | > loader_time: 0.0114 (0.010620584286189613) + + + --> TIME: 2024-12-22 21:36:01 -- STEP: 2792/2902 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.05050218105316162 (0.05113514076603469) + | > loss_mel_ce: 3.0662810802459717 (3.1603817380431045) + | > loss: 0.38959789276123047 (0.4014396101108253) + | > current_lr: 5e-06 + | > step_time: 0.49 (0.6065895064683209) + | > loader_time: 0.0107 (0.010630526651967216) + + + --> TIME: 2024-12-22 21:36:47 -- STEP: 2842/2902 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.05209615081548691 (0.05113076055678025) + | > loss_mel_ce: 3.178262710571289 (3.1601328545267693) + | > loss: 0.4037948548793793 (0.40140795211696356) + | > current_lr: 5e-06 + | > step_time: 0.5541 (0.6063526925027234) + | > loader_time: 0.0108 (0.010636977961497937) + + + --> TIME: 2024-12-22 21:37:34 -- STEP: 2892/2902 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.04835975542664528 (0.05111794193039538) + | > loss_mel_ce: 3.1759161949157715 (3.160163748561099) + | > loss: 0.403034508228302 (0.4014102115423353) + | > current_lr: 5e-06 + | > step_time: 0.6244 (0.6062773292984693) + | > loader_time: 0.0108 (0.010642424081535906) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.11487200645485313 (-0.0002208045034696876) + | > avg_loss_text_ce: 0.04858186071494011 (-0.0012198377099603003) + | > avg_loss_mel_ce: 3.122558103667365 (-0.021738741132947492) + | > avg_loss: 3.1711399687661066 (-0.02295857487302877) + + > BEST MODEL : /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000/best_model_14510.pth