|
> Training Environment: |
|
| > Backend: Torch |
|
| > Mixed precision: False |
|
| > Precision: float32 |
|
| > Current device: 0 |
|
| > Num. of GPUs: 1 |
|
| > Num. of CPUs: 96 |
|
| > Num. of Torch Threads: 1 |
|
| > Torch seed: 54321 |
|
| > Torch CUDNN: True |
|
| > Torch CUDNN deterministic: False |
|
| > Torch CUDNN benchmark: False |
|
| > Torch TF32 MatMul: False |
|
> Start Tensorboard: tensorboard --logdir=/checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
> Model has 532793243 parameters |
|
|
|
[4m[1m > EPOCH: 0/5[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-22 17:44:01) [0m |
|
|
|
[1m --> TIME: 2024-12-22 17:44:03 -- STEP: 0/2902 -- GLOBAL_STEP: 0[0m |
|
| > loss_text_ce: 0.10459499806165695 (0.10459499806165695) |
|
| > loss_mel_ce: 3.311910390853882 (3.311910390853882) |
|
| > loss: 0.4270631670951843 (0.4270631670951843) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.8415 (0.8415403366088867) |
|
| > loader_time: 1.0344 (1.0344254970550537) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:44:48 -- STEP: 50/2902 -- GLOBAL_STEP: 50[0m |
|
| > loss_text_ce: 0.08981729298830032 (0.09741745188832283) |
|
| > loss_mel_ce: 3.367861747741699 (3.394744510650635) |
|
| > loss: 0.43220987915992737 (0.4365202432870865) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5525 (0.591734619140625) |
|
| > loader_time: 0.0148 (0.015042276382446288) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:45:34 -- STEP: 100/2902 -- GLOBAL_STEP: 100[0m |
|
| > loss_text_ce: 0.08311788737773895 (0.0911357863992453) |
|
| > loss_mel_ce: 3.351332426071167 (3.394733407497406) |
|
| > loss: 0.429306298494339 (0.4357336476445197) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6436 (0.5889876151084902) |
|
| > loader_time: 0.015 (0.014310407638549804) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:46:21 -- STEP: 150/2902 -- GLOBAL_STEP: 150[0m |
|
| > loss_text_ce: 0.07448040693998337 (0.08718039671579998) |
|
| > loss_mel_ce: 3.401618719100952 (3.3927350441614785) |
|
| > loss: 0.4345123767852783 (0.43498942931493106) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6071 (0.5917791159947713) |
|
| > loader_time: 0.0119 (0.013932906786600748) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:47:08 -- STEP: 200/2902 -- GLOBAL_STEP: 200[0m |
|
| > loss_text_ce: 0.0731874480843544 (0.08409653138369327) |
|
| > loss_mel_ce: 3.291797637939453 (3.390307182073593) |
|
| > loss: 0.42062312364578247 (0.43430046319961535) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5773 (0.5956235480308533) |
|
| > loader_time: 0.0131 (0.01358080267906189) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:47:56 -- STEP: 250/2902 -- GLOBAL_STEP: 250[0m |
|
| > loss_text_ce: 0.06732229888439178 (0.08167070341110234) |
|
| > loss_mel_ce: 3.3784215450286865 (3.3851796493530273) |
|
| > loss: 0.43071797490119934 (0.4333562933206557) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6582 (0.5984164400100709) |
|
| > loader_time: 0.012 (0.013197577476501465) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:48:43 -- STEP: 300/2902 -- GLOBAL_STEP: 300[0m |
|
| > loss_text_ce: 0.07035832107067108 (0.0798381366580725) |
|
| > loss_mel_ce: 3.4118311405181885 (3.3834915582338967) |
|
| > loss: 0.435273677110672 (0.43291621098915717) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6434 (0.5990857418378196) |
|
| > loader_time: 0.0125 (0.01297310829162598) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:49:30 -- STEP: 350/2902 -- GLOBAL_STEP: 350[0m |
|
| > loss_text_ce: 0.06829076260328293 (0.0782227069245918) |
|
| > loss_mel_ce: 3.2637181282043457 (3.381631039210728) |
|
| > loss: 0.41650110483169556 (0.4324817172970089) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6559 (0.6008020734786992) |
|
| > loader_time: 0.01 (0.01282343115125384) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:50:17 -- STEP: 400/2902 -- GLOBAL_STEP: 400[0m |
|
| > loss_text_ce: 0.07181409001350403 (0.07684497562237082) |
|
| > loss_mel_ce: 3.338163137435913 (3.378450380563736) |
|
| > loss: 0.42624714970588684 (0.4319119187444447) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5614 (0.6016418385505681) |
|
| > loader_time: 0.0103 (0.012634595632553106) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:51:04 -- STEP: 450/2902 -- GLOBAL_STEP: 450[0m |
|
| > loss_text_ce: 0.0650668740272522 (0.07570000054107771) |
|
| > loss_mel_ce: 3.341148853302002 (3.374886935022142) |
|
| > loss: 0.42577695846557617 (0.43132336596647886) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6586 (0.6017436684502502) |
|
| > loader_time: 0.0104 (0.012495169109768342) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:51:51 -- STEP: 500/2902 -- GLOBAL_STEP: 500[0m |
|
| > loss_text_ce: 0.06929623335599899 (0.07468750177323819) |
|
| > loss_mel_ce: 3.3347978591918945 (3.3740318789482115) |
|
| > loss: 0.42551174759864807 (0.43108992183208455) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6249 (0.6018227620124823) |
|
| > loader_time: 0.0123 (0.01235325384140015) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:52:39 -- STEP: 550/2902 -- GLOBAL_STEP: 550[0m |
|
| > loss_text_ce: 0.06491728872060776 (0.07386755391278053) |
|
| > loss_mel_ce: 3.290853977203369 (3.372003945870833) |
|
| > loss: 0.41947141289711 (0.430733936862512) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5923 (0.6032628406177872) |
|
| > loader_time: 0.01 (0.012248729792508213) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:53:26 -- STEP: 600/2902 -- GLOBAL_STEP: 600[0m |
|
| > loss_text_ce: 0.0667271837592125 (0.07305075627441214) |
|
| > loss_mel_ce: 3.284752607345581 (3.3701720937093085) |
|
| > loss: 0.41893497109413147 (0.4304028554260729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4854 (0.6030249472459164) |
|
| > loader_time: 0.01 (0.012161032358805342) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:54:14 -- STEP: 650/2902 -- GLOBAL_STEP: 650[0m |
|
| > loss_text_ce: 0.06883151084184647 (0.07234381749652904) |
|
| > loss_mel_ce: 3.3112661838531494 (3.367854071763844) |
|
| > loss: 0.4225122034549713 (0.4300247354232346) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5225 (0.6042447258875925) |
|
| > loader_time: 0.0096 (0.012069188264700085) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:55:01 -- STEP: 700/2902 -- GLOBAL_STEP: 700[0m |
|
| > loss_text_ce: 0.06538775563240051 (0.07166379387357402) |
|
| > loss_mel_ce: 3.388728380203247 (3.3648004106112874) |
|
| > loss: 0.43176451325416565 (0.4295580248321804) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.656 (0.60397415297372) |
|
| > loader_time: 0.0132 (0.012077730042593824) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:55:47 -- STEP: 750/2902 -- GLOBAL_STEP: 750[0m |
|
| > loss_text_ce: 0.058807797729969025 (0.0710751066207886) |
|
| > loss_mel_ce: 3.3790392875671387 (3.364291217168171) |
|
| > loss: 0.4297308921813965 (0.42942078975836423) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6502 (0.6036211016972853) |
|
| > loader_time: 0.0099 (0.011993129412333174) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:56:34 -- STEP: 800/2902 -- GLOBAL_STEP: 800[0m |
|
| > loss_text_ce: 0.062027063220739365 (0.07054349425248804) |
|
| > loss_mel_ce: 3.3717057704925537 (3.363725082874297) |
|
| > loss: 0.429216593503952 (0.4292835713922976) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6413 (0.6034053999185554) |
|
| > loader_time: 0.0159 (0.011959144175052647) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:57:21 -- STEP: 850/2902 -- GLOBAL_STEP: 850[0m |
|
| > loss_text_ce: 0.059973880648612976 (0.07002651334685442) |
|
| > loss_mel_ce: 3.4005930423736572 (3.362588267606846) |
|
| > loss: 0.4325708746910095 (0.42907684687305886) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6503 (0.6033904190624451) |
|
| > loader_time: 0.0131 (0.011911653350381295) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:58:09 -- STEP: 900/2902 -- GLOBAL_STEP: 900[0m |
|
| > loss_text_ce: 0.059261124581098557 (0.06958188860780673) |
|
| > loss_mel_ce: 3.326876640319824 (3.3624700564808307) |
|
| > loss: 0.4232672154903412 (0.42900649242930927) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5363 (0.6036524536874549) |
|
| > loader_time: 0.0099 (0.011872834629482697) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:58:56 -- STEP: 950/2902 -- GLOBAL_STEP: 950[0m |
|
| > loss_text_ce: 0.05842612311244011 (0.06918677367662138) |
|
| > loss_mel_ce: 3.364814043045044 (3.361136853820398) |
|
| > loss: 0.42790502309799194 (0.4287904527626538) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5597 (0.603434893959446) |
|
| > loader_time: 0.0105 (0.01180140896847374) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 17:59:43 -- STEP: 1000/2902 -- GLOBAL_STEP: 1000[0m |
|
| > loss_text_ce: 0.061046820133924484 (0.06879791953414686) |
|
| > loss_mel_ce: 3.3686323165893555 (3.360496532917022) |
|
| > loss: 0.42870989441871643 (0.428661805897951) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5852 (0.6037456159591665) |
|
| > loader_time: 0.0103 (0.01176662635803223) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:00:31 -- STEP: 1050/2902 -- GLOBAL_STEP: 1050[0m |
|
| > loss_text_ce: 0.06030765920877457 (0.06841222434881197) |
|
| > loss_mel_ce: 3.262373685836792 (3.3596309841246823) |
|
| > loss: 0.41533517837524414 (0.42850540047600144) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6931 (0.6041640540531695) |
|
| > loader_time: 0.0133 (0.011732036953880679) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:01:17 -- STEP: 1100/2902 -- GLOBAL_STEP: 1100[0m |
|
| > loss_text_ce: 0.06249762699007988 (0.06807924053547065) |
|
| > loss_mel_ce: 3.444939374923706 (3.3585651447556235) |
|
| > loss: 0.4384296238422394 (0.42833054761994965) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6096 (0.6037762509692791) |
|
| > loader_time: 0.01 (0.011697546135295524) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:02:05 -- STEP: 1150/2902 -- GLOBAL_STEP: 1150[0m |
|
| > loss_text_ce: 0.05816451832652092 (0.06773634191764444) |
|
| > loss_mel_ce: 3.32035756111145 (3.3562073423551486) |
|
| > loss: 0.4223152697086334 (0.4279929601109548) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6184 (0.6039227991518759) |
|
| > loader_time: 0.0113 (0.01168110660884692) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:02:51 -- STEP: 1200/2902 -- GLOBAL_STEP: 1200[0m |
|
| > loss_text_ce: 0.061584342271089554 (0.06741090319119396) |
|
| > loss_mel_ce: 3.287616491317749 (3.35481663107872) |
|
| > loss: 0.4186500906944275 (0.4277784414092702) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5726 (0.6035150116682043) |
|
| > loader_time: 0.0105 (0.011648295323054) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:03:38 -- STEP: 1250/2902 -- GLOBAL_STEP: 1250[0m |
|
| > loss_text_ce: 0.062285855412483215 (0.06711042538881305) |
|
| > loss_mel_ce: 3.24466609954834 (3.352927883148192) |
|
| > loss: 0.41336899995803833 (0.42750478818416615) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6217 (0.6035623750686641) |
|
| > loader_time: 0.0109 (0.011627090263366705) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:04:25 -- STEP: 1300/2902 -- GLOBAL_STEP: 1300[0m |
|
| > loss_text_ce: 0.060854967683553696 (0.0668398949761804) |
|
| > loss_mel_ce: 3.3223676681518555 (3.351386453921977) |
|
| > loss: 0.42290282249450684 (0.4272782932107267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6664 (0.6035717503841104) |
|
| > loader_time: 0.0106 (0.01160200100678665) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:05:13 -- STEP: 1350/2902 -- GLOBAL_STEP: 1350[0m |
|
| > loss_text_ce: 0.058691028505563736 (0.0665982692633515) |
|
| > loss_mel_ce: 3.4332950115203857 (3.351136484146117) |
|
| > loss: 0.4364982545375824 (0.4272168437639874) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6591 (0.6039325431541159) |
|
| > loader_time: 0.0106 (0.01157924245904994) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:05:59 -- STEP: 1400/2902 -- GLOBAL_STEP: 1400[0m |
|
| > loss_text_ce: 0.0591551810503006 (0.06633740439478851) |
|
| > loss_mel_ce: 3.4388298988342285 (3.349815227644783) |
|
| > loss: 0.4372481405735016 (0.42701907856123805) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5407 (0.6033196180207391) |
|
| > loader_time: 0.0105 (0.011555085863385888) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:06:46 -- STEP: 1450/2902 -- GLOBAL_STEP: 1450[0m |
|
| > loss_text_ce: 0.06207405775785446 (0.06609510640132035) |
|
| > loss_mel_ce: 3.350438356399536 (3.348956268409202) |
|
| > loss: 0.4265640377998352 (0.42688142141391505) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6589 (0.6032376984892222) |
|
| > loader_time: 0.0106 (0.011531794975543852) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:07:33 -- STEP: 1500/2902 -- GLOBAL_STEP: 1500[0m |
|
| > loss_text_ce: 0.05779704079031944 (0.0658860725015404) |
|
| > loss_mel_ce: 3.186833381652832 (3.3477389181454966) |
|
| > loss: 0.4055787920951843 (0.4267031234502794) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6797 (0.6032957612673445) |
|
| > loader_time: 0.0106 (0.01151966810226441) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:08:20 -- STEP: 1550/2902 -- GLOBAL_STEP: 1550[0m |
|
| > loss_text_ce: 0.06286037713289261 (0.0656808182981708) |
|
| > loss_mel_ce: 3.3885998725891113 (3.3471834834929424) |
|
| > loss: 0.4314325451850891 (0.42660803737178943) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4494 (0.6032871904680812) |
|
| > loader_time: 0.0097 (0.011501272724520782) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:09:06 -- STEP: 1600/2902 -- GLOBAL_STEP: 1600[0m |
|
| > loss_text_ce: 0.056488897651433945 (0.06548411942552791) |
|
| > loss_mel_ce: 3.230045795440674 (3.3461627419292914) |
|
| > loss: 0.41081684827804565 (0.42645585726946605) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4419 (0.6029248094558723) |
|
| > loader_time: 0.0102 (0.011486575603485115) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:09:52 -- STEP: 1650/2902 -- GLOBAL_STEP: 1650[0m |
|
| > loss_text_ce: 0.05772990733385086 (0.0652738300598031) |
|
| > loss_mel_ce: 3.189204454421997 (3.344762894890524) |
|
| > loss: 0.405866801738739 (0.4262545901898183) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5127 (0.602524100650441) |
|
| > loader_time: 0.0102 (0.011477450457486245) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:10:40 -- STEP: 1700/2902 -- GLOBAL_STEP: 1700[0m |
|
| > loss_text_ce: 0.05655720457434654 (0.065094419630135) |
|
| > loss_mel_ce: 3.2922372817993164 (3.3436829961047443) |
|
| > loss: 0.41859930753707886 (0.4260971765483128) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6106 (0.6027139033990753) |
|
| > loader_time: 0.0104 (0.011463808171889368) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:11:27 -- STEP: 1750/2902 -- GLOBAL_STEP: 1750[0m |
|
| > loss_text_ce: 0.05985289812088013 (0.06492117699129257) |
|
| > loss_mel_ce: 3.4752132892608643 (3.342860545022146) |
|
| > loss: 0.44188326597213745 (0.4259727147987912) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6503 (0.6031666985920503) |
|
| > loader_time: 0.0117 (0.011454036848885678) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:12:14 -- STEP: 1800/2902 -- GLOBAL_STEP: 1800[0m |
|
| > loss_text_ce: 0.060121070593595505 (0.06474052529368152) |
|
| > loss_mel_ce: 3.321464776992798 (3.3419743609428396) |
|
| > loss: 0.4226982295513153 (0.42583936037288783) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5076 (0.6028450124793587) |
|
| > loader_time: 0.0107 (0.011442316770553594) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:13:01 -- STEP: 1850/2902 -- GLOBAL_STEP: 1850[0m |
|
| > loss_text_ce: 0.05790895223617554 (0.06457843535856637) |
|
| > loss_mel_ce: 3.2426202297210693 (3.340656888936016) |
|
| > loss: 0.4125661551952362 (0.4256544151016185) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6492 (0.6027948614068939) |
|
| > loader_time: 0.0105 (0.01143438712970631) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:13:48 -- STEP: 1900/2902 -- GLOBAL_STEP: 1900[0m |
|
| > loss_text_ce: 0.05779070779681206 (0.06441327211103959) |
|
| > loss_mel_ce: 3.4901840686798096 (3.3398845655039726) |
|
| > loss: 0.44349685311317444 (0.4255372292744487) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6231 (0.6030444751287767) |
|
| > loader_time: 0.0109 (0.011421437263488775) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:14:36 -- STEP: 1950/2902 -- GLOBAL_STEP: 1950[0m |
|
| > loss_text_ce: 0.057499103248119354 (0.06425976193104062) |
|
| > loss_mel_ce: 3.2813806533813477 (3.339388028902884) |
|
| > loss: 0.41735997796058655 (0.42545597343872765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6133 (0.6034406099563995) |
|
| > loader_time: 0.0109 (0.01141151843926846) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:15:24 -- STEP: 2000/2902 -- GLOBAL_STEP: 2000[0m |
|
| > loss_text_ce: 0.06024341285228729 (0.06411207320168634) |
|
| > loss_mel_ce: 3.2993335723876953 (3.3388525167703618) |
|
| > loss: 0.4199471175670624 (0.42537057338655004) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5848 (0.6038204357624058) |
|
| > loader_time: 0.011 (0.011401470899581915) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:16:12 -- STEP: 2050/2902 -- GLOBAL_STEP: 2050[0m |
|
| > loss_text_ce: 0.05515234172344208 (0.0639634985585767) |
|
| > loss_mel_ce: 3.3303256034851074 (3.338357712699145) |
|
| > loss: 0.42318475246429443 (0.4252901510785267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6663 (0.6038993442349322) |
|
| > loader_time: 0.0111 (0.011391908250203952) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:17:00 -- STEP: 2100/2902 -- GLOBAL_STEP: 2100[0m |
|
| > loss_text_ce: 0.05871572345495224 (0.06382640003980644) |
|
| > loss_mel_ce: 3.335447311401367 (3.337795670827229) |
|
| > loss: 0.4242703914642334 (0.4252027585534823) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6248 (0.6043373265720553) |
|
| > loader_time: 0.0109 (0.011387435368129188) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:17:48 -- STEP: 2150/2902 -- GLOBAL_STEP: 2150[0m |
|
| > loss_text_ce: 0.058568090200424194 (0.06369089260177577) |
|
| > loss_mel_ce: 3.3574507236480713 (3.3373832586199734) |
|
| > loss: 0.42700234055519104 (0.42513426865256115) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6186 (0.6046538991706321) |
|
| > loader_time: 0.0107 (0.011380267919496052) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:18:34 -- STEP: 2200/2902 -- GLOBAL_STEP: 2200[0m |
|
| > loss_text_ce: 0.05417061224579811 (0.06356014532629754) |
|
| > loss_mel_ce: 3.2483224868774414 (3.336935182593085) |
|
| > loss: 0.41281163692474365 (0.42506191578778346) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3652 (0.604359569874677) |
|
| > loader_time: 0.0147 (0.011376231258565735) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:19:21 -- STEP: 2250/2902 -- GLOBAL_STEP: 2250[0m |
|
| > loss_text_ce: 0.05976838991045952 (0.06343722468614592) |
|
| > loss_mel_ce: 3.3819050788879395 (3.3364632027943935) |
|
| > loss: 0.4302091896533966 (0.4249875532388684) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6488 (0.6042012519836432) |
|
| > loader_time: 0.0112 (0.011366784837510855) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:20:08 -- STEP: 2300/2902 -- GLOBAL_STEP: 2300[0m |
|
| > loss_text_ce: 0.05424568057060242 (0.06331661111958663) |
|
| > loss_mel_ce: 3.2424044609069824 (3.335641909993212) |
|
| > loss: 0.4120812714099884 (0.42486981498158466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6175 (0.6043975578183717) |
|
| > loader_time: 0.0108 (0.011360070187112565) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:20:56 -- STEP: 2350/2902 -- GLOBAL_STEP: 2350[0m |
|
| > loss_text_ce: 0.05863575264811516 (0.06319462328356651) |
|
| > loss_mel_ce: 3.1610910892486572 (3.334841926757325) |
|
| > loss: 0.4024658501148224 (0.4247545686173943) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5305 (0.6044632607317989) |
|
| > loader_time: 0.0111 (0.011355122505350318) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:21:43 -- STEP: 2400/2902 -- GLOBAL_STEP: 2400[0m |
|
| > loss_text_ce: 0.05514562875032425 (0.0630755390987423) |
|
| > loss_mel_ce: 3.3872921466827393 (3.33422359963258) |
|
| > loss: 0.43030473589897156 (0.4246623922015226) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.608 (0.6043834758798285) |
|
| > loader_time: 0.0113 (0.011349765062332156) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:22:31 -- STEP: 2450/2902 -- GLOBAL_STEP: 2450[0m |
|
| > loss_text_ce: 0.05645584315061569 (0.06295087482552163) |
|
| > loss_mel_ce: 3.2656829357147217 (3.33336971107794) |
|
| > loss: 0.41526734828948975 (0.4245400731417593) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6592 (0.6047812546515954) |
|
| > loader_time: 0.0114 (0.011347482253094114) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:23:18 -- STEP: 2500/2902 -- GLOBAL_STEP: 2500[0m |
|
| > loss_text_ce: 0.05541292577981949 (0.06283749222308417) |
|
| > loss_mel_ce: 3.3705124855041504 (3.3327593852043123) |
|
| > loss: 0.42824068665504456 (0.4244496095895763) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6591 (0.6047453528404236) |
|
| > loader_time: 0.0115 (0.011346870326995856) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:24:05 -- STEP: 2550/2902 -- GLOBAL_STEP: 2550[0m |
|
| > loss_text_ce: 0.05708501115441322 (0.06273307577652111) |
|
| > loss_mel_ce: 3.210287570953369 (3.3319304627063198) |
|
| > loss: 0.4084215760231018 (0.42433294222635337) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6514 (0.6046757781271843) |
|
| > loader_time: 0.0109 (0.011345180156184185) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:24:51 -- STEP: 2600/2902 -- GLOBAL_STEP: 2600[0m |
|
| > loss_text_ce: 0.055396050214767456 (0.06263674674269117) |
|
| > loss_mel_ce: 3.3255748748779297 (3.331115982624198) |
|
| > loss: 0.42262136936187744 (0.42421909111050415) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5778 (0.6045672522141382) |
|
| > loader_time: 0.0115 (0.011345681593968331) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:25:38 -- STEP: 2650/2902 -- GLOBAL_STEP: 2650[0m |
|
| > loss_text_ce: 0.054033081978559494 (0.06253566949575595) |
|
| > loss_mel_ce: 3.202986717224121 (3.3304516717622836) |
|
| > loss: 0.4071274697780609 (0.42412341760014544) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5688 (0.6044498934835748) |
|
| > loader_time: 0.011 (0.011339990417912336) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:26:26 -- STEP: 2700/2902 -- GLOBAL_STEP: 2700[0m |
|
| > loss_text_ce: 0.056942086666822433 (0.06243673637647341) |
|
| > loss_mel_ce: 3.1709163188934326 (3.3299163836903016) |
|
| > loss: 0.4034822881221771 (0.42404413991504203) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6471 (0.6046004780133557) |
|
| > loader_time: 0.011 (0.011335281795925578) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:27:12 -- STEP: 2750/2902 -- GLOBAL_STEP: 2750[0m |
|
| > loss_text_ce: 0.056967683136463165 (0.06234114655310478) |
|
| > loss_mel_ce: 3.262450933456421 (3.32917199074138) |
|
| > loss: 0.41492733359336853 (0.4239391420754515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6701 (0.6045005709908213) |
|
| > loader_time: 0.0113 (0.01133127125826751) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:28:00 -- STEP: 2800/2902 -- GLOBAL_STEP: 2800[0m |
|
| > loss_text_ce: 0.05768461897969246 (0.06225454275762417) |
|
| > loss_mel_ce: 3.3250513076782227 (3.3285182338952994) |
|
| > loss: 0.42284199595451355 (0.4238465970115998) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4433 (0.6045015747206539) |
|
| > loader_time: 0.0102 (0.011324909329414391) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:28:47 -- STEP: 2850/2902 -- GLOBAL_STEP: 2850[0m |
|
| > loss_text_ce: 0.05957743152976036 (0.06216866457279334) |
|
| > loss_mel_ce: 3.287506580352783 (3.32793902003974) |
|
| > loss: 0.41838550567626953 (0.4237634605064723) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6157 (0.6046459482427213) |
|
| > loader_time: 0.0111 (0.011319453674450277) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:29:34 -- STEP: 2900/2902 -- GLOBAL_STEP: 2900[0m |
|
| > loss_text_ce: 0.057000596076250076 (0.06207586748718207) |
|
| > loss_mel_ce: 3.371854782104492 (3.327414204334388) |
|
| > loss: 0.42860692739486694 (0.42368625892647344) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6715 (0.6046704241325099) |
|
| > loader_time: 0.0111 (0.011315281719997034) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time: 0.11512538640186042 [0m(+0) |
|
| > avg_loss_text_ce: 0.05481784877301467 [0m(+0) |
|
| > avg_loss_mel_ce: 3.244625385361489 [0m(+0) |
|
| > avg_loss: 3.299443240117545 [0m(+0) |
|
|
|
|
|
[4m[1m > EPOCH: 1/5[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-22 18:31:16) [0m |
|
|
|
[1m --> TIME: 2024-12-22 18:32:02 -- STEP: 48/2902 -- GLOBAL_STEP: 2950[0m |
|
| > loss_text_ce: 0.05890386551618576 (0.05689182776647309) |
|
| > loss_mel_ce: 3.2426769733428955 (3.2889040211836496) |
|
| > loss: 0.41269761323928833 (0.41822448062400025) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4931 (0.6041691402594247) |
|
| > loader_time: 0.0098 (0.010133753220240274) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:32:49 -- STEP: 98/2902 -- GLOBAL_STEP: 3000[0m |
|
| > loss_text_ce: 0.05669895187020302 (0.05675384869837031) |
|
| > loss_mel_ce: 3.316187858581543 (3.2876880923096015) |
|
| > loss: 0.42161086201667786 (0.41805524302988634) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6463 (0.6044869422912595) |
|
| > loader_time: 0.0098 (0.010064643256518305) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:33:36 -- STEP: 148/2902 -- GLOBAL_STEP: 3050[0m |
|
| > loss_text_ce: 0.05863890051841736 (0.056781802452295214) |
|
| > loss_mel_ce: 3.3207154273986816 (3.284461807560276) |
|
| > loss: 0.4224192798137665 (0.41765545167633006) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5437 (0.6032516183079898) |
|
| > loader_time: 0.0098 (0.010022640228271484) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:34:23 -- STEP: 198/2902 -- GLOBAL_STEP: 3100[0m |
|
| > loss_text_ce: 0.05307390168309212 (0.05668377612877374) |
|
| > loss_mel_ce: 3.1541969776153564 (3.281082079868124) |
|
| > loss: 0.40090885758399963 (0.4172207321783509) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6491 (0.6029641315190476) |
|
| > loader_time: 0.0103 (0.010148859987355242) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:35:11 -- STEP: 248/2902 -- GLOBAL_STEP: 3150[0m |
|
| > loss_text_ce: 0.056664470583200455 (0.05664264377687247) |
|
| > loss_mel_ce: 3.267181873321533 (3.284639359481873) |
|
| > loss: 0.4154807925224304 (0.4176602506589505) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5497 (0.6068478284343594) |
|
| > loader_time: 0.0098 (0.010098123742688087) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:35:57 -- STEP: 298/2902 -- GLOBAL_STEP: 3200[0m |
|
| > loss_text_ce: 0.05854443460702896 (0.056619707457471205) |
|
| > loss_mel_ce: 3.436552047729492 (3.283927426242189) |
|
| > loss: 0.43688705563545227 (0.41756839210155033) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.54 (0.6054019759965423) |
|
| > loader_time: 0.0097 (0.0100771468757783) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:36:45 -- STEP: 348/2902 -- GLOBAL_STEP: 3250[0m |
|
| > loss_text_ce: 0.05434746667742729 (0.05662827220499172) |
|
| > loss_mel_ce: 3.4035985469818115 (3.2863656522213733) |
|
| > loss: 0.4322432577610016 (0.41787424092662756) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6651 (0.6075942810924571) |
|
| > loader_time: 0.0103 (0.010104454111778875) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:37:33 -- STEP: 398/2902 -- GLOBAL_STEP: 3300[0m |
|
| > loss_text_ce: 0.055893950164318085 (0.05664088315887367) |
|
| > loss_mel_ce: 3.214393138885498 (3.2846952245463084) |
|
| > loss: 0.408785879611969 (0.4176670137661786) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5718 (0.6087509783069092) |
|
| > loader_time: 0.0097 (0.010112432978261063) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:38:20 -- STEP: 448/2902 -- GLOBAL_STEP: 3350[0m |
|
| > loss_text_ce: 0.055915918201208115 (0.05663545990994732) |
|
| > loss_mel_ce: 3.395284652709961 (3.2834518684872562) |
|
| > loss: 0.4314000606536865 (0.4175109162128398) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5086 (0.6075441422207012) |
|
| > loader_time: 0.0099 (0.010116907102721085) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:39:06 -- STEP: 498/2902 -- GLOBAL_STEP: 3400[0m |
|
| > loss_text_ce: 0.05525524914264679 (0.05658867202669262) |
|
| > loss_mel_ce: 3.2936763763427734 (3.2832315762837734) |
|
| > loss: 0.4186164438724518 (0.41747753112670416) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6469 (0.60625598756185) |
|
| > loader_time: 0.0103 (0.010127688986230572) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:39:54 -- STEP: 548/2902 -- GLOBAL_STEP: 3450[0m |
|
| > loss_text_ce: 0.05756908282637596 (0.05664774154170152) |
|
| > loss_mel_ce: 3.1639130115509033 (3.28317973996601) |
|
| > loss: 0.4026852548122406 (0.41747843528533507) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6413 (0.6067891307990916) |
|
| > loader_time: 0.0103 (0.010137229505246578) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:40:40 -- STEP: 598/2902 -- GLOBAL_STEP: 3500[0m |
|
| > loss_text_ce: 0.054091762751340866 (0.05661526037049931) |
|
| > loss_mel_ce: 3.264618158340454 (3.2823297997382173) |
|
| > loss: 0.4148387312889099 (0.4173681325836726) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6618 (0.6058910582775254) |
|
| > loader_time: 0.0104 (0.010126844297683365) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:41:28 -- STEP: 648/2902 -- GLOBAL_STEP: 3550[0m |
|
| > loss_text_ce: 0.05606652796268463 (0.056563418749489905) |
|
| > loss_mel_ce: 3.352484703063965 (3.2804698237666376) |
|
| > loss: 0.42606890201568604 (0.417129155294395) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4843 (0.6063799847055359) |
|
| > loader_time: 0.0097 (0.010125751112714238) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:42:15 -- STEP: 698/2902 -- GLOBAL_STEP: 3600[0m |
|
| > loss_text_ce: 0.05525405332446098 (0.05655210185529168) |
|
| > loss_mel_ce: 3.2965216636657715 (3.2807158672365553) |
|
| > loss: 0.4189719557762146 (0.41715849598704247) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6065 (0.6062579441890324) |
|
| > loader_time: 0.01 (0.010130226782878017) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:43:03 -- STEP: 748/2902 -- GLOBAL_STEP: 3650[0m |
|
| > loss_text_ce: 0.058175548911094666 (0.05650098778725786) |
|
| > loss_mel_ce: 3.269888162612915 (3.280304757987752) |
|
| > loss: 0.41600796580314636 (0.4171007181552644) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6982 (0.6072237536231458) |
|
| > loader_time: 0.0106 (0.010144633724090253) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:43:49 -- STEP: 798/2902 -- GLOBAL_STEP: 3700[0m |
|
| > loss_text_ce: 0.056960977613925934 (0.056491326578054814) |
|
| > loss_mel_ce: 3.241190195083618 (3.2806498221586224) |
|
| > loss: 0.41226890683174133 (0.4171426434415326) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6506 (0.6060144674210314) |
|
| > loader_time: 0.0101 (0.010138096964747698) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:44:36 -- STEP: 848/2902 -- GLOBAL_STEP: 3750[0m |
|
| > loss_text_ce: 0.05707242339849472 (0.05649265555998767) |
|
| > loss_mel_ce: 3.3955237865448 (3.28043776075795) |
|
| > loss: 0.4315745234489441 (0.4171163019074023) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5557 (0.6060532727893788) |
|
| > loader_time: 0.0099 (0.010141247004832874) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:45:23 -- STEP: 898/2902 -- GLOBAL_STEP: 3800[0m |
|
| > loss_text_ce: 0.05551436170935631 (0.05646666096635412) |
|
| > loss_mel_ce: 3.2455344200134277 (3.279986635613813) |
|
| > loss: 0.412631094455719 (0.4170566619371785) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5661 (0.605561435355375) |
|
| > loader_time: 0.0097 (0.010150369397781486) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:46:10 -- STEP: 948/2902 -- GLOBAL_STEP: 3850[0m |
|
| > loss_text_ce: 0.055860064923763275 (0.05645387268424788) |
|
| > loss_mel_ce: 3.196481227874756 (3.2784469580348534) |
|
| > loss: 0.40654265880584717 (0.4168626036384942) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6979 (0.6058803857127317) |
|
| > loader_time: 0.0103 (0.010132792881269507) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:46:57 -- STEP: 998/2902 -- GLOBAL_STEP: 3900[0m |
|
| > loss_text_ce: 0.05521142855286598 (0.05644223373196646) |
|
| > loss_mel_ce: 3.2823712825775146 (3.2776020464295135) |
|
| > loss: 0.4171978533267975 (0.41675553484287925) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5983 (0.6054262338515987) |
|
| > loader_time: 0.0113 (0.01014956730401111) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:47:43 -- STEP: 1048/2902 -- GLOBAL_STEP: 3950[0m |
|
| > loss_text_ce: 0.0541345551609993 (0.05641971721548502) |
|
| > loss_mel_ce: 3.2423479557037354 (3.2767769084631944) |
|
| > loss: 0.41206032037734985 (0.4166495781249673) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5016 (0.6050974510553224) |
|
| > loader_time: 0.0105 (0.010205373281740973) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:48:30 -- STEP: 1098/2902 -- GLOBAL_STEP: 4000[0m |
|
| > loss_text_ce: 0.05483408272266388 (0.056407355969747996) |
|
| > loss_mel_ce: 3.241896867752075 (3.2765870491663613) |
|
| > loss: 0.41209137439727783 (0.4166243005482878) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6277 (0.6046719931079604) |
|
| > loader_time: 0.01 (0.010229592983405675) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:49:18 -- STEP: 1148/2902 -- GLOBAL_STEP: 4050[0m |
|
| > loss_text_ce: 0.05799783021211624 (0.05640103503451633) |
|
| > loss_mel_ce: 3.125845432281494 (3.2763871357416035) |
|
| > loss: 0.3979804217815399 (0.41659852130280145) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6496 (0.6050292589938597) |
|
| > loader_time: 0.0108 (0.010255601763309925) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:50:04 -- STEP: 1198/2902 -- GLOBAL_STEP: 4100[0m |
|
| > loss_text_ce: 0.055139560252428055 (0.05637540748165624) |
|
| > loss_mel_ce: 3.3101248741149902 (3.275573732259876) |
|
| > loss: 0.42065805196762085 (0.4164936424346519) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5471 (0.6046004488392543) |
|
| > loader_time: 0.0106 (0.01028222253605202) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:50:51 -- STEP: 1248/2902 -- GLOBAL_STEP: 4150[0m |
|
| > loss_text_ce: 0.05775659531354904 (0.05636790161952375) |
|
| > loss_mel_ce: 3.269582986831665 (3.275295549669328) |
|
| > loss: 0.41591745615005493 (0.41645793134394365) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4634 (0.6042926706946812) |
|
| > loader_time: 0.0102 (0.010291477999626075) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:51:38 -- STEP: 1298/2902 -- GLOBAL_STEP: 4200[0m |
|
| > loss_text_ce: 0.05557722970843315 (0.056327807755770586) |
|
| > loss_mel_ce: 3.2988786697387695 (3.274222171361716) |
|
| > loss: 0.41930699348449707 (0.4163187472680684) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6939 (0.6040190265432156) |
|
| > loader_time: 0.0109 (0.010298490891654985) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:52:26 -- STEP: 1348/2902 -- GLOBAL_STEP: 4250[0m |
|
| > loss_text_ce: 0.05759159103035927 (0.05631046397583862) |
|
| > loss_mel_ce: 3.2881672382354736 (3.273693052703621) |
|
| > loss: 0.4182198643684387 (0.41625043948786195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.7071 (0.6047909778020497) |
|
| > loader_time: 0.0107 (0.010317974699003411) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:53:13 -- STEP: 1398/2902 -- GLOBAL_STEP: 4300[0m |
|
| > loss_text_ce: 0.05378188565373421 (0.056270858414140715) |
|
| > loss_mel_ce: 3.1348822116851807 (3.2728953762286395) |
|
| > loss: 0.3985830247402191 (0.4161457792244244) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5446 (0.6049843866255487) |
|
| > loader_time: 0.0102 (0.010335659605580175) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:54:00 -- STEP: 1448/2902 -- GLOBAL_STEP: 4350[0m |
|
| > loss_text_ce: 0.05448736995458603 (0.056238353370755736) |
|
| > loss_mel_ce: 3.215205430984497 (3.272332094486248) |
|
| > loss: 0.40871161222457886 (0.41607130596733227) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4742 (0.6047209682385556) |
|
| > loader_time: 0.0101 (0.010342140882713364) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:54:48 -- STEP: 1498/2902 -- GLOBAL_STEP: 4400[0m |
|
| > loss_text_ce: 0.05498555675148964 (0.056216870995365244) |
|
| > loss_mel_ce: 3.271206855773926 (3.272168317847005) |
|
| > loss: 0.41577404737472534 (0.4160481485847797) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6284 (0.6049662217278979) |
|
| > loader_time: 0.0101 (0.01034619254327426) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:55:35 -- STEP: 1548/2902 -- GLOBAL_STEP: 4450[0m |
|
| > loss_text_ce: 0.057937052100896835 (0.056200356920579446) |
|
| > loss_mel_ce: 3.3727030754089355 (3.2725960398213205) |
|
| > loss: 0.42883002758026123 (0.4160995496104854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6974 (0.6050552325347286) |
|
| > loader_time: 0.0115 (0.010363647796078863) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:56:22 -- STEP: 1598/2902 -- GLOBAL_STEP: 4500[0m |
|
| > loss_text_ce: 0.05774551257491112 (0.056181561288065986) |
|
| > loss_mel_ce: 3.3820343017578125 (3.271953099361798) |
|
| > loss: 0.42997246980667114 (0.4160168326654482) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6635 (0.6048786176161103) |
|
| > loader_time: 0.0117 (0.010390232889464262) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:57:09 -- STEP: 1648/2902 -- GLOBAL_STEP: 4550[0m |
|
| > loss_text_ce: 0.053543396294116974 (0.05615658306706588) |
|
| > loss_mel_ce: 3.2377357482910156 (3.271739111073967) |
|
| > loss: 0.4114098846912384 (0.41598696182075057) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5607 (0.6048709005117405) |
|
| > loader_time: 0.0114 (0.010413509549446476) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:57:56 -- STEP: 1698/2902 -- GLOBAL_STEP: 4600[0m |
|
| > loss_text_ce: 0.05661720037460327 (0.05613417721609063) |
|
| > loss_mel_ce: 3.2408382892608643 (3.27091189141268) |
|
| > loss: 0.41218194365501404 (0.41588075864441404) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6238 (0.6048471832163056) |
|
| > loader_time: 0.0106 (0.010442374030608591) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:58:44 -- STEP: 1748/2902 -- GLOBAL_STEP: 4650[0m |
|
| > loss_text_ce: 0.05818689614534378 (0.05609983744342689) |
|
| > loss_mel_ce: 3.2191317081451416 (3.2704604229883425) |
|
| > loss: 0.4096648395061493 (0.4158200326139103) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6999 (0.6053034963269524) |
|
| > loader_time: 0.0111 (0.010458539225144026) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 18:59:31 -- STEP: 1798/2902 -- GLOBAL_STEP: 4700[0m |
|
| > loss_text_ce: 0.05465058237314224 (0.056072617156868246) |
|
| > loss_mel_ce: 3.252976655960083 (3.27008914642525) |
|
| > loss: 0.4134534001350403 (0.4157702205443674) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6463 (0.6055046300071227) |
|
| > loader_time: 0.0109 (0.01047546341103627) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:00:19 -- STEP: 1848/2902 -- GLOBAL_STEP: 4750[0m |
|
| > loss_text_ce: 0.052592262625694275 (0.05605650214817035) |
|
| > loss_mel_ce: 3.2938497066497803 (3.2699186003311382) |
|
| > loss: 0.41830524802207947 (0.41574688796034626) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6226 (0.6058445509636036) |
|
| > loader_time: 0.0114 (0.01049227864195258) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:01:07 -- STEP: 1898/2902 -- GLOBAL_STEP: 4800[0m |
|
| > loss_text_ce: 0.05527101084589958 (0.05603910592611333) |
|
| > loss_mel_ce: 3.3503198623657227 (3.269640341573823) |
|
| > loss: 0.425698846578598 (0.41570993103832793) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6871 (0.6060072334347831) |
|
| > loader_time: 0.0114 (0.010505890193050855) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:01:54 -- STEP: 1948/2902 -- GLOBAL_STEP: 4850[0m |
|
| > loss_text_ce: 0.05501137673854828 (0.056014667678188364) |
|
| > loss_mel_ce: 3.2952277660369873 (3.269260302216611) |
|
| > loss: 0.4187798798084259 (0.41565937137334497) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.7268 (0.6060542482615002) |
|
| > loader_time: 0.0113 (0.010517563908007111) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:02:41 -- STEP: 1998/2902 -- GLOBAL_STEP: 4900[0m |
|
| > loss_text_ce: 0.05698436126112938 (0.05600299724788815) |
|
| > loss_mel_ce: 3.2746849060058594 (3.268788534122426) |
|
| > loss: 0.4164586663246155 (0.41559894159212485) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6491 (0.6059473711210442) |
|
| > loader_time: 0.0108 (0.010526264155352558) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:03:29 -- STEP: 2048/2902 -- GLOBAL_STEP: 4950[0m |
|
| > loss_text_ce: 0.05508394539356232 (0.05598831629686173) |
|
| > loss_mel_ce: 3.2314789295196533 (3.2680820173118272) |
|
| > loss: 0.4108203649520874 (0.41550879187707324) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4947 (0.6060070462990548) |
|
| > loader_time: 0.0106 (0.010532886837609112) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:04:17 -- STEP: 2098/2902 -- GLOBAL_STEP: 5000[0m |
|
| > loss_text_ce: 0.05668158456683159 (0.05596930274581829) |
|
| > loss_mel_ce: 3.2369794845581055 (3.2679047324296064) |
|
| > loss: 0.41170763969421387 (0.4154842545920263) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5536 (0.6062503820152025) |
|
| > loader_time: 0.0104 (0.010541449624998894) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:05:04 -- STEP: 2148/2902 -- GLOBAL_STEP: 5050[0m |
|
| > loss_text_ce: 0.05353817716240883 (0.0559521284911719) |
|
| > loss_mel_ce: 3.2572579383850098 (3.267594929513985) |
|
| > loss: 0.41384950280189514 (0.4154433824219073) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6125 (0.6061867110334069) |
|
| > loader_time: 0.0107 (0.010544300412332551) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:05:52 -- STEP: 2198/2902 -- GLOBAL_STEP: 5100[0m |
|
| > loss_text_ce: 0.05389314889907837 (0.055934098555119174) |
|
| > loss_mel_ce: 3.159824848175049 (3.2674999072185087) |
|
| > loss: 0.4017147421836853 (0.41542925089733296) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6574 (0.6065022010386691) |
|
| > loader_time: 0.011 (0.010550862990041777) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:06:38 -- STEP: 2248/2902 -- GLOBAL_STEP: 5150[0m |
|
| > loss_text_ce: 0.05729653686285019 (0.05592138946281738) |
|
| > loss_mel_ce: 3.3485758304595947 (3.2666910624801035) |
|
| > loss: 0.4257340431213379 (0.41532655668799556) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5645 (0.6061298031611797) |
|
| > loader_time: 0.0105 (0.010557977436276102) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:07:25 -- STEP: 2298/2902 -- GLOBAL_STEP: 5200[0m |
|
| > loss_text_ce: 0.05610555410385132 (0.055886846526912524) |
|
| > loss_mel_ce: 3.221963405609131 (3.2663745807501616) |
|
| > loss: 0.40975862741470337 (0.41528267858856327) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5627 (0.6061801232909823) |
|
| > loader_time: 0.0105 (0.010561991713169462) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:08:12 -- STEP: 2348/2902 -- GLOBAL_STEP: 5250[0m |
|
| > loss_text_ce: 0.054220084100961685 (0.05587315915098484) |
|
| > loss_mel_ce: 3.2626988887786865 (3.266057619853564) |
|
| > loss: 0.41461488604545593 (0.41524134751479097) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6679 (0.6060712704057382) |
|
| > loader_time: 0.011 (0.01056742150292081) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:08:59 -- STEP: 2398/2902 -- GLOBAL_STEP: 5300[0m |
|
| > loss_text_ce: 0.05538155138492584 (0.05585878610523964) |
|
| > loss_mel_ce: 3.1319658756256104 (3.2655375949336256) |
|
| > loss: 0.3984184265136719 (0.41517454782210345) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.534 (0.6057613339396293) |
|
| > loader_time: 0.0107 (0.01057357794051374) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:09:46 -- STEP: 2448/2902 -- GLOBAL_STEP: 5350[0m |
|
| > loss_text_ce: 0.054799508303403854 (0.05584115826439573) |
|
| > loss_mel_ce: 3.246894359588623 (3.265253891547521) |
|
| > loss: 0.4127117395401001 (0.4151368814404883) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5644 (0.6058000731312372) |
|
| > loader_time: 0.0108 (0.010580366538241025) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:10:32 -- STEP: 2498/2902 -- GLOBAL_STEP: 5400[0m |
|
| > loss_text_ce: 0.05230073258280754 (0.0558200412056296) |
|
| > loss_mel_ce: 3.3292624950408936 (3.264872017147257) |
|
| > loss: 0.4226953983306885 (0.4150865075347708) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5618 (0.6056058732293148) |
|
| > loader_time: 0.0108 (0.010589490612570631) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:11:20 -- STEP: 2548/2902 -- GLOBAL_STEP: 5450[0m |
|
| > loss_text_ce: 0.05251738056540489 (0.05579752388324007) |
|
| > loss_mel_ce: 3.153108596801758 (3.2645589915137836) |
|
| > loss: 0.4007032513618469 (0.4150445646777445) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5492 (0.6057547577883424) |
|
| > loader_time: 0.0105 (0.010596270651046307) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:12:06 -- STEP: 2598/2902 -- GLOBAL_STEP: 5500[0m |
|
| > loss_text_ce: 0.052732329815626144 (0.05577686714658008) |
|
| > loss_mel_ce: 3.3292698860168457 (3.264208226042403) |
|
| > loss: 0.4227502644062042 (0.41499813687948195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5166 (0.6055064683147716) |
|
| > loader_time: 0.0106 (0.010603652264724981) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:12:53 -- STEP: 2648/2902 -- GLOBAL_STEP: 5550[0m |
|
| > loss_text_ce: 0.05599590763449669 (0.05576192359566642) |
|
| > loss_mel_ce: 3.354013204574585 (3.263891820370971) |
|
| > loss: 0.4262511432170868 (0.41495671823305663) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.594 (0.6053740501043655) |
|
| > loader_time: 0.0115 (0.010613989884039431) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:13:39 -- STEP: 2698/2902 -- GLOBAL_STEP: 5600[0m |
|
| > loss_text_ce: 0.05634847655892372 (0.05574999848246749) |
|
| > loss_mel_ce: 3.174736976623535 (3.2637718163922242) |
|
| > loss: 0.40388569235801697 (0.41494022710752804) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6399 (0.6048974610859585) |
|
| > loader_time: 0.0112 (0.010621621486785422) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:14:26 -- STEP: 2748/2902 -- GLOBAL_STEP: 5650[0m |
|
| > loss_text_ce: 0.052691757678985596 (0.05573155193994961) |
|
| > loss_mel_ce: 3.3672266006469727 (3.2633937294479716) |
|
| > loss: 0.4274897873401642 (0.4148906604258081) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6579 (0.6049603481480119) |
|
| > loader_time: 0.0111 (0.010630740001906142) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:15:14 -- STEP: 2798/2902 -- GLOBAL_STEP: 5700[0m |
|
| > loss_text_ce: 0.05469852685928345 (0.05571502604900596) |
|
| > loss_mel_ce: 3.3810243606567383 (3.263473064836389) |
|
| > loss: 0.4294653534889221 (0.4148985116267903) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6643 (0.6051521071201562) |
|
| > loader_time: 0.0122 (0.010641849236968946) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:16:01 -- STEP: 2848/2902 -- GLOBAL_STEP: 5750[0m |
|
| > loss_text_ce: 0.05531623214483261 (0.05570412047071318) |
|
| > loss_mel_ce: 3.2582409381866455 (3.2633204451772606) |
|
| > loss: 0.41419464349746704 (0.4148780709823196) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3849 (0.6051703050230323) |
|
| > loader_time: 0.0101 (0.010653467241967678) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:16:48 -- STEP: 2898/2902 -- GLOBAL_STEP: 5800[0m |
|
| > loss_text_ce: 0.05267462506890297 (0.05568916210486938) |
|
| > loss_mel_ce: 3.2610042095184326 (3.2627478573880087) |
|
| > loss: 0.41420984268188477 (0.41480462774480104) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6969 (0.6050742793856527) |
|
| > loader_time: 0.0117 (0.010668510317884718) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.11507114858338323 [0m(-5.423781847718745e-05) |
|
| > avg_loss_text_ce:[92m 0.05247822260916836 [0m(-0.002339626163846309) |
|
| > avg_loss_mel_ce:[92m 3.202105414987815 [0m(-0.04251997037367383) |
|
| > avg_loss:[92m 3.254583633307255 [0m(-0.04485960681029022) |
|
|
|
|
|
[4m[1m > EPOCH: 2/5[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-22 19:18:12) [0m |
|
|
|
[1m --> TIME: 2024-12-22 19:18:56 -- STEP: 46/2902 -- GLOBAL_STEP: 5850[0m |
|
| > loss_text_ce: 0.05449153855443001 (0.054341975761496505) |
|
| > loss_mel_ce: 3.34686017036438 (3.262047643246858) |
|
| > loss: 0.4251689612865448 (0.4145487028619517) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.607 (0.5991597175598146) |
|
| > loader_time: 0.0104 (0.010427303936170496) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:19:43 -- STEP: 96/2902 -- GLOBAL_STEP: 5900[0m |
|
| > loss_text_ce: 0.055128224194049835 (0.05449626039868841) |
|
| > loss_mel_ce: 3.243983268737793 (3.2608101293444633) |
|
| > loss: 0.41238895058631897 (0.41441329941153526) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5481 (0.597640375296275) |
|
| > loader_time: 0.0102 (0.010360459486643466) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:20:31 -- STEP: 146/2902 -- GLOBAL_STEP: 5950[0m |
|
| > loss_text_ce: 0.05616119131445885 (0.05466639781243181) |
|
| > loss_mel_ce: 3.2564191818237305 (3.253628141259494) |
|
| > loss: 0.41407254338264465 (0.4135368174477799) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6546 (0.6053444686001294) |
|
| > loader_time: 0.0103 (0.010333493964312823) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:21:17 -- STEP: 196/2902 -- GLOBAL_STEP: 6000[0m |
|
| > loss_text_ce: 0.05680551752448082 (0.05473522758301424) |
|
| > loss_mel_ce: 3.142683982849121 (3.2478924880222397) |
|
| > loss: 0.3999361991882324 (0.41282846444115345) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6581 (0.6036743071614478) |
|
| > loader_time: 0.0103 (0.010324716567993155) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:22:05 -- STEP: 246/2902 -- GLOBAL_STEP: 6050[0m |
|
| > loss_text_ce: 0.053551316261291504 (0.05457976252991494) |
|
| > loss_mel_ce: 3.1189703941345215 (3.245878595646804) |
|
| > loss: 0.39656519889831543 (0.41255729506170846) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6213 (0.6064855470889952) |
|
| > loader_time: 0.0099 (0.01028650562937666) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:22:53 -- STEP: 296/2902 -- GLOBAL_STEP: 6100[0m |
|
| > loss_text_ce: 0.05468261241912842 (0.05454644439993679) |
|
| > loss_mel_ce: 3.2226357460021973 (3.244215094559901) |
|
| > loss: 0.4096648097038269 (0.4123451929237391) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5772 (0.6066698633335732) |
|
| > loader_time: 0.0104 (0.010362294880119523) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:23:39 -- STEP: 346/2902 -- GLOBAL_STEP: 6150[0m |
|
| > loss_text_ce: 0.056551966816186905 (0.05455692140439343) |
|
| > loss_mel_ce: 3.2596166133880615 (3.2438248712892466) |
|
| > loss: 0.41452106833457947 (0.41229772490228533) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5327 (0.6049295891227059) |
|
| > loader_time: 0.0097 (0.010348508812788594) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:24:26 -- STEP: 396/2902 -- GLOBAL_STEP: 6200[0m |
|
| > loss_text_ce: 0.057674016803503036 (0.05454431032065792) |
|
| > loss_mel_ce: 3.2081027030944824 (3.243972488726027) |
|
| > loss: 0.4082220792770386 (0.41231460033944156) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6618 (0.6051323636613708) |
|
| > loader_time: 0.0102 (0.010334551936448218) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:25:13 -- STEP: 446/2902 -- GLOBAL_STEP: 6250[0m |
|
| > loss_text_ce: 0.053182203322649 (0.05448943492643235) |
|
| > loss_mel_ce: 3.3238401412963867 (3.2430883688777006) |
|
| > loss: 0.42212778329849243 (0.4121972257647279) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5257 (0.6037687779542039) |
|
| > loader_time: 0.0097 (0.010317636712249615) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:26:00 -- STEP: 496/2902 -- GLOBAL_STEP: 6300[0m |
|
| > loss_text_ce: 0.05793963000178337 (0.0545288963588856) |
|
| > loss_mel_ce: 3.1546380519866943 (3.2435592682130867) |
|
| > loss: 0.40157219767570496 (0.41226102081277677) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5773 (0.6035887161570208) |
|
| > loader_time: 0.0103 (0.010317006899464512) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:26:46 -- STEP: 546/2902 -- GLOBAL_STEP: 6350[0m |
|
| > loss_text_ce: 0.05591798573732376 (0.05447477271486988) |
|
| > loss_mel_ce: 3.255568742752075 (3.2395439558413432) |
|
| > loss: 0.4139358401298523 (0.4117523413765561) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6578 (0.6032865440452491) |
|
| > loader_time: 0.0105 (0.010307735575861107) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:27:34 -- STEP: 596/2902 -- GLOBAL_STEP: 6400[0m |
|
| > loss_text_ce: 0.056878942996263504 (0.05450224877653906) |
|
| > loss_mel_ce: 3.32279109954834 (3.2391228695843832) |
|
| > loss: 0.4224587678909302 (0.41170314004357217) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6144 (0.6037989190760873) |
|
| > loader_time: 0.0104 (0.01030485742044128) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:28:21 -- STEP: 646/2902 -- GLOBAL_STEP: 6450[0m |
|
| > loss_text_ce: 0.05395687371492386 (0.05448623928284682) |
|
| > loss_mel_ce: 3.2676842212677 (3.238311486716608) |
|
| > loss: 0.4152051508426666 (0.41159971593518735) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6494 (0.6037543484301021) |
|
| > loader_time: 0.0105 (0.010310063052103604) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:29:08 -- STEP: 696/2902 -- GLOBAL_STEP: 6500[0m |
|
| > loss_text_ce: 0.05541800707578659 (0.05446035207795172) |
|
| > loss_mel_ce: 3.185835123062134 (3.2377473331730924) |
|
| > loss: 0.4051566421985626 (0.41152596079755094) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4914 (0.6042240799843576) |
|
| > loader_time: 0.0098 (0.010309737989272186) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:29:55 -- STEP: 746/2902 -- GLOBAL_STEP: 6550[0m |
|
| > loss_text_ce: 0.05779316648840904 (0.05444739832375548) |
|
| > loss_mel_ce: 3.315509557723999 (3.2381504930695626) |
|
| > loss: 0.421662837266922 (0.41157473659707133) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6138 (0.6041072274660613) |
|
| > loader_time: 0.016 (0.010313577690329045) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:30:42 -- STEP: 796/2902 -- GLOBAL_STEP: 6600[0m |
|
| > loss_text_ce: 0.051639851182699203 (0.05440891647814357) |
|
| > loss_mel_ce: 3.1917884349823 (3.237836667640723) |
|
| > loss: 0.40542852878570557 (0.41153069817690385) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.7215 (0.6042170922959863) |
|
| > loader_time: 0.0113 (0.010306383197631065) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:31:30 -- STEP: 846/2902 -- GLOBAL_STEP: 6650[0m |
|
| > loss_text_ce: 0.0565703846514225 (0.05434837889357803) |
|
| > loss_mel_ce: 3.3043394088745117 (3.2371244619360473) |
|
| > loss: 0.4201137125492096 (0.4114341052429613) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6267 (0.6048971769657534) |
|
| > loader_time: 0.0102 (0.010311559018795649) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:32:17 -- STEP: 896/2902 -- GLOBAL_STEP: 6700[0m |
|
| > loss_text_ce: 0.05269205942749977 (0.05433550530246326) |
|
| > loss_mel_ce: 3.2086503505706787 (3.236831199909958) |
|
| > loss: 0.4076678156852722 (0.41139583825133735) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5897 (0.6048591102340385) |
|
| > loader_time: 0.0105 (0.010325204581022259) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:33:05 -- STEP: 946/2902 -- GLOBAL_STEP: 6750[0m |
|
| > loss_text_ce: 0.05327130854129791 (0.05433601268171382) |
|
| > loss_mel_ce: 3.1741154193878174 (3.236998047435509) |
|
| > loss: 0.40342333912849426 (0.4114167576864456) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5356 (0.6051080012674306) |
|
| > loader_time: 0.0099 (0.010325364532228752) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:33:52 -- STEP: 996/2902 -- GLOBAL_STEP: 6800[0m |
|
| > loss_text_ce: 0.05429665371775627 (0.05431595303671786) |
|
| > loss_mel_ce: 3.098682165145874 (3.2360697728563013) |
|
| > loss: 0.3941223621368408 (0.41129821604753586) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6609 (0.6053404793681862) |
|
| > loader_time: 0.0107 (0.010332633931952783) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:34:38 -- STEP: 1046/2902 -- GLOBAL_STEP: 6850[0m |
|
| > loss_text_ce: 0.0514579601585865 (0.05428189642186033) |
|
| > loss_mel_ce: 3.1680314540863037 (3.2349629500172092) |
|
| > loss: 0.40243616700172424 (0.4111556061481198) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5487 (0.604390150502352) |
|
| > loader_time: 0.0102 (0.010338225966429842) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:35:25 -- STEP: 1096/2902 -- GLOBAL_STEP: 6900[0m |
|
| > loss_text_ce: 0.05452772602438927 (0.05426657630171437) |
|
| > loss_mel_ce: 3.296816825866699 (3.234400193839176) |
|
| > loss: 0.41891807317733765 (0.4110833465726704) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6502 (0.6042889208689218) |
|
| > loader_time: 0.012 (0.010351204524075022) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:36:12 -- STEP: 1146/2902 -- GLOBAL_STEP: 6950[0m |
|
| > loss_text_ce: 0.054743699729442596 (0.054242047489365564) |
|
| > loss_mel_ce: 3.1826770305633545 (3.2341083564891435) |
|
| > loss: 0.4046775996685028 (0.4110438008426994) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6152 (0.6043572690058332) |
|
| > loader_time: 0.0101 (0.010360107579988538) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:36:59 -- STEP: 1196/2902 -- GLOBAL_STEP: 7000[0m |
|
| > loss_text_ce: 0.04921133816242218 (0.05422345994594804) |
|
| > loss_mel_ce: 3.323059320449829 (3.2337135487176893) |
|
| > loss: 0.42153382301330566 (0.41099212650072603) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6726 (0.6041809440057805) |
|
| > loader_time: 0.011 (0.010368075857194368) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:37:47 -- STEP: 1246/2902 -- GLOBAL_STEP: 7050[0m |
|
| > loss_text_ce: 0.05187971889972687 (0.05421067672642622) |
|
| > loss_mel_ce: 3.2639362812042236 (3.2332898234861807) |
|
| > loss: 0.41447699069976807 (0.4109375628958158) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.639 (0.6047971242886284) |
|
| > loader_time: 0.0106 (0.01038035209833331) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:38:34 -- STEP: 1296/2902 -- GLOBAL_STEP: 7100[0m |
|
| > loss_text_ce: 0.055725544691085815 (0.05422138063708482) |
|
| > loss_mel_ce: 3.212235927581787 (3.2331566674473833) |
|
| > loss: 0.4084951877593994 (0.41092225642483876) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4821 (0.6050901449756845) |
|
| > loader_time: 0.0104 (0.010385573278238736) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:39:21 -- STEP: 1346/2902 -- GLOBAL_STEP: 7150[0m |
|
| > loss_text_ce: 0.054803065955638885 (0.05419990306866825) |
|
| > loss_mel_ce: 3.3109726905822754 (3.2331086253694745) |
|
| > loss: 0.42072197794914246 (0.41091356646784316) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.653 (0.6051710766102181) |
|
| > loader_time: 0.0105 (0.010394215760932447) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:40:07 -- STEP: 1396/2902 -- GLOBAL_STEP: 7200[0m |
|
| > loss_text_ce: 0.05577743798494339 (0.05419548767963078) |
|
| > loss_mel_ce: 3.1629726886749268 (3.2319712032561303) |
|
| > loss: 0.4023437798023224 (0.4107708367799275) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6159 (0.6044754990533964) |
|
| > loader_time: 0.0104 (0.010397913804368499) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:40:54 -- STEP: 1446/2902 -- GLOBAL_STEP: 7250[0m |
|
| > loss_text_ce: 0.05135389044880867 (0.05418163613721544) |
|
| > loss_mel_ce: 3.1880338191986084 (3.231893382296686) |
|
| > loss: 0.4049234688282013 (0.41075937767328863) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6488 (0.604256208177093) |
|
| > loader_time: 0.0108 (0.010400912738604823) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:41:40 -- STEP: 1496/2902 -- GLOBAL_STEP: 7300[0m |
|
| > loss_text_ce: 0.055541761219501495 (0.05417019395994392) |
|
| > loss_mel_ce: 3.259507417678833 (3.2317391237154354) |
|
| > loss: 0.41438114643096924 (0.4107386650925967) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5431 (0.6038286035711105) |
|
| > loader_time: 0.0104 (0.010405490742647704) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:42:27 -- STEP: 1546/2902 -- GLOBAL_STEP: 7350[0m |
|
| > loss_text_ce: 0.05562349036335945 (0.05414642839852774) |
|
| > loss_mel_ce: 3.2322540283203125 (3.230981718368085) |
|
| > loss: 0.41098469495773315 (0.41064101872052416) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6215 (0.603749968564525) |
|
| > loader_time: 0.0106 (0.010414998867533335) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:43:15 -- STEP: 1596/2902 -- GLOBAL_STEP: 7400[0m |
|
| > loss_text_ce: 0.05427920073270798 (0.05413814238914497) |
|
| > loss_mel_ce: 3.1687004566192627 (3.2307760164253683) |
|
| > loss: 0.4028724431991577 (0.4106142702407404) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6616 (0.6040683033173532) |
|
| > loader_time: 0.0114 (0.0104308143295441) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:44:02 -- STEP: 1646/2902 -- GLOBAL_STEP: 7450[0m |
|
| > loss_text_ce: 0.05014851316809654 (0.054124010054627726) |
|
| > loss_mel_ce: 3.308337926864624 (3.2305475654462987) |
|
| > loss: 0.41981080174446106 (0.41058394731104997) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6745 (0.6039762705528275) |
|
| > loader_time: 0.0109 (0.010435233718714472) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:44:50 -- STEP: 1696/2902 -- GLOBAL_STEP: 7500[0m |
|
| > loss_text_ce: 0.05018627643585205 (0.05409667860576005) |
|
| > loss_mel_ce: 3.0870144367218018 (3.229944757836044) |
|
| > loss: 0.3921501040458679 (0.4105051799308295) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6483 (0.604342738271883) |
|
| > loader_time: 0.0112 (0.010440538233181211) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:45:36 -- STEP: 1746/2902 -- GLOBAL_STEP: 7550[0m |
|
| > loss_text_ce: 0.054639771580696106 (0.05408420456618174) |
|
| > loss_mel_ce: 3.236097574234009 (3.2295827566553217) |
|
| > loss: 0.41134217381477356 (0.4104583704969998) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6497 (0.6042213617458091) |
|
| > loader_time: 0.0105 (0.010453733381946467) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:46:24 -- STEP: 1796/2902 -- GLOBAL_STEP: 7600[0m |
|
| > loss_text_ce: 0.0547931008040905 (0.05406024138378872) |
|
| > loss_mel_ce: 3.2425832748413086 (3.2288573825014186) |
|
| > loss: 0.4121720492839813 (0.4103647032897826) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.654 (0.6042706026531794) |
|
| > loader_time: 0.0114 (0.01046769297733604) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:47:11 -- STEP: 1846/2902 -- GLOBAL_STEP: 7650[0m |
|
| > loss_text_ce: 0.05211947485804558 (0.0540442640871972) |
|
| > loss_mel_ce: 3.1296088695526123 (3.2281167809392355) |
|
| > loss: 0.39771604537963867 (0.4102701309055313) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5564 (0.6042208054212929) |
|
| > loader_time: 0.0106 (0.010471764778393445) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:47:58 -- STEP: 1896/2902 -- GLOBAL_STEP: 7700[0m |
|
| > loss_text_ce: 0.056086037307977676 (0.05404073305483044) |
|
| > loss_mel_ce: 3.290001392364502 (3.2276765414934108) |
|
| > loss: 0.41826093196868896 (0.4102146595810785) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.578 (0.6043190175219416) |
|
| > loader_time: 0.0106 (0.010481623280400462) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:48:45 -- STEP: 1946/2902 -- GLOBAL_STEP: 7750[0m |
|
| > loss_text_ce: 0.05454717576503754 (0.054029209030081904) |
|
| > loss_mel_ce: 3.259446620941162 (3.2277184737059454) |
|
| > loss: 0.4142492115497589 (0.41021846057148387) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6551 (0.604525358311808) |
|
| > loader_time: 0.0114 (0.01049390493414507) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:49:33 -- STEP: 1996/2902 -- GLOBAL_STEP: 7800[0m |
|
| > loss_text_ce: 0.05354653671383858 (0.0540268157099286) |
|
| > loss_mel_ce: 3.304413080215454 (3.22762779996008) |
|
| > loss: 0.41974493861198425 (0.41020682715402074) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6331 (0.6046111153696238) |
|
| > loader_time: 0.0113 (0.010508355850686048) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:50:20 -- STEP: 2046/2902 -- GLOBAL_STEP: 7850[0m |
|
| > loss_text_ce: 0.055073026567697525 (0.05401429145770805) |
|
| > loss_mel_ce: 3.216965675354004 (3.2274121992282727) |
|
| > loss: 0.4090048372745514 (0.4101783115405838) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6854 (0.604751432746036) |
|
| > loader_time: 0.0113 (0.010519989657844845) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:51:08 -- STEP: 2096/2902 -- GLOBAL_STEP: 7900[0m |
|
| > loss_text_ce: 0.05682572349905968 (0.05401011059089131) |
|
| > loss_mel_ce: 3.143216848373413 (3.2272642189992284) |
|
| > loss: 0.4000053107738495 (0.4101592913724992) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6136 (0.6048859395371127) |
|
| > loader_time: 0.0113 (0.01052777946905325) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:51:55 -- STEP: 2146/2902 -- GLOBAL_STEP: 7950[0m |
|
| > loss_text_ce: 0.052709128707647324 (0.053994231788790474) |
|
| > loss_mel_ce: 3.2841806411743164 (3.2269571499633423) |
|
| > loss: 0.41711121797561646 (0.41011892291061897) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5301 (0.6049023500026538) |
|
| > loader_time: 0.0102 (0.010533556689309487) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:52:42 -- STEP: 2196/2902 -- GLOBAL_STEP: 8000[0m |
|
| > loss_text_ce: 0.05199521407485008 (0.053986151086909545) |
|
| > loss_mel_ce: 3.114041566848755 (3.2264509198878404) |
|
| > loss: 0.3957546055316925 (0.4100546340283364) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6118 (0.6049591221444579) |
|
| > loader_time: 0.011 (0.010538012820731954) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:53:29 -- STEP: 2246/2902 -- GLOBAL_STEP: 8050[0m |
|
| > loss_text_ce: 0.05536806583404541 (0.05397705866300891) |
|
| > loss_mel_ce: 3.229199171066284 (3.226278807155169) |
|
| > loss: 0.4105709195137024 (0.41003198335187707) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6545 (0.6048499229011621) |
|
| > loader_time: 0.0109 (0.010542242100274884) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:54:16 -- STEP: 2296/2902 -- GLOBAL_STEP: 8100[0m |
|
| > loss_text_ce: 0.05085474252700806 (0.05395412064102261) |
|
| > loss_mel_ce: 3.0547423362731934 (3.2258496954258287) |
|
| > loss: 0.3881996273994446 (0.4099754771081411) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5028 (0.6047402797883378) |
|
| > loader_time: 0.0106 (0.010546214281474247) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:55:04 -- STEP: 2346/2902 -- GLOBAL_STEP: 8150[0m |
|
| > loss_text_ce: 0.055554307997226715 (0.053945425656470924) |
|
| > loss_mel_ce: 3.1857361793518066 (3.225720517267451) |
|
| > loss: 0.4051613211631775 (0.4099582429508417) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6962 (0.6051811104845186) |
|
| > loader_time: 0.0113 (0.01055516936488366) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:55:51 -- STEP: 2396/2902 -- GLOBAL_STEP: 8200[0m |
|
| > loss_text_ce: 0.05265699699521065 (0.053927266866105536) |
|
| > loss_mel_ce: 3.222463846206665 (3.225377675388409) |
|
| > loss: 0.40939009189605713 (0.4099131178900672) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6853 (0.6051519718114415) |
|
| > loader_time: 0.0114 (0.010562321379507113) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:56:37 -- STEP: 2446/2902 -- GLOBAL_STEP: 8250[0m |
|
| > loss_text_ce: 0.05529283732175827 (0.05392556344802422) |
|
| > loss_mel_ce: 3.248408555984497 (3.2250945345846596) |
|
| > loss: 0.4129626750946045 (0.40987751231462516) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6224 (0.6047231179227202) |
|
| > loader_time: 0.0109 (0.010566833754201832) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:57:24 -- STEP: 2496/2902 -- GLOBAL_STEP: 8300[0m |
|
| > loss_text_ce: 0.05385390669107437 (0.05390973614815337) |
|
| > loss_mel_ce: 3.188469171524048 (3.224661102279637) |
|
| > loss: 0.4052903950214386 (0.4098213548676515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5605 (0.6046044805493085) |
|
| > loader_time: 0.0112 (0.010573913558171335) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:58:11 -- STEP: 2546/2902 -- GLOBAL_STEP: 8350[0m |
|
| > loss_text_ce: 0.05382354557514191 (0.05389312902891253) |
|
| > loss_mel_ce: 3.26172137260437 (3.2242961726784207) |
|
| > loss: 0.41444310545921326 (0.40977366277999216) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6391 (0.6046588743004547) |
|
| > loader_time: 0.0113 (0.010582529459052066) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:58:58 -- STEP: 2596/2902 -- GLOBAL_STEP: 8400[0m |
|
| > loss_text_ce: 0.05468406900763512 (0.053886171581025585) |
|
| > loss_mel_ce: 3.3646421432495117 (3.2239668191122135) |
|
| > loss: 0.42741578817367554 (0.4097316239620392) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6687 (0.6046174602075035) |
|
| > loader_time: 0.0109 (0.010590319181627414) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 19:59:46 -- STEP: 2646/2902 -- GLOBAL_STEP: 8450[0m |
|
| > loss_text_ce: 0.051759347319602966 (0.05386880528849135) |
|
| > loss_mel_ce: 3.094008207321167 (3.223380103853732) |
|
| > loss: 0.3932209312915802 (0.40965611376684885) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6131 (0.6047673099255417) |
|
| > loader_time: 0.0108 (0.010597522837234462) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:00:34 -- STEP: 2696/2902 -- GLOBAL_STEP: 8500[0m |
|
| > loss_text_ce: 0.054853543639183044 (0.05384999631492295) |
|
| > loss_mel_ce: 3.108388662338257 (3.2230357261545906) |
|
| > loss: 0.39540526270866394 (0.40961071541560495) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4407 (0.6050107980517307) |
|
| > loader_time: 0.01 (0.010605064980707816) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:01:20 -- STEP: 2746/2902 -- GLOBAL_STEP: 8550[0m |
|
| > loss_text_ce: 0.05507945269346237 (0.053841534410505) |
|
| > loss_mel_ce: 3.3420369625091553 (3.2224513486798254) |
|
| > loss: 0.4246395528316498 (0.40953661047752443) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6173 (0.6048634139100554) |
|
| > loader_time: 0.0111 (0.010611069176375911) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:02:08 -- STEP: 2796/2902 -- GLOBAL_STEP: 8600[0m |
|
| > loss_text_ce: 0.051820311695337296 (0.053828303700843266) |
|
| > loss_mel_ce: 3.2003228664398193 (3.2219526886257808) |
|
| > loss: 0.406517893075943 (0.40947262414108854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6264 (0.6049439221492642) |
|
| > loader_time: 0.0108 (0.010617568598625836) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:02:55 -- STEP: 2846/2902 -- GLOBAL_STEP: 8650[0m |
|
| > loss_text_ce: 0.05038287490606308 (0.0538107970950684) |
|
| > loss_mel_ce: 3.0824766159057617 (3.22168692776793) |
|
| > loss: 0.3916074335575104 (0.4094372156926299) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5422 (0.6049977293001112) |
|
| > loader_time: 0.0104 (0.010621224585671881) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:03:42 -- STEP: 2896/2902 -- GLOBAL_STEP: 8700[0m |
|
| > loss_text_ce: 0.05103882774710655 (0.053802881721736494) |
|
| > loss_mel_ce: 3.1896164417266846 (3.221427488063578) |
|
| > loss: 0.4050818979740143 (0.4094037962820153) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5014 (0.60504338013533) |
|
| > loader_time: 0.0104 (0.010624508749055609) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.11469822218923863 [0m(-0.0003729263941446015) |
|
| > avg_loss_text_ce:[92m 0.05099275098605589 [0m(-0.0014854716231124665) |
|
| > avg_loss_mel_ce:[92m 3.170435007172402 [0m(-0.03167040781541308) |
|
| > avg_loss:[92m 3.2214277597388823 [0m(-0.033155873568372574) |
|
|
|
|
|
[4m[1m > EPOCH: 3/5[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-22 20:05:08) [0m |
|
|
|
[1m --> TIME: 2024-12-22 20:05:50 -- STEP: 44/2902 -- GLOBAL_STEP: 8750[0m |
|
| > loss_text_ce: 0.05118555575609207 (0.053070816888727924) |
|
| > loss_mel_ce: 3.2175662517547607 (3.2166340080174534) |
|
| > loss: 0.4085939824581146 (0.40871310301802377) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6559 (0.6008733131668783) |
|
| > loader_time: 0.0103 (0.010414784604852848) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:06:38 -- STEP: 94/2902 -- GLOBAL_STEP: 8800[0m |
|
| > loss_text_ce: 0.05431003123521805 (0.05314608088674697) |
|
| > loss_mel_ce: 3.2067341804504395 (3.222269390491729) |
|
| > loss: 0.4076305329799652 (0.409426933590402) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5549 (0.6045856957739972) |
|
| > loader_time: 0.0099 (0.010405466911640572) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:07:25 -- STEP: 144/2902 -- GLOBAL_STEP: 8850[0m |
|
| > loss_text_ce: 0.057936280965805054 (0.05344269391045803) |
|
| > loss_mel_ce: 3.121586322784424 (3.2203697181410256) |
|
| > loss: 0.3974403142929077 (0.409226551031073) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5058 (0.6065922164254718) |
|
| > loader_time: 0.0099 (0.010360252526071334) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:08:11 -- STEP: 194/2902 -- GLOBAL_STEP: 8900[0m |
|
| > loss_text_ce: 0.051768817007541656 (0.05326584741934059) |
|
| > loss_mel_ce: 3.128735303878784 (3.2146200224296333) |
|
| > loss: 0.39756301045417786 (0.4084857332030523) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4981 (0.6034614535951123) |
|
| > loader_time: 0.0094 (0.010401970332430815) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:08:59 -- STEP: 244/2902 -- GLOBAL_STEP: 8950[0m |
|
| > loss_text_ce: 0.0531751923263073 (0.05316113559987213) |
|
| > loss_mel_ce: 3.196899890899658 (3.213496774923606) |
|
| > loss: 0.4062593877315521 (0.40833223892039944) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6879 (0.6043678590508755) |
|
| > loader_time: 0.0105 (0.010358643336374256) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:09:46 -- STEP: 294/2902 -- GLOBAL_STEP: 9000[0m |
|
| > loss_text_ce: 0.053236380219459534 (0.05315211574946131) |
|
| > loss_mel_ce: 3.101793050765991 (3.210114073591167) |
|
| > loss: 0.3943786919116974 (0.4079082736960887) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6839 (0.6065582020753093) |
|
| > loader_time: 0.0105 (0.010334769884745283) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:10:33 -- STEP: 344/2902 -- GLOBAL_STEP: 9050[0m |
|
| > loss_text_ce: 0.05260626599192619 (0.05303334600703661) |
|
| > loss_mel_ce: 3.1087124347686768 (3.20666435984678) |
|
| > loss: 0.3951648473739624 (0.4074622132750446) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4613 (0.6061097487460737) |
|
| > loader_time: 0.0096 (0.010324067154596021) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:11:20 -- STEP: 394/2902 -- GLOBAL_STEP: 9100[0m |
|
| > loss_text_ce: 0.052920322865247726 (0.053000804972905796) |
|
| > loss_mel_ce: 3.29287052154541 (3.2060848898088867) |
|
| > loss: 0.41822385787963867 (0.40738571181817723) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6541 (0.6054662000104255) |
|
| > loader_time: 0.0105 (0.010301449577215363) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:12:07 -- STEP: 444/2902 -- GLOBAL_STEP: 9150[0m |
|
| > loss_text_ce: 0.049667783081531525 (0.052979113911656106) |
|
| > loss_mel_ce: 3.1026852130889893 (3.20582131061468) |
|
| > loss: 0.3940441310405731 (0.40735005298713317) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4883 (0.6050507517548295) |
|
| > loader_time: 0.0094 (0.01028660664687286) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:12:54 -- STEP: 494/2902 -- GLOBAL_STEP: 9200[0m |
|
| > loss_text_ce: 0.04981204494833946 (0.052981702980362935) |
|
| > loss_mel_ce: 3.082789182662964 (3.206828746718433) |
|
| > loss: 0.3915751576423645 (0.40747630620292336) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6535 (0.6049649213489733) |
|
| > loader_time: 0.0106 (0.010281438286970505) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:13:41 -- STEP: 544/2902 -- GLOBAL_STEP: 9250[0m |
|
| > loss_text_ce: 0.054514117538928986 (0.05301835546133055) |
|
| > loss_mel_ce: 3.2055437564849854 (3.2056176938554812) |
|
| > loss: 0.4075072407722473 (0.40732950602165047) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3799 (0.6048707545680159) |
|
| > loader_time: 0.0091 (0.01027688208748313) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:14:27 -- STEP: 594/2902 -- GLOBAL_STEP: 9300[0m |
|
| > loss_text_ce: 0.05195319652557373 (0.05302652499014486) |
|
| > loss_mel_ce: 3.207526922225952 (3.204179445099749) |
|
| > loss: 0.40743499994277954 (0.4071507459437406) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5395 (0.6040624107174604) |
|
| > loader_time: 0.01 (0.01028768141261656) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:15:15 -- STEP: 644/2902 -- GLOBAL_STEP: 9350[0m |
|
| > loss_text_ce: 0.05081777274608612 (0.05300852938653526) |
|
| > loss_mel_ce: 3.2010488510131836 (3.204077336358725) |
|
| > loss: 0.40648332238197327 (0.4071357330272656) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6665 (0.604685469813969) |
|
| > loader_time: 0.0108 (0.010288058600810744) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:16:01 -- STEP: 694/2902 -- GLOBAL_STEP: 9400[0m |
|
| > loss_text_ce: 0.053493183106184006 (0.0530088954156947) |
|
| > loss_mel_ce: 3.2406392097473145 (3.2033663084596284) |
|
| > loss: 0.41176655888557434 (0.4070469001442279) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5094 (0.6040704161014617) |
|
| > loader_time: 0.0095 (0.010290941175191477) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:16:48 -- STEP: 744/2902 -- GLOBAL_STEP: 9450[0m |
|
| > loss_text_ce: 0.05268382281064987 (0.05297802821783135) |
|
| > loss_mel_ce: 3.216268301010132 (3.202414394386351) |
|
| > loss: 0.4086190164089203 (0.4069240525845558) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6145 (0.6038787175891229) |
|
| > loader_time: 0.0111 (0.0103129840666248) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:17:35 -- STEP: 794/2902 -- GLOBAL_STEP: 9500[0m |
|
| > loss_text_ce: 0.05258984863758087 (0.05294387003111304) |
|
| > loss_mel_ce: 3.1116855144500732 (3.201499309587837) |
|
| > loss: 0.3955344259738922 (0.4068053971309805) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6243 (0.6037636656004484) |
|
| > loader_time: 0.0104 (0.010329933550856275) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:18:22 -- STEP: 844/2902 -- GLOBAL_STEP: 9550[0m |
|
| > loss_text_ce: 0.04872863367199898 (0.052906146908611534) |
|
| > loss_mel_ce: 3.1472809314727783 (3.2004845972874696) |
|
| > loss: 0.3995012044906616 (0.4066738426261603) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6961 (0.6039798511148075) |
|
| > loader_time: 0.0112 (0.010345070565481321) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:19:10 -- STEP: 894/2902 -- GLOBAL_STEP: 9600[0m |
|
| > loss_text_ce: 0.05376376956701279 (0.052866009224447945) |
|
| > loss_mel_ce: 3.118699550628662 (3.2001385982137913) |
|
| > loss: 0.39655792713165283 (0.4066255754339081) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6157 (0.6042345548902859) |
|
| > loader_time: 0.0104 (0.010358344788519328) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:19:56 -- STEP: 944/2902 -- GLOBAL_STEP: 9650[0m |
|
| > loss_text_ce: 0.05543249472975731 (0.05286841211698449) |
|
| > loss_mel_ce: 3.1901323795318604 (3.199281110349346) |
|
| > loss: 0.4056956171989441 (0.4065186898460832) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3571 (0.6036462536302674) |
|
| > loader_time: 0.0096 (0.010371917637728028) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:20:43 -- STEP: 994/2902 -- GLOBAL_STEP: 9700[0m |
|
| > loss_text_ce: 0.05254547297954559 (0.05283306972301706) |
|
| > loss_mel_ce: 3.26566481590271 (3.198850807769436) |
|
| > loss: 0.4147762954235077 (0.40646048432864407) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6149 (0.6033839968368571) |
|
| > loader_time: 0.0107 (0.010391604252744248) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:21:29 -- STEP: 1044/2902 -- GLOBAL_STEP: 9750[0m |
|
| > loss_text_ce: 0.051356155425310135 (0.05281972371298695) |
|
| > loss_mel_ce: 3.2884762287139893 (3.1981828160212853) |
|
| > loss: 0.4174790382385254 (0.4063753171099556) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6524 (0.603219235993894) |
|
| > loader_time: 0.0111 (0.010409164702755281) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:22:17 -- STEP: 1094/2902 -- GLOBAL_STEP: 9800[0m |
|
| > loss_text_ce: 0.05341365188360214 (0.052800002707069756) |
|
| > loss_mel_ce: 3.23124623298645 (3.198679782354851) |
|
| > loss: 0.4105824828147888 (0.40643497279945634) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.638 (0.6037313306135502) |
|
| > loader_time: 0.0117 (0.010436894034990664) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:23:04 -- STEP: 1144/2902 -- GLOBAL_STEP: 9850[0m |
|
| > loss_text_ce: 0.05161362886428833 (0.05279735522995987) |
|
| > loss_mel_ce: 3.176171064376831 (3.1983392276130336) |
|
| > loss: 0.4034730792045593 (0.4063920725423555) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5035 (0.603374870715442) |
|
| > loader_time: 0.0107 (0.010455987878612703) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:23:50 -- STEP: 1194/2902 -- GLOBAL_STEP: 9900[0m |
|
| > loss_text_ce: 0.053719062358140945 (0.05276699765106444) |
|
| > loss_mel_ce: 3.240354299545288 (3.1977713854868) |
|
| > loss: 0.4117591679096222 (0.40631729763756236) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6053 (0.6026648664394623) |
|
| > loader_time: 0.0107 (0.010469512124756475) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:24:37 -- STEP: 1244/2902 -- GLOBAL_STEP: 9950[0m |
|
| > loss_text_ce: 0.05204193294048309 (0.0527393467712345) |
|
| > loss_mel_ce: 3.062506914138794 (3.197535026878406) |
|
| > loss: 0.3893186151981354 (0.4062842964598985) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6637 (0.6031234285455812) |
|
| > loader_time: 0.0112 (0.010495160170306727) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:25:25 -- STEP: 1294/2902 -- GLOBAL_STEP: 10000[0m |
|
| > loss_text_ce: 0.053739190101623535 (0.052719865308087074) |
|
| > loss_mel_ce: 3.235835313796997 (3.1971923310016006) |
|
| > loss: 0.41119682788848877 (0.4062390243389874) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6554 (0.6032189197112826) |
|
| > loader_time: 0.0112 (0.010511806243354049) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:26:12 -- STEP: 1344/2902 -- GLOBAL_STEP: 10050[0m |
|
| > loss_text_ce: 0.05264301598072052 (0.05269939124506589) |
|
| > loss_mel_ce: 3.1769728660583496 (3.1967695257848225) |
|
| > loss: 0.4037019908428192 (0.40618361446208173) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5749 (0.6032951499025029) |
|
| > loader_time: 0.0109 (0.010520114075569876) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:26:59 -- STEP: 1394/2902 -- GLOBAL_STEP: 10100[0m |
|
| > loss_text_ce: 0.05526312068104744 (0.05267769771185805) |
|
| > loss_mel_ce: 3.0526609420776367 (3.1962102015357114) |
|
| > loss: 0.3884904980659485 (0.40611098721687544) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5445 (0.6036030539823234) |
|
| > loader_time: 0.0101 (0.010525740715147253) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:27:46 -- STEP: 1444/2902 -- GLOBAL_STEP: 10150[0m |
|
| > loss_text_ce: 0.05535884574055672 (0.052661094868715626) |
|
| > loss_mel_ce: 3.212167739868164 (3.195650814973087) |
|
| > loss: 0.40844082832336426 (0.4060389885983307) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5163 (0.6032200695074831) |
|
| > loader_time: 0.0104 (0.010530813083754342) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:28:32 -- STEP: 1494/2902 -- GLOBAL_STEP: 10200[0m |
|
| > loss_text_ce: 0.0518801212310791 (0.05264195443007602) |
|
| > loss_mel_ce: 3.2703099250793457 (3.195997038519527) |
|
| > loss: 0.4152737557888031 (0.4060798740011939) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6595 (0.6027859074364076) |
|
| > loader_time: 0.0111 (0.01054092423823304) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:29:18 -- STEP: 1544/2902 -- GLOBAL_STEP: 10250[0m |
|
| > loss_text_ce: 0.049537673592567444 (0.052636940536509066) |
|
| > loss_mel_ce: 3.1364898681640625 (3.195794070477313) |
|
| > loss: 0.3982534408569336 (0.4060538762630266) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4407 (0.6026072866558418) |
|
| > loader_time: 0.0101 (0.010545256224320955) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:30:05 -- STEP: 1594/2902 -- GLOBAL_STEP: 10300[0m |
|
| > loss_text_ce: 0.05353932827711105 (0.05263680126868572) |
|
| > loss_mel_ce: 3.1715071201324463 (3.1952996852257325) |
|
| > loss: 0.40313079953193665 (0.4059920606753756) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5432 (0.6026972758126834) |
|
| > loader_time: 0.0105 (0.010556090132355535) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:30:52 -- STEP: 1644/2902 -- GLOBAL_STEP: 10350[0m |
|
| > loss_text_ce: 0.0515349842607975 (0.05263029766062824) |
|
| > loss_mel_ce: 3.0010695457458496 (3.1949205655251114) |
|
| > loss: 0.3815755546092987 (0.40594385780049635) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6357 (0.6025581902251055) |
|
| > loader_time: 0.0106 (0.010555189311359335) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:31:39 -- STEP: 1694/2902 -- GLOBAL_STEP: 10400[0m |
|
| > loss_text_ce: 0.050352662801742554 (0.05262529455705897) |
|
| > loss_mel_ce: 3.185516834259033 (3.1946503501011607) |
|
| > loss: 0.4044836759567261 (0.4059094554769949) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6068 (0.6024062957223231) |
|
| > loader_time: 0.0111 (0.010554100856308105) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:32:26 -- STEP: 1744/2902 -- GLOBAL_STEP: 10450[0m |
|
| > loss_text_ce: 0.052729882299900055 (0.0525995482361679) |
|
| > loss_mel_ce: 3.1776161193847656 (3.194501636908689) |
|
| > loss: 0.40379324555397034 (0.4058876480531254) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5961 (0.6025670185821876) |
|
| > loader_time: 0.0107 (0.01056057829922492) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:33:13 -- STEP: 1794/2902 -- GLOBAL_STEP: 10500[0m |
|
| > loss_text_ce: 0.05042431131005287 (0.05259022799630961) |
|
| > loss_mel_ce: 3.1463003158569336 (3.1944169817428) |
|
| > loss: 0.3995905816555023 (0.40587590109876687) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5049 (0.6027706765809595) |
|
| > loader_time: 0.01 (0.010568745160182578) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:34:01 -- STEP: 1844/2902 -- GLOBAL_STEP: 10550[0m |
|
| > loss_text_ce: 0.05469150468707085 (0.05258134340184566) |
|
| > loss_mel_ce: 3.293409824371338 (3.194005653553051) |
|
| > loss: 0.4185126721858978 (0.40582337445395616) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5551 (0.6029681765076406) |
|
| > loader_time: 0.0106 (0.010575436847587466) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:34:48 -- STEP: 1894/2902 -- GLOBAL_STEP: 10600[0m |
|
| > loss_text_ce: 0.05276293680071831 (0.052574189625672584) |
|
| > loss_mel_ce: 3.239795446395874 (3.19401058891381) |
|
| > loss: 0.4115698039531708 (0.40582309715221637) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6411 (0.6032363728963344) |
|
| > loader_time: 0.011 (0.010578809217515436) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:35:36 -- STEP: 1944/2902 -- GLOBAL_STEP: 10650[0m |
|
| > loss_text_ce: 0.049916088581085205 (0.05257261406683935) |
|
| > loss_mel_ce: 3.2820544242858887 (3.193959850955893) |
|
| > loss: 0.41649630665779114 (0.40581655793597166) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4468 (0.6033671674905009) |
|
| > loader_time: 0.0091 (0.010581779504509125) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:36:23 -- STEP: 1994/2902 -- GLOBAL_STEP: 10700[0m |
|
| > loss_text_ce: 0.053316082805395126 (0.05255969537005442) |
|
| > loss_mel_ce: 3.1072206497192383 (3.193483378368252) |
|
| > loss: 0.39506709575653076 (0.4057553840377026) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6845 (0.60350839544084) |
|
| > loader_time: 0.0105 (0.0105668879081397) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:37:10 -- STEP: 2044/2902 -- GLOBAL_STEP: 10750[0m |
|
| > loss_text_ce: 0.04926644265651703 (0.05255933851550412) |
|
| > loss_mel_ce: 3.3206586837768555 (3.193047124462352) |
|
| > loss: 0.4212406277656555 (0.4057008076291494) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5429 (0.6033082252145989) |
|
| > loader_time: 0.0109 (0.010563473062272637) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:37:57 -- STEP: 2094/2902 -- GLOBAL_STEP: 10800[0m |
|
| > loss_text_ce: 0.0534113310277462 (0.05254988536174627) |
|
| > loss_mel_ce: 3.1378417015075684 (3.192547170116203) |
|
| > loss: 0.3989066183567047 (0.4056371316925729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4968 (0.6034088593613234) |
|
| > loader_time: 0.0105 (0.010568689594068405) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:38:44 -- STEP: 2144/2902 -- GLOBAL_STEP: 10850[0m |
|
| > loss_text_ce: 0.05344215780496597 (0.05255435538867406) |
|
| > loss_mel_ce: 3.1524195671081543 (3.1925554472341475) |
|
| > loss: 0.40073272585868835 (0.40563872507981835) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5771 (0.6033291484215376) |
|
| > loader_time: 0.0115 (0.010582063958716036) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:39:30 -- STEP: 2194/2902 -- GLOBAL_STEP: 10900[0m |
|
| > loss_text_ce: 0.051915958523750305 (0.052540452967490436) |
|
| > loss_mel_ce: 3.1636173725128174 (3.1927434840632665) |
|
| > loss: 0.4019416570663452 (0.40566049191554) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5515 (0.6030407171414578) |
|
| > loader_time: 0.0109 (0.0105995729823708) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:40:18 -- STEP: 2244/2902 -- GLOBAL_STEP: 10950[0m |
|
| > loss_text_ce: 0.052167460322380066 (0.0525299172994553) |
|
| > loss_mel_ce: 3.199925184249878 (3.1927137905051146) |
|
| > loss: 0.4065115749835968 (0.4056554632709626) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6492 (0.6033633877249335) |
|
| > loader_time: 0.0117 (0.01061397854657097) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:41:04 -- STEP: 2294/2902 -- GLOBAL_STEP: 11000[0m |
|
| > loss_text_ce: 0.04624050855636597 (0.05252224427129046) |
|
| > loss_mel_ce: 3.2247681617736816 (3.1923680543483797) |
|
| > loss: 0.40887609124183655 (0.40561128713664757) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5472 (0.6029953439025756) |
|
| > loader_time: 0.0107 (0.01062412925045076) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:41:50 -- STEP: 2344/2902 -- GLOBAL_STEP: 11050[0m |
|
| > loss_text_ce: 0.05163954943418503 (0.052516960615209565) |
|
| > loss_mel_ce: 3.2304461002349854 (3.1922061061899796) |
|
| > loss: 0.41026070713996887 (0.4055903831388763) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5826 (0.602789519996774) |
|
| > loader_time: 0.0112 (0.010636577528898213) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:42:37 -- STEP: 2394/2902 -- GLOBAL_STEP: 11100[0m |
|
| > loss_text_ce: 0.053269702941179276 (0.05250816199066448) |
|
| > loss_mel_ce: 3.2108967304229736 (3.1919024122089956) |
|
| > loss: 0.4080207943916321 (0.40555132153687523) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5341 (0.6027643641732401) |
|
| > loader_time: 0.0111 (0.010651086705270757) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:43:24 -- STEP: 2444/2902 -- GLOBAL_STEP: 11150[0m |
|
| > loss_text_ce: 0.052502501755952835 (0.05249957667294825) |
|
| > loss_mel_ce: 3.2452847957611084 (3.1916058965868737) |
|
| > loss: 0.4122233986854553 (0.4055131839042599) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6571 (0.6027423118023176) |
|
| > loader_time: 0.0114 (0.010663800953648083) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:44:11 -- STEP: 2494/2902 -- GLOBAL_STEP: 11200[0m |
|
| > loss_text_ce: 0.05313148349523544 (0.05248425879128094) |
|
| > loss_mel_ce: 3.137269973754883 (3.191296629553905) |
|
| > loss: 0.39880019426345825 (0.4054726108418718) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5558 (0.6027711930041723) |
|
| > loader_time: 0.0112 (0.010675928931094774) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:44:59 -- STEP: 2544/2902 -- GLOBAL_STEP: 11250[0m |
|
| > loss_text_ce: 0.05024084821343422 (0.05247156317902251) |
|
| > loss_mel_ce: 3.179081439971924 (3.1910943921257107) |
|
| > loss: 0.4036652743816376 (0.40544574424249563) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5751 (0.6028636446538974) |
|
| > loader_time: 0.0115 (0.010689315181108381) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:45:46 -- STEP: 2594/2902 -- GLOBAL_STEP: 11300[0m |
|
| > loss_text_ce: 0.05368289723992348 (0.05246220678579332) |
|
| > loss_mel_ce: 3.191403388977051 (3.1909259006254644) |
|
| > loss: 0.4056357741355896 (0.40542351324096854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5432 (0.602958301350809) |
|
| > loader_time: 0.0108 (0.010702524078563248) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:46:33 -- STEP: 2644/2902 -- GLOBAL_STEP: 11350[0m |
|
| > loss_text_ce: 0.053973011672496796 (0.052453115493701664) |
|
| > loss_mel_ce: 3.285907506942749 (3.190714084615143) |
|
| > loss: 0.4174850583076477 (0.4053958998302648) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.658 (0.6029926955970453) |
|
| > loader_time: 0.0119 (0.01071605108869959) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:47:21 -- STEP: 2694/2902 -- GLOBAL_STEP: 11400[0m |
|
| > loss_text_ce: 0.05342477560043335 (0.05244395500650519) |
|
| > loss_mel_ce: 3.285519599914551 (3.190686175446201) |
|
| > loss: 0.4173680543899536 (0.4053912661366756) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6651 (0.6032455267513263) |
|
| > loader_time: 0.0117 (0.010733391146528872) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:48:08 -- STEP: 2744/2902 -- GLOBAL_STEP: 11450[0m |
|
| > loss_text_ce: 0.047638945281505585 (0.05242956084343159) |
|
| > loss_mel_ce: 3.2169618606567383 (3.1906095680620497) |
|
| > loss: 0.40807509422302246 (0.4053798909394108) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5872 (0.6032850741471233) |
|
| > loader_time: 0.0117 (0.01075017017803804) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:48:56 -- STEP: 2794/2902 -- GLOBAL_STEP: 11500[0m |
|
| > loss_text_ce: 0.05027558282017708 (0.05241230539524224) |
|
| > loss_mel_ce: 3.273003339767456 (3.190310692053311) |
|
| > loss: 0.4154098629951477 (0.4053403744974047) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6852 (0.6034290926236311) |
|
| > loader_time: 0.0123 (0.010766002904882409) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:49:44 -- STEP: 2844/2902 -- GLOBAL_STEP: 11550[0m |
|
| > loss_text_ce: 0.05199229717254639 (0.05240542293197455) |
|
| > loss_mel_ce: 3.114150047302246 (3.19000683672485) |
|
| > loss: 0.39576780796051025 (0.4053015322666798) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.709 (0.6036963121632833) |
|
| > loader_time: 0.0122 (0.0107818767323608) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:50:31 -- STEP: 2894/2902 -- GLOBAL_STEP: 11600[0m |
|
| > loss_text_ce: 0.054400309920310974 (0.05239259631970916) |
|
| > loss_mel_ce: 3.325909376144409 (3.189384226808896) |
|
| > loss: 0.422538697719574 (0.405222102701046) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6677 (0.6037184401224148) |
|
| > loader_time: 0.0114 (0.01079611524517321) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.11509281095832281 [0m(+0.0003945887690841865) |
|
| > avg_loss_text_ce:[92m 0.04980169842490041 [0m(-0.0011910525611554812) |
|
| > avg_loss_mel_ce:[92m 3.1442968448003126 [0m(-0.026138162372089546) |
|
| > avg_loss:[92m 3.1940985436391354 [0m(-0.027329216099746922) |
|
|
|
> BEST MODEL : /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000/best_model_11608.pth |
|
|
|
[4m[1m > EPOCH: 4/5[0m |
|
--> /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000 |
|
|
|
[1m > TRAINING (2024-12-22 20:52:03) [0m |
|
|
|
[1m --> TIME: 2024-12-22 20:52:44 -- STEP: 42/2902 -- GLOBAL_STEP: 11650[0m |
|
| > loss_text_ce: 0.05242399126291275 (0.05177849477955273) |
|
| > loss_mel_ce: 3.0292856693267822 (3.157657742500305) |
|
| > loss: 0.385213702917099 (0.40117952866213663) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6336 (0.6046405576524281) |
|
| > loader_time: 0.0103 (0.01020863510313488) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:53:31 -- STEP: 92/2902 -- GLOBAL_STEP: 11700[0m |
|
| > loss_text_ce: 0.052793268114328384 (0.05141946446636449) |
|
| > loss_mel_ce: 3.267806053161621 (3.165517534898675) |
|
| > loss: 0.4150749146938324 (0.4021171247181685) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6089 (0.6012791395187376) |
|
| > loader_time: 0.0104 (0.010277999484020733) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:54:18 -- STEP: 142/2902 -- GLOBAL_STEP: 11750[0m |
|
| > loss_text_ce: 0.050963904708623886 (0.05146324702761542) |
|
| > loss_mel_ce: 3.2492852210998535 (3.1661211789493824) |
|
| > loss: 0.41253113746643066 (0.40219805274211184) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6675 (0.6018522910668817) |
|
| > loader_time: 0.0105 (0.010207364256952853) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:55:05 -- STEP: 192/2902 -- GLOBAL_STEP: 11800[0m |
|
| > loss_text_ce: 0.04937686026096344 (0.05135606706608087) |
|
| > loss_mel_ce: 3.152395248413086 (3.1661600942413006) |
|
| > loss: 0.4002215266227722 (0.4021895203428964) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5679 (0.6053141777714087) |
|
| > loader_time: 0.0099 (0.010187438378731413) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:55:52 -- STEP: 242/2902 -- GLOBAL_STEP: 11850[0m |
|
| > loss_text_ce: 0.04982029274106026 (0.051418416833212556) |
|
| > loss_mel_ce: 3.183880567550659 (3.1688949736681846) |
|
| > loss: 0.4042125940322876 (0.4025391741725039) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6461 (0.6039103515877204) |
|
| > loader_time: 0.011 (0.010216754330091244) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:56:39 -- STEP: 292/2902 -- GLOBAL_STEP: 11900[0m |
|
| > loss_text_ce: 0.05215315520763397 (0.0514818048487379) |
|
| > loss_mel_ce: 3.3319084644317627 (3.1706454353789764) |
|
| > loss: 0.42300769686698914 (0.4027659054080101) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5685 (0.6028565535806625) |
|
| > loader_time: 0.0114 (0.010344660445435411) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:57:26 -- STEP: 342/2902 -- GLOBAL_STEP: 11950[0m |
|
| > loss_text_ce: 0.05081651732325554 (0.05145429278457024) |
|
| > loss_mel_ce: 3.119997262954712 (3.1713683577308878) |
|
| > loss: 0.39635172486305237 (0.4028528318767659) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.62 (0.6045072148417866) |
|
| > loader_time: 0.0111 (0.010437014507271393) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:58:14 -- STEP: 392/2902 -- GLOBAL_STEP: 12000[0m |
|
| > loss_text_ce: 0.050988949835300446 (0.05142504952809943) |
|
| > loss_mel_ce: 3.1205058097839355 (3.16914310382337) |
|
| > loss: 0.3964368402957916 (0.40257101977358056) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5909 (0.6051843859711471) |
|
| > loader_time: 0.0115 (0.010557485478264945) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:59:01 -- STEP: 442/2902 -- GLOBAL_STEP: 12050[0m |
|
| > loss_text_ce: 0.05060780048370361 (0.051467611888482555) |
|
| > loss_mel_ce: 3.14922833442688 (3.1712528445602004) |
|
| > loss: 0.39997953176498413 (0.40284005764922404) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5165 (0.6043184364543239) |
|
| > loader_time: 0.0101 (0.010559999565193556) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 20:59:48 -- STEP: 492/2902 -- GLOBAL_STEP: 12100[0m |
|
| > loss_text_ce: 0.052262645214796066 (0.05145950533631371) |
|
| > loss_mel_ce: 3.0933821201324463 (3.1706015079971253) |
|
| > loss: 0.39320558309555054 (0.40275762751092753) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6487 (0.6053038733761483) |
|
| > loader_time: 0.0103 (0.010541675536613154) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:00:36 -- STEP: 542/2902 -- GLOBAL_STEP: 12150[0m |
|
| > loss_text_ce: 0.05357429012656212 (0.05147910170339569) |
|
| > loss_mel_ce: 3.2249159812927246 (3.1703968549566506) |
|
| > loss: 0.4098112881183624 (0.4027344952870119) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.543 (0.6057071637403478) |
|
| > loader_time: 0.01 (0.01051323088332736) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:01:23 -- STEP: 592/2902 -- GLOBAL_STEP: 12200[0m |
|
| > loss_text_ce: 0.04849829524755478 (0.051450166826111234) |
|
| > loss_mel_ce: 3.157019853591919 (3.1691637586902925) |
|
| > loss: 0.4006897807121277 (0.4025767413848961) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6251 (0.606176301031499) |
|
| > loader_time: 0.0106 (0.010490711073617685) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:02:11 -- STEP: 642/2902 -- GLOBAL_STEP: 12250[0m |
|
| > loss_text_ce: 0.05318274348974228 (0.05145047221483657) |
|
| > loss_mel_ce: 3.197951555252075 (3.1695703424025923) |
|
| > loss: 0.40639179944992065 (0.4026276024516871) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6083 (0.6072793760775032) |
|
| > loader_time: 0.01 (0.010462552216194134) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:02:58 -- STEP: 692/2902 -- GLOBAL_STEP: 12300[0m |
|
| > loss_text_ce: 0.04917445406317711 (0.05141469234912899) |
|
| > loss_mel_ce: 3.0380802154541016 (3.16752684770981) |
|
| > loss: 0.385906845331192 (0.40236769310828596) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5506 (0.6069841088587149) |
|
| > loader_time: 0.0104 (0.010431977365747355) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:03:46 -- STEP: 742/2902 -- GLOBAL_STEP: 12350[0m |
|
| > loss_text_ce: 0.0537167526781559 (0.051418948411499715) |
|
| > loss_mel_ce: 3.1005213260650635 (3.1670297853387592) |
|
| > loss: 0.39427974820137024 (0.40230609211478285) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5324 (0.607316173954473) |
|
| > loader_time: 0.0098 (0.010424615237912084) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:04:33 -- STEP: 792/2902 -- GLOBAL_STEP: 12400[0m |
|
| > loss_text_ce: 0.05441666394472122 (0.051431453706828334) |
|
| > loss_mel_ce: 3.1533031463623047 (3.1675818153101987) |
|
| > loss: 0.40096497535705566 (0.4023766589887216) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3769 (0.60757312028095) |
|
| > loader_time: 0.0092 (0.01040240610488738) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:05:21 -- STEP: 842/2902 -- GLOBAL_STEP: 12450[0m |
|
| > loss_text_ce: 0.051836397498846054 (0.051435001375874825) |
|
| > loss_mel_ce: 3.162421464920044 (3.167984573121875) |
|
| > loss: 0.40178224444389343 (0.4024274471678248) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.608 (0.6077996503712163) |
|
| > loader_time: 0.0101 (0.010387159016806273) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:06:08 -- STEP: 892/2902 -- GLOBAL_STEP: 12500[0m |
|
| > loss_text_ce: 0.050409939140081406 (0.05143925132054518) |
|
| > loss_mel_ce: 3.1199047565460205 (3.1676170697661257) |
|
| > loss: 0.3962893486022949 (0.40238204050491755) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6262 (0.6074925163936188) |
|
| > loader_time: 0.0109 (0.01040899967398879) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:06:54 -- STEP: 942/2902 -- GLOBAL_STEP: 12550[0m |
|
| > loss_text_ce: 0.054377250373363495 (0.05144039717465323) |
|
| > loss_mel_ce: 3.2185378074645996 (3.1676341611615157) |
|
| > loss: 0.40911439061164856 (0.4023843201355824) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5024 (0.6071113489742238) |
|
| > loader_time: 0.0099 (0.010440987133422985) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:07:42 -- STEP: 992/2902 -- GLOBAL_STEP: 12600[0m |
|
| > loss_text_ce: 0.04666450247168541 (0.05143451606929903) |
|
| > loss_mel_ce: 3.063586473464966 (3.167725384956406) |
|
| > loss: 0.3887813687324524 (0.4023949879732345) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5517 (0.607087874124127) |
|
| > loader_time: 0.0109 (0.010459479064710682) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:08:29 -- STEP: 1042/2902 -- GLOBAL_STEP: 12650[0m |
|
| > loss_text_ce: 0.050158604979515076 (0.05141120305331335) |
|
| > loss_mel_ce: 3.0921571254730225 (3.167083098426204) |
|
| > loss: 0.39278945326805115 (0.4023117879530749) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6452 (0.6070227060116642) |
|
| > loader_time: 0.0119 (0.01050893679232607) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:09:16 -- STEP: 1092/2902 -- GLOBAL_STEP: 12700[0m |
|
| > loss_text_ce: 0.053189586848020554 (0.05139344183885025) |
|
| > loss_mel_ce: 3.1477901935577393 (3.1663511314671555) |
|
| > loss: 0.4001224637031555 (0.40221807196900083) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6439 (0.6070524135352056) |
|
| > loader_time: 0.0118 (0.010553024190686133) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:10:04 -- STEP: 1142/2902 -- GLOBAL_STEP: 12750[0m |
|
| > loss_text_ce: 0.05265435203909874 (0.051385245982381374) |
|
| > loss_mel_ce: 3.2024030685424805 (3.165868102147158) |
|
| > loss: 0.4068821668624878 (0.4021566687396445) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6971 (0.6074808042646498) |
|
| > loader_time: 0.011 (0.010573815130310598) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:10:52 -- STEP: 1192/2902 -- GLOBAL_STEP: 12800[0m |
|
| > loss_text_ce: 0.05063112452626228 (0.051397483155206335) |
|
| > loss_mel_ce: 3.160621166229248 (3.1657690583859512) |
|
| > loss: 0.40140652656555176 (0.402145817875862) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5908 (0.6079006377082538) |
|
| > loader_time: 0.0106 (0.01057791729901461) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:11:39 -- STEP: 1242/2902 -- GLOBAL_STEP: 12850[0m |
|
| > loss_text_ce: 0.049502015113830566 (0.05138907669750962) |
|
| > loss_mel_ce: 3.150031805038452 (3.165537585766804) |
|
| > loss: 0.39994174242019653 (0.40211583297225795) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5612 (0.6078281895933898) |
|
| > loader_time: 0.01 (0.010567935000679155) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:12:25 -- STEP: 1292/2902 -- GLOBAL_STEP: 12900[0m |
|
| > loss_text_ce: 0.05091576278209686 (0.05138804009270739) |
|
| > loss_mel_ce: 3.257941484451294 (3.165761600337899) |
|
| > loss: 0.4136071503162384 (0.4021437052095272) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6488 (0.6072963262120997) |
|
| > loader_time: 0.0104 (0.010560713869868423) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:13:12 -- STEP: 1342/2902 -- GLOBAL_STEP: 12950[0m |
|
| > loss_text_ce: 0.05174795538187027 (0.05137953288220256) |
|
| > loss_mel_ce: 3.145700693130493 (3.1660235461108552) |
|
| > loss: 0.39968109130859375 (0.4021753851107801) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6661 (0.6069889692187127) |
|
| > loader_time: 0.0101 (0.010547872094212637) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:14:00 -- STEP: 1392/2902 -- GLOBAL_STEP: 13000[0m |
|
| > loss_text_ce: 0.051908206194639206 (0.05137049605074369) |
|
| > loss_mel_ce: 3.218794107437134 (3.165184286819107) |
|
| > loss: 0.40883779525756836 (0.4020693480497465) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6109 (0.6072547183982253) |
|
| > loader_time: 0.0106 (0.010541640650266897) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:14:46 -- STEP: 1442/2902 -- GLOBAL_STEP: 13050[0m |
|
| > loss_text_ce: 0.04997578635811806 (0.05138166409228693) |
|
| > loss_mel_ce: 3.3202295303344727 (3.16502617648835) |
|
| > loss: 0.42127567529678345 (0.40205098029313935) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6776 (0.6069498730102618) |
|
| > loader_time: 0.0105 (0.010534495156614863) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:15:34 -- STEP: 1492/2902 -- GLOBAL_STEP: 13100[0m |
|
| > loss_text_ce: 0.050916023552417755 (0.0513936553010592) |
|
| > loss_mel_ce: 3.1688055992126465 (3.1647604515341587) |
|
| > loss: 0.40246519446372986 (0.4020192635922269) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6073 (0.6068949724969521) |
|
| > loader_time: 0.0098 (0.010515906855503937) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:16:21 -- STEP: 1542/2902 -- GLOBAL_STEP: 13150[0m |
|
| > loss_text_ce: 0.05477416515350342 (0.05138281299629949) |
|
| > loss_mel_ce: 3.264493942260742 (3.164743644718066) |
|
| > loss: 0.4149085283279419 (0.4020158074090765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6556 (0.6069266871874115) |
|
| > loader_time: 0.0101 (0.0104985054674161) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:17:09 -- STEP: 1592/2902 -- GLOBAL_STEP: 13200[0m |
|
| > loss_text_ce: 0.05062420293688774 (0.05138683669390764) |
|
| > loss_mel_ce: 3.109247922897339 (3.1645478925213744) |
|
| > loss: 0.39498400688171387 (0.4019918413505186) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.586 (0.6073737449981449) |
|
| > loader_time: 0.0101 (0.010484467499220199) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:17:56 -- STEP: 1642/2902 -- GLOBAL_STEP: 13250[0m |
|
| > loss_text_ce: 0.04851849004626274 (0.051371706259435754) |
|
| > loss_mel_ce: 3.08614444732666 (3.164616929918491) |
|
| > loss: 0.3918328583240509 (0.40199857971990616) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6683 (0.6072920318352599) |
|
| > loader_time: 0.0103 (0.010469061593626077) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:18:44 -- STEP: 1692/2902 -- GLOBAL_STEP: 13300[0m |
|
| > loss_text_ce: 0.051790542900562286 (0.05136777905760503) |
|
| > loss_mel_ce: 3.2155044078826904 (3.164545286349935) |
|
| > loss: 0.4084118604660034 (0.40198913338125203) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.624 (0.6075981038118357) |
|
| > loader_time: 0.0102 (0.010463569603913226) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:19:30 -- STEP: 1742/2902 -- GLOBAL_STEP: 13350[0m |
|
| > loss_text_ce: 0.05148990824818611 (0.05135159586886655) |
|
| > loss_mel_ce: 3.149303674697876 (3.1640812554124027) |
|
| > loss: 0.4000991880893707 (0.4019291066274854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6116 (0.6072361833329305) |
|
| > loader_time: 0.011 (0.010460974702057696) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:20:18 -- STEP: 1792/2902 -- GLOBAL_STEP: 13400[0m |
|
| > loss_text_ce: 0.05156005173921585 (0.051346307745136835) |
|
| > loss_mel_ce: 3.0656654834747314 (3.1633247026641453) |
|
| > loss: 0.38965320587158203 (0.4018338766181844) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6007 (0.6074373084785673) |
|
| > loader_time: 0.0105 (0.010466490872204309) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:21:05 -- STEP: 1842/2902 -- GLOBAL_STEP: 13450[0m |
|
| > loss_text_ce: 0.05219785124063492 (0.051339257172795236) |
|
| > loss_mel_ce: 3.2246766090393066 (3.1631171858660174) |
|
| > loss: 0.409609317779541 (0.4018070556538892) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6783 (0.6072862473165811) |
|
| > loader_time: 0.0112 (0.010471460744172307) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:21:53 -- STEP: 1892/2902 -- GLOBAL_STEP: 13500[0m |
|
| > loss_text_ce: 0.04937213659286499 (0.051324877614277474) |
|
| > loss_mel_ce: 3.2181973457336426 (3.162961993736646) |
|
| > loss: 0.40844619274139404 (0.40178585920412013) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6131 (0.6075245814898039) |
|
| > loader_time: 0.0101 (0.010493385993401046) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:22:39 -- STEP: 1942/2902 -- GLOBAL_STEP: 13550[0m |
|
| > loss_text_ce: 0.04959951713681221 (0.05130738955486016) |
|
| > loss_mel_ce: 3.0643162727355957 (3.1629433719063393) |
|
| > loss: 0.38923946022987366 (0.4017813454787839) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.682 (0.6071882382971364) |
|
| > loader_time: 0.0106 (0.01049361550599245) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:23:26 -- STEP: 1992/2902 -- GLOBAL_STEP: 13600[0m |
|
| > loss_text_ce: 0.05019519850611687 (0.05130434854060829) |
|
| > loss_mel_ce: 2.967393398284912 (3.162763780140015) |
|
| > loss: 0.37719857692718506 (0.40175851636442816) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5847 (0.6066965500274327) |
|
| > loader_time: 0.0108 (0.010494193877561031) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:24:13 -- STEP: 2042/2902 -- GLOBAL_STEP: 13650[0m |
|
| > loss_text_ce: 0.04995539039373398 (0.05128760228296567) |
|
| > loss_mel_ce: 3.0459234714508057 (3.1625825939636156) |
|
| > loss: 0.38698485493659973 (0.4017337748049291) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5468 (0.6069906450275334) |
|
| > loader_time: 0.0106 (0.010499159572874069) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:25:00 -- STEP: 2092/2902 -- GLOBAL_STEP: 13700[0m |
|
| > loss_text_ce: 0.04973485693335533 (0.051272201517429436) |
|
| > loss_mel_ce: 3.0299553871154785 (3.1628303902782857) |
|
| > loss: 0.3849612772464752 (0.4017628242420195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5548 (0.6068838057508879) |
|
| > loader_time: 0.0104 (0.010503764020325355) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:25:48 -- STEP: 2142/2902 -- GLOBAL_STEP: 13750[0m |
|
| > loss_text_ce: 0.05021126568317413 (0.051260509819529745) |
|
| > loss_mel_ce: 2.955406904220581 (3.1625539609809317) |
|
| > loss: 0.37570226192474365 (0.40172680913288955) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5523 (0.6070832329081319) |
|
| > loader_time: 0.0112 (0.010521110366372504) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:26:35 -- STEP: 2192/2902 -- GLOBAL_STEP: 13800[0m |
|
| > loss_text_ce: 0.04956125468015671 (0.051253811561631654) |
|
| > loss_mel_ce: 3.1815543174743652 (3.1624464504692664) |
|
| > loss: 0.4038894474506378 (0.40171253303215454) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5867 (0.6070984596536095) |
|
| > loader_time: 0.011 (0.010532227310821093) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:27:23 -- STEP: 2242/2902 -- GLOBAL_STEP: 13850[0m |
|
| > loss_text_ce: 0.050890352576971054 (0.05124137310990786) |
|
| > loss_mel_ce: 3.058953285217285 (3.1622287470990247) |
|
| > loss: 0.3887304663658142 (0.4016837653148187) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5351 (0.6072680375918427) |
|
| > loader_time: 0.0102 (0.010540215273609555) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:28:10 -- STEP: 2292/2902 -- GLOBAL_STEP: 13900[0m |
|
| > loss_text_ce: 0.05306136608123779 (0.05123496174578263) |
|
| > loss_mel_ce: 3.2562832832336426 (3.1622462749065092) |
|
| > loss: 0.41366809606552124 (0.4016851548749116) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5083 (0.6071446213422633) |
|
| > loader_time: 0.0101 (0.010546645152839275) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:28:57 -- STEP: 2342/2902 -- GLOBAL_STEP: 13950[0m |
|
| > loss_text_ce: 0.04956350103020668 (0.05122417288837929) |
|
| > loss_mel_ce: 3.125633955001831 (3.1618559527458223) |
|
| > loss: 0.39689967036247253 (0.4016350159703107) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6593 (0.6069975350271614) |
|
| > loader_time: 0.0107 (0.010552020586015414) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:29:44 -- STEP: 2392/2902 -- GLOBAL_STEP: 14000[0m |
|
| > loss_text_ce: 0.048557110130786896 (0.0512131784301263) |
|
| > loss_mel_ce: 3.2124428749084473 (3.161652133217623) |
|
| > loss: 0.4076249897480011 (0.40160816420729306) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5845 (0.6070272774002609) |
|
| > loader_time: 0.0109 (0.010562337860215857) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:30:31 -- STEP: 2442/2902 -- GLOBAL_STEP: 14050[0m |
|
| > loss_text_ce: 0.05165933817625046 (0.051207817576819456) |
|
| > loss_mel_ce: 3.187281847000122 (3.161279260379971) |
|
| > loss: 0.40486764907836914 (0.4015608849728525) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6548 (0.6070269353071448) |
|
| > loader_time: 0.0117 (0.010570833177277526) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:31:19 -- STEP: 2492/2902 -- GLOBAL_STEP: 14100[0m |
|
| > loss_text_ce: 0.049455221742391586 (0.05119555806785959) |
|
| > loss_mel_ce: 3.148606300354004 (3.1611325667145564) |
|
| > loss: 0.3997576832771301 (0.40154101581194596) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6502 (0.6071065388751442) |
|
| > loader_time: 0.0112 (0.010577304692367864) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:32:05 -- STEP: 2542/2902 -- GLOBAL_STEP: 14150[0m |
|
| > loss_text_ce: 0.04723748564720154 (0.05118038502824431) |
|
| > loss_mel_ce: 3.230195999145508 (3.1610278290711853) |
|
| > loss: 0.4096791744232178 (0.40152602700808215) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.691 (0.6068531577742812) |
|
| > loader_time: 0.0111 (0.01058123244909111) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:32:52 -- STEP: 2592/2902 -- GLOBAL_STEP: 14200[0m |
|
| > loss_text_ce: 0.04967782273888588 (0.05117524785915222) |
|
| > loss_mel_ce: 3.1262459754943848 (3.160997681779626) |
|
| > loss: 0.39699047803878784 (0.4015216164773813) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5813 (0.6068024520336834) |
|
| > loader_time: 0.0114 (0.010590111307891791) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:33:40 -- STEP: 2642/2902 -- GLOBAL_STEP: 14250[0m |
|
| > loss_text_ce: 0.05493231117725372 (0.051161204823585306) |
|
| > loss_mel_ce: 3.163623809814453 (3.1608336640523294) |
|
| > loss: 0.4023195207118988 (0.401499358870873) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4954 (0.6067724356048478) |
|
| > loader_time: 0.0103 (0.010597871344347599) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:34:26 -- STEP: 2692/2902 -- GLOBAL_STEP: 14300[0m |
|
| > loss_text_ce: 0.04973088577389717 (0.051152218153968636) |
|
| > loss_mel_ce: 3.069262981414795 (3.1608455970241267) |
|
| > loss: 0.3898742198944092 (0.4014997271624398) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6973 (0.606688487760024) |
|
| > loader_time: 0.0115 (0.010608202103874123) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:35:14 -- STEP: 2742/2902 -- GLOBAL_STEP: 14350[0m |
|
| > loss_text_ce: 0.05162692070007324 (0.05113960035895226) |
|
| > loss_mel_ce: 3.1889121532440186 (3.1607107881383247) |
|
| > loss: 0.4050673842430115 (0.40148129882080374) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6184 (0.606666887093424) |
|
| > loader_time: 0.0114 (0.010620584286189613) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:36:01 -- STEP: 2792/2902 -- GLOBAL_STEP: 14400[0m |
|
| > loss_text_ce: 0.05050218105316162 (0.05113514076603469) |
|
| > loss_mel_ce: 3.0662810802459717 (3.1603817380431045) |
|
| > loss: 0.38959789276123047 (0.4014396101108253) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.49 (0.6065895064683209) |
|
| > loader_time: 0.0107 (0.010630526651967216) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:36:47 -- STEP: 2842/2902 -- GLOBAL_STEP: 14450[0m |
|
| > loss_text_ce: 0.05209615081548691 (0.05113076055678025) |
|
| > loss_mel_ce: 3.178262710571289 (3.1601328545267693) |
|
| > loss: 0.4037948548793793 (0.40140795211696356) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5541 (0.6063526925027234) |
|
| > loader_time: 0.0108 (0.010636977961497937) |
|
|
|
|
|
[1m --> TIME: 2024-12-22 21:37:34 -- STEP: 2892/2902 -- GLOBAL_STEP: 14500[0m |
|
| > loss_text_ce: 0.04835975542664528 (0.05111794193039538) |
|
| > loss_mel_ce: 3.1759161949157715 (3.160163748561099) |
|
| > loss: 0.403034508228302 (0.4014102115423353) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.6244 (0.6062773292984693) |
|
| > loader_time: 0.0108 (0.010642424081535906) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.11487200645485313 [0m(-0.0002208045034696876) |
|
| > avg_loss_text_ce:[92m 0.04858186071494011 [0m(-0.0012198377099603003) |
|
| > avg_loss_mel_ce:[92m 3.122558103667365 [0m(-0.021738741132947492) |
|
| > avg_loss:[92m 3.1711399687661066 [0m(-0.02295857487302877) |
|
|
|
> BEST MODEL : /checkpoints/GPT_XTTS_FT-December-22-2024_05+43PM-0000000/best_model_14510.pth |
|
|