|
> Training Environment: |
|
| > Backend: Torch |
|
| > Mixed precision: False |
|
| > Precision: float32 |
|
| > Current device: 0 |
|
| > Num. of GPUs: 1 |
|
| > Num. of CPUs: 2 |
|
| > Num. of Torch Threads: 1 |
|
| > Torch seed: 54321 |
|
| > Torch CUDNN: True |
|
| > Torch CUDNN deterministic: False |
|
| > Torch CUDNN benchmark: False |
|
| > Torch TF32 MatMul: False |
|
> Start Tensorboard: tensorboard --logdir=/content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
> Model has 524933279 parameters |
|
|
|
[4m[1m > EPOCH: 0/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 07:31:48) [0m |
|
|
|
[1m --> TIME: 2024-12-05 07:31:54 -- STEP: 0/1882 -- GLOBAL_STEP: 0[0m |
|
| > loss_text_ce: 0.09924504905939102 (0.09924504905939102) |
|
| > loss_mel_ce: 4.85481595993042 (4.85481595993042) |
|
| > loss: 1.2385152578353882 (1.2385152578353882) |
|
| > current_lr: 5e-06 |
|
| > step_time: 1.5443 (1.5443494319915771) |
|
| > loader_time: 4.5711 (4.571079969406128) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:32:29 -- STEP: 50/1882 -- GLOBAL_STEP: 50[0m |
|
| > loss_text_ce: 0.083099365234375 (0.08492921724915505) |
|
| > loss_mel_ce: 2.3642194271087646 (4.104769711494445) |
|
| > loss: 0.6118296980857849 (1.0474247276782989) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.7637 (0.38436351776123057) |
|
| > loader_time: 0.0217 (0.027569952011108404) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:33:04 -- STEP: 100/1882 -- GLOBAL_STEP: 100[0m |
|
| > loss_text_ce: 0.07482599467039108 (0.08061948306858537) |
|
| > loss_mel_ce: 3.2283222675323486 (3.8622838258743273) |
|
| > loss: 0.8257870674133301 (0.9857258236408236) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3145 (0.38407387971878065) |
|
| > loader_time: 0.011 (0.025744540691375738) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:33:38 -- STEP: 150/1882 -- GLOBAL_STEP: 150[0m |
|
| > loss_text_ce: 0.0712834894657135 (0.07766069752474625) |
|
| > loss_mel_ce: 3.8669068813323975 (3.745108982721964) |
|
| > loss: 0.9845476150512695 (0.9556924180189771) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3577 (0.37902108510335286) |
|
| > loader_time: 0.0261 (0.023449031511942546) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:34:13 -- STEP: 200/1882 -- GLOBAL_STEP: 200[0m |
|
| > loss_text_ce: 0.06880829483270645 (0.07488819662481547) |
|
| > loss_mel_ce: 3.5049221515655518 (3.648459727764129) |
|
| > loss: 0.8934326171875 (0.9308369803428651) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3095 (0.3778447484970093) |
|
| > loader_time: 0.0341 (0.022308769226074228) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:34:48 -- STEP: 250/1882 -- GLOBAL_STEP: 250[0m |
|
| > loss_text_ce: 0.06479701399803162 (0.07286127062141894) |
|
| > loss_mel_ce: 3.3651700019836426 (3.600287427902221) |
|
| > loss: 0.8574917316436768 (0.9182871744632722) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.373 (0.3783556795120239) |
|
| > loader_time: 0.0121 (0.021318180084228523) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:35:22 -- STEP: 300/1882 -- GLOBAL_STEP: 300[0m |
|
| > loss_text_ce: 0.0698036253452301 (0.07115643757085002) |
|
| > loss_mel_ce: 2.7742412090301514 (3.531399564743042) |
|
| > loss: 0.7110112309455872 (0.9006390005350113) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3133 (0.3780966591835022) |
|
| > loader_time: 0.0105 (0.021106003920237235) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:35:56 -- STEP: 350/1882 -- GLOBAL_STEP: 350[0m |
|
| > loss_text_ce: 0.05505380406975746 (0.06972291081079415) |
|
| > loss_mel_ce: 3.3067102432250977 (3.4816977507727485) |
|
| > loss: 0.8404409885406494 (0.8878551653453282) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.373 (0.37720673152378603) |
|
| > loader_time: 0.0115 (0.02087914943695069) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:36:31 -- STEP: 400/1882 -- GLOBAL_STEP: 400[0m |
|
| > loss_text_ce: 0.05936818942427635 (0.0686929952166974) |
|
| > loss_mel_ce: 3.000779628753662 (3.4523355239629745) |
|
| > loss: 0.765036940574646 (0.8802571296691895) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3047 (0.3775877100229261) |
|
| > loader_time: 0.0108 (0.020512846708297748) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:37:05 -- STEP: 450/1882 -- GLOBAL_STEP: 450[0m |
|
| > loss_text_ce: 0.06834416836500168 (0.06756721379028421) |
|
| > loss_mel_ce: 3.202094793319702 (3.4054960611131455) |
|
| > loss: 0.8176097273826599 (0.868265818754832) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4062 (0.37602972242567256) |
|
| > loader_time: 0.0132 (0.02014299286736384) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:37:38 -- STEP: 500/1882 -- GLOBAL_STEP: 500[0m |
|
| > loss_text_ce: 0.06273879110813141 (0.06670551944524049) |
|
| > loss_mel_ce: 3.4992682933807373 (3.3772657043933867) |
|
| > loss: 0.8905017971992493 (0.8609928060173988) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3533 (0.3748016214370726) |
|
| > loader_time: 0.013 (0.01976001119613649) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:38:13 -- STEP: 550/1882 -- GLOBAL_STEP: 550[0m |
|
| > loss_text_ce: 0.06922796368598938 (0.06607147215442222) |
|
| > loss_mel_ce: 3.276764154434204 (3.349526229121469) |
|
| > loss: 0.8364980220794678 (0.8538994256474757) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3513 (0.37584898515181087) |
|
| > loader_time: 0.013 (0.01939521876248447) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:38:48 -- STEP: 600/1882 -- GLOBAL_STEP: 600[0m |
|
| > loss_text_ce: 0.06333325058221817 (0.06546429000794883) |
|
| > loss_mel_ce: 3.3643157482147217 (3.3281260595719013) |
|
| > loss: 0.8569122552871704 (0.8483975877861183) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3499 (0.37678679386774677) |
|
| > loader_time: 0.0104 (0.019182165463765465) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:39:23 -- STEP: 650/1882 -- GLOBAL_STEP: 650[0m |
|
| > loss_text_ce: 0.05429554730653763 (0.0648687771134651) |
|
| > loss_mel_ce: 2.7892813682556152 (3.312888774688426) |
|
| > loss: 0.7108942270278931 (0.8444393881009177) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4184 (0.37725360980400646) |
|
| > loader_time: 0.011 (0.019111327391404375) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:39:57 -- STEP: 700/1882 -- GLOBAL_STEP: 700[0m |
|
| > loss_text_ce: 0.039287667721509933 (0.06432954653033182) |
|
| > loss_mel_ce: 1.8357176780700684 (3.29177328790937) |
|
| > loss: 0.46875134110450745 (0.8390257089904378) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2243 (0.37670540571212735) |
|
| > loader_time: 0.0129 (0.01908094644546509) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:40:32 -- STEP: 750/1882 -- GLOBAL_STEP: 750[0m |
|
| > loss_text_ce: 0.060255978256464005 (0.06386621005833142) |
|
| > loss_mel_ce: 2.884608507156372 (3.275246043682097) |
|
| > loss: 0.736216127872467 (0.8347780639330548) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3529 (0.37677554194132457) |
|
| > loader_time: 0.0109 (0.019001229604085294) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:41:07 -- STEP: 800/1882 -- GLOBAL_STEP: 800[0m |
|
| > loss_text_ce: 0.05972779542207718 (0.06339100702200082) |
|
| > loss_mel_ce: 3.1456761360168457 (3.260235311388968) |
|
| > loss: 0.801351010799408 (0.8309065800905229) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3419 (0.37698441177606556) |
|
| > loader_time: 0.0316 (0.018770188987255118) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:41:42 -- STEP: 850/1882 -- GLOBAL_STEP: 850[0m |
|
| > loss_text_ce: 0.06344010680913925 (0.06301001760012959) |
|
| > loss_mel_ce: 3.1476807594299316 (3.250657018633449) |
|
| > loss: 0.8027802109718323 (0.8284167594068192) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4377 (0.378033638000488) |
|
| > loader_time: 0.0126 (0.01861985430997963) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:42:17 -- STEP: 900/1882 -- GLOBAL_STEP: 900[0m |
|
| > loss_text_ce: 0.061851028352975845 (0.06264245327148166) |
|
| > loss_mel_ce: 3.5121958255767822 (3.2372269405259018) |
|
| > loss: 0.8935117125511169 (0.8249673487742744) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3494 (0.3779749936527674) |
|
| > loader_time: 0.0117 (0.01844337701797488) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:42:51 -- STEP: 950/1882 -- GLOBAL_STEP: 950[0m |
|
| > loss_text_ce: 0.05493725836277008 (0.062243758255713794) |
|
| > loss_mel_ce: 2.922435760498047 (3.22293208799864) |
|
| > loss: 0.7443432807922363 (0.8212939620645424) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3631 (0.37769459147202317) |
|
| > loader_time: 0.0107 (0.018393406366047128) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:43:26 -- STEP: 1000/1882 -- GLOBAL_STEP: 1000[0m |
|
| > loss_text_ce: 0.048630405217409134 (0.06188541639223689) |
|
| > loss_mel_ce: 2.2146332263946533 (3.2112817206382744) |
|
| > loss: 0.5658159255981445 (0.818291784644127) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2181 (0.37746165561676004) |
|
| > loader_time: 0.0103 (0.018366051197052022) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:44:00 -- STEP: 1050/1882 -- GLOBAL_STEP: 1050[0m |
|
| > loss_text_ce: 0.05343294143676758 (0.06159354990791702) |
|
| > loss_mel_ce: 3.1901137828826904 (3.198328064736865) |
|
| > loss: 0.8108866810798645 (0.8149804040931522) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.414 (0.3775623217083156) |
|
| > loader_time: 0.0111 (0.01830981731414797) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:44:35 -- STEP: 1100/1882 -- GLOBAL_STEP: 1100[0m |
|
| > loss_text_ce: 0.06230022758245468 (0.061290736581114155) |
|
| > loss_mel_ce: 3.0784037113189697 (3.186055079265074) |
|
| > loss: 0.7851759791374207 (0.8118364542722705) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3085 (0.37754894646731263) |
|
| > loader_time: 0.0303 (0.018272532766515578) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:45:09 -- STEP: 1150/1882 -- GLOBAL_STEP: 1150[0m |
|
| > loss_text_ce: 0.054398152977228165 (0.060990911100221666) |
|
| > loss_mel_ce: 3.0849740505218506 (3.1741876318143767) |
|
| > loss: 0.7848430275917053 (0.8087946360525882) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4081 (0.3771058677590408) |
|
| > loader_time: 0.0275 (0.018180749520011596) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:45:43 -- STEP: 1200/1882 -- GLOBAL_STEP: 1200[0m |
|
| > loss_text_ce: 0.05580421909689903 (0.06071773615665729) |
|
| > loss_mel_ce: 2.895834445953369 (3.160872702697913) |
|
| > loss: 0.7379096746444702 (0.8053976102421686) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3348 (0.37693971693515754) |
|
| > loader_time: 0.0109 (0.018112674156824774) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:46:18 -- STEP: 1250/1882 -- GLOBAL_STEP: 1250[0m |
|
| > loss_text_ce: 0.05592736229300499 (0.06042716381549831) |
|
| > loss_mel_ce: 3.0615696907043457 (3.1520899143218983) |
|
| > loss: 0.7793742418289185 (0.8031292700529106) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4085 (0.37685755290985073) |
|
| > loader_time: 0.0104 (0.018071360015869172) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:46:52 -- STEP: 1300/1882 -- GLOBAL_STEP: 1300[0m |
|
| > loss_text_ce: 0.05919681489467621 (0.06020007626941566) |
|
| > loss_mel_ce: 2.867332696914673 (3.1424857934621655) |
|
| > loss: 0.7316323518753052 (0.8006714678498423) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4445 (0.3767912892194894) |
|
| > loader_time: 0.0106 (0.018009135356316226) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:47:28 -- STEP: 1350/1882 -- GLOBAL_STEP: 1350[0m |
|
| > loss_text_ce: 0.0473044328391552 (0.05997705526098053) |
|
| > loss_mel_ce: 2.486260414123535 (3.1343662619590744) |
|
| > loss: 0.6333912014961243 (0.798585829845182) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3274 (0.3772480600851554) |
|
| > loader_time: 0.0307 (0.01803452597724068) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:48:02 -- STEP: 1400/1882 -- GLOBAL_STEP: 1400[0m |
|
| > loss_text_ce: 0.04989030212163925 (0.05974180312295041) |
|
| > loss_mel_ce: 2.585953950881958 (3.1218532574176776) |
|
| > loss: 0.6589610576629639 (0.7953987656533726) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2627 (0.3770433352674757) |
|
| > loader_time: 0.0259 (0.017918350185666766) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:48:36 -- STEP: 1450/1882 -- GLOBAL_STEP: 1450[0m |
|
| > loss_text_ce: 0.05371575057506561 (0.059506403146118915) |
|
| > loss_mel_ce: 3.012324333190918 (3.114612379238522) |
|
| > loss: 0.766510009765625 (0.7935296961562395) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4006 (0.3768588291365525) |
|
| > loader_time: 0.0313 (0.017907943232306132) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:49:10 -- STEP: 1500/1882 -- GLOBAL_STEP: 1500[0m |
|
| > loss_text_ce: 0.050513703376054764 (0.059286245917280475) |
|
| > loss_mel_ce: 2.9979100227355957 (3.1046217261950164) |
|
| > loss: 0.7621059417724609 (0.7909769935210553) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3373 (0.37677624241511026) |
|
| > loader_time: 0.0106 (0.017874268849690768) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:49:44 -- STEP: 1550/1882 -- GLOBAL_STEP: 1550[0m |
|
| > loss_text_ce: 0.050273776054382324 (0.0590715786213836) |
|
| > loss_mel_ce: 2.662717580795288 (3.0963486574542127) |
|
| > loss: 0.6782478094100952 (0.7888550596083371) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4074 (0.3764988136291504) |
|
| > loader_time: 0.0109 (0.017961703731167723) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:50:19 -- STEP: 1600/1882 -- GLOBAL_STEP: 1600[0m |
|
| > loss_text_ce: 0.05000990629196167 (0.05885403142077845) |
|
| > loss_mel_ce: 2.9688661098480225 (3.085899863541125) |
|
| > loss: 0.7547190189361572 (0.786188474409283) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3568 (0.376675209403038) |
|
| > loader_time: 0.03 (0.017942187488079086) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:50:54 -- STEP: 1650/1882 -- GLOBAL_STEP: 1650[0m |
|
| > loss_text_ce: 0.053765278309583664 (0.058666774167707436) |
|
| > loss_mel_ce: 3.0707948207855225 (3.079608861027341) |
|
| > loss: 0.7811400294303894 (0.7845689095150348) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.398 (0.3767626125162298) |
|
| > loader_time: 0.0257 (0.017964726505857526) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:51:28 -- STEP: 1700/1882 -- GLOBAL_STEP: 1700[0m |
|
| > loss_text_ce: 0.05041825771331787 (0.058450627826592466) |
|
| > loss_mel_ce: 2.9377691745758057 (3.072894616127013) |
|
| > loss: 0.7470468282699585 (0.7828363118101576) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3346 (0.3767284283918493) |
|
| > loader_time: 0.0126 (0.017906559355118694) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:52:03 -- STEP: 1750/1882 -- GLOBAL_STEP: 1750[0m |
|
| > loss_text_ce: 0.05107766389846802 (0.058320554975952385) |
|
| > loss_mel_ce: 2.4425883293151855 (3.067847148145947) |
|
| > loss: 0.6234164834022522 (0.7815419266394213) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.34 (0.3768365471703666) |
|
| > loader_time: 0.0103 (0.017820667266845686) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:52:37 -- STEP: 1800/1882 -- GLOBAL_STEP: 1800[0m |
|
| > loss_text_ce: 0.053609699010849 (0.05816975363633696) |
|
| > loss_mel_ce: 2.662489414215088 (3.061090404656197) |
|
| > loss: 0.6790247559547424 (0.7798150404459907) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3071 (0.3767030643092261) |
|
| > loader_time: 0.01 (0.017780634032355396) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:53:12 -- STEP: 1850/1882 -- GLOBAL_STEP: 1850[0m |
|
| > loss_text_ce: 0.04811837524175644 (0.05801803833729509) |
|
| > loss_mel_ce: 2.968719959259033 (3.05341012722737) |
|
| > loss: 0.754209578037262 (0.7778570422449633) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4012 (0.3768521142650295) |
|
| > loader_time: 0.031 (0.017816945668813325) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time: 0.6419341839276828 [0m(+0) |
|
| > avg_loss_text_ce: 0.05174965955890142 [0m(+0) |
|
| > avg_loss_mel_ce: 2.752386340728173 [0m(+0) |
|
| > avg_loss: 2.8041359736369205 [0m(+0) |
|
|
|
|
|
[4m[1m > EPOCH: 1/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 07:55:03) [0m |
|
|
|
[1m --> TIME: 2024-12-05 07:55:17 -- STEP: 18/1882 -- GLOBAL_STEP: 1900[0m |
|
| > loss_text_ce: 0.052426598966121674 (0.05318810749385092) |
|
| > loss_mel_ce: 3.072071075439453 (2.9160577721065946) |
|
| > loss: 0.7811244130134583 (0.7423114710383945) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.402 (0.385830110973782) |
|
| > loader_time: 0.0106 (0.015610151820712619) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:55:52 -- STEP: 68/1882 -- GLOBAL_STEP: 1950[0m |
|
| > loss_text_ce: 0.05308894068002701 (0.05306620463071501) |
|
| > loss_mel_ce: 2.2859935760498047 (2.791519594543121) |
|
| > loss: 0.5847706198692322 (0.7111464510945713) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3487 (0.38308564705007214) |
|
| > loader_time: 0.0324 (0.015714024796205407) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:56:25 -- STEP: 118/1882 -- GLOBAL_STEP: 2000[0m |
|
| > loss_text_ce: 0.056313175708055496 (0.052033257433923626) |
|
| > loss_mel_ce: 2.8023486137390137 (2.751858439485906) |
|
| > loss: 0.7146654725074768 (0.7009729247982219) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3072 (0.3757195533332178) |
|
| > loader_time: 0.015 (0.016030378260854954) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:57:00 -- STEP: 168/1882 -- GLOBAL_STEP: 2050[0m |
|
| > loss_text_ce: 0.0493742972612381 (0.052271621412641946) |
|
| > loss_mel_ce: 2.820385694503784 (2.7783693990537106) |
|
| > loss: 0.7174400091171265 (0.7076602546232084) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3187 (0.37569107186226613) |
|
| > loader_time: 0.0104 (0.015897887093680257) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:57:34 -- STEP: 218/1882 -- GLOBAL_STEP: 2100[0m |
|
| > loss_text_ce: 0.05055192485451698 (0.05206616950828001) |
|
| > loss_mel_ce: 2.7067816257476807 (2.7965132714411545) |
|
| > loss: 0.6893333792686462 (0.7121448593402123) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3801 (0.375874151877307) |
|
| > loader_time: 0.0121 (0.015850636937202664) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:58:09 -- STEP: 268/1882 -- GLOBAL_STEP: 2150[0m |
|
| > loss_text_ce: 0.05510799214243889 (0.052181149040585136) |
|
| > loss_mel_ce: 2.57230544090271 (2.811071665874169) |
|
| > loss: 0.6568533778190613 (0.715813203645286) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3043 (0.37736053164325545) |
|
| > loader_time: 0.0107 (0.016066730912051977) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:58:43 -- STEP: 318/1882 -- GLOBAL_STEP: 2200[0m |
|
| > loss_text_ce: 0.05795169621706009 (0.052025459613065275) |
|
| > loss_mel_ce: 2.7621166706085205 (2.8055856029192605) |
|
| > loss: 0.70501708984375 (0.7144027660665271) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4103 (0.3771416908540066) |
|
| > loader_time: 0.0113 (0.01622715461179146) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:59:19 -- STEP: 368/1882 -- GLOBAL_STEP: 2250[0m |
|
| > loss_text_ce: 0.056500621140003204 (0.0519521195826161) |
|
| > loss_mel_ce: 3.0766406059265137 (2.787438908348915) |
|
| > loss: 0.7832853198051453 (0.7098477572688587) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3904 (0.3791620835014012) |
|
| > loader_time: 0.0109 (0.01631724704866825) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 07:59:54 -- STEP: 418/1882 -- GLOBAL_STEP: 2300[0m |
|
| > loss_text_ce: 0.056084319949150085 (0.05199400746401701) |
|
| > loss_mel_ce: 3.1689751148223877 (2.791928973779726) |
|
| > loss: 0.8062648773193359 (0.7109807453800043) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4911 (0.38020207494069513) |
|
| > loader_time: 0.0126 (0.01634194291949843) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:00:29 -- STEP: 468/1882 -- GLOBAL_STEP: 2350[0m |
|
| > loss_text_ce: 0.0479445606470108 (0.05203480649198221) |
|
| > loss_mel_ce: 2.7748751640319824 (2.7954741474400238) |
|
| > loss: 0.705704927444458 (0.7118772386103611) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3362 (0.38064892067868483) |
|
| > loader_time: 0.0292 (0.016122652934147753) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:01:04 -- STEP: 518/1882 -- GLOBAL_STEP: 2400[0m |
|
| > loss_text_ce: 0.03471546620130539 (0.051932221120331266) |
|
| > loss_mel_ce: 2.1587975025177 (2.788418019830492) |
|
| > loss: 0.5483782291412354 (0.7100875605487453) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.289 (0.38062922466675747) |
|
| > loader_time: 0.0304 (0.01612333303252702) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:01:37 -- STEP: 568/1882 -- GLOBAL_STEP: 2450[0m |
|
| > loss_text_ce: 0.05483030527830124 (0.05182629030092919) |
|
| > loss_mel_ce: 3.130516529083252 (2.7842224177340413) |
|
| > loss: 0.7963367104530334 (0.7090121771038418) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.353 (0.37929383507916625) |
|
| > loader_time: 0.0161 (0.01603563024964131) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:02:12 -- STEP: 618/1882 -- GLOBAL_STEP: 2500[0m |
|
| > loss_text_ce: 0.045925211161375046 (0.05174133179763565) |
|
| > loss_mel_ce: 2.507418394088745 (2.781639666040356) |
|
| > loss: 0.6383358836174011 (0.7083452497774728) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3552 (0.3789926323690075) |
|
| > loader_time: 0.0101 (0.015988354929828332) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:02:47 -- STEP: 668/1882 -- GLOBAL_STEP: 2550[0m |
|
| > loss_text_ce: 0.04139676317572594 (0.05162946184439036) |
|
| > loss_mel_ce: 2.4429500102996826 (2.786367932657999) |
|
| > loss: 0.6210867166519165 (0.709499348914195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2895 (0.3788235448077767) |
|
| > loader_time: 0.0108 (0.01603364123555713) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:03:21 -- STEP: 718/1882 -- GLOBAL_STEP: 2600[0m |
|
| > loss_text_ce: 0.04883828014135361 (0.051658471179788806) |
|
| > loss_mel_ce: 2.886007308959961 (2.7876507495439196) |
|
| > loss: 0.7337114214897156 (0.7098273055467096) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3601 (0.3791780986493676) |
|
| > loader_time: 0.0104 (0.016071820325506088) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:03:56 -- STEP: 768/1882 -- GLOBAL_STEP: 2650[0m |
|
| > loss_text_ce: 0.048046793788671494 (0.051593684615606115) |
|
| > loss_mel_ce: 3.0345120429992676 (2.7869685672534046) |
|
| > loss: 0.7706397175788879 (0.709640563349239) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3394 (0.37934845406562084) |
|
| > loader_time: 0.011 (0.016018653598924463) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:04:30 -- STEP: 818/1882 -- GLOBAL_STEP: 2700[0m |
|
| > loss_text_ce: 0.052566930651664734 (0.05156577778820739) |
|
| > loss_mel_ce: 2.9754605293273926 (2.7821632859759378) |
|
| > loss: 0.7570068836212158 (0.7084322663201674) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3893 (0.3790102083700501) |
|
| > loader_time: 0.0114 (0.015968910347861565) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:05:05 -- STEP: 868/1882 -- GLOBAL_STEP: 2750[0m |
|
| > loss_text_ce: 0.05958146974444389 (0.051572226777604084) |
|
| > loss_mel_ce: 2.8045504093170166 (2.7812668818482638) |
|
| > loss: 0.7160329818725586 (0.708209777619981) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3003 (0.3788354064462371) |
|
| > loader_time: 0.0249 (0.015955776662870468) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:05:39 -- STEP: 918/1882 -- GLOBAL_STEP: 2800[0m |
|
| > loss_text_ce: 0.051034167408943176 (0.051592070037146005) |
|
| > loss_mel_ce: 3.2114169597625732 (2.7778165724802153) |
|
| > loss: 0.81561279296875 (0.7073521609529686) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4196 (0.3784267240100436) |
|
| > loader_time: 0.0246 (0.015977746520946204) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:06:13 -- STEP: 968/1882 -- GLOBAL_STEP: 2850[0m |
|
| > loss_text_ce: 0.050729382783174515 (0.051509361360068145) |
|
| > loss_mel_ce: 3.2121450901031494 (2.7718632823425904) |
|
| > loss: 0.8157185912132263 (0.7058431611758121) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5307 (0.3781180972895346) |
|
| > loader_time: 0.0205 (0.016011584642504826) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:06:47 -- STEP: 1018/1882 -- GLOBAL_STEP: 2900[0m |
|
| > loss_text_ce: 0.050678227096796036 (0.051493015992940046) |
|
| > loss_mel_ce: 2.9751203060150146 (2.7701922917412896) |
|
| > loss: 0.7564496397972107 (0.7054213272565002) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3665 (0.3782419263731051) |
|
| > loader_time: 0.01 (0.01598816044438326) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:07:23 -- STEP: 1068/1882 -- GLOBAL_STEP: 2950[0m |
|
| > loss_text_ce: 0.048151563853025436 (0.05150367478292421) |
|
| > loss_mel_ce: 2.958618402481079 (2.7705959701136282) |
|
| > loss: 0.7516924738883972 (0.7055249115668429) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3122 (0.37875039389963894) |
|
| > loader_time: 0.0106 (0.01599704348639152) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:07:58 -- STEP: 1118/1882 -- GLOBAL_STEP: 3000[0m |
|
| > loss_text_ce: 0.04912221431732178 (0.051462226475660636) |
|
| > loss_mel_ce: 2.8820700645446777 (2.7674506838718362) |
|
| > loss: 0.7327980995178223 (0.7047282278484435) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4024 (0.3788043139019251) |
|
| > loader_time: 0.0286 (0.016086539983322844) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:08:33 -- STEP: 1168/1882 -- GLOBAL_STEP: 3050[0m |
|
| > loss_text_ce: 0.05568931624293327 (0.05143110151402653) |
|
| > loss_mel_ce: 2.769667148590088 (2.7671326462128416) |
|
| > loss: 0.7063391208648682 (0.7046409369994925) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4393 (0.3792408868058088) |
|
| > loader_time: 0.0509 (0.01610229276630974) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:09:07 -- STEP: 1218/1882 -- GLOBAL_STEP: 3100[0m |
|
| > loss_text_ce: 0.050573624670505524 (0.05139864373138579) |
|
| > loss_mel_ce: 2.5638158321380615 (2.7636984976250183) |
|
| > loss: 0.653597354888916 (0.7037742854583832) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3738 (0.3792227483148059) |
|
| > loader_time: 0.0109 (0.016005233395080046) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:09:42 -- STEP: 1268/1882 -- GLOBAL_STEP: 3150[0m |
|
| > loss_text_ce: 0.053142108023166656 (0.051395164467746855) |
|
| > loss_mel_ce: 3.3399930000305176 (2.76471219811154) |
|
| > loss: 0.8482837677001953 (0.7040268407212071) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3463 (0.37946628659305537) |
|
| > loader_time: 0.0106 (0.01594377297332233) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:10:16 -- STEP: 1318/1882 -- GLOBAL_STEP: 3200[0m |
|
| > loss_text_ce: 0.057923998683691025 (0.051349750775080666) |
|
| > loss_mel_ce: 3.20357346534729 (2.762712666858133) |
|
| > loss: 0.8153743743896484 (0.7035156042740455) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.382 (0.37896739170294463) |
|
| > loader_time: 0.0116 (0.015908112294395343) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:10:51 -- STEP: 1368/1882 -- GLOBAL_STEP: 3250[0m |
|
| > loss_text_ce: 0.056495342403650284 (0.05136022446077988) |
|
| > loss_mel_ce: 3.181690216064453 (2.763092335814626) |
|
| > loss: 0.8095464110374451 (0.7036131400375347) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4278 (0.3792242395598989) |
|
| > loader_time: 0.0108 (0.01593674954615141) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:11:26 -- STEP: 1418/1882 -- GLOBAL_STEP: 3300[0m |
|
| > loss_text_ce: 0.04920697584748268 (0.051337273357953565) |
|
| > loss_mel_ce: 2.9405927658081055 (2.7584835826425906) |
|
| > loss: 0.7474499344825745 (0.7024552139686102) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.498 (0.37907766673729715) |
|
| > loader_time: 0.0102 (0.01602147059312828) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:12:02 -- STEP: 1468/1882 -- GLOBAL_STEP: 3350[0m |
|
| > loss_text_ce: 0.04622867330908775 (0.05133850907536387) |
|
| > loss_mel_ce: 2.693631172180176 (2.7577209517319163) |
|
| > loss: 0.6849649548530579 (0.702264865225928) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3093 (0.37964519980168177) |
|
| > loader_time: 0.0115 (0.016024299148642715) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:12:36 -- STEP: 1518/1882 -- GLOBAL_STEP: 3400[0m |
|
| > loss_text_ce: 0.04980720952153206 (0.051331195599928325) |
|
| > loss_mel_ce: 2.882662057876587 (2.757214652141252) |
|
| > loss: 0.7331173419952393 (0.7021364619138222) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3694 (0.37962048908153256) |
|
| > loader_time: 0.0115 (0.015942557841264526) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:13:11 -- STEP: 1568/1882 -- GLOBAL_STEP: 3450[0m |
|
| > loss_text_ce: 0.051159992814064026 (0.051327158951162494) |
|
| > loss_mel_ce: 2.966050863265991 (2.7565830905674695) |
|
| > loss: 0.7543027400970459 (0.701977562463406) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3494 (0.3798028840398305) |
|
| > loader_time: 0.0298 (0.015960918093214226) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:13:44 -- STEP: 1618/1882 -- GLOBAL_STEP: 3500[0m |
|
| > loss_text_ce: 0.05334069952368736 (0.051235701025397984) |
|
| > loss_mel_ce: 3.0583243370056152 (2.7509684688670695) |
|
| > loss: 0.7779162526130676 (0.7005510426348611) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4188 (0.3791428578946148) |
|
| > loader_time: 0.011 (0.016025065049547493) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:14:19 -- STEP: 1668/1882 -- GLOBAL_STEP: 3550[0m |
|
| > loss_text_ce: 0.045789025723934174 (0.05119294725242347) |
|
| > loss_mel_ce: 2.7834532260894775 (2.7484110863231663) |
|
| > loss: 0.7073105573654175 (0.699901008598811) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3359 (0.3790379471058471) |
|
| > loader_time: 0.0247 (0.016047407397263343) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:14:54 -- STEP: 1718/1882 -- GLOBAL_STEP: 3600[0m |
|
| > loss_text_ce: 0.050995614379644394 (0.05116577481805442) |
|
| > loss_mel_ce: 3.046502113342285 (2.7465488802429148) |
|
| > loss: 0.7743744254112244 (0.6994286640108125) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4041 (0.3791320944276486) |
|
| > loader_time: 0.0331 (0.016139872970625458) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:15:29 -- STEP: 1768/1882 -- GLOBAL_STEP: 3650[0m |
|
| > loss_text_ce: 0.04487369582056999 (0.051151294659452305) |
|
| > loss_mel_ce: 2.7761974334716797 (2.745545053441602) |
|
| > loss: 0.7052677869796753 (0.6991740872923343) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2618 (0.37935197690493405) |
|
| > loader_time: 0.0199 (0.016097893542294052) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:16:03 -- STEP: 1818/1882 -- GLOBAL_STEP: 3700[0m |
|
| > loss_text_ce: 0.062418144196271896 (0.05109705030508295) |
|
| > loss_mel_ce: 3.1166765689849854 (2.7409057253264772) |
|
| > loss: 0.7947736978530884 (0.6980006942490679) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4819 (0.3791677052169483) |
|
| > loader_time: 0.0229 (0.016056420254890943) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:16:37 -- STEP: 1868/1882 -- GLOBAL_STEP: 3750[0m |
|
| > loss_text_ce: 0.04787027835845947 (0.051040213585011554) |
|
| > loss_mel_ce: 2.422337770462036 (2.737861978292978) |
|
| > loss: 0.6175520420074463 (0.6972255482950616) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2592 (0.3790902531121479) |
|
| > loader_time: 0.01 (0.016046678994401386) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.6322868237128625 [0m(-0.009647360214820266) |
|
| > avg_loss_text_ce:[92m 0.04977148857254248 [0m(-0.0019781709863589403) |
|
| > avg_loss_mel_ce:[92m 2.6167104702729445 [0m(-0.13567587045522833) |
|
| > avg_loss:[92m 2.6664819625707774 [0m(-0.13765401106614306) |
|
|
|
|
|
[4m[1m > EPOCH: 2/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 08:17:57) [0m |
|
|
|
[1m --> TIME: 2024-12-05 08:18:22 -- STEP: 36/1882 -- GLOBAL_STEP: 3800[0m |
|
| > loss_text_ce: 0.04396151378750801 (0.04866313779105743) |
|
| > loss_mel_ce: 2.078479766845703 (2.6083439124955072) |
|
| > loss: 0.5306103229522705 (0.6642517670989037) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2909 (0.3667616777949863) |
|
| > loader_time: 0.0111 (0.013416058487362333) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:18:57 -- STEP: 86/1882 -- GLOBAL_STEP: 3850[0m |
|
| > loss_text_ce: 0.049731578677892685 (0.04951499034325745) |
|
| > loss_mel_ce: 2.7855894565582275 (2.6597119001455085) |
|
| > loss: 0.7088302373886108 (0.6773067244956658) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.416 (0.3719912240671557) |
|
| > loader_time: 0.0103 (0.015634509020073465) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:19:31 -- STEP: 136/1882 -- GLOBAL_STEP: 3900[0m |
|
| > loss_text_ce: 0.05067215859889984 (0.049482095329200525) |
|
| > loss_mel_ce: 3.062451124191284 (2.6761255536009294) |
|
| > loss: 0.7782807946205139 (0.681401913437773) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3403 (0.36965448540799756) |
|
| > loader_time: 0.0301 (0.015461252016179698) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:20:06 -- STEP: 186/1882 -- GLOBAL_STEP: 3950[0m |
|
| > loss_text_ce: 0.05557019263505936 (0.04975081263210184) |
|
| > loss_mel_ce: 3.060039758682251 (2.678567587688405) |
|
| > loss: 0.7789024710655212 (0.6820796002303395) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3939 (0.3740362121212867) |
|
| > loader_time: 0.0326 (0.015293064937796641) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:20:40 -- STEP: 236/1882 -- GLOBAL_STEP: 4000[0m |
|
| > loss_text_ce: 0.055366601794958115 (0.04978977366334806) |
|
| > loss_mel_ce: 2.762542486190796 (2.661134036920839) |
|
| > loss: 0.7044772505760193 (0.6777309534155711) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3349 (0.3727318614216174) |
|
| > loader_time: 0.0115 (0.015356387122202728) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:21:14 -- STEP: 286/1882 -- GLOBAL_STEP: 4050[0m |
|
| > loss_text_ce: 0.043853115290403366 (0.04962179278368716) |
|
| > loss_mel_ce: 2.3044681549072266 (2.63877770200476) |
|
| > loss: 0.5870802998542786 (0.6720998740487995) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3465 (0.37325567465562076) |
|
| > loader_time: 0.0105 (0.015555235055776747) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:21:48 -- STEP: 336/1882 -- GLOBAL_STEP: 4100[0m |
|
| > loss_text_ce: 0.04620029777288437 (0.049648616961868744) |
|
| > loss_mel_ce: 2.6945016384124756 (2.6448376664803144) |
|
| > loss: 0.6851754784584045 (0.6736215711349522) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3082 (0.37353802224000293) |
|
| > loader_time: 0.0111 (0.015485447077524098) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:22:23 -- STEP: 386/1882 -- GLOBAL_STEP: 4150[0m |
|
| > loss_text_ce: 0.04761936888098717 (0.049643642573081764) |
|
| > loss_mel_ce: 3.095506429672241 (2.664492744238267) |
|
| > loss: 0.7857814431190491 (0.6785340962323488) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3726 (0.3744067509557299) |
|
| > loader_time: 0.0106 (0.015520748079131924) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:22:57 -- STEP: 436/1882 -- GLOBAL_STEP: 4200[0m |
|
| > loss_text_ce: 0.051781296730041504 (0.049623084587788364) |
|
| > loss_mel_ce: 2.9125301837921143 (2.643042788319633) |
|
| > loss: 0.7410778999328613 (0.6731664677141993) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3371 (0.3742498618747116) |
|
| > loader_time: 0.0258 (0.015591199244927925) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:23:32 -- STEP: 486/1882 -- GLOBAL_STEP: 4250[0m |
|
| > loss_text_ce: 0.048189252614974976 (0.04967508251192393) |
|
| > loss_mel_ce: 2.733698844909668 (2.6489237662205496) |
|
| > loss: 0.695472002029419 (0.674649711928249) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3606 (0.3744087439996225) |
|
| > loader_time: 0.0309 (0.01569517845970123) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:24:07 -- STEP: 536/1882 -- GLOBAL_STEP: 4300[0m |
|
| > loss_text_ce: 0.054472416639328 (0.049744358091656844) |
|
| > loss_mel_ce: 2.6780200004577637 (2.6460944465292044) |
|
| > loss: 0.6831231117248535 (0.6739597010857126) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3372 (0.3752097652029636) |
|
| > loader_time: 0.021 (0.015619610227755654) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:24:41 -- STEP: 586/1882 -- GLOBAL_STEP: 4350[0m |
|
| > loss_text_ce: 0.05555795505642891 (0.049656997704973824) |
|
| > loss_mel_ce: 2.9654433727264404 (2.643276162318403) |
|
| > loss: 0.7552503347396851 (0.6732332901234508) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.405 (0.37582844149944317) |
|
| > loader_time: 0.011 (0.0157037200374408) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:25:16 -- STEP: 636/1882 -- GLOBAL_STEP: 4400[0m |
|
| > loss_text_ce: 0.05496556684374809 (0.04971582201198213) |
|
| > loss_mel_ce: 2.6257359981536865 (2.6423071495392048) |
|
| > loss: 0.6701753735542297 (0.6730057428467945) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.29 (0.37637479612662356) |
|
| > loader_time: 0.0105 (0.015692521940987064) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:25:51 -- STEP: 686/1882 -- GLOBAL_STEP: 4450[0m |
|
| > loss_text_ce: 0.05185301601886749 (0.04970588413971334) |
|
| > loss_mel_ce: 3.158379077911377 (2.646189088724098) |
|
| > loss: 0.8025580048561096 (0.6739737431874423) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3742 (0.3769932387521595) |
|
| > loader_time: 0.0109 (0.01566414061510181) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:26:27 -- STEP: 736/1882 -- GLOBAL_STEP: 4500[0m |
|
| > loss_text_ce: 0.05595708265900612 (0.0497044946026543) |
|
| > loss_mel_ce: 2.792696714401245 (2.6474051388061564) |
|
| > loss: 0.7121634483337402 (0.6742774083319564) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3426 (0.37795760417762025) |
|
| > loader_time: 0.0114 (0.01568709249081819) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:27:01 -- STEP: 786/1882 -- GLOBAL_STEP: 4550[0m |
|
| > loss_text_ce: 0.051778897643089294 (0.04971885636126571) |
|
| > loss_mel_ce: 2.8943517208099365 (2.646304918912833) |
|
| > loss: 0.7365326285362244 (0.6740059438173399) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.415 (0.37815914475584167) |
|
| > loader_time: 0.011 (0.015749499088025268) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:27:36 -- STEP: 836/1882 -- GLOBAL_STEP: 4600[0m |
|
| > loss_text_ce: 0.0513005405664444 (0.049698672542553494) |
|
| > loss_mel_ce: 2.6725316047668457 (2.6428720316533254) |
|
| > loss: 0.6809580326080322 (0.673142676255064) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3561 (0.3779092756184666) |
|
| > loader_time: 0.0116 (0.015789403584585218) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:28:11 -- STEP: 886/1882 -- GLOBAL_STEP: 4650[0m |
|
| > loss_text_ce: 0.04504093900322914 (0.04975730192092822) |
|
| > loss_mel_ce: 1.617553949356079 (2.642547463859445) |
|
| > loss: 0.4156487286090851 (0.6730761917530816) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3071 (0.3785348630112936) |
|
| > loader_time: 0.0174 (0.01570989046893323) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:28:46 -- STEP: 936/1882 -- GLOBAL_STEP: 4700[0m |
|
| > loss_text_ce: 0.04732833430171013 (0.0497812971743381) |
|
| > loss_mel_ce: 2.6624865531921387 (2.645591570908187) |
|
| > loss: 0.6774536967277527 (0.6738432172017218) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3073 (0.3789049407355807) |
|
| > loader_time: 0.0196 (0.015737243187733178) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:29:21 -- STEP: 986/1882 -- GLOBAL_STEP: 4750[0m |
|
| > loss_text_ce: 0.04641490802168846 (0.04970783865339979) |
|
| > loss_mel_ce: 2.5365262031555176 (2.641840008040228) |
|
| > loss: 0.6457352638244629 (0.6728869617952535) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4027 (0.3787740195740552) |
|
| > loader_time: 0.0111 (0.01566123599696593) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:29:55 -- STEP: 1036/1882 -- GLOBAL_STEP: 4800[0m |
|
| > loss_text_ce: 0.050602611154317856 (0.04970844202894272) |
|
| > loss_mel_ce: 2.785438299179077 (2.6403259027418473) |
|
| > loss: 0.7090102434158325 (0.672508586500142) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3482 (0.37882833545272426) |
|
| > loader_time: 0.011 (0.01567263170558972) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:30:29 -- STEP: 1086/1882 -- GLOBAL_STEP: 4850[0m |
|
| > loss_text_ce: 0.047948259860277176 (0.0497099486853701) |
|
| > loss_mel_ce: 2.451852798461914 (2.635956272126122) |
|
| > loss: 0.6249502897262573 (0.6714165556222255) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4076 (0.3784328940604055) |
|
| > loader_time: 0.0135 (0.015710877011054992) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:31:02 -- STEP: 1136/1882 -- GLOBAL_STEP: 4900[0m |
|
| > loss_text_ce: 0.05294486880302429 (0.049638034024892336) |
|
| > loss_mel_ce: 2.659789800643921 (2.630311516703854) |
|
| > loss: 0.6781836748123169 (0.6699873883487054) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3558 (0.37763865040221684) |
|
| > loader_time: 0.0117 (0.01569398880844383) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:31:36 -- STEP: 1186/1882 -- GLOBAL_STEP: 4950[0m |
|
| > loss_text_ce: 0.03661234304308891 (0.04955620053860327) |
|
| > loss_mel_ce: 2.33951473236084 (2.626886891172867) |
|
| > loss: 0.5940317511558533 (0.6691107736872182) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.314 (0.37749844437706925) |
|
| > loader_time: 0.0122 (0.015726188584081622) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:32:11 -- STEP: 1236/1882 -- GLOBAL_STEP: 5000[0m |
|
| > loss_text_ce: 0.04913272708654404 (0.04956040827973373) |
|
| > loss_mel_ce: 1.5647969245910645 (2.6248206926395192) |
|
| > loss: 0.4034824073314667 (0.6685952761219544) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2993 (0.3776542337966969) |
|
| > loader_time: 0.0112 (0.01569538189755286) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:32:44 -- STEP: 1286/1882 -- GLOBAL_STEP: 5050[0m |
|
| > loss_text_ce: 0.048719245940446854 (0.04951207913910247) |
|
| > loss_mel_ce: 2.7043356895446777 (2.624142702222797) |
|
| > loss: 0.6882637143135071 (0.6684136961704094) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4007 (0.37698472714164605) |
|
| > loader_time: 0.01 (0.015653597993390896) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:33:19 -- STEP: 1336/1882 -- GLOBAL_STEP: 5100[0m |
|
| > loss_text_ce: 0.04365723952651024 (0.04949361920735971) |
|
| > loss_mel_ce: 2.7681798934936523 (2.623566473494033) |
|
| > loss: 0.7029592990875244 (0.6682650239435503) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2967 (0.3769312941742517) |
|
| > loader_time: 0.0111 (0.01573007531508712) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:33:53 -- STEP: 1386/1882 -- GLOBAL_STEP: 5150[0m |
|
| > loss_text_ce: 0.050255049020051956 (0.049511118242342396) |
|
| > loss_mel_ce: 2.5173380374908447 (2.622880516716241) |
|
| > loss: 0.6418982744216919 (0.6680979094808068) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4012 (0.3767458917774682) |
|
| > loader_time: 0.0104 (0.015827679530882703) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:34:28 -- STEP: 1436/1882 -- GLOBAL_STEP: 5200[0m |
|
| > loss_text_ce: 0.044951338320970535 (0.049491060132415006) |
|
| > loss_mel_ce: 2.6681361198425293 (2.621126489552948) |
|
| > loss: 0.6782718896865845 (0.6676543880679485) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3068 (0.3769616223975479) |
|
| > loader_time: 0.0213 (0.01578966621568939) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:35:03 -- STEP: 1486/1882 -- GLOBAL_STEP: 5250[0m |
|
| > loss_text_ce: 0.04580783098936081 (0.049499457391181986) |
|
| > loss_mel_ce: 2.5428309440612793 (2.622302750880116) |
|
| > loss: 0.6471596956253052 (0.6679505526337619) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3719 (0.3772269394323934) |
|
| > loader_time: 0.0215 (0.015804226273322324) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:35:38 -- STEP: 1536/1882 -- GLOBAL_STEP: 5300[0m |
|
| > loss_text_ce: 0.05881780385971069 (0.04948446750252817) |
|
| > loss_mel_ce: 3.1165318489074707 (2.6197290655691234) |
|
| > loss: 0.7938374280929565 (0.6673033839130461) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.346 (0.3772500827908518) |
|
| > loader_time: 0.0108 (0.015781774806479628) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:36:11 -- STEP: 1586/1882 -- GLOBAL_STEP: 5350[0m |
|
| > loss_text_ce: 0.0410224013030529 (0.04948438748172372) |
|
| > loss_mel_ce: 2.6241729259490967 (2.6155365547646854) |
|
| > loss: 0.6662988066673279 (0.6662552361605595) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3806 (0.3769837144344183) |
|
| > loader_time: 0.0107 (0.015810806393473036) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:36:46 -- STEP: 1636/1882 -- GLOBAL_STEP: 5400[0m |
|
| > loss_text_ce: 0.05113517493009567 (0.049501450755121065) |
|
| > loss_mel_ce: 2.638089895248413 (2.6144659108958272) |
|
| > loss: 0.67230623960495 (0.6659918410708091) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3241 (0.37699349308363694) |
|
| > loader_time: 0.0291 (0.01583868833509227) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:37:22 -- STEP: 1686/1882 -- GLOBAL_STEP: 5450[0m |
|
| > loss_text_ce: 0.04728825390338898 (0.04949558057191462) |
|
| > loss_mel_ce: 2.419800281524658 (2.6135795293188098) |
|
| > loss: 0.6167721152305603 (0.6657687781438282) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3898 (0.3774661224507778) |
|
| > loader_time: 0.0276 (0.01581919037715963) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:37:56 -- STEP: 1736/1882 -- GLOBAL_STEP: 5500[0m |
|
| > loss_text_ce: 0.047781601548194885 (0.04948301703673422) |
|
| > loss_mel_ce: 2.624194383621216 (2.613832816550265) |
|
| > loss: 0.6679940223693848 (0.6658289590979224) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4139 (0.3775494046474933) |
|
| > loader_time: 0.0119 (0.01582600045863386) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:38:30 -- STEP: 1786/1882 -- GLOBAL_STEP: 5550[0m |
|
| > loss_text_ce: 0.04921665042638779 (0.04946683039356852) |
|
| > loss_mel_ce: 2.4106242656707764 (2.6109384227573327) |
|
| > loss: 0.614960253238678 (0.6651013140448795) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2944 (0.3771254423617785) |
|
| > loader_time: 0.0103 (0.015801034075148573) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:39:04 -- STEP: 1836/1882 -- GLOBAL_STEP: 5600[0m |
|
| > loss_text_ce: 0.050528090447187424 (0.04947670218017365) |
|
| > loss_mel_ce: 2.8891496658325195 (2.6085964333517673) |
|
| > loss: 0.7349194288253784 (0.6645182846580713) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3445 (0.37699494915070897) |
|
| > loader_time: 0.0115 (0.015739263142895348) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.6424625928585346 [0m(+0.010175769145672087) |
|
| > avg_loss_text_ce:[92m 0.04824803640636114 [0m(-0.001523452166181341) |
|
| > avg_loss_mel_ce:[92m 2.5397496865345883 [0m(-0.07696078373835613) |
|
| > avg_loss:[92m 2.5879977299616885 [0m(-0.07848423260908888) |
|
|
|
|
|
[4m[1m > EPOCH: 3/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 08:40:46) [0m |
|
|
|
[1m --> TIME: 2024-12-05 08:40:50 -- STEP: 4/1882 -- GLOBAL_STEP: 5650[0m |
|
| > loss_text_ce: 0.049121372401714325 (0.05122512113302946) |
|
| > loss_mel_ce: 2.962510824203491 (2.6322966814041138) |
|
| > loss: 0.7529080510139465 (0.670880451798439) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3422 (0.36920344829559326) |
|
| > loader_time: 0.0123 (0.010863304138183594) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:41:23 -- STEP: 54/1882 -- GLOBAL_STEP: 5700[0m |
|
| > loss_text_ce: 0.050911203026771545 (0.04957079852896708) |
|
| > loss_mel_ce: 2.581885576248169 (2.527686492160515) |
|
| > loss: 0.6581991910934448 (0.6443143255180781) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.398 (0.3682806094487509) |
|
| > loader_time: 0.0106 (0.014908689039724844) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:41:59 -- STEP: 104/1882 -- GLOBAL_STEP: 5750[0m |
|
| > loss_text_ce: 0.04941564425826073 (0.04957584972278431) |
|
| > loss_mel_ce: 2.484434127807617 (2.5587594738373407) |
|
| > loss: 0.6334624290466309 (0.6520838339168289) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3461 (0.3810229278527774) |
|
| > loader_time: 0.0134 (0.015246129952944242) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:42:33 -- STEP: 154/1882 -- GLOBAL_STEP: 5800[0m |
|
| > loss_text_ce: 0.050299081951379776 (0.04917074328699671) |
|
| > loss_mel_ce: 2.363583564758301 (2.545053271504193) |
|
| > loss: 0.6034706830978394 (0.6485560044839782) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4052 (0.37562913089603583) |
|
| > loader_time: 0.0294 (0.014881912764016684) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:43:08 -- STEP: 204/1882 -- GLOBAL_STEP: 5850[0m |
|
| > loss_text_ce: 0.047523383051157 (0.049135756952797684) |
|
| > loss_mel_ce: 2.6699559688568115 (2.5619408745391716) |
|
| > loss: 0.6793698668479919 (0.6527691591604075) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3416 (0.37767959692898945) |
|
| > loader_time: 0.0303 (0.014794604451048608) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:43:43 -- STEP: 254/1882 -- GLOBAL_STEP: 5900[0m |
|
| > loss_text_ce: 0.05222291871905327 (0.04919937723792915) |
|
| > loss_mel_ce: 3.004953145980835 (2.591289051874418) |
|
| > loss: 0.7642940282821655 (0.6601221085064047) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3977 (0.3799668146869331) |
|
| > loader_time: 0.0106 (0.015339515340609813) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:44:18 -- STEP: 304/1882 -- GLOBAL_STEP: 5950[0m |
|
| > loss_text_ce: 0.04581185802817345 (0.04914820370705506) |
|
| > loss_mel_ce: 2.5018625259399414 (2.582158493368251) |
|
| > loss: 0.6369186043739319 (0.6578266751021138) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3478 (0.3800723819356218) |
|
| > loader_time: 0.011 (0.015579516166134885) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:44:52 -- STEP: 354/1882 -- GLOBAL_STEP: 6000[0m |
|
| > loss_text_ce: 0.0525100938975811 (0.049162510214215645) |
|
| > loss_mel_ce: 2.5959599018096924 (2.5801285131502976) |
|
| > loss: 0.6621174812316895 (0.6573227560621188) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.401 (0.3800444333566786) |
|
| > loader_time: 0.0113 (0.01567159668873933) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:45:26 -- STEP: 404/1882 -- GLOBAL_STEP: 6050[0m |
|
| > loss_text_ce: 0.04864883795380592 (0.048999713051437174) |
|
| > loss_mel_ce: 2.7685468196868896 (2.563722385333317) |
|
| > loss: 0.7042989134788513 (0.6531805246330713) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3597 (0.37836378517717434) |
|
| > loader_time: 0.031 (0.01580539198205023) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:46:01 -- STEP: 454/1882 -- GLOBAL_STEP: 6100[0m |
|
| > loss_text_ce: 0.05506376549601555 (0.04902115346051525) |
|
| > loss_mel_ce: 2.720313787460327 (2.5639894326352874) |
|
| > loss: 0.6938443779945374 (0.6532526465239494) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4076 (0.37906514863085655) |
|
| > loader_time: 0.0286 (0.0157353646954776) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:46:35 -- STEP: 504/1882 -- GLOBAL_STEP: 6150[0m |
|
| > loss_text_ce: 0.051236242055892944 (0.048985472946826925) |
|
| > loss_mel_ce: 3.1658830642700195 (2.559079632399575) |
|
| > loss: 0.8042798042297363 (0.6520162764641019) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4036 (0.37843994914539286) |
|
| > loader_time: 0.0131 (0.01581268320007931) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:47:09 -- STEP: 554/1882 -- GLOBAL_STEP: 6200[0m |
|
| > loss_text_ce: 0.044484693557024 (0.04897548303357747) |
|
| > loss_mel_ce: 2.457536458969116 (2.5597776361751223) |
|
| > loss: 0.625505268573761 (0.652188279729887) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3443 (0.37816043666123494) |
|
| > loader_time: 0.0117 (0.015692522379465494) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:47:45 -- STEP: 604/1882 -- GLOBAL_STEP: 6250[0m |
|
| > loss_text_ce: 0.05064237117767334 (0.048994787154558905) |
|
| > loss_mel_ce: 2.80884051322937 (2.5557671045230723) |
|
| > loss: 0.7148706912994385 (0.651190472606396) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3485 (0.3789450976232819) |
|
| > loader_time: 0.0105 (0.015854891562303974) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:48:20 -- STEP: 654/1882 -- GLOBAL_STEP: 6300[0m |
|
| > loss_text_ce: 0.04432835429906845 (0.048918348124723586) |
|
| > loss_mel_ce: 2.659001588821411 (2.5543489299418374) |
|
| > loss: 0.6758325099945068 (0.6508168195821457) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4 (0.3793055522697053) |
|
| > loader_time: 0.0105 (0.01586981322787225) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:48:54 -- STEP: 704/1882 -- GLOBAL_STEP: 6350[0m |
|
| > loss_text_ce: 0.051667679101228714 (0.04889696785672148) |
|
| > loss_mel_ce: 2.8343920707702637 (2.554960313507101) |
|
| > loss: 0.7215149402618408 (0.6509643201716239) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3206 (0.378859396007928) |
|
| > loader_time: 0.0333 (0.015781355513767775) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:49:29 -- STEP: 754/1882 -- GLOBAL_STEP: 6400[0m |
|
| > loss_text_ce: 0.04925987496972084 (0.048902071073690864) |
|
| > loss_mel_ce: 2.9799604415893555 (2.55484176203806) |
|
| > loss: 0.7573050856590271 (0.6509359583693408) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3663 (0.37905553639409395) |
|
| > loader_time: 0.0129 (0.015802731880774877) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:50:03 -- STEP: 804/1882 -- GLOBAL_STEP: 6450[0m |
|
| > loss_text_ce: 0.05016998201608658 (0.048873451218676235) |
|
| > loss_mel_ce: 2.5528671741485596 (2.5528148280743914) |
|
| > loss: 0.6507592797279358 (0.6504220700456727) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2939 (0.3786232047413119) |
|
| > loader_time: 0.031 (0.015834732731776466) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:50:39 -- STEP: 854/1882 -- GLOBAL_STEP: 6500[0m |
|
| > loss_text_ce: 0.04777722433209419 (0.04890195370610344) |
|
| > loss_mel_ce: 2.775980234146118 (2.5594324944170053) |
|
| > loss: 0.7059393525123596 (0.652083612472447) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4064 (0.3798302774965344) |
|
| > loader_time: 0.0105 (0.015791727452423323) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:51:13 -- STEP: 904/1882 -- GLOBAL_STEP: 6550[0m |
|
| > loss_text_ce: 0.05136550962924957 (0.048859566458360836) |
|
| > loss_mel_ce: 2.531508445739746 (2.5584512929209544) |
|
| > loss: 0.645718514919281 (0.6518277153702435) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3167 (0.3794704860290595) |
|
| > loader_time: 0.0111 (0.015848972101127172) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:51:48 -- STEP: 954/1882 -- GLOBAL_STEP: 6600[0m |
|
| > loss_text_ce: 0.0438341461122036 (0.04885724032262589) |
|
| > loss_mel_ce: 2.5621273517608643 (2.5555261730897834) |
|
| > loss: 0.6514903903007507 (0.6510958538177877) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3792 (0.37976916206207917) |
|
| > loader_time: 0.0116 (0.01593471323169254) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:52:23 -- STEP: 1004/1882 -- GLOBAL_STEP: 6650[0m |
|
| > loss_text_ce: 0.043358754366636276 (0.048850006746699634) |
|
| > loss_mel_ce: 2.1759748458862305 (2.5487437735040808) |
|
| > loss: 0.5548334121704102 (0.6493984455784477) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3128 (0.37999664953505374) |
|
| > loader_time: 0.0118 (0.01595075529410071) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:52:58 -- STEP: 1054/1882 -- GLOBAL_STEP: 6700[0m |
|
| > loss_text_ce: 0.03950514644384384 (0.04880029810998994) |
|
| > loss_mel_ce: 2.2964468002319336 (2.5471071854261997) |
|
| > loss: 0.5839880108833313 (0.6489768713938896) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2697 (0.3797533607120984) |
|
| > loader_time: 0.0106 (0.01600253559380375) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:53:32 -- STEP: 1104/1882 -- GLOBAL_STEP: 6750[0m |
|
| > loss_text_ce: 0.04605181887745857 (0.04876185118463702) |
|
| > loss_mel_ce: 2.615833282470703 (2.5441272522228338) |
|
| > loss: 0.6654712557792664 (0.6482222762120806) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3394 (0.37942193923653034) |
|
| > loader_time: 0.0115 (0.015990896933320647) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:54:06 -- STEP: 1154/1882 -- GLOBAL_STEP: 6800[0m |
|
| > loss_text_ce: 0.047021329402923584 (0.04876055384129029) |
|
| > loss_mel_ce: 1.8479975461959839 (2.5406500710565045) |
|
| > loss: 0.4737547039985657 (0.6473526566755835) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2708 (0.3794766629382685) |
|
| > loader_time: 0.0105 (0.015927887871938946) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:54:41 -- STEP: 1204/1882 -- GLOBAL_STEP: 6850[0m |
|
| > loss_text_ce: 0.057348836213350296 (0.04879402918462642) |
|
| > loss_mel_ce: 2.783555030822754 (2.539784596805952) |
|
| > loss: 0.7102259397506714 (0.6471446568635217) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3465 (0.37920259261052086) |
|
| > loader_time: 0.0166 (0.01607902006453454) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:55:15 -- STEP: 1254/1882 -- GLOBAL_STEP: 6900[0m |
|
| > loss_text_ce: 0.04044807329773903 (0.04875509680339408) |
|
| > loss_mel_ce: 2.6356589794158936 (2.540165751079242) |
|
| > loss: 0.669026792049408 (0.6472302123880465) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3412 (0.37926269661296497) |
|
| > loader_time: 0.0119 (0.01599356033014909) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:55:50 -- STEP: 1304/1882 -- GLOBAL_STEP: 6950[0m |
|
| > loss_text_ce: 0.04935640096664429 (0.04875815781747551) |
|
| > loss_mel_ce: 2.5002806186676025 (2.540909856557845) |
|
| > loss: 0.6374092698097229 (0.6474170040544924) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3515 (0.3790716894565183) |
|
| > loader_time: 0.011 (0.01600838018341299) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:56:23 -- STEP: 1354/1882 -- GLOBAL_STEP: 7000[0m |
|
| > loss_text_ce: 0.04472864419221878 (0.0487314589690291) |
|
| > loss_mel_ce: 2.6232564449310303 (2.5381843948645955) |
|
| > loss: 0.6669963002204895 (0.646728964024492) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3682 (0.3786764336829699) |
|
| > loader_time: 0.0111 (0.015994453711911178) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:56:57 -- STEP: 1404/1882 -- GLOBAL_STEP: 7050[0m |
|
| > loss_text_ce: 0.04938598349690437 (0.04871402503100051) |
|
| > loss_mel_ce: 2.6298165321350098 (2.537040158129821) |
|
| > loss: 0.6698006391525269 (0.646438546139121) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3361 (0.3783379253159221) |
|
| > loader_time: 0.0106 (0.016000602320048882) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:57:31 -- STEP: 1454/1882 -- GLOBAL_STEP: 7100[0m |
|
| > loss_text_ce: 0.04531159624457359 (0.048686583163895715) |
|
| > loss_mel_ce: 2.292116165161133 (2.5349443831174834) |
|
| > loss: 0.584356963634491 (0.6459077419572224) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3592 (0.37794307288310214) |
|
| > loader_time: 0.0107 (0.01613394430283831) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:58:06 -- STEP: 1504/1882 -- GLOBAL_STEP: 7150[0m |
|
| > loss_text_ce: 0.051953066140413284 (0.048673752559091385) |
|
| > loss_mel_ce: 2.7978878021240234 (2.5313807801204753) |
|
| > loss: 0.7124602198600769 (0.6450136334417349) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4085 (0.3780278395465079) |
|
| > loader_time: 0.0159 (0.01612569106386061) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:58:40 -- STEP: 1554/1882 -- GLOBAL_STEP: 7200[0m |
|
| > loss_text_ce: 0.04479723423719406 (0.04867785789571141) |
|
| > loss_mel_ce: 2.439084768295288 (2.529370542351952) |
|
| > loss: 0.6209704875946045 (0.6445121003202186) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3678 (0.37770386245413323) |
|
| > loader_time: 0.0105 (0.01604192950397229) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:59:15 -- STEP: 1604/1882 -- GLOBAL_STEP: 7250[0m |
|
| > loss_text_ce: 0.05068426951766014 (0.04867166835824009) |
|
| > loss_mel_ce: 2.4282007217407227 (2.5303747455526713) |
|
| > loss: 0.6197212338447571 (0.6447616037622356) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3577 (0.3778936104881495) |
|
| > loader_time: 0.0261 (0.0159840106666831) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 08:59:50 -- STEP: 1654/1882 -- GLOBAL_STEP: 7300[0m |
|
| > loss_text_ce: 0.04876066744327545 (0.04863785180244081) |
|
| > loss_mel_ce: 2.958799362182617 (2.527376046636224) |
|
| > loss: 0.7518900036811829 (0.64400347479886) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4748 (0.378019683594571) |
|
| > loader_time: 0.0102 (0.016067179255502804) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:00:25 -- STEP: 1704/1882 -- GLOBAL_STEP: 7350[0m |
|
| > loss_text_ce: 0.04525046795606613 (0.04860666874979797) |
|
| > loss_mel_ce: 2.537158489227295 (2.5268725908837926) |
|
| > loss: 0.6456022262573242 (0.6438698151756621) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3415 (0.37838891414409503) |
|
| > loader_time: 0.0179 (0.01609663378464781) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:00:59 -- STEP: 1754/1882 -- GLOBAL_STEP: 7400[0m |
|
| > loss_text_ce: 0.045673150569200516 (0.048588684977732104) |
|
| > loss_mel_ce: 1.8871983289718628 (2.5244734230851598) |
|
| > loss: 0.48321786522865295 (0.6432655273236871) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2361 (0.37817932969488055) |
|
| > loader_time: 0.0096 (0.016064146361573894) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:01:34 -- STEP: 1804/1882 -- GLOBAL_STEP: 7450[0m |
|
| > loss_text_ce: 0.051513269543647766 (0.048573705342963805) |
|
| > loss_mel_ce: 2.575340986251831 (2.5233861445191126) |
|
| > loss: 0.6567135453224182 (0.6429899627313911) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3646 (0.3781173892666125) |
|
| > loader_time: 0.0113 (0.016067548099483943) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:02:08 -- STEP: 1854/1882 -- GLOBAL_STEP: 7500[0m |
|
| > loss_text_ce: 0.04701970890164375 (0.048571568305936744) |
|
| > loss_mel_ce: 2.812221050262451 (2.5237322886913427) |
|
| > loss: 0.7148101925849915 (0.6430759644527653) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3925 (0.3781353937383608) |
|
| > loader_time: 0.0327 (0.01602796237702762) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.6578387297116793 [0m(+0.015376136853144762) |
|
| > avg_loss_text_ce:[92m 0.04757265775249554 [0m(-0.0006753786538655979) |
|
| > avg_loss_mel_ce:[92m 2.4795724612015944 [0m(-0.06017722533299397) |
|
| > avg_loss:[92m 2.527145110643827 [0m(-0.06085261931786157) |
|
|
|
|
|
[4m[1m > EPOCH: 4/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 09:03:39) [0m |
|
|
|
[1m --> TIME: 2024-12-05 09:03:55 -- STEP: 22/1882 -- GLOBAL_STEP: 7550[0m |
|
| > loss_text_ce: 0.05057277902960777 (0.046882199462164535) |
|
| > loss_mel_ce: 1.5044759511947632 (2.4312130863016304) |
|
| > loss: 0.3887621760368347 (0.619523825970563) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3455 (0.37903809547424333) |
|
| > loader_time: 0.0121 (0.01476928320798007) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:04:30 -- STEP: 72/1882 -- GLOBAL_STEP: 7600[0m |
|
| > loss_text_ce: 0.04465623199939728 (0.047699472608251706) |
|
| > loss_mel_ce: 2.3176302909851074 (2.452460668153233) |
|
| > loss: 0.5905716419219971 (0.6250400381783644) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3427 (0.37597524457507664) |
|
| > loader_time: 0.0114 (0.014694283405939737) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:05:04 -- STEP: 122/1882 -- GLOBAL_STEP: 7650[0m |
|
| > loss_text_ce: 0.045687198638916016 (0.047689007717322145) |
|
| > loss_mel_ce: 2.2801923751831055 (2.459037862840246) |
|
| > loss: 0.5814698934555054 (0.6266817201356416) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3422 (0.3773533457615336) |
|
| > loader_time: 0.0294 (0.015290801642371005) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:05:39 -- STEP: 172/1882 -- GLOBAL_STEP: 7700[0m |
|
| > loss_text_ce: 0.04894641786813736 (0.047705010424346425) |
|
| > loss_mel_ce: 2.570587158203125 (2.484527068775752) |
|
| > loss: 0.6548833847045898 (0.6330580219279887) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3413 (0.3776136304056922) |
|
| > loader_time: 0.0113 (0.0158437781555708) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:06:14 -- STEP: 222/1882 -- GLOBAL_STEP: 7750[0m |
|
| > loss_text_ce: 0.051667220890522 (0.04786004238501863) |
|
| > loss_mel_ce: 2.538835287094116 (2.488678243246164) |
|
| > loss: 0.6476256251335144 (0.6341345727712182) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3998 (0.37935221839595484) |
|
| > loader_time: 0.0114 (0.015658769521627328) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:06:49 -- STEP: 272/1882 -- GLOBAL_STEP: 7800[0m |
|
| > loss_text_ce: 0.05200611799955368 (0.04803181421833441) |
|
| > loss_mel_ce: 2.2503397464752197 (2.490648247301578) |
|
| > loss: 0.5755864381790161 (0.6346700158627591) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4097 (0.380382883198121) |
|
| > loader_time: 0.0109 (0.01574803976451648) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:07:23 -- STEP: 322/1882 -- GLOBAL_STEP: 7850[0m |
|
| > loss_text_ce: 0.05018450692296028 (0.04815005284288654) |
|
| > loss_mel_ce: 2.7924141883850098 (2.486298128314641) |
|
| > loss: 0.7106496691703796 (0.6336120453125195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4054 (0.37886133253204146) |
|
| > loader_time: 0.0306 (0.015517477663407404) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:07:57 -- STEP: 372/1882 -- GLOBAL_STEP: 7900[0m |
|
| > loss_text_ce: 0.04812903329730034 (0.048044439715643705) |
|
| > loss_mel_ce: 2.510529041290283 (2.478351413242278) |
|
| > loss: 0.6396645307540894 (0.6315989628914861) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4236 (0.37897993608187597) |
|
| > loader_time: 0.0144 (0.015609586751589204) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:08:32 -- STEP: 422/1882 -- GLOBAL_STEP: 7950[0m |
|
| > loss_text_ce: 0.05191934108734131 (0.047947141019653916) |
|
| > loss_mel_ce: 2.6443026065826416 (2.4811964238424444) |
|
| > loss: 0.6740554571151733 (0.6322858907035173) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.405 (0.3778612619327707) |
|
| > loader_time: 0.0111 (0.01588597907839227) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:09:06 -- STEP: 472/1882 -- GLOBAL_STEP: 8000[0m |
|
| > loss_text_ce: 0.05194719508290291 (0.0479894223922895) |
|
| > loss_mel_ce: 2.5598716735839844 (2.479964058025409) |
|
| > loss: 0.6529546976089478 (0.6319883697492588) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3087 (0.37804564075954883) |
|
| > loader_time: 0.0106 (0.015911170991800585) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:09:40 -- STEP: 522/1882 -- GLOBAL_STEP: 8050[0m |
|
| > loss_text_ce: 0.05657385662198067 (0.04797177308620848) |
|
| > loss_mel_ce: 3.0420632362365723 (2.4829665116423394) |
|
| > loss: 0.774659276008606 (0.632734571037621) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.371 (0.3776036135538327) |
|
| > loader_time: 0.011 (0.015889523129810304) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:10:15 -- STEP: 572/1882 -- GLOBAL_STEP: 8100[0m |
|
| > loss_text_ce: 0.03817710652947426 (0.047942815895576585) |
|
| > loss_mel_ce: 1.8464139699935913 (2.4804512672907832) |
|
| > loss: 0.4711477756500244 (0.6320985209170751) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2466 (0.37793762700541034) |
|
| > loader_time: 0.0108 (0.015915423006444537) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:10:49 -- STEP: 622/1882 -- GLOBAL_STEP: 8150[0m |
|
| > loss_text_ce: 0.04426265135407448 (0.04794384367835871) |
|
| > loss_mel_ce: 2.2361340522766113 (2.4786617207757162) |
|
| > loss: 0.5700991749763489 (0.6316513911704158) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3382 (0.37757706412165115) |
|
| > loader_time: 0.0116 (0.015963887861687266) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:11:24 -- STEP: 672/1882 -- GLOBAL_STEP: 8200[0m |
|
| > loss_text_ce: 0.05336824804544449 (0.04790545231662696) |
|
| > loss_mel_ce: 2.485259771347046 (2.4738379574957357) |
|
| > loss: 0.6346570253372192 (0.6304358525556466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2838 (0.37795790071998336) |
|
| > loader_time: 0.0303 (0.015975214895747963) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:12:00 -- STEP: 722/1882 -- GLOBAL_STEP: 8250[0m |
|
| > loss_text_ce: 0.047276921570301056 (0.04797087433822768) |
|
| > loss_mel_ce: 2.544235944747925 (2.4764030067214033) |
|
| > loss: 0.6478782296180725 (0.6310934700959272) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4109 (0.378940225638181) |
|
| > loader_time: 0.0286 (0.016113029622635354) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:12:33 -- STEP: 772/1882 -- GLOBAL_STEP: 8300[0m |
|
| > loss_text_ce: 0.050416722893714905 (0.04791126837423124) |
|
| > loss_mel_ce: 2.2028439044952393 (2.468350847922459) |
|
| > loss: 0.5633151531219482 (0.6290655292840818) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3096 (0.37767170871477684) |
|
| > loader_time: 0.0112 (0.016031793979783132) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:13:07 -- STEP: 822/1882 -- GLOBAL_STEP: 8350[0m |
|
| > loss_text_ce: 0.04733885079622269 (0.04791254548858991) |
|
| > loss_mel_ce: 1.964216947555542 (2.4636104144609163) |
|
| > loss: 0.5028889775276184 (0.6278807400859473) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2912 (0.37719913352725004) |
|
| > loader_time: 0.0105 (0.016194436962007022) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:13:42 -- STEP: 872/1882 -- GLOBAL_STEP: 8400[0m |
|
| > loss_text_ce: 0.04477662593126297 (0.04791952127061037) |
|
| > loss_mel_ce: 2.864856243133545 (2.464259639314007) |
|
| > loss: 0.727408230304718 (0.6280447899816777) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.319 (0.3775689325748235) |
|
| > loader_time: 0.0335 (0.016249103557079207) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:14:17 -- STEP: 922/1882 -- GLOBAL_STEP: 8450[0m |
|
| > loss_text_ce: 0.05519567057490349 (0.04797166786328318) |
|
| > loss_mel_ce: 2.9778971672058105 (2.464957452453914) |
|
| > loss: 0.758273184299469 (0.6282322800792993) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4028 (0.37783796668311276) |
|
| > loader_time: 0.012 (0.01629442920395194) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:14:50 -- STEP: 972/1882 -- GLOBAL_STEP: 8500[0m |
|
| > loss_text_ce: 0.04868988320231438 (0.04792550671079158) |
|
| > loss_mel_ce: 2.8081624507904053 (2.462418920157376) |
|
| > loss: 0.7142130732536316 (0.6275861067965686) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.448 (0.377251060901846) |
|
| > loader_time: 0.0213 (0.01615331300492148) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:15:24 -- STEP: 1022/1882 -- GLOBAL_STEP: 8550[0m |
|
| > loss_text_ce: 0.052141427993774414 (0.04788254907832224) |
|
| > loss_mel_ce: 2.8489911556243896 (2.4630194080669106) |
|
| > loss: 0.725283145904541 (0.6277254893646779) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3739 (0.37708844130753083) |
|
| > loader_time: 0.0107 (0.016194686964300032) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:16:00 -- STEP: 1072/1882 -- GLOBAL_STEP: 8600[0m |
|
| > loss_text_ce: 0.05540282651782036 (0.04791814524249469) |
|
| > loss_mel_ce: 3.204578161239624 (2.463945080484471) |
|
| > loss: 0.8149952292442322 (0.6279658065247) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3339 (0.3773039933016052) |
|
| > loader_time: 0.0106 (0.016276975843443786) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:16:34 -- STEP: 1122/1882 -- GLOBAL_STEP: 8650[0m |
|
| > loss_text_ce: 0.047773320227861404 (0.04789626204247679) |
|
| > loss_mel_ce: 2.4677274227142334 (2.465289601614556) |
|
| > loss: 0.628875195980072 (0.6282964659831524) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3827 (0.37734448187279) |
|
| > loader_time: 0.0213 (0.01629258710857805) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:17:09 -- STEP: 1172/1882 -- GLOBAL_STEP: 8700[0m |
|
| > loss_text_ce: 0.04913204163312912 (0.04791933194171554) |
|
| > loss_mel_ce: 3.1828510761260986 (2.466031861742609) |
|
| > loss: 0.8079957962036133 (0.6284877984139288) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3393 (0.37757914607435394) |
|
| > loader_time: 0.0321 (0.01626363020301269) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:17:43 -- STEP: 1222/1882 -- GLOBAL_STEP: 8750[0m |
|
| > loss_text_ce: 0.04570939764380455 (0.047926138827360854) |
|
| > loss_mel_ce: 2.3451316356658936 (2.4646515856495474) |
|
| > loss: 0.5977102518081665 (0.6281444311580946) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3543 (0.3774895506092453) |
|
| > loader_time: 0.0102 (0.0163114363550164) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:18:17 -- STEP: 1272/1882 -- GLOBAL_STEP: 8800[0m |
|
| > loss_text_ce: 0.04378071799874306 (0.047882151558597014) |
|
| > loss_mel_ce: 2.531548500061035 (2.460941933095455) |
|
| > loss: 0.6438323259353638 (0.6272060213458234) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3136 (0.37719598731154935) |
|
| > loader_time: 0.011 (0.016286048124421304) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:18:51 -- STEP: 1322/1882 -- GLOBAL_STEP: 8850[0m |
|
| > loss_text_ce: 0.04715706780552864 (0.04786694873795033) |
|
| > loss_mel_ce: 2.638216733932495 (2.45700869820843) |
|
| > loss: 0.6713434457778931 (0.626218911852129) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4125 (0.377039449478242) |
|
| > loader_time: 0.0107 (0.01627159046513589) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:19:26 -- STEP: 1372/1882 -- GLOBAL_STEP: 8900[0m |
|
| > loss_text_ce: 0.04640312120318413 (0.047859452436773205) |
|
| > loss_mel_ce: 1.9429771900177002 (2.456297054973705) |
|
| > loss: 0.49734508991241455 (0.6260391269585129) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2811 (0.37688711050995616) |
|
| > loader_time: 0.0243 (0.016262428058479658) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:20:00 -- STEP: 1422/1882 -- GLOBAL_STEP: 8950[0m |
|
| > loss_text_ce: 0.050793398171663284 (0.04785347207018433) |
|
| > loss_mel_ce: 2.724759101867676 (2.4515728186370582) |
|
| > loss: 0.6938881278038025 (0.6248565728654332) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4057 (0.3768368483260382) |
|
| > loader_time: 0.0316 (0.016255220951074935) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:20:34 -- STEP: 1472/1882 -- GLOBAL_STEP: 9000[0m |
|
| > loss_text_ce: 0.050074271857738495 (0.047849800189673636) |
|
| > loss_mel_ce: 2.8413913249969482 (2.449363167114231) |
|
| > loss: 0.722866415977478 (0.6243032419892102) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4201 (0.3769218189560852) |
|
| > loader_time: 0.0114 (0.016253326898035787) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:21:10 -- STEP: 1522/1882 -- GLOBAL_STEP: 9050[0m |
|
| > loss_text_ce: 0.04927993565797806 (0.04787573953123669) |
|
| > loss_mel_ce: 1.2563700675964355 (2.4518535551511027) |
|
| > loss: 0.32641249895095825 (0.6249323238015332) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4035 (0.3776059792953474) |
|
| > loader_time: 0.0115 (0.016261755871866748) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:21:45 -- STEP: 1572/1882 -- GLOBAL_STEP: 9100[0m |
|
| > loss_text_ce: 0.03691606596112251 (0.047867529236405915) |
|
| > loss_mel_ce: 2.123967409133911 (2.451530055614525) |
|
| > loss: 0.5402208566665649 (0.6248493962388005) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.279 (0.37753493473426714) |
|
| > loader_time: 0.0107 (0.01629338585996748) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:22:20 -- STEP: 1622/1882 -- GLOBAL_STEP: 9150[0m |
|
| > loss_text_ce: 0.04652049392461777 (0.04786846443376843) |
|
| > loss_mel_ce: 2.8418939113616943 (2.4517690345332612) |
|
| > loss: 0.7221035957336426 (0.6249093746837646) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4064 (0.3778319447019982) |
|
| > loader_time: 0.0103 (0.016258205203327953) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:22:55 -- STEP: 1672/1882 -- GLOBAL_STEP: 9200[0m |
|
| > loss_text_ce: 0.05008281022310257 (0.04786041312133586) |
|
| > loss_mel_ce: 2.472480535507202 (2.452564732666222) |
|
| > loss: 0.6306408643722534 (0.625106286346627) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3186 (0.3780465933124417) |
|
| > loader_time: 0.0378 (0.016250133799593964) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:23:28 -- STEP: 1722/1882 -- GLOBAL_STEP: 9250[0m |
|
| > loss_text_ce: 0.04776207357645035 (0.04782822934973397) |
|
| > loss_mel_ce: 2.506317377090454 (2.4496480431429335) |
|
| > loss: 0.6385198831558228 (0.6243690679436084) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3709 (0.37778098034941804) |
|
| > loader_time: 0.0213 (0.01623797264387104) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:24:02 -- STEP: 1772/1882 -- GLOBAL_STEP: 9300[0m |
|
| > loss_text_ce: 0.04409496858716011 (0.04781307849578951) |
|
| > loss_mel_ce: 1.499693751335144 (2.4484492555832498) |
|
| > loss: 0.38594716787338257 (0.6240655832816729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2813 (0.37751171578133913) |
|
| > loader_time: 0.0281 (0.016209045597446667) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:24:37 -- STEP: 1822/1882 -- GLOBAL_STEP: 9350[0m |
|
| > loss_text_ce: 0.04363618791103363 (0.047801451133942414) |
|
| > loss_mel_ce: 1.8268128633499146 (2.4479537571562644) |
|
| > loss: 0.46761226654052734 (0.623938801815952) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3427 (0.3773794192514098) |
|
| > loader_time: 0.0103 (0.016229916482542236) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:25:12 -- STEP: 1872/1882 -- GLOBAL_STEP: 9400[0m |
|
| > loss_text_ce: 0.045876555144786835 (0.04782410430658263) |
|
| > loss_mel_ce: 2.3220767974853516 (2.448616170985068) |
|
| > loss: 0.5919883251190186 (0.6241100686642094) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3075 (0.3776661114305515) |
|
| > loader_time: 0.0101 (0.01624821954303314) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.6516944078298713 [0m(-0.006144321881808024) |
|
| > avg_loss_text_ce:[92m 0.0468339054630353 [0m(-0.0007387522894602427) |
|
| > avg_loss_mel_ce:[92m 2.4417033929091234 [0m(-0.03786906829247094) |
|
| > avg_loss:[92m 2.488537284044119 [0m(-0.03860782659970807) |
|
|
|
|
|
[4m[1m > EPOCH: 5/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 09:26:30) [0m |
|
|
|
[1m --> TIME: 2024-12-05 09:26:59 -- STEP: 40/1882 -- GLOBAL_STEP: 9450[0m |
|
| > loss_text_ce: 0.04425692558288574 (0.04745454899966717) |
|
| > loss_mel_ce: 2.296861171722412 (2.5129141062498097) |
|
| > loss: 0.5852795243263245 (0.6400921635329724) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2865 (0.3822966992855072) |
|
| > loader_time: 0.0109 (0.014165860414505006) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:27:34 -- STEP: 90/1882 -- GLOBAL_STEP: 9500[0m |
|
| > loss_text_ce: 0.04577038064599037 (0.047873995991216764) |
|
| > loss_mel_ce: 2.176891326904297 (2.463674171765645) |
|
| > loss: 0.5556654334068298 (0.6278870423634844) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3435 (0.38164394431644016) |
|
| > loader_time: 0.0111 (0.015800062815348313) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:28:09 -- STEP: 140/1882 -- GLOBAL_STEP: 9550[0m |
|
| > loss_text_ce: 0.04921982064843178 (0.04803577443318707) |
|
| > loss_mel_ce: 2.6119556427001953 (2.4669877801622664) |
|
| > loss: 0.6652938723564148 (0.6287558881299834) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3313 (0.38220484937940324) |
|
| > loader_time: 0.0122 (0.016607897622244693) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:28:43 -- STEP: 190/1882 -- GLOBAL_STEP: 9600[0m |
|
| > loss_text_ce: 0.05202653259038925 (0.04760476370391091) |
|
| > loss_mel_ce: 2.19246244430542 (2.460836834029148) |
|
| > loss: 0.5611222386360168 (0.6271103984431219) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3789 (0.3811253823732075) |
|
| > loader_time: 0.0121 (0.01666875011042545) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:29:18 -- STEP: 240/1882 -- GLOBAL_STEP: 9650[0m |
|
| > loss_text_ce: 0.04512608051300049 (0.047553793449575686) |
|
| > loss_mel_ce: 2.1611220836639404 (2.46036764283975) |
|
| > loss: 0.5515620708465576 (0.6269803582380216) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2936 (0.3813148746887843) |
|
| > loader_time: 0.012 (0.01609154840310416) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:29:52 -- STEP: 290/1882 -- GLOBAL_STEP: 9700[0m |
|
| > loss_text_ce: 0.044929202646017075 (0.04743245556179819) |
|
| > loss_mel_ce: 2.359145164489746 (2.448165933017074) |
|
| > loss: 0.6010186076164246 (0.623899595696351) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4098 (0.3807274020951369) |
|
| > loader_time: 0.0284 (0.01583275712769609) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:30:26 -- STEP: 340/1882 -- GLOBAL_STEP: 9750[0m |
|
| > loss_text_ce: 0.047092255204916 (0.047338487131192375) |
|
| > loss_mel_ce: 2.423640251159668 (2.4356652754194603) |
|
| > loss: 0.6176831126213074 (0.620750939232462) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.305 (0.37950651154798604) |
|
| > loader_time: 0.0105 (0.01579403316273409) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:31:01 -- STEP: 390/1882 -- GLOBAL_STEP: 9800[0m |
|
| > loss_text_ce: 0.04951821267604828 (0.04740494500177026) |
|
| > loss_mel_ce: 2.76869535446167 (2.4299635379742366) |
|
| > loss: 0.7045533657073975 (0.6193421196479072) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4825 (0.37931478268060914) |
|
| > loader_time: 0.0119 (0.015981193689199598) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:31:36 -- STEP: 440/1882 -- GLOBAL_STEP: 9850[0m |
|
| > loss_text_ce: 0.04465514421463013 (0.047445283102040915) |
|
| > loss_mel_ce: 2.6185152530670166 (2.4353132578459644) |
|
| > loss: 0.6657925844192505 (0.6206896345723765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3432 (0.3793785138563675) |
|
| > loader_time: 0.0118 (0.016245429624210713) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:32:10 -- STEP: 490/1882 -- GLOBAL_STEP: 9900[0m |
|
| > loss_text_ce: 0.04978286474943161 (0.04741802740918128) |
|
| > loss_mel_ce: 2.4103379249572754 (2.431377379748285) |
|
| > loss: 0.6150301694869995 (0.6196988509625809) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4062 (0.3793740652045424) |
|
| > loader_time: 0.0114 (0.01631446517243678) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:32:44 -- STEP: 540/1882 -- GLOBAL_STEP: 9950[0m |
|
| > loss_text_ce: 0.049010854214429855 (0.047419511664796736) |
|
| > loss_mel_ce: 2.746603488922119 (2.4305540338710494) |
|
| > loss: 0.6989035606384277 (0.6194933857078909) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3433 (0.3787009283348365) |
|
| > loader_time: 0.0108 (0.0162298012662817) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:33:19 -- STEP: 590/1882 -- GLOBAL_STEP: 10000[0m |
|
| > loss_text_ce: 0.04333464428782463 (0.04746639139445151) |
|
| > loss_mel_ce: 2.2359249591827393 (2.4313715354870937) |
|
| > loss: 0.569814920425415 (0.6197094809705929) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3066 (0.3787445860393975) |
|
| > loader_time: 0.0105 (0.016306335239087123) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:33:53 -- STEP: 640/1882 -- GLOBAL_STEP: 10050[0m |
|
| > loss_text_ce: 0.026934992522001266 (0.04740678615635262) |
|
| > loss_mel_ce: 1.5695183277130127 (2.430899931117893) |
|
| > loss: 0.39911332726478577 (0.6195766784250736) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2267 (0.37802942991256694) |
|
| > loader_time: 0.0102 (0.01628230959177018) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:34:27 -- STEP: 690/1882 -- GLOBAL_STEP: 10100[0m |
|
| > loss_text_ce: 0.048027053475379944 (0.047299909921012066) |
|
| > loss_mel_ce: 2.785146474838257 (2.4296146748722474) |
|
| > loss: 0.7082933783531189 (0.6192286452953368) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4971 (0.3776278969170389) |
|
| > loader_time: 0.0113 (0.016315899379011525) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:35:01 -- STEP: 740/1882 -- GLOBAL_STEP: 10150[0m |
|
| > loss_text_ce: 0.05640196055173874 (0.04732247423884031) |
|
| > loss_mel_ce: 2.5407285690307617 (2.4284333585081885) |
|
| > loss: 0.6492826342582703 (0.6189389576380316) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.404 (0.3777460439785105) |
|
| > loader_time: 0.029 (0.016184663450395764) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:35:35 -- STEP: 790/1882 -- GLOBAL_STEP: 10200[0m |
|
| > loss_text_ce: 0.05446901544928551 (0.04736574425633195) |
|
| > loss_mel_ce: 2.484647512435913 (2.4272771174394627) |
|
| > loss: 0.634779155254364 (0.6186607150714603) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4082 (0.37733150101915175) |
|
| > loader_time: 0.0292 (0.016094074068190214) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:36:10 -- STEP: 840/1882 -- GLOBAL_STEP: 10250[0m |
|
| > loss_text_ce: 0.04767652228474617 (0.04735688224789642) |
|
| > loss_mel_ce: 2.268815755844116 (2.423652352037882) |
|
| > loss: 0.5791230797767639 (0.6177523085049212) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3169 (0.3773012606870559) |
|
| > loader_time: 0.0124 (0.016123421703066184) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:36:44 -- STEP: 890/1882 -- GLOBAL_STEP: 10300[0m |
|
| > loss_text_ce: 0.042929451912641525 (0.04730946937722436) |
|
| > loss_mel_ce: 2.1457552909851074 (2.4218885903947793) |
|
| > loss: 0.5471711754798889 (0.6172995148749826) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3704 (0.3769685718450652) |
|
| > loader_time: 0.0109 (0.01621776430794366) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:37:18 -- STEP: 940/1882 -- GLOBAL_STEP: 10350[0m |
|
| > loss_text_ce: 0.04758644104003906 (0.047285851915466026) |
|
| > loss_mel_ce: 2.1129696369171143 (2.4191436062467844) |
|
| > loss: 0.5401390194892883 (0.6166073644097808) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3008 (0.3764154642186265) |
|
| > loader_time: 0.0107 (0.016161755044409594) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:37:52 -- STEP: 990/1882 -- GLOBAL_STEP: 10400[0m |
|
| > loss_text_ce: 0.0490913987159729 (0.04725715500688312) |
|
| > loss_mel_ce: 2.680724620819092 (2.4189597081656395) |
|
| > loss: 0.682453989982605 (0.6165542156106287) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4045 (0.3768198461243599) |
|
| > loader_time: 0.0346 (0.01621019767992428) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:38:26 -- STEP: 1040/1882 -- GLOBAL_STEP: 10450[0m |
|
| > loss_text_ce: 0.04625069350004196 (0.04728194806509866) |
|
| > loss_mel_ce: 2.1374728679656982 (2.4159605942093396) |
|
| > loss: 0.5459308624267578 (0.6158106351414545) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2919 (0.37627725578271415) |
|
| > loader_time: 0.0121 (0.016247074879132815) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:39:01 -- STEP: 1090/1882 -- GLOBAL_STEP: 10500[0m |
|
| > loss_text_ce: 0.04763149097561836 (0.04728112616191763) |
|
| > loss_mel_ce: 2.648810386657715 (2.418134759434865) |
|
| > loss: 0.674110472202301 (0.6163539710941659) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3583 (0.37658745586325254) |
|
| > loader_time: 0.0301 (0.016220406217312612) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:39:35 -- STEP: 1140/1882 -- GLOBAL_STEP: 10550[0m |
|
| > loss_text_ce: 0.04611315578222275 (0.04725188457717498) |
|
| > loss_mel_ce: 2.622199058532715 (2.4192704101403537) |
|
| > loss: 0.6670780777931213 (0.6166305735184432) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3143 (0.37651253102118487) |
|
| > loader_time: 0.0113 (0.016291751987055746) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:40:09 -- STEP: 1190/1882 -- GLOBAL_STEP: 10600[0m |
|
| > loss_text_ce: 0.041046224534511566 (0.0472477882161361) |
|
| > loss_mel_ce: 2.145897388458252 (2.4188948819617244) |
|
| > loss: 0.5467358827590942 (0.616535667261156) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2721 (0.37631508001760255) |
|
| > loader_time: 0.0105 (0.01630410907649195) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:40:44 -- STEP: 1240/1882 -- GLOBAL_STEP: 10650[0m |
|
| > loss_text_ce: 0.052357178181409836 (0.04727134676650171) |
|
| > loss_mel_ce: 2.6584110260009766 (2.4185847186273133) |
|
| > loss: 0.6776920557022095 (0.6164640160097237) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4465 (0.37657723772910323) |
|
| > loader_time: 0.0108 (0.016197205743482043) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:41:19 -- STEP: 1290/1882 -- GLOBAL_STEP: 10700[0m |
|
| > loss_text_ce: 0.04933706670999527 (0.04726912830775794) |
|
| > loss_mel_ce: 2.26413893699646 (2.4172117510507283) |
|
| > loss: 0.5783690214157104 (0.6161202195771905) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2967 (0.3765886168147243) |
|
| > loader_time: 0.0245 (0.016203812665717555) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:41:53 -- STEP: 1340/1882 -- GLOBAL_STEP: 10750[0m |
|
| > loss_text_ce: 0.0490921288728714 (0.04725222238269984) |
|
| > loss_mel_ce: 2.7186968326568604 (2.415236151574264) |
|
| > loss: 0.6919472217559814 (0.6156220932505018) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3374 (0.37655265402438054) |
|
| > loader_time: 0.016 (0.016241452231335992) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:42:27 -- STEP: 1390/1882 -- GLOBAL_STEP: 10800[0m |
|
| > loss_text_ce: 0.0510031059384346 (0.047254980981296386) |
|
| > loss_mel_ce: 3.0744411945343018 (2.416923801675976) |
|
| > loss: 0.7813611030578613 (0.61604469557889) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4101 (0.37642808046272347) |
|
| > loader_time: 0.0114 (0.016322661989884402) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:43:02 -- STEP: 1440/1882 -- GLOBAL_STEP: 10850[0m |
|
| > loss_text_ce: 0.04925103858113289 (0.047219333993012334) |
|
| > loss_mel_ce: 2.8562724590301514 (2.418808634579183) |
|
| > loss: 0.7263808846473694 (0.6165069921356102) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4111 (0.3764080527755951) |
|
| > loader_time: 0.0114 (0.01634302321407529) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:43:37 -- STEP: 1490/1882 -- GLOBAL_STEP: 10900[0m |
|
| > loss_text_ce: 0.04541463777422905 (0.04720494438012778) |
|
| > loss_mel_ce: 2.4924042224884033 (2.4168856554383407) |
|
| > loss: 0.6344547271728516 (0.6160226499474281) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3799 (0.37662630705225403) |
|
| > loader_time: 0.0114 (0.01645200220530465) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:44:10 -- STEP: 1540/1882 -- GLOBAL_STEP: 10950[0m |
|
| > loss_text_ce: 0.047439806163311005 (0.04719182741163036) |
|
| > loss_mel_ce: 1.8699241876602173 (2.4146377535609482) |
|
| > loss: 0.4793410003185272 (0.6154573953770962) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2465 (0.37636858364204323) |
|
| > loader_time: 0.0302 (0.016444113657071983) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:44:45 -- STEP: 1590/1882 -- GLOBAL_STEP: 11000[0m |
|
| > loss_text_ce: 0.04906311258673668 (0.04715626434827192) |
|
| > loss_mel_ce: 2.2783844470977783 (2.414177258359565) |
|
| > loss: 0.5818619132041931 (0.6153333807328948) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3696 (0.37667568149806585) |
|
| > loader_time: 0.0106 (0.016395393707467322) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:45:19 -- STEP: 1640/1882 -- GLOBAL_STEP: 11050[0m |
|
| > loss_text_ce: 0.0445123054087162 (0.04714398187267166) |
|
| > loss_mel_ce: 2.1594033241271973 (2.413446443473426) |
|
| > loss: 0.5509788990020752 (0.6151476063742854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.303 (0.3765579389362802) |
|
| > loader_time: 0.0106 (0.016387702197563348) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:45:54 -- STEP: 1690/1882 -- GLOBAL_STEP: 11100[0m |
|
| > loss_text_ce: 0.053023576736450195 (0.047141228751802974) |
|
| > loss_mel_ce: 2.6317620277404785 (2.412810781157232) |
|
| > loss: 0.6711964011192322 (0.614988002480838) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4026 (0.37658498639891147) |
|
| > loader_time: 0.0106 (0.016420707617991075) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:46:28 -- STEP: 1740/1882 -- GLOBAL_STEP: 11150[0m |
|
| > loss_text_ce: 0.05173889547586441 (0.04709871545241314) |
|
| > loss_mel_ce: 2.4501407146453857 (2.410136652061312) |
|
| > loss: 0.6254699230194092 (0.6143088418348064) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3135 (0.3763887512272803) |
|
| > loader_time: 0.0277 (0.0164786349767926) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:47:02 -- STEP: 1790/1882 -- GLOBAL_STEP: 11200[0m |
|
| > loss_text_ce: 0.03475270792841911 (0.047075510236262975) |
|
| > loss_mel_ce: 2.390998125076294 (2.4084807248754907) |
|
| > loss: 0.6064376831054688 (0.6138890587584249) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3355 (0.3762511521078356) |
|
| > loader_time: 0.0107 (0.01644501872568822) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:47:37 -- STEP: 1840/1882 -- GLOBAL_STEP: 11250[0m |
|
| > loss_text_ce: 0.04448926821351051 (0.04708144332626433) |
|
| > loss_mel_ce: 2.711236000061035 (2.4073781295962986) |
|
| > loss: 0.6889313459396362 (0.6136148932187436) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3382 (0.3762621420881024) |
|
| > loader_time: 0.0109 (0.016478625846945715) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.6464404784716091 [0m(-0.005253929358262188) |
|
| > avg_loss_text_ce:[92m 0.046453144401311874 [0m(-0.0003807610617234253) |
|
| > avg_loss_mel_ce:[92m 2.4144917176320004 [0m(-0.027211675277122982) |
|
| > avg_loss:[92m 2.460944863466116 [0m(-0.02759242057800293) |
|
|
|
> BEST MODEL : /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb/best_model_11292.pth |
|
|
|
[4m[1m > EPOCH: 6/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 09:51:24) [0m |
|
|
|
[1m --> TIME: 2024-12-05 09:51:31 -- STEP: 8/1882 -- GLOBAL_STEP: 11300[0m |
|
| > loss_text_ce: 0.04613903909921646 (0.04611108498647809) |
|
| > loss_mel_ce: 2.4604389667510986 (2.364427387714386) |
|
| > loss: 0.626644492149353 (0.6026346161961555) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2652 (0.35470736026763916) |
|
| > loader_time: 0.0104 (0.010754942893981934) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:52:06 -- STEP: 58/1882 -- GLOBAL_STEP: 11350[0m |
|
| > loss_text_ce: 0.04855888709425926 (0.04634132936339954) |
|
| > loss_mel_ce: 2.656344175338745 (2.3630492933865246) |
|
| > loss: 0.6762257814407349 (0.6023476570844648) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4933 (0.38172562780051394) |
|
| > loader_time: 0.0113 (0.016830588209217987) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:52:41 -- STEP: 108/1882 -- GLOBAL_STEP: 11400[0m |
|
| > loss_text_ce: 0.04155217856168747 (0.04614335219203322) |
|
| > loss_mel_ce: 2.391383171081543 (2.3658708400196504) |
|
| > loss: 0.6082338094711304 (0.6030035490791003) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3286 (0.3798437537970366) |
|
| > loader_time: 0.0292 (0.016057992423022235) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:53:15 -- STEP: 158/1882 -- GLOBAL_STEP: 11450[0m |
|
| > loss_text_ce: 0.039950717240571976 (0.046202818002504635) |
|
| > loss_mel_ce: 1.7486661672592163 (2.3614744996722745) |
|
| > loss: 0.4471542239189148 (0.6019193305622174) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2754 (0.37807868553113333) |
|
| > loader_time: 0.0273 (0.016250756722462335) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:53:49 -- STEP: 208/1882 -- GLOBAL_STEP: 11500[0m |
|
| > loss_text_ce: 0.0511976033449173 (0.046407076458518284) |
|
| > loss_mel_ce: 2.5985212326049805 (2.365861657147224) |
|
| > loss: 0.662429690361023 (0.6030671848700598) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4092 (0.37772997869895053) |
|
| > loader_time: 0.0109 (0.01610605189433464) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:54:24 -- STEP: 258/1882 -- GLOBAL_STEP: 11550[0m |
|
| > loss_text_ce: 0.05071314051747322 (0.046567114676500446) |
|
| > loss_mel_ce: 2.439138889312744 (2.368063319561094) |
|
| > loss: 0.6224629878997803 (0.6036576101946278) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4182 (0.37764799410058547) |
|
| > loader_time: 0.011 (0.015717973080716388) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:54:57 -- STEP: 308/1882 -- GLOBAL_STEP: 11600[0m |
|
| > loss_text_ce: 0.05125904828310013 (0.04651156509319292) |
|
| > loss_mel_ce: 1.9027681350708008 (2.36283624946297) |
|
| > loss: 0.4885067939758301 (0.602336954857622) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3492 (0.37620456884433695) |
|
| > loader_time: 0.0107 (0.01596067323313131) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:55:32 -- STEP: 358/1882 -- GLOBAL_STEP: 11650[0m |
|
| > loss_text_ce: 0.05237085744738579 (0.046550221460241785) |
|
| > loss_mel_ce: 2.304797649383545 (2.3681031182491568) |
|
| > loss: 0.5892921090126038 (0.6036633359653327) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4133 (0.37637950055426067) |
|
| > loader_time: 0.0111 (0.016083237179164794) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:56:06 -- STEP: 408/1882 -- GLOBAL_STEP: 11700[0m |
|
| > loss_text_ce: 0.04584602639079094 (0.046510933569687204) |
|
| > loss_mel_ce: 2.867380142211914 (2.3665544916601733) |
|
| > loss: 0.7283065319061279 (0.6032663574259657) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3537 (0.3757155783036177) |
|
| > loader_time: 0.0268 (0.01622877752079684) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:56:40 -- STEP: 458/1882 -- GLOBAL_STEP: 11750[0m |
|
| > loss_text_ce: 0.04478771612048149 (0.046517997882511895) |
|
| > loss_mel_ce: 2.413888454437256 (2.36107953094499) |
|
| > loss: 0.6146690249443054 (0.6018993833700121) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3638 (0.3761655420195068) |
|
| > loader_time: 0.0255 (0.016161632850180553) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:57:14 -- STEP: 508/1882 -- GLOBAL_STEP: 11800[0m |
|
| > loss_text_ce: 0.04717489704489708 (0.04655152337816283) |
|
| > loss_mel_ce: 2.4350192546844482 (2.3666477602297857) |
|
| > loss: 0.6205485463142395 (0.6032998222769714) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3301 (0.37512256780008635) |
|
| > loader_time: 0.0376 (0.0161886942668224) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:57:49 -- STEP: 558/1882 -- GLOBAL_STEP: 11850[0m |
|
| > loss_text_ce: 0.05763954669237137 (0.046554998177567655) |
|
| > loss_mel_ce: 2.623936414718628 (2.363752322384961) |
|
| > loss: 0.6703940033912659 (0.6025768314424805) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4014 (0.37544823632872665) |
|
| > loader_time: 0.0235 (0.01618009143405491) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:58:23 -- STEP: 608/1882 -- GLOBAL_STEP: 11900[0m |
|
| > loss_text_ce: 0.03906853497028351 (0.04651970524144797) |
|
| > loss_mel_ce: 1.7225269079208374 (2.3617050037964384) |
|
| > loss: 0.4403988718986511 (0.6020561788525238) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2963 (0.3755870947712346) |
|
| > loader_time: 0.011 (0.016142005198880253) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:58:58 -- STEP: 658/1882 -- GLOBAL_STEP: 11950[0m |
|
| > loss_text_ce: 0.03191494569182396 (0.046536600432361484) |
|
| > loss_mel_ce: 1.76835036277771 (2.364017470991719) |
|
| > loss: 0.45006632804870605 (0.6026385193266042) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2346 (0.37585373707455333) |
|
| > loader_time: 0.0103 (0.01618837440629861) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 09:59:32 -- STEP: 708/1882 -- GLOBAL_STEP: 12000[0m |
|
| > loss_text_ce: 0.04768934100866318 (0.046539008580569134) |
|
| > loss_mel_ce: 2.7405364513397217 (2.361223746996139) |
|
| > loss: 0.6970564723014832 (0.601940690166196) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4115 (0.375816861114933) |
|
| > loader_time: 0.0305 (0.01621402926364189) |
|
|
|
|
|
> CHECKPOINT : /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb/checkpoint_12000.pth |
|
|
|
[1m --> TIME: 2024-12-05 10:01:05 -- STEP: 758/1882 -- GLOBAL_STEP: 12050[0m |
|
| > loss_text_ce: 0.048268940299749374 (0.046571931961461845) |
|
| > loss_mel_ce: 2.4962260723114014 (2.358676607816388) |
|
| > loss: 0.636123776435852 (0.6013121362333247) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4082 (0.37569563558674113) |
|
| > loader_time: 0.0111 (0.01632869652519127) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:01:39 -- STEP: 808/1882 -- GLOBAL_STEP: 12100[0m |
|
| > loss_text_ce: 0.04226656258106232 (0.04662105499646892) |
|
| > loss_mel_ce: 2.232948064804077 (2.36118545936476) |
|
| > loss: 0.5688036680221558 (0.6019516298824021) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2795 (0.3756405773139236) |
|
| > loader_time: 0.0114 (0.016325791283409198) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:02:13 -- STEP: 858/1882 -- GLOBAL_STEP: 12150[0m |
|
| > loss_text_ce: 0.04711628705263138 (0.046682129607542394) |
|
| > loss_mel_ce: 2.308320999145508 (2.363566861702845) |
|
| > loss: 0.5888593196868896 (0.6025622491931028) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.399 (0.375478765625498) |
|
| > loader_time: 0.0117 (0.016240794341880974) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:02:48 -- STEP: 908/1882 -- GLOBAL_STEP: 12200[0m |
|
| > loss_text_ce: 0.036448586732149124 (0.04669319757871689) |
|
| > loss_mel_ce: 1.9169631004333496 (2.3661283255936287) |
|
| > loss: 0.4883529245853424 (0.6032053821346834) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2505 (0.37595951005750816) |
|
| > loader_time: 0.0103 (0.016224349122740636) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:03:22 -- STEP: 958/1882 -- GLOBAL_STEP: 12250[0m |
|
| > loss_text_ce: 0.04734441265463829 (0.046699894444398676) |
|
| > loss_mel_ce: 2.685298442840576 (2.362466196550954) |
|
| > loss: 0.6831607222557068 (0.6022915238813967) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3987 (0.375500070277236) |
|
| > loader_time: 0.0115 (0.01626751965421228) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:03:56 -- STEP: 1008/1882 -- GLOBAL_STEP: 12300[0m |
|
| > loss_text_ce: 0.043045468628406525 (0.0467418657933613) |
|
| > loss_mel_ce: 2.0488595962524414 (2.363599302276732) |
|
| > loss: 0.522976279258728 (0.6025852930569461) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3408 (0.3756100168776891) |
|
| > loader_time: 0.0106 (0.01623107705797468) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:04:31 -- STEP: 1058/1882 -- GLOBAL_STEP: 12350[0m |
|
| > loss_text_ce: 0.04879369959235191 (0.046758418286567814) |
|
| > loss_mel_ce: 2.7980237007141113 (2.3650150127807494) |
|
| > loss: 0.7117043733596802 (0.6029433586928706) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4088 (0.3757602557793457) |
|
| > loader_time: 0.0116 (0.016181549638340738) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:05:06 -- STEP: 1108/1882 -- GLOBAL_STEP: 12400[0m |
|
| > loss_text_ce: 0.03986736387014389 (0.04673943850161366) |
|
| > loss_mel_ce: 2.0544791221618652 (2.3675018063736286) |
|
| > loss: 0.523586630821228 (0.6035603122913451) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3052 (0.37614212354598064) |
|
| > loader_time: 0.0314 (0.016161591137359287) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:05:40 -- STEP: 1158/1882 -- GLOBAL_STEP: 12450[0m |
|
| > loss_text_ce: 0.047534964978694916 (0.04673764874614992) |
|
| > loss_mel_ce: 2.1532084941864014 (2.3660754428626327) |
|
| > loss: 0.5501858592033386 (0.6032032738335187) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2933 (0.3760675776601865) |
|
| > loader_time: 0.0304 (0.016141133810777943) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:06:14 -- STEP: 1208/1882 -- GLOBAL_STEP: 12500[0m |
|
| > loss_text_ce: 0.05173400789499283 (0.0467115971537319) |
|
| > loss_mel_ce: 2.7228615283966064 (2.3618912266579675) |
|
| > loss: 0.6936488747596741 (0.602150706981389) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3355 (0.37558211119759155) |
|
| > loader_time: 0.0109 (0.01619641196648806) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:06:48 -- STEP: 1258/1882 -- GLOBAL_STEP: 12550[0m |
|
| > loss_text_ce: 0.0429232157766819 (0.04670037877033737) |
|
| > loss_mel_ce: 2.3669755458831787 (2.3600365449400487) |
|
| > loss: 0.6024746894836426 (0.6016842319211064) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4031 (0.3754477377726277) |
|
| > loader_time: 0.0105 (0.016210942465474366) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:07:22 -- STEP: 1308/1882 -- GLOBAL_STEP: 12600[0m |
|
| > loss_text_ce: 0.05015582963824272 (0.04667425054532764) |
|
| > loss_mel_ce: 2.025806427001953 (2.35625941290403) |
|
| > loss: 0.5189905762672424 (0.6007334167986469) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2442 (0.37523406598181525) |
|
| > loader_time: 0.0107 (0.01623617946554761) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:07:56 -- STEP: 1358/1882 -- GLOBAL_STEP: 12650[0m |
|
| > loss_text_ce: 0.044032830744981766 (0.046681335213729955) |
|
| > loss_mel_ce: 1.7083603143692017 (2.355985580382533) |
|
| > loss: 0.438098281621933 (0.6006667297306043) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3691 (0.3751503964762623) |
|
| > loader_time: 0.0203 (0.016239402220421226) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:08:31 -- STEP: 1408/1882 -- GLOBAL_STEP: 12700[0m |
|
| > loss_text_ce: 0.045526932924985886 (0.04670999427087903) |
|
| > loss_mel_ce: 2.146852731704712 (2.3574042660607493) |
|
| > loss: 0.5480949282646179 (0.6010285658465526) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3117 (0.37550782090560947) |
|
| > loader_time: 0.0313 (0.016260190104896377) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:09:07 -- STEP: 1458/1882 -- GLOBAL_STEP: 12750[0m |
|
| > loss_text_ce: 0.038733821362257004 (0.046695179587028256) |
|
| > loss_mel_ce: 1.4812157154083252 (2.357788214549439) |
|
| > loss: 0.3799873888492584 (0.6011208492255824) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4273 (0.3759046487193375) |
|
| > loader_time: 0.011 (0.016255373653888062) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:09:41 -- STEP: 1508/1882 -- GLOBAL_STEP: 12800[0m |
|
| > loss_text_ce: 0.0425475649535656 (0.04668167202464191) |
|
| > loss_mel_ce: 2.0973517894744873 (2.355582501037682) |
|
| > loss: 0.5349748134613037 (0.6005660439557351) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2521 (0.3759270607634627) |
|
| > loader_time: 0.0105 (0.01624078744286253) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:10:16 -- STEP: 1558/1882 -- GLOBAL_STEP: 12850[0m |
|
| > loss_text_ce: 0.04189208894968033 (0.04666963190830222) |
|
| > loss_mel_ce: 1.7931029796600342 (2.3548898810905703) |
|
| > loss: 0.4587487578392029 (0.6003898788477848) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3328 (0.3760430027798908) |
|
| > loader_time: 0.0111 (0.01621476783678988) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:10:50 -- STEP: 1608/1882 -- GLOBAL_STEP: 12900[0m |
|
| > loss_text_ce: 0.053014468401670456 (0.04665017223323869) |
|
| > loss_mel_ce: 2.8838601112365723 (2.355752730769893) |
|
| > loss: 0.7342186570167542 (0.6006007263650038) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.404 (0.3759644643880834) |
|
| > loader_time: 0.0111 (0.0161591280455613) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:11:24 -- STEP: 1658/1882 -- GLOBAL_STEP: 12950[0m |
|
| > loss_text_ce: 0.0443340428173542 (0.04663907094275529) |
|
| > loss_mel_ce: 2.4457571506500244 (2.354295918464081) |
|
| > loss: 0.62252277135849 (0.6002337477642884) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4051 (0.3760835572232383) |
|
| > loader_time: 0.0107 (0.01610994569067501) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:12:00 -- STEP: 1708/1882 -- GLOBAL_STEP: 13000[0m |
|
| > loss_text_ce: 0.039211150258779526 (0.0466370582198877) |
|
| > loss_mel_ce: 2.339411973953247 (2.357708045903074) |
|
| > loss: 0.5946557521820068 (0.6010862764416023) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3186 (0.3764711343432477) |
|
| > loader_time: 0.0118 (0.016175203636044362) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:12:33 -- STEP: 1758/1882 -- GLOBAL_STEP: 13050[0m |
|
| > loss_text_ce: 0.05208699032664299 (0.0466220347418432) |
|
| > loss_mel_ce: 2.81113338470459 (2.356935722768234) |
|
| > loss: 0.7158051133155823 (0.6008894398079518) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.413 (0.3761425807614375) |
|
| > loader_time: 0.0121 (0.01610413292026628) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:13:08 -- STEP: 1808/1882 -- GLOBAL_STEP: 13100[0m |
|
| > loss_text_ce: 0.03811686113476753 (0.04664110557477) |
|
| > loss_mel_ce: 2.0309271812438965 (2.3560043766567094) |
|
| > loss: 0.5172610282897949 (0.60066137093622) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2253 (0.3761068662974686) |
|
| > loader_time: 0.0311 (0.016118086544813295) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:13:42 -- STEP: 1858/1882 -- GLOBAL_STEP: 13150[0m |
|
| > loss_text_ce: 0.044217657297849655 (0.04662979053046688) |
|
| > loss_mel_ce: 2.3050618171691895 (2.3556108903961834) |
|
| > loss: 0.5873198509216309 (0.6005601708098549) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3421 (0.3761661141244664) |
|
| > loader_time: 0.0109 (0.016140092817908083) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[91m 0.6698607298044058 [0m(+0.023420251332796682) |
|
| > avg_loss_text_ce:[92m 0.04589031894619648 [0m(-0.0005628254551153927) |
|
| > avg_loss_mel_ce:[92m 2.4014567778660703 [0m(-0.013034939765930176) |
|
| > avg_loss:[92m 2.447347099964435 [0m(-0.013597763501680848) |
|
|
|
> BEST MODEL : /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb/best_model_13174.pth |
|
|
|
[4m[1m > EPOCH: 7/8[0m |
|
--> /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb |
|
|
|
[1m > TRAINING (2024-12-05 10:17:12) [0m |
|
|
|
[1m --> TIME: 2024-12-05 10:17:32 -- STEP: 26/1882 -- GLOBAL_STEP: 13200[0m |
|
| > loss_text_ce: 0.044245634227991104 (0.045481983285683855) |
|
| > loss_mel_ce: 2.3962671756744385 (2.293517612493955) |
|
| > loss: 0.6101282238960266 (0.5847499003777137) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4119 (0.3762317162293654) |
|
| > loader_time: 0.0109 (0.020151395064133864) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:18:07 -- STEP: 76/1882 -- GLOBAL_STEP: 13250[0m |
|
| > loss_text_ce: 0.04839194566011429 (0.046384176631507124) |
|
| > loss_mel_ce: 2.6909663677215576 (2.3396946897632205) |
|
| > loss: 0.6848396062850952 (0.5965197196132259) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.338 (0.3800739740070544) |
|
| > loader_time: 0.0114 (0.018625168423903615) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:18:42 -- STEP: 126/1882 -- GLOBAL_STEP: 13300[0m |
|
| > loss_text_ce: 0.044796913862228394 (0.04640725541800734) |
|
| > loss_mel_ce: 2.3293001651763916 (2.331923804585896) |
|
| > loss: 0.5935242772102356 (0.5945827658214267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3646 (0.3823176357481215) |
|
| > loader_time: 0.0108 (0.01840065774463472) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:19:17 -- STEP: 176/1882 -- GLOBAL_STEP: 13350[0m |
|
| > loss_text_ce: 0.04943159595131874 (0.04643003466877748) |
|
| > loss_mel_ce: 2.480738639831543 (2.334629558026791) |
|
| > loss: 0.6325425505638123 (0.595264898443764) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3508 (0.38304426724260504) |
|
| > loader_time: 0.0109 (0.017293795943260196) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:19:53 -- STEP: 226/1882 -- GLOBAL_STEP: 13400[0m |
|
| > loss_text_ce: 0.05271992087364197 (0.046801271329147624) |
|
| > loss_mel_ce: 2.1553120613098145 (2.341856836217697) |
|
| > loss: 0.5520079731941223 (0.597164526713633) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2909 (0.385179242201611) |
|
| > loader_time: 0.0222 (0.01724207084790796) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:20:27 -- STEP: 276/1882 -- GLOBAL_STEP: 13450[0m |
|
| > loss_text_ce: 0.046385083347558975 (0.046776235697494034) |
|
| > loss_mel_ce: 2.2362403869628906 (2.338895834010582) |
|
| > loss: 0.5706563591957092 (0.5964180173865263) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.321 (0.3837436290754788) |
|
| > loader_time: 0.0122 (0.016972190228061405) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:21:02 -- STEP: 326/1882 -- GLOBAL_STEP: 13500[0m |
|
| > loss_text_ce: 0.047588739544153214 (0.04671309863222889) |
|
| > loss_mel_ce: 2.6642825603485107 (2.337723762345462) |
|
| > loss: 0.6779678463935852 (0.5961092158329274) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4117 (0.38229015122162063) |
|
| > loader_time: 0.0107 (0.016756130873791294) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:21:37 -- STEP: 376/1882 -- GLOBAL_STEP: 13550[0m |
|
| > loss_text_ce: 0.05018116533756256 (0.04669325153759502) |
|
| > loss_mel_ce: 2.1846437454223633 (2.3347461597716577) |
|
| > loss: 0.5587062239646912 (0.5953598538453281) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2876 (0.3837052082761805) |
|
| > loader_time: 0.0104 (0.016695241978827956) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:22:11 -- STEP: 426/1882 -- GLOBAL_STEP: 13600[0m |
|
| > loss_text_ce: 0.049154914915561676 (0.04672370323727668) |
|
| > loss_mel_ce: 2.3988547325134277 (2.333198427034657) |
|
| > loss: 0.612002432346344 (0.5949805333440855) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3761 (0.3825947471627607) |
|
| > loader_time: 0.0106 (0.01685593273718035) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:22:46 -- STEP: 476/1882 -- GLOBAL_STEP: 13650[0m |
|
| > loss_text_ce: 0.04724135249853134 (0.0466301188796383) |
|
| > loss_mel_ce: 2.6310462951660156 (2.3381817964445655) |
|
| > loss: 0.6695719361305237 (0.5962029795686741) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3535 (0.3830583300911078) |
|
| > loader_time: 0.011 (0.016809341286410834) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:23:22 -- STEP: 526/1882 -- GLOBAL_STEP: 13700[0m |
|
| > loss_text_ce: 0.04947245121002197 (0.046650527845430736) |
|
| > loss_mel_ce: 2.5421087741851807 (2.336106734470724) |
|
| > loss: 0.647895336151123 (0.5956893164395382) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4129 (0.38387768214193135) |
|
| > loader_time: 0.0115 (0.016589535506505958) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:23:57 -- STEP: 576/1882 -- GLOBAL_STEP: 13750[0m |
|
| > loss_text_ce: 0.04409770295023918 (0.04671040058989699) |
|
| > loss_mel_ce: 1.7867435216903687 (2.342741262064213) |
|
| > loss: 0.45771029591560364 (0.597362916554428) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2546 (0.3844384310974014) |
|
| > loader_time: 0.0108 (0.016543516268332826) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:24:32 -- STEP: 626/1882 -- GLOBAL_STEP: 13800[0m |
|
| > loss_text_ce: 0.04188084602355957 (0.046681604306466465) |
|
| > loss_mel_ce: 1.536620855331421 (2.343054916435918) |
|
| > loss: 0.3946254253387451 (0.5974341310536903) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3242 (0.38427082532510953) |
|
| > loader_time: 0.0114 (0.016339244553075456) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:25:06 -- STEP: 676/1882 -- GLOBAL_STEP: 13850[0m |
|
| > loss_text_ce: 0.04611041769385338 (0.046592429442849186) |
|
| > loss_mel_ce: 2.711399555206299 (2.345143354062498) |
|
| > loss: 0.68937748670578 (0.5979339465410755) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.341 (0.3836713440319487) |
|
| > loader_time: 0.0119 (0.016312667251338658) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:25:40 -- STEP: 726/1882 -- GLOBAL_STEP: 13900[0m |
|
| > loss_text_ce: 0.048677459359169006 (0.04651538380896337) |
|
| > loss_mel_ce: 2.289421558380127 (2.3386691339909214) |
|
| > loss: 0.5845247507095337 (0.5962961301741199) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3443 (0.38265197631741327) |
|
| > loader_time: 0.0104 (0.016438053659171135) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:26:15 -- STEP: 776/1882 -- GLOBAL_STEP: 13950[0m |
|
| > loss_text_ce: 0.04703879728913307 (0.046543903924889614) |
|
| > loss_mel_ce: 2.6527795791625977 (2.3393370712694437) |
|
| > loss: 0.6749545931816101 (0.5964702443824609) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3091 (0.38267989041878975) |
|
| > loader_time: 0.0112 (0.016520702040072586) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:26:50 -- STEP: 826/1882 -- GLOBAL_STEP: 14000[0m |
|
| > loss_text_ce: 0.03920504078269005 (0.046570184784907535) |
|
| > loss_mel_ce: 1.6489046812057495 (2.3404959985043745) |
|
| > loss: 0.42202743887901306 (0.5967665462964387) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2614 (0.38242953250829564) |
|
| > loader_time: 0.0106 (0.01667083608613467) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:27:24 -- STEP: 876/1882 -- GLOBAL_STEP: 14050[0m |
|
| > loss_text_ce: 0.043858274817466736 (0.046602175815549735) |
|
| > loss_mel_ce: 2.1654295921325684 (2.33776504447743) |
|
| > loss: 0.5523219704627991 (0.5960918055277442) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3115 (0.3821804259465707) |
|
| > loader_time: 0.0117 (0.016648390521741916) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:27:58 -- STEP: 926/1882 -- GLOBAL_STEP: 14100[0m |
|
| > loss_text_ce: 0.049139827489852905 (0.04656905708050044) |
|
| > loss_mel_ce: 1.937494158744812 (2.3362429714228607) |
|
| > loss: 0.4966585040092468 (0.5957030075930112) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.356 (0.38165486734320203) |
|
| > loader_time: 0.0264 (0.016554912017179645) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:28:32 -- STEP: 976/1882 -- GLOBAL_STEP: 14150[0m |
|
| > loss_text_ce: 0.05618557333946228 (0.04659720409287474) |
|
| > loss_mel_ce: 2.540794610977173 (2.334154286162285) |
|
| > loss: 0.649245023727417 (0.5951878729841265) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3388 (0.38107679833154184) |
|
| > loader_time: 0.0122 (0.01654404988054373) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:29:07 -- STEP: 1026/1882 -- GLOBAL_STEP: 14200[0m |
|
| > loss_text_ce: 0.04484308138489723 (0.04653988783129764) |
|
| > loss_mel_ce: 2.2472152709960938 (2.3320755739541994) |
|
| > loss: 0.5730146169662476 (0.5946538659215675) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3751 (0.380835399060686) |
|
| > loader_time: 0.012 (0.016573496496933047) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:29:42 -- STEP: 1076/1882 -- GLOBAL_STEP: 14250[0m |
|
| > loss_text_ce: 0.04618416354060173 (0.04654242758739558) |
|
| > loss_mel_ce: 2.7834622859954834 (2.3333683858684435) |
|
| > loss: 0.7074115872383118 (0.5949777037071475) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4595 (0.3810822487320596) |
|
| > loader_time: 0.0285 (0.016701387428439712) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:30:17 -- STEP: 1126/1882 -- GLOBAL_STEP: 14300[0m |
|
| > loss_text_ce: 0.04525912553071976 (0.046554541031682786) |
|
| > loss_mel_ce: 2.8172640800476074 (2.338035514839588) |
|
| > loss: 0.715630829334259 (0.5961475143900563) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3775 (0.3811668060177384) |
|
| > loader_time: 0.012 (0.016602469379906146) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:30:52 -- STEP: 1176/1882 -- GLOBAL_STEP: 14350[0m |
|
| > loss_text_ce: 0.04119035601615906 (0.04652090618849357) |
|
| > loss_mel_ce: 1.972036361694336 (2.3334658371449315) |
|
| > loss: 0.5033066868782043 (0.5949966864419634) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2999 (0.3812634164378754) |
|
| > loader_time: 0.0301 (0.01657105749156201) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:31:27 -- STEP: 1226/1882 -- GLOBAL_STEP: 14400[0m |
|
| > loss_text_ce: 0.05364225432276726 (0.0465261857625443) |
|
| > loss_mel_ce: 2.40470814704895 (2.3336384063541007) |
|
| > loss: 0.6145876049995422 (0.5950411485954752) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.369 (0.3814054922022981) |
|
| > loader_time: 0.0118 (0.016528560909128125) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:32:02 -- STEP: 1276/1882 -- GLOBAL_STEP: 14450[0m |
|
| > loss_text_ce: 0.048330120742321014 (0.04650181955380538) |
|
| > loss_mel_ce: 2.571920156478882 (2.3331291438737254) |
|
| > loss: 0.6550625562667847 (0.5949077413980859) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3442 (0.38168395032703295) |
|
| > loader_time: 0.0112 (0.01651595341374508) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:32:37 -- STEP: 1326/1882 -- GLOBAL_STEP: 14500[0m |
|
| > loss_text_ce: 0.04357806220650673 (0.046520420906399365) |
|
| > loss_mel_ce: 2.4230384826660156 (2.33444997785138) |
|
| > loss: 0.6166541576385498 (0.5952426001407075) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3389 (0.38135524698691825) |
|
| > loader_time: 0.0108 (0.016587985227190714) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:33:12 -- STEP: 1376/1882 -- GLOBAL_STEP: 14550[0m |
|
| > loss_text_ce: 0.04576686769723892 (0.04653402820613961) |
|
| > loss_mel_ce: 1.9518007040023804 (2.333926411995362) |
|
| > loss: 0.4993918836116791 (0.5951151104297392) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2883 (0.38166912506486067) |
|
| > loader_time: 0.0105 (0.01665988460529682) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:33:46 -- STEP: 1426/1882 -- GLOBAL_STEP: 14600[0m |
|
| > loss_text_ce: 0.05240388587117195 (0.046507883502162506) |
|
| > loss_mel_ce: 2.5006113052368164 (2.331595989046059) |
|
| > loss: 0.6382538080215454 (0.5945259685854083) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3703 (0.3812482972298897) |
|
| > loader_time: 0.0304 (0.016614373235180866) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:34:21 -- STEP: 1476/1882 -- GLOBAL_STEP: 14650[0m |
|
| > loss_text_ce: 0.049637001007795334 (0.0465104358364148) |
|
| > loss_mel_ce: 2.4328866004943848 (2.3321986477872536) |
|
| > loss: 0.6206309199333191 (0.5946772712759854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4325 (0.3811064902036814) |
|
| > loader_time: 0.0287 (0.016623972554193935) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:34:55 -- STEP: 1526/1882 -- GLOBAL_STEP: 14700[0m |
|
| > loss_text_ce: 0.04330560564994812 (0.04647470138805908) |
|
| > loss_mel_ce: 2.1197612285614014 (2.3287030967003695) |
|
| > loss: 0.540766716003418 (0.5937944499880736) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4022 (0.3807676205628819) |
|
| > loader_time: 0.0111 (0.016632199912364976) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:35:29 -- STEP: 1576/1882 -- GLOBAL_STEP: 14750[0m |
|
| > loss_text_ce: 0.04308868944644928 (0.046448283551052724) |
|
| > loss_mel_ce: 2.4323582649230957 (2.327508991631456) |
|
| > loss: 0.618861734867096 (0.5934893192089903) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3417 (0.3804198958849542) |
|
| > loader_time: 0.0103 (0.016609468435878093) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:36:03 -- STEP: 1626/1882 -- GLOBAL_STEP: 14800[0m |
|
| > loss_text_ce: 0.04802737757563591 (0.04647459809955883) |
|
| > loss_mel_ce: 2.5959150791168213 (2.3272614365794118) |
|
| > loss: 0.6609855890274048 (0.5934340090915872) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4105 (0.38049376612073216) |
|
| > loader_time: 0.0112 (0.016574647593762176) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:36:39 -- STEP: 1676/1882 -- GLOBAL_STEP: 14850[0m |
|
| > loss_text_ce: 0.047295261174440384 (0.046457787953441555) |
|
| > loss_mel_ce: 2.16266131401062 (2.328164359870276) |
|
| > loss: 0.552489161491394 (0.593655537413811) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3208 (0.38073396554710165) |
|
| > loader_time: 0.0115 (0.01656258035104429) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:37:13 -- STEP: 1726/1882 -- GLOBAL_STEP: 14900[0m |
|
| > loss_text_ce: 0.04552800953388214 (0.04644287559112066) |
|
| > loss_mel_ce: 1.9304524660110474 (2.330145907602334) |
|
| > loss: 0.49399513006210327 (0.5941471962014325) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2949 (0.3807050459354527) |
|
| > loader_time: 0.0227 (0.016489442640770895) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:37:47 -- STEP: 1776/1882 -- GLOBAL_STEP: 14950[0m |
|
| > loss_text_ce: 0.039535827934741974 (0.04642298796029569) |
|
| > loss_mel_ce: 1.9357730150222778 (2.3283269839982212) |
|
| > loss: 0.493827223777771 (0.59368749330374) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2868 (0.3805394736496176) |
|
| > loader_time: 0.0284 (0.01642560838042077) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:38:22 -- STEP: 1826/1882 -- GLOBAL_STEP: 15000[0m |
|
| > loss_text_ce: 0.03666478022933006 (0.046431184021056206) |
|
| > loss_mel_ce: 1.9831334352493286 (2.3275039236817165) |
|
| > loss: 0.5049495697021484 (0.5934837774046147) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3334 (0.3805520977143519) |
|
| > loader_time: 0.0335 (0.016372569288899304) |
|
|
|
|
|
[1m --> TIME: 2024-12-05 10:38:57 -- STEP: 1876/1882 -- GLOBAL_STEP: 15050[0m |
|
| > loss_text_ce: 0.04517654329538345 (0.04642884262096773) |
|
| > loss_mel_ce: 2.3960533142089844 (2.327856908061867) |
|
| > loss: 0.6103074550628662 (0.593571438130412) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3291 (0.38062136780732714) |
|
| > loader_time: 0.0111 (0.016346962213007868) |
|
|
|
|
|
[1m > EVALUATION [0m |
|
|
|
|
|
[1m--> EVAL PERFORMANCE[0m |
|
| > avg_loader_time:[92m 0.633646928347074 [0m(-0.036213801457331773) |
|
| > avg_loss_text_ce:[92m 0.04556017827529175 [0m(-0.00033014067090473415) |
|
| > avg_loss_mel_ce:[92m 2.375472408074599 [0m(-0.02598436979147145) |
|
| > avg_loss:[92m 2.42103257546058 [0m(-0.026314524503855186) |
|
|
|
> BEST MODEL : /content/checkpoints/GPT_XTTS_FT-December-05-2024_07+31AM-9b9aedb/best_model_15056.pth |
|
|