diff --git "a/run.log" "b/run.log"
--- "a/run.log"
+++ "b/run.log"
@@ -38728,3 +38728,1347 @@ Time to load utils op: 0.00040650367736816406 seconds
 [2022-12-21 00:18:12,524] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-4000/global_step4048/zero_pp_rank_0_mp_rank_00_optim_states.pt
 [2022-12-21 00:18:12,524] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step4048 is ready now!
 [INFO|feature_extraction_utils.py:368] 2022-12-21 00:18:17,037 >> Feature extractor saved in ./preprocessor_config.json
+[2022-12-21 00:20:25,579] [INFO] [timer.py:197:stop] 0/8098, RunningAvgSamplesPerSec=5.879106476468289, CurrSamplesPerSec=5.093867850109623, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4001/5000 [13:58:06<121:46:12, 438.81s/it][2022-12-21 00:20:36,552] [INFO] [logging.py:68:log_dist] [Rank 0] step=4050, skipped=5, lr=[2.1244444444444443e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:20:36,553] [INFO] [timer.py:197:stop] 0/8100, RunningAvgSamplesPerSec=5.879111191728076, CurrSamplesPerSec=5.344233386168689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4002/5000 [13:58:17<86:03:58, 310.46s/it] [2022-12-21 00:20:47,528] [INFO] [timer.py:197:stop] 0/8102, RunningAvgSamplesPerSec=5.879115681926362, CurrSamplesPerSec=5.340438972256464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4003/5000 [13:58:28<61:06:00, 220.62s/it][2022-12-21 00:20:58,499] [INFO] [timer.py:197:stop] 0/8104, RunningAvgSamplesPerSec=5.879120604860978, CurrSamplesPerSec=5.368241653516812, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4004/5000 [13:58:38<43:37:59, 157.71s/it][2022-12-21 00:21:09,452] [INFO] [timer.py:197:stop] 0/8106, RunningAvgSamplesPerSec=5.879128004984679, CurrSamplesPerSec=5.334036813902272, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4005/5000 [13:58:49<31:25:16, 113.68s/it][2022-12-21 00:21:20,438] [INFO] [timer.py:197:stop] 0/8108, RunningAvgSamplesPerSec=5.879131214988293, CurrSamplesPerSec=5.310336205125673, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4006/5000 [13:59:00<22:53:14, 82.89s/it] [2022-12-21 00:21:31,473] [INFO] [timer.py:197:stop] 0/8110, RunningAvgSamplesPerSec=5.879127594218704, CurrSamplesPerSec=5.315880958213466, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4007/5000 [13:59:12<16:55:04, 61.33s/it][2022-12-21 00:21:42,451] [INFO] [timer.py:197:stop] 0/8112, RunningAvgSamplesPerSec=5.879131823148013, CurrSamplesPerSec=5.365864567875278, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4008/5000 [13:59:22<12:44:07, 46.22s/it][2022-12-21 00:21:53,404] [INFO] [timer.py:197:stop] 0/8114, RunningAvgSamplesPerSec=5.879139519458495, CurrSamplesPerSec=5.358554872858129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4009/5000 [13:59:33<9:48:47, 35.65s/it] [2022-12-21 00:22:04,426] [INFO] [timer.py:197:stop] 0/8116, RunningAvgSamplesPerSec=5.879137829975435, CurrSamplesPerSec=5.326190144079811, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4010/5000 [13:59:44<7:46:14, 28.26s/it][2022-12-21 00:22:15,421] [INFO] [timer.py:197:stop] 0/8118, RunningAvgSamplesPerSec=5.87913966067019, CurrSamplesPerSec=5.339037950493925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4011/5000 [13:59:55<6:20:19, 23.07s/it][2022-12-21 00:22:26,384] [INFO] [logging.py:68:log_dist] [Rank 0] step=4060, skipped=5, lr=[2.1022222222222224e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:22:26,385] [INFO] [timer.py:197:stop] 0/8120, RunningAvgSamplesPerSec=5.8791455071835035, CurrSamplesPerSec=5.351907982309296, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4012/5000 [14:00:06<5:20:15, 19.45s/it][2022-12-21 00:22:37,424] [INFO] [timer.py:197:stop] 0/8122, RunningAvgSamplesPerSec=5.879141627214186, CurrSamplesPerSec=5.309427661146448, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4013/5000 [14:00:17<4:38:26, 16.93s/it][2022-12-21 00:22:48,408] [INFO] [timer.py:197:stop] 0/8124, RunningAvgSamplesPerSec=5.879145044906297, CurrSamplesPerSec=5.361578649322693, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4014/5000 [14:00:28<4:08:47, 15.14s/it][2022-12-21 00:22:59,393] [INFO] [timer.py:197:stop] 0/8126, RunningAvgSamplesPerSec=5.879148138633306, CurrSamplesPerSec=5.346817538017743, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4015/5000 [14:00:39<3:47:59, 13.89s/it][2022-12-21 00:23:10,359] [INFO] [timer.py:197:stop] 0/8128, RunningAvgSamplesPerSec=5.879154032939573, CurrSamplesPerSec=5.350251181444112, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4016/5000 [14:00:50<3:33:33, 13.02s/it][2022-12-21 00:23:21,399] [INFO] [timer.py:197:stop] 0/8130, RunningAvgSamplesPerSec=5.8791499275397605, CurrSamplesPerSec=5.315678634182523, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4017/5000 [14:01:01<3:23:25, 12.42s/it][2022-12-21 00:23:32,345] [INFO] [timer.py:197:stop] 0/8132, RunningAvgSamplesPerSec=5.879158380112219, CurrSamplesPerSec=5.367233345519316, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4018/5000 [14:01:12<3:16:10, 11.99s/it][2022-12-21 00:23:42,466] [INFO] [timer.py:197:stop] 0/8134, RunningAvgSamplesPerSec=5.879275089343199, CurrSamplesPerSec=6.260939885516436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 00:23:53,499] [INFO] [timer.py:197:stop] 0/8136, RunningAvgSamplesPerSec=5.879271499843242, CurrSamplesPerSec=5.310662096302032, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4019/5000 [14:01:29<3:36:38, 13.25s/it][2022-12-21 00:24:04,489] [INFO] [timer.py:197:stop] 0/8138, RunningAvgSamplesPerSec=5.8792741510805, CurrSamplesPerSec=5.306878886370058, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4020/5000 [14:01:40<3:25:20, 12.57s/it][2022-12-21 00:24:15,441] [INFO] [logging.py:68:log_dist] [Rank 0] step=4070, skipped=5, lr=[2.08e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:24:15,442] [INFO] [timer.py:197:stop] 0/8140, RunningAvgSamplesPerSec=5.879281779396381, CurrSamplesPerSec=5.359638039810382, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4021/5000 [14:01:51<3:17:12, 12.09s/it][2022-12-21 00:24:26,443] [INFO] [timer.py:197:stop] 0/8142, RunningAvgSamplesPerSec=5.879282888934946, CurrSamplesPerSec=5.350252247815665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4022/5000 [14:02:02<3:11:41, 11.76s/it][2022-12-21 00:24:37,436] [INFO] [timer.py:197:stop] 0/8144, RunningAvgSamplesPerSec=5.879285093361157, CurrSamplesPerSec=5.3417701458130535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4023/5000 [14:02:13<3:07:45, 11.53s/it][2022-12-21 00:24:48,414] [INFO] [timer.py:197:stop] 0/8146, RunningAvgSamplesPerSec=5.879289455792325, CurrSamplesPerSec=5.340900121248683, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4024/5000 [14:02:24<3:04:51, 11.36s/it][2022-12-21 00:24:59,386] [INFO] [timer.py:197:stop] 0/8148, RunningAvgSamplesPerSec=5.87929447814198, CurrSamplesPerSec=5.3434887082552285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4025/5000 [14:02:34<3:02:45, 11.25s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.0711111111111114e-06, 'epoch': 98.17}
+
 80%|████████  | 4025/5000 [14:02:34<3:02:45, 11.25s/it][2022-12-21 00:25:10,344] [INFO] [timer.py:197:stop] 0/8150, RunningAvgSamplesPerSec=5.879301661537801, CurrSamplesPerSec=5.3441580578820895, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4026/5000 [14:02:45<3:01:09, 11.16s/it][2022-12-21 00:25:21,282] [INFO] [timer.py:197:stop] 0/8152, RunningAvgSamplesPerSec=5.879311095349123, CurrSamplesPerSec=5.331857665831148, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4027/5000 [14:02:56<2:59:53, 11.09s/it][2022-12-21 00:25:32,309] [INFO] [timer.py:197:stop] 0/8154, RunningAvgSamplesPerSec=5.879308716138182, CurrSamplesPerSec=5.311412155148097, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4028/5000 [14:03:07<2:59:23, 11.07s/it][2022-12-21 00:25:43,295] [INFO] [timer.py:197:stop] 0/8156, RunningAvgSamplesPerSec=5.879311798430926, CurrSamplesPerSec=5.346016562735164, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4029/5000 [14:03:18<2:58:47, 11.05s/it][2022-12-21 00:25:54,241] [INFO] [timer.py:197:stop] 0/8158, RunningAvgSamplesPerSec=5.879320160512912, CurrSamplesPerSec=5.369269886546951, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4030/5000 [14:03:29<2:58:06, 11.02s/it][2022-12-21 00:26:05,221] [INFO] [logging.py:68:log_dist] [Rank 0] step=4080, skipped=5, lr=[2.057777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:26:05,222] [INFO] [timer.py:197:stop] 0/8160, RunningAvgSamplesPerSec=5.879323786823011, CurrSamplesPerSec=5.358107354464163, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4031/5000 [14:03:40<2:57:44, 11.01s/it][2022-12-21 00:26:16,182] [INFO] [timer.py:197:stop] 0/8162, RunningAvgSamplesPerSec=5.8793302130287195, CurrSamplesPerSec=5.320550265681192, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4032/5000 [14:03:51<2:57:20, 10.99s/it][2022-12-21 00:26:27,180] [INFO] [timer.py:197:stop] 0/8164, RunningAvgSamplesPerSec=5.8793317793904345, CurrSamplesPerSec=5.3537840549629365, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4033/5000 [14:04:02<2:57:11, 10.99s/it][2022-12-21 00:26:38,202] [INFO] [timer.py:197:stop] 0/8166, RunningAvgSamplesPerSec=5.879330078091181, CurrSamplesPerSec=5.338407677414076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4034/5000 [14:04:13<2:57:08, 11.00s/it][2022-12-21 00:26:49,257] [INFO] [timer.py:197:stop] 0/8168, RunningAvgSamplesPerSec=5.879324068320064, CurrSamplesPerSec=5.33001766893835, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4035/5000 [14:04:24<2:57:12, 11.02s/it][2022-12-21 00:27:00,230] [INFO] [timer.py:197:stop] 0/8170, RunningAvgSamplesPerSec=5.879328832563151, CurrSamplesPerSec=5.327826364064929, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4036/5000 [14:04:35<2:56:48, 11.00s/it][2022-12-21 00:27:11,278] [INFO] [timer.py:197:stop] 0/8172, RunningAvgSamplesPerSec=5.879323737897583, CurrSamplesPerSec=5.297826113814562, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4037/5000 [14:04:46<2:56:49, 11.02s/it][2022-12-21 00:27:22,289] [INFO] [timer.py:197:stop] 0/8174, RunningAvgSamplesPerSec=5.879323514095325, CurrSamplesPerSec=5.340851027407004, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4038/5000 [14:04:57<2:56:37, 11.02s/it][2022-12-21 00:27:33,253] [INFO] [timer.py:197:stop] 0/8176, RunningAvgSamplesPerSec=5.879329714417915, CurrSamplesPerSec=5.343612097677564, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4039/5000 [14:05:08<2:56:11, 11.00s/it][2022-12-21 00:27:44,280] [INFO] [timer.py:197:stop] 0/8178, RunningAvgSamplesPerSec=5.879327269015308, CurrSamplesPerSec=5.326658348334905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4040/5000 [14:05:19<2:56:07, 11.01s/it][2022-12-21 00:27:55,347] [INFO] [logging.py:68:log_dist] [Rank 0] step=4090, skipped=5, lr=[2.0355555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:27:55,349] [INFO] [timer.py:197:stop] 0/8180, RunningAvgSamplesPerSec=5.879319206799394, CurrSamplesPerSec=5.294461080259542, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4041/5000 [14:05:30<2:56:14, 11.03s/it][2022-12-21 00:28:06,378] [INFO] [timer.py:197:stop] 0/8182, RunningAvgSamplesPerSec=5.879316601799705, CurrSamplesPerSec=5.31617552405933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4042/5000 [14:05:41<2:56:04, 11.03s/it][2022-12-21 00:28:17,379] [INFO] [timer.py:197:stop] 0/8184, RunningAvgSamplesPerSec=5.8793174795792424, CurrSamplesPerSec=5.343528702823113, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4043/5000 [14:05:52<2:55:45, 11.02s/it][2022-12-21 00:28:28,416] [INFO] [timer.py:197:stop] 0/8186, RunningAvgSamplesPerSec=5.879313575772309, CurrSamplesPerSec=5.317720897653772, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4044/5000 [14:06:04<2:55:39, 11.02s/it][2022-12-21 00:28:39,425] [INFO] [timer.py:197:stop] 0/8188, RunningAvgSamplesPerSec=5.879313460482193, CurrSamplesPerSec=5.318454195355415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4045/5000 [14:06:15<2:55:24, 11.02s/it][2022-12-21 00:28:50,469] [INFO] [timer.py:197:stop] 0/8190, RunningAvgSamplesPerSec=5.87930858672184, CurrSamplesPerSec=5.3081671381953415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4046/5000 [14:06:26<2:55:19, 11.03s/it][2022-12-21 00:29:01,511] [INFO] [timer.py:197:stop] 0/8192, RunningAvgSamplesPerSec=5.8793040365653395, CurrSamplesPerSec=5.304365259826807, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4047/5000 [14:06:37<2:55:13, 11.03s/it][2022-12-21 00:29:12,539] [INFO] [timer.py:197:stop] 0/8194, RunningAvgSamplesPerSec=5.879301497132757, CurrSamplesPerSec=5.34051207074838, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4048/5000 [14:06:48<2:55:00, 11.03s/it][2022-12-21 00:29:23,529] [INFO] [timer.py:197:stop] 0/8196, RunningAvgSamplesPerSec=5.879303695644854, CurrSamplesPerSec=5.357045332338806, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4049/5000 [14:06:59<2:54:38, 11.02s/it][2022-12-21 00:29:34,535] [INFO] [timer.py:197:stop] 0/8198, RunningAvgSamplesPerSec=5.879304068603482, CurrSamplesPerSec=5.320401365315319, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4050/5000 [14:07:10<2:54:23, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.0155555555555554e-06, 'epoch': 98.77}
+
 81%|████████  | 4050/5000 [14:07:10<2:54:23, 11.01s/it][2022-12-21 00:29:45,588] [INFO] [logging.py:68:log_dist] [Rank 0] step=4100, skipped=5, lr=[2.0133333333333337e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:29:45,589] [INFO] [timer.py:197:stop] 0/8200, RunningAvgSamplesPerSec=5.8792980656306755, CurrSamplesPerSec=5.3027859456804896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4051/5000 [14:07:21<2:54:24, 11.03s/it][2022-12-21 00:29:56,586] [INFO] [timer.py:197:stop] 0/8202, RunningAvgSamplesPerSec=5.879299507086128, CurrSamplesPerSec=5.367553163750655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4052/5000 [14:07:32<2:54:04, 11.02s/it][2022-12-21 00:30:07,603] [INFO] [timer.py:197:stop] 0/8204, RunningAvgSamplesPerSec=5.8792981870773104, CurrSamplesPerSec=5.330421343142767, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4053/5000 [14:07:43<2:53:53, 11.02s/it][2022-12-21 00:30:18,603] [INFO] [timer.py:197:stop] 0/8206, RunningAvgSamplesPerSec=5.879299247332314, CurrSamplesPerSec=5.3136585546421955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4054/5000 [14:07:54<2:53:37, 11.01s/it][2022-12-21 00:30:29,649] [INFO] [timer.py:197:stop] 0/8208, RunningAvgSamplesPerSec=5.879294174526971, CurrSamplesPerSec=5.313461237818312, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4055/5000 [14:08:05<2:53:36, 11.02s/it][2022-12-21 00:30:40,708] [INFO] [timer.py:197:stop] 0/8210, RunningAvgSamplesPerSec=5.879287511915992, CurrSamplesPerSec=5.297798092531466, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4056/5000 [14:08:16<2:53:35, 11.03s/it][2022-12-21 00:30:51,780] [INFO] [timer.py:197:stop] 0/8212, RunningAvgSamplesPerSec=5.879279172400807, CurrSamplesPerSec=5.2937452406464445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4057/5000 [14:08:27<2:53:35, 11.04s/it][2022-12-21 00:31:02,790] [INFO] [timer.py:197:stop] 0/8214, RunningAvgSamplesPerSec=5.8792789194315, CurrSamplesPerSec=5.346420747331694, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4058/5000 [14:08:38<2:53:14, 11.03s/it][2022-12-21 00:31:13,849] [INFO] [timer.py:197:stop] 0/8216, RunningAvgSamplesPerSec=5.879272233361858, CurrSamplesPerSec=5.2794328545431055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4059/5000 [14:08:49<2:53:10, 11.04s/it][2022-12-21 00:31:23,981] [INFO] [timer.py:197:stop] 0/8218, RunningAvgSamplesPerSec=5.879386338403868, CurrSamplesPerSec=5.298430107245878, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4060/5000 [14:09:04<3:12:02, 12.26s/it][2022-12-21 00:31:35,010] [INFO] [logging.py:68:log_dist] [Rank 0] step=4110, skipped=5, lr=[1.9911111111111113e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:31:35,012] [INFO] [timer.py:197:stop] 0/8220, RunningAvgSamplesPerSec=5.879383407324619, CurrSamplesPerSec=5.330578849817791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4061/5000 [14:09:15<3:06:01, 11.89s/it][2022-12-21 00:31:46,063] [INFO] [timer.py:197:stop] 0/8222, RunningAvgSamplesPerSec=5.8793774035216995, CurrSamplesPerSec=5.30016568135569, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████  | 4062/5000 [14:09:26<3:01:50, 11.63s/it][2022-12-21 00:31:57,096] [INFO] [timer.py:197:stop] 0/8224, RunningAvgSamplesPerSec=5.879374101981544, CurrSamplesPerSec=5.306484014885291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4063/5000 [14:09:37<2:58:48, 11.45s/it][2022-12-21 00:32:08,137] [INFO] [timer.py:197:stop] 0/8226, RunningAvgSamplesPerSec=5.879369774379468, CurrSamplesPerSec=5.293551070163772, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4064/5000 [14:09:48<2:56:47, 11.33s/it][2022-12-21 00:32:19,139] [INFO] [timer.py:197:stop] 0/8228, RunningAvgSamplesPerSec=5.879370417958987, CurrSamplesPerSec=5.343381917183105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4065/5000 [14:09:59<2:54:59, 11.23s/it][2022-12-21 00:32:30,157] [INFO] [timer.py:197:stop] 0/8230, RunningAvgSamplesPerSec=5.879368973390279, CurrSamplesPerSec=5.317729114524873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4066/5000 [14:10:10<2:53:55, 11.17s/it][2022-12-21 00:32:41,200] [INFO] [timer.py:197:stop] 0/8232, RunningAvgSamplesPerSec=5.879364288039955, CurrSamplesPerSec=5.315493587498121, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4067/5000 [14:10:21<2:52:55, 11.12s/it][2022-12-21 00:32:52,212] [INFO] [timer.py:197:stop] 0/8234, RunningAvgSamplesPerSec=5.879363602434334, CurrSamplesPerSec=5.305016875195725, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4068/5000 [14:10:32<2:52:30, 11.11s/it][2022-12-21 00:33:03,337] [INFO] [timer.py:197:stop] 0/8236, RunningAvgSamplesPerSec=5.879348210411852, CurrSamplesPerSec=5.25656555054468, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4069/5000 [14:10:43<2:52:14, 11.10s/it][2022-12-21 00:33:14,317] [INFO] [timer.py:197:stop] 0/8238, RunningAvgSamplesPerSec=5.879351928844178, CurrSamplesPerSec=5.3506662453673215, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4070/5000 [14:10:54<2:51:25, 11.06s/it][2022-12-21 00:33:25,334] [INFO] [logging.py:68:log_dist] [Rank 0] step=4120, skipped=5, lr=[1.968888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:33:25,336] [INFO] [timer.py:197:stop] 0/8240, RunningAvgSamplesPerSec=5.879350439568546, CurrSamplesPerSec=5.303852340653922, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4071/5000 [14:11:05<2:51:08, 11.05s/it][2022-12-21 00:33:36,379] [INFO] [timer.py:197:stop] 0/8242, RunningAvgSamplesPerSec=5.879345799719313, CurrSamplesPerSec=5.298996580782541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4072/5000 [14:11:16<2:50:53, 11.05s/it][2022-12-21 00:33:47,395] [INFO] [timer.py:197:stop] 0/8244, RunningAvgSamplesPerSec=5.879344885673582, CurrSamplesPerSec=5.319105061078207, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4073/5000 [14:11:27<2:50:40, 11.05s/it][2022-12-21 00:33:58,431] [INFO] [timer.py:197:stop] 0/8246, RunningAvgSamplesPerSec=5.879341245952769, CurrSamplesPerSec=5.325692225726671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 81%|████████▏ | 4074/5000 [14:11:38<2:50:24, 11.04s/it][2022-12-21 00:34:09,464] [INFO] [timer.py:197:stop] 0/8248, RunningAvgSamplesPerSec=5.879338013802974, CurrSamplesPerSec=5.3221258299209016, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4075/5000 [14:11:50<2:50:07, 11.04s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.9600000000000003e-06, 'epoch': 99.39}
+
 82%|████████▏ | 4075/5000 [14:11:50<2:50:07, 11.04s/it][2022-12-21 00:34:20,516] [INFO] [timer.py:197:stop] 0/8250, RunningAvgSamplesPerSec=5.879332539982383, CurrSamplesPerSec=5.295277808145547, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4076/5000 [14:12:01<2:50:03, 11.04s/it][2022-12-21 00:34:31,583] [INFO] [timer.py:197:stop] 0/8252, RunningAvgSamplesPerSec=5.879324855602666, CurrSamplesPerSec=5.287708615297361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4077/5000 [14:12:12<2:49:55, 11.05s/it][2022-12-21 00:34:42,582] [INFO] [timer.py:197:stop] 0/8254, RunningAvgSamplesPerSec=5.879325835855229, CurrSamplesPerSec=5.334560252616491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4078/5000 [14:12:23<2:49:39, 11.04s/it][2022-12-21 00:34:53,600] [INFO] [timer.py:197:stop] 0/8256, RunningAvgSamplesPerSec=5.879324640207131, CurrSamplesPerSec=5.345490660586304, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4079/5000 [14:12:34<2:49:22, 11.03s/it][2022-12-21 00:35:04,637] [INFO] [timer.py:197:stop] 0/8258, RunningAvgSamplesPerSec=5.87932080682032, CurrSamplesPerSec=5.329677757907739, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4080/5000 [14:12:45<2:49:10, 11.03s/it][2022-12-21 00:35:15,671] [INFO] [logging.py:68:log_dist] [Rank 0] step=4130, skipped=5, lr=[1.9466666666666665e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:35:15,672] [INFO] [timer.py:197:stop] 0/8260, RunningAvgSamplesPerSec=5.87931726467643, CurrSamplesPerSec=5.325379700604006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4081/5000 [14:12:56<2:48:53, 11.03s/it][2022-12-21 00:35:26,678] [INFO] [timer.py:197:stop] 0/8262, RunningAvgSamplesPerSec=5.879317527167356, CurrSamplesPerSec=5.331451232721348, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4082/5000 [14:13:07<2:48:35, 11.02s/it][2022-12-21 00:35:37,726] [INFO] [timer.py:197:stop] 0/8264, RunningAvgSamplesPerSec=5.879312260945233, CurrSamplesPerSec=5.2907326097628795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4083/5000 [14:13:18<2:48:28, 11.02s/it][2022-12-21 00:35:48,737] [INFO] [timer.py:197:stop] 0/8266, RunningAvgSamplesPerSec=5.8793119604205035, CurrSamplesPerSec=5.310472355955648, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4084/5000 [14:13:29<2:48:30, 11.04s/it][2022-12-21 00:35:59,757] [INFO] [timer.py:197:stop] 0/8268, RunningAvgSamplesPerSec=5.879310575610749, CurrSamplesPerSec=5.354943060774805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4085/5000 [14:13:40<2:48:09, 11.03s/it][2022-12-21 00:36:10,823] [INFO] [timer.py:197:stop] 0/8270, RunningAvgSamplesPerSec=5.879303357668457, CurrSamplesPerSec=5.30105661144027, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4086/5000 [14:13:51<2:48:10, 11.04s/it][2022-12-21 00:36:21,874] [INFO] [timer.py:197:stop] 0/8272, RunningAvgSamplesPerSec=5.87929781934806, CurrSamplesPerSec=5.315562846921397, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4087/5000 [14:14:02<2:47:47, 11.03s/it][2022-12-21 00:36:32,898] [INFO] [timer.py:197:stop] 0/8274, RunningAvgSamplesPerSec=5.879295931707599, CurrSamplesPerSec=5.293162771927432, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4088/5000 [14:14:13<2:47:49, 11.04s/it][2022-12-21 00:36:43,939] [INFO] [timer.py:197:stop] 0/8276, RunningAvgSamplesPerSec=5.8792915841199065, CurrSamplesPerSec=5.319952183831361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4089/5000 [14:14:24<2:47:35, 11.04s/it][2022-12-21 00:36:54,982] [INFO] [timer.py:197:stop] 0/8278, RunningAvgSamplesPerSec=5.879287020187908, CurrSamplesPerSec=5.310920778571532, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4090/5000 [14:14:35<2:47:12, 11.03s/it][2022-12-21 00:37:05,982] [INFO] [logging.py:68:log_dist] [Rank 0] step=4140, skipped=5, lr=[1.9244444444444446e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:37:05,984] [INFO] [timer.py:197:stop] 0/8280, RunningAvgSamplesPerSec=5.879287821846341, CurrSamplesPerSec=5.304253109237295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4091/5000 [14:14:46<2:47:15, 11.04s/it][2022-12-21 00:37:17,017] [INFO] [timer.py:197:stop] 0/8282, RunningAvgSamplesPerSec=5.879284536773077, CurrSamplesPerSec=5.340084770406883, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4092/5000 [14:14:57<2:46:46, 11.02s/it][2022-12-21 00:37:27,999] [INFO] [timer.py:197:stop] 0/8284, RunningAvgSamplesPerSec=5.879287845445242, CurrSamplesPerSec=5.332195101098891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4093/5000 [14:15:08<2:46:21, 11.01s/it][2022-12-21 00:37:38,981] [INFO] [timer.py:197:stop] 0/8286, RunningAvgSamplesPerSec=5.87929115476188, CurrSamplesPerSec=5.322443038743695, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4094/5000 [14:15:19<2:46:09, 11.00s/it][2022-12-21 00:37:49,984] [INFO] [timer.py:197:stop] 0/8288, RunningAvgSamplesPerSec=5.879291947687337, CurrSamplesPerSec=5.321762658531498, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4095/5000 [14:15:30<2:45:55, 11.00s/it][2022-12-21 00:38:00,993] [INFO] [timer.py:197:stop] 0/8290, RunningAvgSamplesPerSec=5.879291927533418, CurrSamplesPerSec=5.307245065734518, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4096/5000 [14:15:41<2:45:55, 11.01s/it][2022-12-21 00:38:12,036] [INFO] [timer.py:197:stop] 0/8292, RunningAvgSamplesPerSec=5.8792872350944245, CurrSamplesPerSec=5.305441937117551, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4097/5000 [14:15:52<2:46:04, 11.03s/it][2022-12-21 00:38:23,122] [INFO] [timer.py:197:stop] 0/8294, RunningAvgSamplesPerSec=5.879277192539105, CurrSamplesPerSec=5.305190708272735, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4098/5000 [14:16:03<2:45:57, 11.04s/it][2022-12-21 00:38:34,132] [INFO] [timer.py:197:stop] 0/8296, RunningAvgSamplesPerSec=5.879276935406706, CurrSamplesPerSec=5.340533108185188, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4099/5000 [14:16:14<2:45:25, 11.02s/it][2022-12-21 00:38:45,118] [INFO] [timer.py:197:stop] 0/8298, RunningAvgSamplesPerSec=5.879280015720075, CurrSamplesPerSec=5.3210259168716565, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4100/5000 [14:16:25<2:45:10, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.9044444444444445e-06, 'epoch': 99.99}
+
 82%|████████▏ | 4100/5000 [14:16:25<2:45:10, 11.01s/it][2022-12-21 00:38:55,176] [INFO] [logging.py:68:log_dist] [Rank 0] step=4150, skipped=5, lr=[1.9022222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:38:55,178] [INFO] [timer.py:197:stop] 0/8300, RunningAvgSamplesPerSec=5.879402408551279, CurrSamplesPerSec=6.295832685812883, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 00:39:06,098] [INFO] [timer.py:197:stop] 0/8302, RunningAvgSamplesPerSec=5.879413597084445, CurrSamplesPerSec=5.374073305010129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4101/5000 [14:16:41<3:07:40, 12.53s/it][2022-12-21 00:39:17,119] [INFO] [timer.py:197:stop] 0/8304, RunningAvgSamplesPerSec=5.879411955153533, CurrSamplesPerSec=5.337322037043201, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4102/5000 [14:16:52<3:00:42, 12.07s/it][2022-12-21 00:39:28,195] [INFO] [timer.py:197:stop] 0/8306, RunningAvgSamplesPerSec=5.87940329020361, CurrSamplesPerSec=5.281855554900549, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4103/5000 [14:17:03<2:56:01, 11.77s/it][2022-12-21 00:39:39,195] [INFO] [timer.py:197:stop] 0/8308, RunningAvgSamplesPerSec=5.879404274059555, CurrSamplesPerSec=5.344067411403323, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4104/5000 [14:17:14<2:52:21, 11.54s/it][2022-12-21 00:39:50,224] [INFO] [timer.py:197:stop] 0/8310, RunningAvgSamplesPerSec=5.8794015580083, CurrSamplesPerSec=5.3158142169505185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4105/5000 [14:17:25<2:49:52, 11.39s/it][2022-12-21 00:40:01,195] [INFO] [timer.py:197:stop] 0/8312, RunningAvgSamplesPerSec=5.879406500998291, CurrSamplesPerSec=5.342292126509466, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████�� | 4106/5000 [14:17:36<2:47:49, 11.26s/it][2022-12-21 00:40:12,226] [INFO] [timer.py:197:stop] 0/8314, RunningAvgSamplesPerSec=5.879403612138365, CurrSamplesPerSec=5.31175310370014, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4107/5000 [14:17:47<2:46:35, 11.19s/it][2022-12-21 00:40:23,193] [INFO] [timer.py:197:stop] 0/8316, RunningAvgSamplesPerSec=5.879408989600911, CurrSamplesPerSec=5.333709107488297, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4108/5000 [14:17:58<2:45:24, 11.13s/it][2022-12-21 00:40:34,182] [INFO] [timer.py:197:stop] 0/8318, RunningAvgSamplesPerSec=5.879411655495076, CurrSamplesPerSec=5.326075800603458, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4109/5000 [14:18:09<2:44:36, 11.08s/it][2022-12-21 00:40:45,214] [INFO] [logging.py:68:log_dist] [Rank 0] step=4160, skipped=5, lr=[1.8800000000000002e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:40:45,216] [INFO] [timer.py:197:stop] 0/8320, RunningAvgSamplesPerSec=5.879408422395517, CurrSamplesPerSec=5.325705961614112, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4110/5000 [14:18:20<2:44:11, 11.07s/it][2022-12-21 00:40:56,193] [INFO] [timer.py:197:stop] 0/8322, RunningAvgSamplesPerSec=5.879412636950251, CurrSamplesPerSec=5.368000329236527, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4111/5000 [14:18:31<2:43:36, 11.04s/it][2022-12-21 00:41:07,164] [INFO] [timer.py:197:stop] 0/8324, RunningAvgSamplesPerSec=5.8794175040949215, CurrSamplesPerSec=5.344241472382163, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4112/5000 [14:18:42<2:43:06, 11.02s/it][2022-12-21 00:41:18,164] [INFO] [timer.py:197:stop] 0/8326, RunningAvgSamplesPerSec=5.879418835814911, CurrSamplesPerSec=5.30564893568786, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4113/5000 [14:18:53<2:42:49, 11.01s/it][2022-12-21 00:41:29,172] [INFO] [timer.py:197:stop] 0/8328, RunningAvgSamplesPerSec=5.87941902936405, CurrSamplesPerSec=5.345746572512703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4114/5000 [14:19:04<2:42:36, 11.01s/it][2022-12-21 00:41:40,191] [INFO] [timer.py:197:stop] 0/8330, RunningAvgSamplesPerSec=5.879417660891862, CurrSamplesPerSec=5.319440462436909, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4115/5000 [14:19:15<2:42:27, 11.01s/it][2022-12-21 00:41:51,211] [INFO] [timer.py:197:stop] 0/8332, RunningAvgSamplesPerSec=5.879416234889478, CurrSamplesPerSec=5.3203868131770555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4116/5000 [14:19:26<2:42:18, 11.02s/it][2022-12-21 00:42:02,285] [INFO] [timer.py:197:stop] 0/8334, RunningAvgSamplesPerSec=5.879407671709497, CurrSamplesPerSec=5.290706331894598, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4117/5000 [14:19:37<2:42:22, 11.03s/it][2022-12-21 00:42:13,297] [INFO] [timer.py:197:stop] 0/8336, RunningAvgSamplesPerSec=5.879407328551862, CurrSamplesPerSec=5.326157806147749, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4118/5000 [14:19:48<2:42:05, 11.03s/it][2022-12-21 00:42:24,320] [INFO] [timer.py:197:stop] 0/8338, RunningAvgSamplesPerSec=5.87940536708284, CurrSamplesPerSec=5.319073863102032, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4119/5000 [14:19:59<2:41:53, 11.03s/it][2022-12-21 00:42:35,345] [INFO] [logging.py:68:log_dist] [Rank 0] step=4170, skipped=5, lr=[1.8577777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:42:35,347] [INFO] [timer.py:197:stop] 0/8340, RunningAvgSamplesPerSec=5.8794029038482085, CurrSamplesPerSec=5.325339554666794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4120/5000 [14:20:10<2:41:42, 11.03s/it][2022-12-21 00:42:46,377] [INFO] [timer.py:197:stop] 0/8342, RunningAvgSamplesPerSec=5.879400008136403, CurrSamplesPerSec=5.328734234667739, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4121/5000 [14:20:21<2:41:32, 11.03s/it][2022-12-21 00:42:57,410] [INFO] [timer.py:197:stop] 0/8344, RunningAvgSamplesPerSec=5.879396917892875, CurrSamplesPerSec=5.3063845719258165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4122/5000 [14:20:33<2:41:23, 11.03s/it][2022-12-21 00:43:08,430] [INFO] [timer.py:197:stop] 0/8346, RunningAvgSamplesPerSec=5.879395533186891, CurrSamplesPerSec=5.320621555208884, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4123/5000 [14:20:44<2:41:10, 11.03s/it][2022-12-21 00:43:19,392] [INFO] [timer.py:197:stop] 0/8348, RunningAvgSamplesPerSec=5.879401568406628, CurrSamplesPerSec=5.337023637625309, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▏ | 4124/5000 [14:20:54<2:40:42, 11.01s/it][2022-12-21 00:43:30,357] [INFO] [timer.py:197:stop] 0/8350, RunningAvgSamplesPerSec=5.879407324596102, CurrSamplesPerSec=5.357073983872946, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 82%|████████▎ | 4125/5000 [14:21:05<2:40:20, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.8466666666666668e-06, 'epoch': 100.6}
+
 82%|████████▎ | 4125/5000 [14:21:05<2:40:20, 10.99s/it][2022-12-21 00:43:41,357] [INFO] [timer.py:197:stop] 0/8352, RunningAvgSamplesPerSec=5.879408571046578, CurrSamplesPerSec=5.328778663116946, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4126/5000 [14:21:16<2:40:10, 11.00s/it][2022-12-21 00:43:52,343] [INFO] [timer.py:197:stop] 0/8354, RunningAvgSamplesPerSec=5.879411531665187, CurrSamplesPerSec=5.34060132162026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4127/5000 [14:21:27<2:39:57, 10.99s/it][2022-12-21 00:44:03,407] [INFO] [timer.py:197:stop] 0/8356, RunningAvgSamplesPerSec=5.879404435660339, CurrSamplesPerSec=5.314817297785836, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4128/5000 [14:21:39<2:40:04, 11.01s/it][2022-12-21 00:44:14,473] [INFO] [timer.py:197:stop] 0/8358, RunningAvgSamplesPerSec=5.879397228381717, CurrSamplesPerSec=5.28983701532112, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4129/5000 [14:21:50<2:40:06, 11.03s/it][2022-12-21 00:44:25,495] [INFO] [logging.py:68:log_dist] [Rank 0] step=4180, skipped=5, lr=[1.8355555555555557e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:44:25,496] [INFO] [timer.py:197:stop] 0/8360, RunningAvgSamplesPerSec=5.879395419203502, CurrSamplesPerSec=5.30878042133571, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4130/5000 [14:22:01<2:39:54, 11.03s/it][2022-12-21 00:44:36,513] [INFO] [timer.py:197:stop] 0/8362, RunningAvgSamplesPerSec=5.879394298349533, CurrSamplesPerSec=5.313073168059205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4131/5000 [14:22:12<2:39:40, 11.02s/it][2022-12-21 00:44:47,583] [INFO] [timer.py:197:stop] 0/8364, RunningAvgSamplesPerSec=5.8793861481877085, CurrSamplesPerSec=5.285998259474284, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4132/5000 [14:22:23<2:39:41, 11.04s/it][2022-12-21 00:44:58,592] [INFO] [timer.py:197:stop] 0/8366, RunningAvgSamplesPerSec=5.879386087523323, CurrSamplesPerSec=5.334557496291045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4133/5000 [14:22:34<2:39:22, 11.03s/it][2022-12-21 00:45:09,601] [INFO] [timer.py:197:stop] 0/8368, RunningAvgSamplesPerSec=5.879386337783563, CurrSamplesPerSec=5.3365389688647715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4134/5000 [14:22:45<2:39:06, 11.02s/it][2022-12-21 00:45:20,549] [INFO] [timer.py:197:stop] 0/8370, RunningAvgSamplesPerSec=5.879394237605077, CurrSamplesPerSec=5.354860807368847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4135/5000 [14:22:56<2:38:35, 11.00s/it][2022-12-21 00:45:31,575] [INFO] [timer.py:197:stop] 0/8372, RunningAvgSamplesPerSec=5.879391975846696, CurrSamplesPerSec=5.295259632686927, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4136/5000 [14:23:07<2:38:31, 11.01s/it][2022-12-21 00:45:41,960] [INFO] [stage_1_and_2.py:1765:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072.0, reducing to 131072.0
+[2022-12-21 00:45:41,962] [INFO] [timer.py:197:stop] 0/8374, RunningAvgSamplesPerSec=5.879472058375945, CurrSamplesPerSec=5.935283442883516, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4137/5000 [14:23:17<2:35:39, 10.82s/it][2022-12-21 00:45:52,281] [INFO] [stage_1_and_2.py:1765:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072.0, reducing to 65536.0
+[2022-12-21 00:45:52,283] [INFO] [timer.py:197:stop] 0/8376, RunningAvgSamplesPerSec=5.879560693339627, CurrSamplesPerSec=5.967002035759316, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4138/5000 [14:23:27<2:33:18, 10.67s/it][2022-12-21 00:46:03,287] [INFO] [timer.py:197:stop] 0/8378, RunningAvgSamplesPerSec=5.879561182430551, CurrSamplesPerSec=5.341895156746999, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4139/5000 [14:23:38<2:34:34, 10.77s/it][2022-12-21 00:46:14,338] [INFO] [logging.py:68:log_dist] [Rank 0] step=4190, skipped=7, lr=[1.817777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:46:14,340] [INFO] [timer.py:197:stop] 0/8380, RunningAvgSamplesPerSec=5.879555372610727, CurrSamplesPerSec=5.308146145046857, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4140/5000 [14:23:49<2:35:35, 10.86s/it][2022-12-21 00:46:25,344] [INFO] [timer.py:197:stop] 0/8382, RunningAvgSamplesPerSec=5.879555805723497, CurrSamplesPerSec=5.324771871398592, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4141/5000 [14:24:00<2:36:03, 10.90s/it][2022-12-21 00:46:35,479] [INFO] [timer.py:197:stop] 0/8384, RunningAvgSamplesPerSec=5.879667104371806, CurrSamplesPerSec=5.316851317016506, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4142/5000 [14:24:16<2:53:51, 12.16s/it][2022-12-21 00:46:46,506] [INFO] [timer.py:197:stop] 0/8386, RunningAvgSamplesPerSec=5.879664441580888, CurrSamplesPerSec=5.327641105094203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4143/5000 [14:24:27<2:48:42, 11.81s/it][2022-12-21 00:46:57,500] [INFO] [timer.py:197:stop] 0/8388, RunningAvgSamplesPerSec=5.8796660646731045, CurrSamplesPerSec=5.33418711446512, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4144/5000 [14:24:38<2:45:03, 11.57s/it][2022-12-21 00:47:08,514] [INFO] [timer.py:197:stop] 0/8390, RunningAvgSamplesPerSec=5.879665109295008, CurrSamplesPerSec=5.328179999145698, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4145/5000 [14:24:49<2:42:24, 11.40s/it][2022-12-21 00:47:19,543] [INFO] [timer.py:197:stop] 0/8392, RunningAvgSamplesPerSec=5.879662307391213, CurrSamplesPerSec=5.298306285736392, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4146/5000 [14:25:00<2:40:44, 11.29s/it][2022-12-21 00:47:30,589] [INFO] [timer.py:197:stop] 0/8394, RunningAvgSamplesPerSec=5.879657297504832, CurrSamplesPerSec=5.301620505539521, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4147/5000 [14:25:11<2:39:30, 11.22s/it][2022-12-21 00:47:41,608] [INFO] [timer.py:197:stop] 0/8396, RunningAvgSamplesPerSec=5.8796558149728195, CurrSamplesPerSec=5.327340192469398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4148/5000 [14:25:22<2:38:21, 11.15s/it][2022-12-21 00:47:52,588] [INFO] [timer.py:197:stop] 0/8398, RunningAvgSamplesPerSec=5.879659317862077, CurrSamplesPerSec=5.339260110504903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4149/5000 [14:25:33<2:37:22, 11.10s/it][2022-12-21 00:48:03,544] [INFO] [logging.py:68:log_dist] [Rank 0] step=4200, skipped=7, lr=[1.7955555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:48:03,546] [INFO] [timer.py:197:stop] 0/8400, RunningAvgSamplesPerSec=5.8796657087285595, CurrSamplesPerSec=5.34709019270009, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4150/5000 [14:25:44<2:36:51, 11.07s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.7955555555555556e-06, 'epoch': 101.22}
+
 83%|████████▎ | 4150/5000 [14:25:44<2:36:51, 11.07s/it][2022-12-21 00:48:14,590] [INFO] [timer.py:197:stop] 0/8402, RunningAvgSamplesPerSec=5.879661096910081, CurrSamplesPerSec=5.324668995723134, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4151/5000 [14:25:55<2:36:21, 11.05s/it][2022-12-21 00:48:25,613] [INFO] [timer.py:197:stop] 0/8404, RunningAvgSamplesPerSec=5.879659187971538, CurrSamplesPerSec=5.301951820472288, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4152/5000 [14:26:06<2:35:57, 11.03s/it][2022-12-21 00:48:36,568] [INFO] [timer.py:197:stop] 0/8406, RunningAvgSamplesPerSec=5.879665717066662, CurrSamplesPerSec=5.339894196532439, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4153/5000 [14:26:17<2:35:40, 11.03s/it][2022-12-21 00:48:47,620] [INFO] [timer.py:197:stop] 0/8408, RunningAvgSamplesPerSec=5.879660047403878, CurrSamplesPerSec=5.305370424648294, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4154/5000 [14:26:28<2:35:35, 11.03s/it][2022-12-21 00:48:58,670] [INFO] [timer.py:197:stop] 0/8410, RunningAvgSamplesPerSec=5.879654527062707, CurrSamplesPerSec=5.305006600738334, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4155/5000 [14:26:39<2:35:24, 11.03s/it][2022-12-21 00:49:09,677] [INFO] [timer.py:197:stop] 0/8412, RunningAvgSamplesPerSec=5.879654713144678, CurrSamplesPerSec=5.330944284215078, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4156/5000 [14:26:50<2:34:54, 11.01s/it][2022-12-21 00:49:20,638] [INFO] [timer.py:197:stop] 0/8414, RunningAvgSamplesPerSec=5.879660606663472, CurrSamplesPerSec=5.331174453145186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4157/5000 [14:27:01<2:34:45, 11.02s/it][2022-12-21 00:49:31,663] [INFO] [timer.py:197:stop] 0/8416, RunningAvgSamplesPerSec=5.879658297128644, CurrSamplesPerSec=5.326898929529991, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4158/5000 [14:27:12<2:34:36, 11.02s/it][2022-12-21 00:49:42,656] [INFO] [timer.py:197:stop] 0/8418, RunningAvgSamplesPerSec=5.879660158556979, CurrSamplesPerSec=5.351426580828682, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4159/5000 [14:27:23<2:34:17, 11.01s/it][2022-12-21 00:49:53,670] [INFO] [logging.py:68:log_dist] [Rank 0] step=4210, skipped=7, lr=[1.7733333333333336e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:49:53,672] [INFO] [timer.py:197:stop] 0/8420, RunningAvgSamplesPerSec=5.879658973887545, CurrSamplesPerSec=5.326159285652409, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4160/5000 [14:27:34<2:34:02, 11.00s/it][2022-12-21 00:50:04,641] [INFO] [timer.py:197:stop] 0/8422, RunningAvgSamplesPerSec=5.8796637542619985, CurrSamplesPerSec=5.343947405302789, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4161/5000 [14:27:45<2:33:42, 10.99s/it][2022-12-21 00:50:15,600] [INFO] [timer.py:197:stop] 0/8424, RunningAvgSamplesPerSec=5.87966984986061, CurrSamplesPerSec=5.351892617094271, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4162/5000 [14:27:56<2:33:31, 10.99s/it][2022-12-21 00:50:26,640] [INFO] [timer.py:197:stop] 0/8426, RunningAvgSamplesPerSec=5.879665595666846, CurrSamplesPerSec=5.308629659383269, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4163/5000 [14:28:07<2:33:34, 11.01s/it][2022-12-21 00:50:37,680] [INFO] [timer.py:197:stop] 0/8428, RunningAvgSamplesPerSec=5.879661471458285, CurrSamplesPerSec=5.319170620180138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4164/5000 [14:28:18<2:33:30, 11.02s/it][2022-12-21 00:50:48,662] [INFO] [timer.py:197:stop] 0/8430, RunningAvgSamplesPerSec=5.879664583778336, CurrSamplesPerSec=5.364266220145827, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4165/5000 [14:28:29<2:33:09, 11.01s/it][2022-12-21 00:50:59,717] [INFO] [timer.py:197:stop] 0/8432, RunningAvgSamplesPerSec=5.879658405520166, CurrSamplesPerSec=5.297931509951434, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4166/5000 [14:28:40<2:33:14, 11.02s/it][2022-12-21 00:51:10,748] [INFO] [timer.py:197:stop] 0/8434, RunningAvgSamplesPerSec=5.879655185345949, CurrSamplesPerSec=5.330481677261155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4167/5000 [14:28:51<2:32:53, 11.01s/it][2022-12-21 00:51:21,689] [INFO] [timer.py:197:stop] 0/8436, RunningAvgSamplesPerSec=5.879663793119303, CurrSamplesPerSec=5.3719054037152105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4168/5000 [14:29:02<2:32:22, 10.99s/it][2022-12-21 00:51:32,656] [INFO] [timer.py:197:stop] 0/8438, RunningAvgSamplesPerSec=5.879668963875951, CurrSamplesPerSec=5.340900333778097, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4169/5000 [14:29:13<2:32:06, 10.98s/it][2022-12-21 00:51:43,595] [INFO] [logging.py:68:log_dist] [Rank 0] step=4220, skipped=7, lr=[1.7511111111111113e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:51:43,597] [INFO] [timer.py:197:stop] 0/8440, RunningAvgSamplesPerSec=5.879677441937503, CurrSamplesPerSec=5.365672364068832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4170/5000 [14:29:24<2:31:51, 10.98s/it][2022-12-21 00:51:54,613] [INFO] [timer.py:197:stop] 0/8442, RunningAvgSamplesPerSec=5.8796763080729, CurrSamplesPerSec=5.32049711615216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4171/5000 [14:29:35<2:31:35, 10.97s/it][2022-12-21 00:52:05,542] [INFO] [timer.py:197:stop] 0/8444, RunningAvgSamplesPerSec=5.87968642508094, CurrSamplesPerSec=5.345220084859095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4172/5000 [14:29:46<2:31:23, 10.97s/it][2022-12-21 00:52:16,533] [INFO] [timer.py:197:stop] 0/8446, RunningAvgSamplesPerSec=5.879688416876027, CurrSamplesPerSec=5.325480702007211, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4173/5000 [14:29:57<2:31:25, 10.99s/it][2022-12-21 00:52:27,553] [INFO] [timer.py:197:stop] 0/8448, RunningAvgSamplesPerSec=5.879686781714583, CurrSamplesPerSec=5.325869107110985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 83%|████████▎ | 4174/5000 [14:30:08<2:31:19, 10.99s/it][2022-12-21 00:52:38,586] [INFO] [timer.py:197:stop] 0/8450, RunningAvgSamplesPerSec=5.879683571104116, CurrSamplesPerSec=5.304717045152238, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4175/5000 [14:30:19<2:31:20, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.74e-06, 'epoch': 101.82}
+
 84%|████████▎ | 4175/5000 [14:30:19<2:31:20, 11.01s/it][2022-12-21 00:52:49,609] [INFO] [timer.py:197:stop] 0/8452, RunningAvgSamplesPerSec=5.8796819432311365, CurrSamplesPerSec=5.320857583579354, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4176/5000 [14:30:30<2:31:02, 11.00s/it][2022-12-21 00:53:00,560] [INFO] [timer.py:197:stop] 0/8454, RunningAvgSamplesPerSec=5.879689247913942, CurrSamplesPerSec=5.343387022635213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4177/5000 [14:30:41<2:30:41, 10.99s/it][2022-12-21 00:53:11,540] [INFO] [timer.py:197:stop] 0/8456, RunningAvgSamplesPerSec=5.87969276761876, CurrSamplesPerSec=5.322449159578507, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4178/5000 [14:30:52<2:30:29, 10.98s/it][2022-12-21 00:53:22,531] [INFO] [timer.py:197:stop] 0/8458, RunningAvgSamplesPerSec=5.879695038829483, CurrSamplesPerSec=5.316233219834995, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4179/5000 [14:31:03<2:30:32, 11.00s/it][2022-12-21 00:53:33,566] [INFO] [logging.py:68:log_dist] [Rank 0] step=4230, skipped=7, lr=[1.728888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:53:33,568] [INFO] [timer.py:197:stop] 0/8460, RunningAvgSamplesPerSec=5.879691356172742, CurrSamplesPerSec=5.320038218355553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4180/5000 [14:31:14<2:30:19, 11.00s/it][2022-12-21 00:53:44,527] [INFO] [timer.py:197:stop] 0/8462, RunningAvgSamplesPerSec=5.879698635061717, CurrSamplesPerSec=5.356279124412723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4181/5000 [14:31:25<2:30:09, 11.00s/it][2022-12-21 00:53:55,518] [INFO] [timer.py:197:stop] 0/8464, RunningAvgSamplesPerSec=5.879700830165782, CurrSamplesPerSec=5.359898732858876, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4182/5000 [14:31:36<2:29:47, 10.99s/it][2022-12-21 00:54:05,592] [INFO] [timer.py:197:stop] 0/8466, RunningAvgSamplesPerSec=5.879818779463994, CurrSamplesPerSec=6.277921683682717, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 00:54:16,609] [INFO] [timer.py:197:stop] 0/8468, RunningAvgSamplesPerSec=5.879817229961785, CurrSamplesPerSec=5.342756998445471, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4183/5000 [14:31:52<2:50:46, 12.54s/it][2022-12-21 00:54:27,614] [INFO] [timer.py:197:stop] 0/8470, RunningAvgSamplesPerSec=5.879817488838075, CurrSamplesPerSec=5.3357682213089275, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4184/5000 [14:32:03<2:44:17, 12.08s/it][2022-12-21 00:54:38,627] [INFO] [timer.py:197:stop] 0/8472, RunningAvgSamplesPerSec=5.879817041016966, CurrSamplesPerSec=5.331419042648102, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4185/5000 [14:32:14<2:39:44, 11.76s/it][2022-12-21 00:54:49,592] [INFO] [timer.py:197:stop] 0/8474, RunningAvgSamplesPerSec=5.879822402718826, CurrSamplesPerSec=5.336799966058936, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4186/5000 [14:32:25<2:36:18, 11.52s/it][2022-12-21 00:55:00,560] [INFO] [timer.py:197:stop] 0/8476, RunningAvgSamplesPerSec=5.87982740175581, CurrSamplesPerSec=5.317093540794607, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▎ | 4187/5000 [14:32:36<2:33:52, 11.36s/it][2022-12-21 00:55:11,612] [INFO] [timer.py:197:stop] 0/8478, RunningAvgSamplesPerSec=5.879821799684799, CurrSamplesPerSec=5.316793607824019, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4188/5000 [14:32:47<2:32:26, 11.26s/it][2022-12-21 00:55:22,627] [INFO] [logging.py:68:log_dist] [Rank 0] step=4240, skipped=7, lr=[1.7066666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:55:22,629] [INFO] [timer.py:197:stop] 0/8480, RunningAvgSamplesPerSec=5.879820597810602, CurrSamplesPerSec=5.312367636312211, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4189/5000 [14:32:58<2:31:15, 11.19s/it][2022-12-21 00:55:33,613] [INFO] [timer.py:197:stop] 0/8482, RunningAvgSamplesPerSec=5.879823615105404, CurrSamplesPerSec=5.323729567729799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4190/5000 [14:33:09<2:30:13, 11.13s/it][2022-12-21 00:55:44,640] [INFO] [timer.py:197:stop] 0/8484, RunningAvgSamplesPerSec=5.879821465635901, CurrSamplesPerSec=5.348264203464134, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4191/5000 [14:33:20<2:29:38, 11.10s/it][2022-12-21 00:55:55,637] [INFO] [timer.py:197:stop] 0/8486, RunningAvgSamplesPerSec=5.879822668531959, CurrSamplesPerSec=5.3570489672179535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4192/5000 [14:33:31<2:29:02, 11.07s/it][2022-12-21 00:56:06,639] [INFO] [timer.py:197:stop] 0/8488, RunningAvgSamplesPerSec=5.879823411453967, CurrSamplesPerSec=5.307147692765277, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4193/5000 [14:33:42<2:28:35, 11.05s/it][2022-12-21 00:56:17,626] [INFO] [timer.py:197:stop] 0/8490, RunningAvgSamplesPerSec=5.879826007732794, CurrSamplesPerSec=5.342149661371552, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4194/5000 [14:33:53<2:28:09, 11.03s/it][2022-12-21 00:56:28,661] [INFO] [timer.py:197:stop] 0/8492, RunningAvgSamplesPerSec=5.879822598078155, CurrSamplesPerSec=5.351265706202805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4195/5000 [14:34:04<2:28:00, 11.03s/it][2022-12-21 00:56:39,667] [INFO] [timer.py:197:stop] 0/8494, RunningAvgSamplesPerSec=5.879822977805749, CurrSamplesPerSec=5.306105564112377, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4196/5000 [14:34:15<2:27:43, 11.02s/it][2022-12-21 00:56:50,730] [INFO] [timer.py:197:stop] 0/8496, RunningAvgSamplesPerSec=5.879816157781824, CurrSamplesPerSec=5.2906237460079195, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4197/5000 [14:34:26<2:27:41, 11.04s/it][2022-12-21 00:57:01,723] [INFO] [timer.py:197:stop] 0/8498, RunningAvgSamplesPerSec=5.879818186033205, CurrSamplesPerSec=5.311964379236667, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4198/5000 [14:34:37<2:27:20, 11.02s/it][2022-12-21 00:57:12,741] [INFO] [logging.py:68:log_dist] [Rank 0] step=4250, skipped=7, lr=[1.6844444444444447e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:57:12,742] [INFO] [timer.py:197:stop] 0/8500, RunningAvgSamplesPerSec=5.8798167700243456, CurrSamplesPerSec=5.341754200961612, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4199/5000 [14:34:48<2:27:08, 11.02s/it][2022-12-21 00:57:23,765] [INFO] [timer.py:197:stop] 0/8502, RunningAvgSamplesPerSec=5.879814940640328, CurrSamplesPerSec=5.3462516552967605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4200/5000 [14:34:59<2:26:57, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.6822222222222224e-06, 'epoch': 102.43}
+
 84%|████████▍ | 4200/5000 [14:34:59<2:26:57, 11.02s/it][2022-12-21 00:57:34,733] [INFO] [timer.py:197:stop] 0/8504, RunningAvgSamplesPerSec=5.879820030056155, CurrSamplesPerSec=5.352814045747434, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4201/5000 [14:35:10<2:26:33, 11.01s/it][2022-12-21 00:57:45,722] [INFO] [timer.py:197:stop] 0/8506, RunningAvgSamplesPerSec=5.879822525254739, CurrSamplesPerSec=5.356367834255162, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4202/5000 [14:35:21<2:26:18, 11.00s/it][2022-12-21 00:57:56,755] [INFO] [timer.py:197:stop] 0/8508, RunningAvgSamplesPerSec=5.8798192972944685, CurrSamplesPerSec=5.329762202491029, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4203/5000 [14:35:32<2:26:15, 11.01s/it][2022-12-21 00:58:07,791] [INFO] [timer.py:197:stop] 0/8510, RunningAvgSamplesPerSec=5.879815689142933, CurrSamplesPerSec=5.313200204709328, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4204/5000 [14:35:43<2:26:10, 11.02s/it][2022-12-21 00:58:18,871] [INFO] [timer.py:197:stop] 0/8512, RunningAvgSamplesPerSec=5.879806381579813, CurrSamplesPerSec=5.282216833830814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4205/5000 [14:35:54<2:26:14, 11.04s/it][2022-12-21 00:58:29,942] [INFO] [timer.py:197:stop] 0/8514, RunningAvgSamplesPerSec=5.879798231727194, CurrSamplesPerSec=5.307425341071713, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4206/5000 [14:36:05<2:26:11, 11.05s/it][2022-12-21 00:58:40,971] [INFO] [timer.py:197:stop] 0/8516, RunningAvgSamplesPerSec=5.879795480167354, CurrSamplesPerSec=5.318413310792897, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4207/5000 [14:36:16<2:25:55, 11.04s/it][2022-12-21 00:58:51,968] [INFO] [timer.py:197:stop] 0/8518, RunningAvgSamplesPerSec=5.879796738958702, CurrSamplesPerSec=5.334033210184575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4208/5000 [14:36:27<2:25:34, 11.03s/it][2022-12-21 00:59:03,042] [INFO] [logging.py:68:log_dist] [Rank 0] step=4260, skipped=7, lr=[1.6622222222222224e-06], mom=[[0.9, 0.999]]
+[2022-12-21 00:59:03,044] [INFO] [timer.py:197:stop] 0/8520, RunningAvgSamplesPerSec=5.879788061594417, CurrSamplesPerSec=5.29999092170661, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4209/5000 [14:36:38<2:25:34, 11.04s/it][2022-12-21 00:59:14,018] [INFO] [timer.py:197:stop] 0/8522, RunningAvgSamplesPerSec=5.879792136754514, CurrSamplesPerSec=5.348200269462686, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4210/5000 [14:36:49<2:25:07, 11.02s/it][2022-12-21 00:59:25,053] [INFO] [timer.py:197:stop] 0/8524, RunningAvgSamplesPerSec=5.8797887582585355, CurrSamplesPerSec=5.326470845372696, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4211/5000 [14:37:00<2:24:59, 11.03s/it][2022-12-21 00:59:36,039] [INFO] [timer.py:197:stop] 0/8526, RunningAvgSamplesPerSec=5.879791592814843, CurrSamplesPerSec=5.347745103554848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4212/5000 [14:37:11<2:24:38, 11.01s/it][2022-12-21 00:59:47,094] [INFO] [timer.py:197:stop] 0/8528, RunningAvgSamplesPerSec=5.879785776145396, CurrSamplesPerSec=5.314856232922785, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4213/5000 [14:37:22<2:24:37, 11.03s/it][2022-12-21 00:59:58,091] [INFO] [timer.py:197:stop] 0/8530, RunningAvgSamplesPerSec=5.87978692428597, CurrSamplesPerSec=5.314828452118117, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4214/5000 [14:37:33<2:24:19, 11.02s/it][2022-12-21 01:00:09,127] [INFO] [timer.py:197:stop] 0/8532, RunningAvgSamplesPerSec=5.879783326940305, CurrSamplesPerSec=5.322858654294942, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4215/5000 [14:37:44<2:24:13, 11.02s/it][2022-12-21 01:00:20,130] [INFO] [timer.py:197:stop] 0/8534, RunningAvgSamplesPerSec=5.879783766407282, CurrSamplesPerSec=5.324480576662671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4216/5000 [14:37:55<2:23:57, 11.02s/it][2022-12-21 01:00:31,146] [INFO] [timer.py:197:stop] 0/8536, RunningAvgSamplesPerSec=5.879782534957491, CurrSamplesPerSec=5.3440806038921265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4217/5000 [14:38:06<2:23:46, 11.02s/it][2022-12-21 01:00:42,204] [INFO] [timer.py:197:stop] 0/8538, RunningAvgSamplesPerSec=5.879776110843483, CurrSamplesPerSec=5.2778996891090655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4218/5000 [14:38:17<2:23:44, 11.03s/it][2022-12-21 01:00:53,248] [INFO] [logging.py:68:log_dist] [Rank 0] step=4270, skipped=7, lr=[1.6400000000000002e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:00:53,250] [INFO] [timer.py:197:stop] 0/8540, RunningAvgSamplesPerSec=5.879771184666082, CurrSamplesPerSec=5.311778960400536, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4219/5000 [14:38:28<2:23:37, 11.03s/it][2022-12-21 01:01:04,276] [INFO] [timer.py:197:stop] 0/8542, RunningAvgSamplesPerSec=5.879768796328558, CurrSamplesPerSec=5.307556515341538, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4220/5000 [14:38:39<2:23:24, 11.03s/it][2022-12-21 01:01:15,285] [INFO] [timer.py:197:stop] 0/8544, RunningAvgSamplesPerSec=5.879768664269369, CurrSamplesPerSec=5.342666399489019, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4221/5000 [14:38:50<2:23:08, 11.02s/it][2022-12-21 01:01:26,303] [INFO] [timer.py:197:stop] 0/8546, RunningAvgSamplesPerSec=5.87976740937338, CurrSamplesPerSec=5.346698899311096, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4222/5000 [14:39:01<2:22:55, 11.02s/it][2022-12-21 01:01:37,335] [INFO] [timer.py:197:stop] 0/8548, RunningAvgSamplesPerSec=5.879764222048964, CurrSamplesPerSec=5.326062274158359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4223/5000 [14:39:12<2:22:46, 11.03s/it][2022-12-21 01:01:47,419] [INFO] [timer.py:197:stop] 0/8550, RunningAvgSamplesPerSec=5.879879769339955, CurrSamplesPerSec=5.356211151148356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4224/5000 [14:39:27<2:38:10, 12.23s/it][2022-12-21 01:01:58,414] [INFO] [timer.py:197:stop] 0/8552, RunningAvgSamplesPerSec=5.8798812802847955, CurrSamplesPerSec=5.354872344057848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 84%|████████▍ | 4225/5000 [14:39:38<2:32:58, 11.84s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.6266666666666666e-06, 'epoch': 103.05}
+
 84%|████████▍ | 4225/5000 [14:39:38<2:32:58, 11.84s/it][2022-12-21 01:02:09,406] [INFO] [timer.py:197:stop] 0/8554, RunningAvgSamplesPerSec=5.879883356304084, CurrSamplesPerSec=5.3117999822937, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4226/5000 [14:39:49<2:29:38, 11.60s/it][2022-12-21 01:02:20,450] [INFO] [timer.py:197:stop] 0/8556, RunningAvgSamplesPerSec=5.879878871536576, CurrSamplesPerSec=5.30135519025362, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4227/5000 [14:40:00<2:27:12, 11.43s/it][2022-12-21 01:02:31,412] [INFO] [timer.py:197:stop] 0/8558, RunningAvgSamplesPerSec=5.879884578652951, CurrSamplesPerSec=5.352786507144388, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4228/5000 [14:40:11<2:25:10, 11.28s/it][2022-12-21 01:02:42,382] [INFO] [logging.py:68:log_dist] [Rank 0] step=4280, skipped=7, lr=[1.6177777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:02:42,383] [INFO] [timer.py:197:stop] 0/8560, RunningAvgSamplesPerSec=5.879888929517951, CurrSamplesPerSec=5.331704743132573, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4229/5000 [14:40:22<2:23:57, 11.20s/it][2022-12-21 01:02:53,417] [INFO] [timer.py:197:stop] 0/8562, RunningAvgSamplesPerSec=5.879885561740268, CurrSamplesPerSec=5.31531802610411, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4230/5000 [14:40:33<2:22:55, 11.14s/it][2022-12-21 01:03:04,371] [INFO] [timer.py:197:stop] 0/8564, RunningAvgSamplesPerSec=5.879892150280649, CurrSamplesPerSec=5.344075497114541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4231/5000 [14:40:44<2:22:12, 11.10s/it][2022-12-21 01:03:15,384] [INFO] [timer.py:197:stop] 0/8566, RunningAvgSamplesPerSec=5.879891268952607, CurrSamplesPerSec=5.331303203956146, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4232/5000 [14:40:55<2:21:43, 11.07s/it][2022-12-21 01:03:26,361] [INFO] [timer.py:197:stop] 0/8568, RunningAvgSamplesPerSec=5.87989502399396, CurrSamplesPerSec=5.3672751987579, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4233/5000 [14:41:06<2:21:10, 11.04s/it][2022-12-21 01:03:37,344] [INFO] [timer.py:197:stop] 0/8570, RunningAvgSamplesPerSec=5.879898038157585, CurrSamplesPerSec=5.362581405479586, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4234/5000 [14:41:17<2:20:32, 11.01s/it][2022-12-21 01:03:48,272] [INFO] [timer.py:197:stop] 0/8572, RunningAvgSamplesPerSec=5.879907968693239, CurrSamplesPerSec=5.359425737115198, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4235/5000 [14:41:28<2:20:17, 11.00s/it][2022-12-21 01:03:59,298] [INFO] [timer.py:197:stop] 0/8574, RunningAvgSamplesPerSec=5.87990554838117, CurrSamplesPerSec=5.329536388003805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4236/5000 [14:41:39<2:20:12, 11.01s/it][2022-12-21 01:04:10,366] [INFO] [timer.py:197:stop] 0/8576, RunningAvgSamplesPerSec=5.879897826390295, CurrSamplesPerSec=5.293361506703441, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4237/5000 [14:41:50<2:20:01, 11.01s/it][2022-12-21 01:04:21,377] [INFO] [timer.py:197:stop] 0/8578, RunningAvgSamplesPerSec=5.879897284138656, CurrSamplesPerSec=5.294852702215552, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4238/5000 [14:42:01<2:19:57, 11.02s/it][2022-12-21 01:04:32,403] [INFO] [logging.py:68:log_dist] [Rank 0] step=4290, skipped=7, lr=[1.5955555555555558e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:04:32,404] [INFO] [timer.py:197:stop] 0/8580, RunningAvgSamplesPerSec=5.879894536827847, CurrSamplesPerSec=5.307809437636336, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4239/5000 [14:42:12<2:19:48, 11.02s/it][2022-12-21 01:04:43,454] [INFO] [timer.py:197:stop] 0/8582, RunningAvgSamplesPerSec=5.879889034920299, CurrSamplesPerSec=5.287650078809855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4240/5000 [14:42:23<2:19:44, 11.03s/it][2022-12-21 01:04:54,484] [INFO] [timer.py:197:stop] 0/8584, RunningAvgSamplesPerSec=5.8798862993144025, CurrSamplesPerSec=5.310039344884665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4241/5000 [14:42:34<2:19:22, 11.02s/it][2022-12-21 01:05:05,448] [INFO] [timer.py:197:stop] 0/8586, RunningAvgSamplesPerSec=5.879891725447864, CurrSamplesPerSec=5.329359262946111, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4242/5000 [14:42:45<2:19:09, 11.02s/it][2022-12-21 01:05:16,488] [INFO] [timer.py:197:stop] 0/8588, RunningAvgSamplesPerSec=5.879887849514769, CurrSamplesPerSec=5.302982470426443, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4243/5000 [14:42:56<2:18:58, 11.02s/it][2022-12-21 01:05:27,431] [INFO] [timer.py:197:stop] 0/8590, RunningAvgSamplesPerSec=5.87989594470536, CurrSamplesPerSec=5.366266824642769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4244/5000 [14:43:07<2:18:37, 11.00s/it][2022-12-21 01:05:38,439] [INFO] [timer.py:197:stop] 0/8592, RunningAvgSamplesPerSec=5.879895621243242, CurrSamplesPerSec=5.333646156856518, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4245/5000 [14:43:18<2:18:21, 11.00s/it][2022-12-21 01:05:49,451] [INFO] [timer.py:197:stop] 0/8594, RunningAvgSamplesPerSec=5.879894858601416, CurrSamplesPerSec=5.303500670002083, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4246/5000 [14:43:29<2:18:20, 11.01s/it][2022-12-21 01:06:00,501] [INFO] [timer.py:197:stop] 0/8596, RunningAvgSamplesPerSec=5.879889543442181, CurrSamplesPerSec=5.2945529756763205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4247/5000 [14:43:41<2:18:20, 11.02s/it][2022-12-21 01:06:11,555] [INFO] [timer.py:197:stop] 0/8598, RunningAvgSamplesPerSec=5.879883701203732, CurrSamplesPerSec=5.296595338869847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4248/5000 [14:43:52<2:18:13, 11.03s/it][2022-12-21 01:06:22,529] [INFO] [logging.py:68:log_dist] [Rank 0] step=4300, skipped=7, lr=[1.5733333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:06:22,530] [INFO] [timer.py:197:stop] 0/8600, RunningAvgSamplesPerSec=5.87988772803385, CurrSamplesPerSec=5.354641192473486, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▍ | 4249/5000 [14:44:03<2:17:55, 11.02s/it][2022-12-21 01:06:33,607] [INFO] [timer.py:197:stop] 0/8602, RunningAvgSamplesPerSec=5.879878853463318, CurrSamplesPerSec=5.285681841067781, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4250/5000 [14:44:14<2:17:57, 11.04s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.5711111111111113e-06, 'epoch': 103.65}
+
 85%|████████▌ | 4250/5000 [14:44:14<2:17:57, 11.04s/it][2022-12-21 01:06:44,675] [INFO] [timer.py:197:stop] 0/8604, RunningAvgSamplesPerSec=5.879871342136044, CurrSamplesPerSec=5.294750352752638, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4251/5000 [14:44:25<2:17:39, 11.03s/it][2022-12-21 01:06:55,660] [INFO] [timer.py:197:stop] 0/8606, RunningAvgSamplesPerSec=5.879874204267828, CurrSamplesPerSec=5.311861156949434, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4252/5000 [14:44:36<2:17:28, 11.03s/it][2022-12-21 01:07:06,675] [INFO] [timer.py:197:stop] 0/8608, RunningAvgSamplesPerSec=5.879873294541577, CurrSamplesPerSec=5.322786882625883, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4253/5000 [14:44:47<2:17:18, 11.03s/it][2022-12-21 01:07:17,774] [INFO] [timer.py:197:stop] 0/8610, RunningAvgSamplesPerSec=5.879861940846503, CurrSamplesPerSec=5.266259724618059, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4254/5000 [14:44:58<2:17:19, 11.04s/it][2022-12-21 01:07:28,764] [INFO] [timer.py:197:stop] 0/8612, RunningAvgSamplesPerSec=5.879864092982874, CurrSamplesPerSec=5.344479813874205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4255/5000 [14:45:09<2:16:59, 11.03s/it][2022-12-21 01:07:39,753] [INFO] [timer.py:197:stop] 0/8614, RunningAvgSamplesPerSec=5.879866575527532, CurrSamplesPerSec=5.360308230528786, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4256/5000 [14:45:20<2:16:32, 11.01s/it][2022-12-21 01:07:50,750] [INFO] [timer.py:197:stop] 0/8616, RunningAvgSamplesPerSec=5.879868026480436, CurrSamplesPerSec=5.330140013655588, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4257/5000 [14:45:31<2:16:14, 11.00s/it][2022-12-21 01:08:01,701] [INFO] [timer.py:197:stop] 0/8618, RunningAvgSamplesPerSec=5.87987494572535, CurrSamplesPerSec=5.352226403567573, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4258/5000 [14:45:42<2:16:01, 11.00s/it][2022-12-21 01:08:12,705] [INFO] [logging.py:68:log_dist] [Rank 0] step=4310, skipped=7, lr=[1.5511111111111113e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:08:12,706] [INFO] [timer.py:197:stop] 0/8620, RunningAvgSamplesPerSec=5.879875058091542, CurrSamplesPerSec=5.340283007164185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4259/5000 [14:45:53<2:15:41, 10.99s/it][2022-12-21 01:08:23,690] [INFO] [timer.py:197:stop] 0/8622, RunningAvgSamplesPerSec=5.879877931862296, CurrSamplesPerSec=5.321532457595625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4260/5000 [14:46:04<2:15:38, 11.00s/it][2022-12-21 01:08:34,684] [INFO] [timer.py:197:stop] 0/8624, RunningAvgSamplesPerSec=5.879879512683643, CurrSamplesPerSec=5.348227760895983, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4261/5000 [14:46:15<2:15:29, 11.00s/it][2022-12-21 01:08:45,736] [INFO] [timer.py:197:stop] 0/8626, RunningAvgSamplesPerSec=5.8798739388340735, CurrSamplesPerSec=5.30644688068591, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4262/5000 [14:46:26<2:15:29, 11.02s/it][2022-12-21 01:08:56,778] [INFO] [timer.py:197:stop] 0/8628, RunningAvgSamplesPerSec=5.879869601827597, CurrSamplesPerSec=5.3136442497315715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4263/5000 [14:46:37<2:15:13, 11.01s/it][2022-12-21 01:09:07,801] [INFO] [timer.py:197:stop] 0/8630, RunningAvgSamplesPerSec=5.879867601905465, CurrSamplesPerSec=5.287326172565773, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4264/5000 [14:46:48<2:15:09, 11.02s/it][2022-12-21 01:09:17,865] [INFO] [timer.py:197:stop] 0/8632, RunningAvgSamplesPerSec=5.879984543501505, CurrSamplesPerSec=6.294213258795681, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 01:09:28,904] [INFO] [timer.py:197:stop] 0/8634, RunningAvgSamplesPerSec=5.879980464743204, CurrSamplesPerSec=5.32522736054657, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4265/5000 [14:47:04<2:33:56, 12.57s/it][2022-12-21 01:09:39,993] [INFO] [timer.py:197:stop] 0/8636, RunningAvgSamplesPerSec=5.879970258240734, CurrSamplesPerSec=5.281802967784431, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4266/5000 [14:47:15<2:28:18, 12.12s/it][2022-12-21 01:09:51,080] [INFO] [timer.py:197:stop] 0/8638, RunningAvgSamplesPerSec=5.879960384968637, CurrSamplesPerSec=5.295443064304069, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4267/5000 [14:47:26<2:24:18, 11.81s/it][2022-12-21 01:10:02,080] [INFO] [logging.py:68:log_dist] [Rank 0] step=4320, skipped=7, lr=[1.5288888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:10:02,082] [INFO] [timer.py:197:stop] 0/8640, RunningAvgSamplesPerSec=5.879961180480427, CurrSamplesPerSec=5.3258177532406314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4268/5000 [14:47:37<2:21:08, 11.57s/it][2022-12-21 01:10:13,111] [INFO] [timer.py:197:stop] 0/8642, RunningAvgSamplesPerSec=5.879958571095166, CurrSamplesPerSec=5.337455117812139, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4269/5000 [14:47:48<2:18:58, 11.41s/it][2022-12-21 01:10:24,103] [INFO] [timer.py:197:stop] 0/8644, RunningAvgSamplesPerSec=5.8799605064346965, CurrSamplesPerSec=5.351816005741557, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4270/5000 [14:47:59<2:17:16, 11.28s/it][2022-12-21 01:10:35,113] [INFO] [timer.py:197:stop] 0/8646, RunningAvgSamplesPerSec=5.879960325666553, CurrSamplesPerSec=5.358436996278135, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4271/5000 [14:48:10<2:16:05, 11.20s/it][2022-12-21 01:10:46,138] [INFO] [timer.py:197:stop] 0/8648, RunningAvgSamplesPerSec=5.879958247187012, CurrSamplesPerSec=5.336512870549401, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4272/5000 [14:48:21<2:15:15, 11.15s/it][2022-12-21 01:10:57,118] [INFO] [timer.py:197:stop] 0/8650, RunningAvgSamplesPerSec=5.879961945320069, CurrSamplesPerSec=5.3632641197350805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4273/5000 [14:48:32<2:14:27, 11.10s/it][2022-12-21 01:11:08,186] [INFO] [timer.py:197:stop] 0/8652, RunningAvgSamplesPerSec=5.879954387250945, CurrSamplesPerSec=5.305256763635032, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 85%|████████▌ | 4274/5000 [14:48:43<2:14:10, 11.09s/it][2022-12-21 01:11:19,139] [INFO] [timer.py:197:stop] 0/8654, RunningAvgSamplesPerSec=5.879961193902788, CurrSamplesPerSec=5.369980301737027, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4275/5000 [14:48:54<2:13:29, 11.05s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.5133333333333334e-06, 'epoch': 104.27}
+
 86%|████████▌ | 4275/5000 [14:48:54<2:13:29, 11.05s/it][2022-12-21 01:11:30,152] [INFO] [timer.py:197:stop] 0/8656, RunningAvgSamplesPerSec=5.879960705135125, CurrSamplesPerSec=5.317077742918941, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4276/5000 [14:49:05<2:13:11, 11.04s/it][2022-12-21 01:11:41,167] [INFO] [timer.py:197:stop] 0/8658, RunningAvgSamplesPerSec=5.879959905847576, CurrSamplesPerSec=5.3146177904674685, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4277/5000 [14:49:16<2:12:55, 11.03s/it][2022-12-21 01:11:52,217] [INFO] [logging.py:68:log_dist] [Rank 0] step=4330, skipped=7, lr=[1.506666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:11:52,219] [INFO] [timer.py:197:stop] 0/8660, RunningAvgSamplesPerSec=5.879954485085444, CurrSamplesPerSec=5.303004470371737, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4278/5000 [14:49:27<2:12:48, 11.04s/it][2022-12-21 01:12:03,232] [INFO] [timer.py:197:stop] 0/8662, RunningAvgSamplesPerSec=5.879953796473493, CurrSamplesPerSec=5.3417982089827865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4279/5000 [14:49:38<2:12:32, 11.03s/it][2022-12-21 01:12:14,278] [INFO] [timer.py:197:stop] 0/8664, RunningAvgSamplesPerSec=5.879949135142992, CurrSamplesPerSec=5.292872003080344, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4280/5000 [14:49:49<2:12:24, 11.03s/it][2022-12-21 01:12:25,285] [INFO] [timer.py:197:stop] 0/8666, RunningAvgSamplesPerSec=5.87994891615027, CurrSamplesPerSec=5.310129260830709, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4281/5000 [14:50:00<2:12:08, 11.03s/it][2022-12-21 01:12:36,312] [INFO] [timer.py:197:stop] 0/8668, RunningAvgSamplesPerSec=5.879946425276437, CurrSamplesPerSec=5.287395324779284, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4282/5000 [14:50:11<2:11:57, 11.03s/it][2022-12-21 01:12:47,373] [INFO] [timer.py:197:stop] 0/8670, RunningAvgSamplesPerSec=5.879939825754816, CurrSamplesPerSec=5.30556105894182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4283/5000 [14:50:22<2:11:53, 11.04s/it][2022-12-21 01:12:58,407] [INFO] [timer.py:197:stop] 0/8672, RunningAvgSamplesPerSec=5.879936613663335, CurrSamplesPerSec=5.297191107198777, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4284/5000 [14:50:34<2:11:41, 11.04s/it][2022-12-21 01:13:09,422] [INFO] [timer.py:197:stop] 0/8674, RunningAvgSamplesPerSec=5.879935544660305, CurrSamplesPerSec=5.33250482504106, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4285/5000 [14:50:45<2:11:26, 11.03s/it][2022-12-21 01:13:20,470] [INFO] [timer.py:197:stop] 0/8676, RunningAvgSamplesPerSec=5.879930340519865, CurrSamplesPerSec=5.322131738987805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4286/5000 [14:50:56<2:11:19, 11.04s/it][2022-12-21 01:13:31,532] [INFO] [timer.py:197:stop] 0/8678, RunningAvgSamplesPerSec=5.879923446591428, CurrSamplesPerSec=5.302201485881988, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4287/5000 [14:51:07<2:11:13, 11.04s/it][2022-12-21 01:13:42,478] [INFO] [logging.py:68:log_dist] [Rank 0] step=4340, skipped=7, lr=[1.4844444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:13:42,479] [INFO] [timer.py:197:stop] 0/8680, RunningAvgSamplesPerSec=5.879930888612194, CurrSamplesPerSec=5.355634516567394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4288/5000 [14:51:18<2:10:42, 11.01s/it][2022-12-21 01:13:53,475] [INFO] [timer.py:197:stop] 0/8682, RunningAvgSamplesPerSec=5.8799323363587686, CurrSamplesPerSec=5.327342095534491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4289/5000 [14:51:29<2:10:27, 11.01s/it][2022-12-21 01:14:04,465] [INFO] [timer.py:197:stop] 0/8684, RunningAvgSamplesPerSec=5.879934408790332, CurrSamplesPerSec=5.327639624766188, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4290/5000 [14:51:40<2:10:12, 11.00s/it][2022-12-21 01:14:15,460] [INFO] [timer.py:197:stop] 0/8686, RunningAvgSamplesPerSec=5.8799360204027735, CurrSamplesPerSec=5.308572968385374, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4291/5000 [14:51:51<2:09:59, 11.00s/it][2022-12-21 01:14:26,507] [INFO] [timer.py:197:stop] 0/8688, RunningAvgSamplesPerSec=5.879931048016617, CurrSamplesPerSec=5.317637044882575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4292/5000 [14:52:02<2:09:58, 11.01s/it][2022-12-21 01:14:37,666] [INFO] [timer.py:197:stop] 0/8690, RunningAvgSamplesPerSec=5.87991216257924, CurrSamplesPerSec=5.208657896838576, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4293/5000 [14:52:13<2:10:18, 11.06s/it][2022-12-21 01:14:48,704] [INFO] [timer.py:197:stop] 0/8692, RunningAvgSamplesPerSec=5.879908475357499, CurrSamplesPerSec=5.3174567071787235, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4294/5000 [14:52:24<2:10:02, 11.05s/it][2022-12-21 01:14:59,687] [INFO] [timer.py:197:stop] 0/8694, RunningAvgSamplesPerSec=5.879911392970317, CurrSamplesPerSec=5.319907058992474, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4295/5000 [14:52:35<2:09:37, 11.03s/it][2022-12-21 01:15:10,692] [INFO] [timer.py:197:stop] 0/8696, RunningAvgSamplesPerSec=5.8799118758698565, CurrSamplesPerSec=5.371700297593712, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4296/5000 [14:52:46<2:09:20, 11.02s/it][2022-12-21 01:15:21,705] [INFO] [timer.py:197:stop] 0/8698, RunningAvgSamplesPerSec=5.879911121242615, CurrSamplesPerSec=5.326974194893718, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4297/5000 [14:52:57<2:09:07, 11.02s/it][2022-12-21 01:15:32,692] [INFO] [logging.py:68:log_dist] [Rank 0] step=4350, skipped=7, lr=[1.4622222222222224e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:15:32,694] [INFO] [timer.py:197:stop] 0/8700, RunningAvgSamplesPerSec=5.879913337321643, CurrSamplesPerSec=5.348110977532254, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4298/5000 [14:53:08<2:08:49, 11.01s/it][2022-12-21 01:15:43,660] [INFO] [timer.py:197:stop] 0/8702, RunningAvgSamplesPerSec=5.8799184575484045, CurrSamplesPerSec=5.343968895360794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4299/5000 [14:53:19<2:08:29, 11.00s/it][2022-12-21 01:15:54,654] [INFO] [timer.py:197:stop] 0/8704, RunningAvgSamplesPerSec=5.879920080703591, CurrSamplesPerSec=5.338558224402548, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4300/5000 [14:53:30<2:08:17, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.457777777777778e-06, 'epoch': 104.87}
+
 86%|████████▌ | 4300/5000 [14:53:30<2:08:17, 11.00s/it][2022-12-21 01:16:05,646] [INFO] [timer.py:197:stop] 0/8706, RunningAvgSamplesPerSec=5.879922194563794, CurrSamplesPerSec=5.361453143773343, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4301/5000 [14:53:41<2:08:05, 11.00s/it][2022-12-21 01:16:16,609] [INFO] [timer.py:197:stop] 0/8708, RunningAvgSamplesPerSec=5.87992776655437, CurrSamplesPerSec=5.348025950517929, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4302/5000 [14:53:52<2:07:47, 10.99s/it][2022-12-21 01:16:27,604] [INFO] [timer.py:197:stop] 0/8710, RunningAvgSamplesPerSec=5.879929449259168, CurrSamplesPerSec=5.3373757355021905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4303/5000 [14:54:03<2:07:38, 10.99s/it][2022-12-21 01:16:38,618] [INFO] [timer.py:197:stop] 0/8712, RunningAvgSamplesPerSec=5.879928735516116, CurrSamplesPerSec=5.325228839534356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4304/5000 [14:54:14<2:07:33, 11.00s/it][2022-12-21 01:16:49,612] [INFO] [timer.py:197:stop] 0/8714, RunningAvgSamplesPerSec=5.879930430650143, CurrSamplesPerSec=5.347734875997547, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4305/5000 [14:54:25<2:07:21, 11.00s/it][2022-12-21 01:16:59,729] [INFO] [timer.py:197:stop] 0/8716, RunningAvgSamplesPerSec=5.880039613304419, CurrSamplesPerSec=5.307166579484311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4306/5000 [14:54:40<2:21:07, 12.20s/it][2022-12-21 01:17:10,667] [INFO] [timer.py:197:stop] 0/8718, RunningAvgSamplesPerSec=5.880048107645109, CurrSamplesPerSec=5.3549135774452425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4307/5000 [14:54:51<2:16:26, 11.81s/it][2022-12-21 01:17:21,601] [INFO] [logging.py:68:log_dist] [Rank 0] step=4360, skipped=7, lr=[1.44e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:17:21,603] [INFO] [timer.py:197:stop] 0/8720, RunningAvgSamplesPerSec=5.880056886831902, CurrSamplesPerSec=5.327961932730861, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4308/5000 [14:55:02<2:13:22, 11.56s/it][2022-12-21 01:17:32,540] [INFO] [timer.py:197:stop] 0/8722, RunningAvgSamplesPerSec=5.880065445682634, CurrSamplesPerSec=5.371005332199622, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4309/5000 [14:55:13<2:11:02, 11.38s/it][2022-12-21 01:17:43,509] [INFO] [timer.py:197:stop] 0/8724, RunningAvgSamplesPerSec=5.880070172640005, CurrSamplesPerSec=5.348568976547431, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4310/5000 [14:55:24<2:09:25, 11.25s/it][2022-12-21 01:17:54,469] [INFO] [timer.py:197:stop] 0/8726, RunningAvgSamplesPerSec=5.8800761015477, CurrSamplesPerSec=5.3544880281562595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4311/5000 [14:55:34<2:08:16, 11.17s/it][2022-12-21 01:18:05,422] [INFO] [timer.py:197:stop] 0/8728, RunningAvgSamplesPerSec=5.8800827255545425, CurrSamplesPerSec=5.37171857161256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▌ | 4312/5000 [14:55:45<2:07:22, 11.11s/it][2022-12-21 01:18:16,450] [INFO] [timer.py:197:stop] 0/8730, RunningAvgSamplesPerSec=5.880080170066392, CurrSamplesPerSec=5.313044143981737, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4313/5000 [14:55:57<2:07:00, 11.09s/it][2022-12-21 01:18:27,491] [INFO] [timer.py:197:stop] 0/8732, RunningAvgSamplesPerSec=5.880075799377713, CurrSamplesPerSec=5.32502052163232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4314/5000 [14:56:08<2:06:31, 11.07s/it][2022-12-21 01:18:38,465] [INFO] [timer.py:197:stop] 0/8734, RunningAvgSamplesPerSec=5.880079823100469, CurrSamplesPerSec=5.354751424935762, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4315/5000 [14:56:19<2:06:09, 11.05s/it][2022-12-21 01:18:49,532] [INFO] [timer.py:197:stop] 0/8736, RunningAvgSamplesPerSec=5.8800723236712535, CurrSamplesPerSec=5.304174292221401, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4316/5000 [14:56:30<2:05:56, 11.05s/it][2022-12-21 01:19:00,545] [INFO] [timer.py:197:stop] 0/8738, RunningAvgSamplesPerSec=5.880071643207836, CurrSamplesPerSec=5.331397653406494, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4317/5000 [14:56:41<2:05:38, 11.04s/it][2022-12-21 01:19:11,558] [INFO] [logging.py:68:log_dist] [Rank 0] step=4370, skipped=7, lr=[1.417777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:19:11,560] [INFO] [timer.py:197:stop] 0/8740, RunningAvgSamplesPerSec=5.8800707027087755, CurrSamplesPerSec=5.32888190923151, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4318/5000 [14:56:52<2:05:23, 11.03s/it][2022-12-21 01:19:22,564] [INFO] [timer.py:197:stop] 0/8742, RunningAvgSamplesPerSec=5.880070989803587, CurrSamplesPerSec=5.342152638177304, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4319/5000 [14:57:03<2:05:00, 11.01s/it][2022-12-21 01:19:33,602] [INFO] [timer.py:197:stop] 0/8744, RunningAvgSamplesPerSec=5.880067259631201, CurrSamplesPerSec=5.286577486558131, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4320/5000 [14:57:14<2:05:01, 11.03s/it][2022-12-21 01:19:44,535] [INFO] [timer.py:197:stop] 0/8746, RunningAvgSamplesPerSec=5.8800763515501755, CurrSamplesPerSec=5.407855504425435, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4321/5000 [14:57:25<2:04:31, 11.00s/it][2022-12-21 01:19:55,512] [INFO] [timer.py:197:stop] 0/8748, RunningAvgSamplesPerSec=5.8800800785827, CurrSamplesPerSec=5.375387931997748, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4322/5000 [14:57:36<2:04:08, 10.99s/it][2022-12-21 01:20:06,545] [INFO] [timer.py:197:stop] 0/8750, RunningAvgSamplesPerSec=5.880076926461079, CurrSamplesPerSec=5.296940868177069, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4323/5000 [14:57:47<2:04:11, 11.01s/it][2022-12-21 01:20:17,530] [INFO] [timer.py:197:stop] 0/8752, RunningAvgSamplesPerSec=5.880080131411976, CurrSamplesPerSec=5.360824205325838, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4324/5000 [14:57:58<2:03:54, 11.00s/it][2022-12-21 01:20:28,548] [INFO] [timer.py:197:stop] 0/8754, RunningAvgSamplesPerSec=5.880078761680582, CurrSamplesPerSec=5.320887747784542, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 86%|████████▋ | 4325/5000 [14:58:09<2:03:52, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.4022222222222223e-06, 'epoch': 105.48}
+
 86%|████████▋ | 4325/5000 [14:58:09<2:03:52, 11.01s/it][2022-12-21 01:20:39,582] [INFO] [timer.py:197:stop] 0/8756, RunningAvgSamplesPerSec=5.880075557846176, CurrSamplesPerSec=5.331276521533824, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4326/5000 [14:58:20<2:03:46, 11.02s/it][2022-12-21 01:20:50,656] [INFO] [timer.py:197:stop] 0/8758, RunningAvgSamplesPerSec=5.8800673407923165, CurrSamplesPerSec=5.295084778614407, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4327/5000 [14:58:31<2:03:45, 11.03s/it][2022-12-21 01:21:01,705] [INFO] [logging.py:68:log_dist] [Rank 0] step=4380, skipped=7, lr=[1.3955555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:21:01,706] [INFO] [timer.py:197:stop] 0/8760, RunningAvgSamplesPerSec=5.8800618286276665, CurrSamplesPerSec=5.309685172906127, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4328/5000 [14:58:42<2:03:25, 11.02s/it][2022-12-21 01:21:12,638] [INFO] [timer.py:197:stop] 0/8762, RunningAvgSamplesPerSec=5.880070949862246, CurrSamplesPerSec=5.3611417611812575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4329/5000 [14:58:53<2:03:02, 11.00s/it][2022-12-21 01:21:23,650] [INFO] [timer.py:197:stop] 0/8764, RunningAvgSamplesPerSec=5.880070233070502, CurrSamplesPerSec=5.315985810648288, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4330/5000 [14:59:04<2:02:51, 11.00s/it][2022-12-21 01:21:34,651] [INFO] [timer.py:197:stop] 0/8766, RunningAvgSamplesPerSec=5.880070864567155, CurrSamplesPerSec=5.316793818439232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4331/5000 [14:59:15<2:02:39, 11.00s/it][2022-12-21 01:21:45,667] [INFO] [timer.py:197:stop] 0/8768, RunningAvgSamplesPerSec=5.880069792628753, CurrSamplesPerSec=5.300094311052882, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4332/5000 [14:59:26<2:02:41, 11.02s/it][2022-12-21 01:21:56,709] [INFO] [timer.py:197:stop] 0/8770, RunningAvgSamplesPerSec=5.880065573477631, CurrSamplesPerSec=5.321472958851297, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4333/5000 [14:59:37<2:02:35, 11.03s/it][2022-12-21 01:22:07,683] [INFO] [timer.py:197:stop] 0/8772, RunningAvgSamplesPerSec=5.880069557703866, CurrSamplesPerSec=5.384811429313304, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4334/5000 [14:59:48<2:02:09, 11.01s/it][2022-12-21 01:22:18,717] [INFO] [timer.py:197:stop] 0/8774, RunningAvgSamplesPerSec=5.880066126120435, CurrSamplesPerSec=5.3133823572609025, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4335/5000 [14:59:59<2:02:00, 11.01s/it][2022-12-21 01:22:29,695] [INFO] [timer.py:197:stop] 0/8776, RunningAvgSamplesPerSec=5.880069581508801, CurrSamplesPerSec=5.34352232067107, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4336/5000 [15:00:10<2:01:34, 10.99s/it][2022-12-21 01:22:40,672] [INFO] [timer.py:197:stop] 0/8778, RunningAvgSamplesPerSec=5.880073123357254, CurrSamplesPerSec=5.30411874440192, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4337/5000 [15:00:21<2:01:27, 10.99s/it][2022-12-21 01:22:51,689] [INFO] [logging.py:68:log_dist] [Rank 0] step=4390, skipped=7, lr=[1.3733333333333335e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:22:51,691] [INFO] [timer.py:197:stop] 0/8780, RunningAvgSamplesPerSec=5.880071600076886, CurrSamplesPerSec=5.293554619388873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4338/5000 [15:00:32<2:01:16, 10.99s/it][2022-12-21 01:23:02,648] [INFO] [timer.py:197:stop] 0/8782, RunningAvgSamplesPerSec=5.880077681704295, CurrSamplesPerSec=5.323924902590398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4339/5000 [15:00:43<2:01:08, 11.00s/it][2022-12-21 01:23:13,684] [INFO] [timer.py:197:stop] 0/8784, RunningAvgSamplesPerSec=5.880074089104433, CurrSamplesPerSec=5.29987958384461, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4340/5000 [15:00:54<2:01:00, 11.00s/it][2022-12-21 01:23:24,682] [INFO] [timer.py:197:stop] 0/8786, RunningAvgSamplesPerSec=5.880075109592547, CurrSamplesPerSec=5.312404222621448, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4341/5000 [15:01:05<2:00:53, 11.01s/it][2022-12-21 01:23:35,706] [INFO] [timer.py:197:stop] 0/8788, RunningAvgSamplesPerSec=5.880072963403767, CurrSamplesPerSec=5.309350580509899, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4342/5000 [15:01:16<2:00:44, 11.01s/it][2022-12-21 01:23:46,726] [INFO] [timer.py:197:stop] 0/8790, RunningAvgSamplesPerSec=5.880071248132091, CurrSamplesPerSec=5.307777952219698, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4343/5000 [15:01:27<2:00:41, 11.02s/it][2022-12-21 01:23:57,722] [INFO] [timer.py:197:stop] 0/8792, RunningAvgSamplesPerSec=5.880072832429559, CurrSamplesPerSec=5.35689951360828, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4344/5000 [15:01:38<2:00:24, 11.01s/it][2022-12-21 01:24:08,783] [INFO] [timer.py:197:stop] 0/8794, RunningAvgSamplesPerSec=5.880066115349277, CurrSamplesPerSec=5.294030885051754, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4345/5000 [15:01:49<2:00:19, 11.02s/it][2022-12-21 01:24:19,765] [INFO] [timer.py:197:stop] 0/8796, RunningAvgSamplesPerSec=5.88006928307179, CurrSamplesPerSec=5.346399237549614, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4346/5000 [15:02:00<2:00:00, 11.01s/it][2022-12-21 01:24:29,864] [INFO] [timer.py:197:stop] 0/8798, RunningAvgSamplesPerSec=5.880179758382183, CurrSamplesPerSec=6.260061498146413, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 01:24:40,866] [INFO] [logging.py:68:log_dist] [Rank 0] step=4400, skipped=7, lr=[1.351111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:24:40,868] [INFO] [timer.py:197:stop] 0/8800, RunningAvgSamplesPerSec=5.880179736182276, CurrSamplesPerSec=5.338708355172277, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4347/5000 [15:02:16<2:16:40, 12.56s/it][2022-12-21 01:24:51,904] [INFO] [timer.py:197:stop] 0/8802, RunningAvgSamplesPerSec=5.88017612778753, CurrSamplesPerSec=5.313580930390437, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4348/5000 [15:02:27<2:11:29, 12.10s/it][2022-12-21 01:25:02,913] [INFO] [timer.py:197:stop] 0/8804, RunningAvgSamplesPerSec=5.880175885743819, CurrSamplesPerSec=5.3382061828149014, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4349/5000 [15:02:38<2:07:44, 11.77s/it][2022-12-21 01:25:13,936] [INFO] [timer.py:197:stop] 0/8806, RunningAvgSamplesPerSec=5.880173870558577, CurrSamplesPerSec=5.310832096676982, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4350/5000 [15:02:49<2:05:06, 11.55s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.3444444444444446e-06, 'epoch': 106.1}
+
 87%|████████▋ | 4350/5000 [15:02:49<2:05:06, 11.55s/it][2022-12-21 01:25:24,927] [INFO] [timer.py:197:stop] 0/8808, RunningAvgSamplesPerSec=5.880176013053391, CurrSamplesPerSec=5.3212305468701615, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4351/5000 [15:03:00<2:03:06, 11.38s/it][2022-12-21 01:25:35,986] [INFO] [timer.py:197:stop] 0/8810, RunningAvgSamplesPerSec=5.880169659993011, CurrSamplesPerSec=5.323630110837856, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4352/5000 [15:03:11<2:01:52, 11.28s/it][2022-12-21 01:25:46,999] [INFO] [timer.py:197:stop] 0/8812, RunningAvgSamplesPerSec=5.8801687924745965, CurrSamplesPerSec=5.322088054410409, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4353/5000 [15:03:22<2:00:48, 11.20s/it][2022-12-21 01:25:58,035] [INFO] [timer.py:197:stop] 0/8814, RunningAvgSamplesPerSec=5.880165118784875, CurrSamplesPerSec=5.332918199429803, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4354/5000 [15:03:33<2:00:04, 11.15s/it][2022-12-21 01:26:09,084] [INFO] [timer.py:197:stop] 0/8816, RunningAvgSamplesPerSec=5.880159984385435, CurrSamplesPerSec=5.290290091909606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4355/5000 [15:03:44<1:59:33, 11.12s/it][2022-12-21 01:26:20,047] [INFO] [timer.py:197:stop] 0/8818, RunningAvgSamplesPerSec=5.880165469730546, CurrSamplesPerSec=5.343558060918867, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4356/5000 [15:03:55<1:58:51, 11.07s/it][2022-12-21 01:26:31,050] [INFO] [logging.py:68:log_dist] [Rank 0] step=4410, skipped=7, lr=[1.3288888888888891e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:26:31,051] [INFO] [timer.py:197:stop] 0/8820, RunningAvgSamplesPerSec=5.880165693390253, CurrSamplesPerSec=5.342179642209602, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4357/5000 [15:04:06<1:58:27, 11.05s/it][2022-12-21 01:26:42,056] [INFO] [timer.py:197:stop] 0/8822, RunningAvgSamplesPerSec=5.880165953633696, CurrSamplesPerSec=5.324351943954912, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4358/5000 [15:04:17<1:58:06, 11.04s/it][2022-12-21 01:26:53,105] [INFO] [timer.py:197:stop] 0/8824, RunningAvgSamplesPerSec=5.880160880174749, CurrSamplesPerSec=5.304343667849356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4359/5000 [15:04:28<1:57:57, 11.04s/it][2022-12-21 01:27:04,138] [INFO] [timer.py:197:stop] 0/8826, RunningAvgSamplesPerSec=5.880157562480806, CurrSamplesPerSec=5.3034748938328855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4360/5000 [15:04:39<1:57:45, 11.04s/it][2022-12-21 01:27:15,137] [INFO] [timer.py:197:stop] 0/8828, RunningAvgSamplesPerSec=5.880158614595609, CurrSamplesPerSec=5.3167371635483684, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4361/5000 [15:04:50<1:57:26, 11.03s/it][2022-12-21 01:27:26,217] [INFO] [timer.py:197:stop] 0/8830, RunningAvgSamplesPerSec=5.880149787502633, CurrSamplesPerSec=5.3169802194450195, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4362/5000 [15:05:01<1:57:25, 11.04s/it][2022-12-21 01:27:37,189] [INFO] [timer.py:197:stop] 0/8832, RunningAvgSamplesPerSec=5.880154016741563, CurrSamplesPerSec=5.3427170154286205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4363/5000 [15:05:12<1:57:00, 11.02s/it][2022-12-21 01:27:48,237] [INFO] [timer.py:197:stop] 0/8834, RunningAvgSamplesPerSec=5.880148927978703, CurrSamplesPerSec=5.325301522231983, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4364/5000 [15:05:23<1:56:54, 11.03s/it][2022-12-21 01:27:59,248] [INFO] [timer.py:197:stop] 0/8836, RunningAvgSamplesPerSec=5.880148352016119, CurrSamplesPerSec=5.304371968049767, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4365/5000 [15:05:34<1:56:40, 11.02s/it][2022-12-21 01:28:10,282] [INFO] [timer.py:197:stop] 0/8838, RunningAvgSamplesPerSec=5.8801450143887655, CurrSamplesPerSec=5.3021426280758535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4366/5000 [15:05:45<1:56:31, 11.03s/it][2022-12-21 01:28:21,234] [INFO] [logging.py:68:log_dist] [Rank 0] step=4420, skipped=7, lr=[1.3066666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:28:21,236] [INFO] [timer.py:197:stop] 0/8840, RunningAvgSamplesPerSec=5.880151304658582, CurrSamplesPerSec=5.371329254410301, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4367/5000 [15:05:56<1:56:06, 11.01s/it][2022-12-21 01:28:32,247] [INFO] [timer.py:197:stop] 0/8842, RunningAvgSamplesPerSec=5.880150740759115, CurrSamplesPerSec=5.354338717403136, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4368/5000 [15:06:07<1:55:56, 11.01s/it][2022-12-21 01:28:43,215] [INFO] [timer.py:197:stop] 0/8844, RunningAvgSamplesPerSec=5.880155449493827, CurrSamplesPerSec=5.39016434576847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4369/5000 [15:06:18<1:55:37, 11.00s/it][2022-12-21 01:28:54,270] [INFO] [timer.py:197:stop] 0/8846, RunningAvgSamplesPerSec=5.88014958794691, CurrSamplesPerSec=5.307182948082875, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4370/5000 [15:06:29<1:55:38, 11.01s/it][2022-12-21 01:29:05,339] [INFO] [timer.py:197:stop] 0/8848, RunningAvgSamplesPerSec=5.8801419687046685, CurrSamplesPerSec=5.292720056191641, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4371/5000 [15:06:40<1:55:37, 11.03s/it][2022-12-21 01:29:16,327] [INFO] [timer.py:197:stop] 0/8850, RunningAvgSamplesPerSec=5.8801443059974146, CurrSamplesPerSec=5.336876360311179, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4372/5000 [15:06:51<1:55:18, 11.02s/it][2022-12-21 01:29:27,354] [INFO] [timer.py:197:stop] 0/8852, RunningAvgSamplesPerSec=5.8801418618885, CurrSamplesPerSec=5.338357992401362, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4373/5000 [15:07:02<1:55:09, 11.02s/it][2022-12-21 01:29:38,347] [INFO] [timer.py:197:stop] 0/8854, RunningAvgSamplesPerSec=5.8801436356633605, CurrSamplesPerSec=5.364568745841959, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 87%|████████▋ | 4374/5000 [15:07:13<1:54:53, 11.01s/it][2022-12-21 01:29:49,329] [INFO] [timer.py:197:stop] 0/8856, RunningAvgSamplesPerSec=5.880146807756508, CurrSamplesPerSec=5.337991753215954, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4375/5000 [15:07:24<1:54:36, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.288888888888889e-06, 'epoch': 106.7}
+
 88%|████████▊ | 4375/5000 [15:07:24<1:54:36, 11.00s/it][2022-12-21 01:30:00,327] [INFO] [timer.py:197:stop] 0/8858, RunningAvgSamplesPerSec=5.880148016976436, CurrSamplesPerSec=5.35105193287398, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4376/5000 [15:07:35<1:54:24, 11.00s/it][2022-12-21 01:30:11,374] [INFO] [logging.py:68:log_dist] [Rank 0] step=4430, skipped=7, lr=[1.2844444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:30:11,375] [INFO] [timer.py:197:stop] 0/8860, RunningAvgSamplesPerSec=5.8801429269202545, CurrSamplesPerSec=5.313219765527942, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4377/5000 [15:07:46<1:54:22, 11.02s/it][2022-12-21 01:30:22,386] [INFO] [timer.py:197:stop] 0/8862, RunningAvgSamplesPerSec=5.880142474086831, CurrSamplesPerSec=5.297064834929426, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4378/5000 [15:07:57<1:54:10, 11.01s/it][2022-12-21 01:30:33,367] [INFO] [timer.py:197:stop] 0/8864, RunningAvgSamplesPerSec=5.880145551747417, CurrSamplesPerSec=5.315032395239184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4379/5000 [15:08:08<1:53:53, 11.00s/it][2022-12-21 01:30:44,430] [INFO] [timer.py:197:stop] 0/8866, RunningAvgSamplesPerSec=5.880138656167615, CurrSamplesPerSec=5.303272255766982, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4380/5000 [15:08:20<1:53:53, 11.02s/it][2022-12-21 01:30:55,377] [INFO] [timer.py:197:stop] 0/8868, RunningAvgSamplesPerSec=5.880145835242804, CurrSamplesPerSec=5.346840755184924, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4381/5000 [15:08:30<1:53:28, 11.00s/it][2022-12-21 01:31:06,379] [INFO] [timer.py:197:stop] 0/8870, RunningAvgSamplesPerSec=5.880146494094314, CurrSamplesPerSec=5.320184567786748, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4382/5000 [15:08:41<1:53:18, 11.00s/it][2022-12-21 01:31:17,377] [INFO] [timer.py:197:stop] 0/8872, RunningAvgSamplesPerSec=5.880147581963907, CurrSamplesPerSec=5.33573216095245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4383/5000 [15:08:52<1:53:06, 11.00s/it][2022-12-21 01:31:28,342] [INFO] [timer.py:197:stop] 0/8874, RunningAvgSamplesPerSec=5.880152690939088, CurrSamplesPerSec=5.335794100218102, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4384/5000 [15:09:03<1:52:49, 10.99s/it][2022-12-21 01:31:39,351] [INFO] [timer.py:197:stop] 0/8876, RunningAvgSamplesPerSec=5.880152326146666, CurrSamplesPerSec=5.343331501367405, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4385/5000 [15:09:14<1:52:42, 11.00s/it][2022-12-21 01:31:50,355] [INFO] [timer.py:197:stop] 0/8878, RunningAvgSamplesPerSec=5.880152626853685, CurrSamplesPerSec=5.329364764861524, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4386/5000 [15:09:25<1:52:32, 11.00s/it][2022-12-21 01:32:01,315] [INFO] [logging.py:68:log_dist] [Rank 0] step=4440, skipped=7, lr=[1.2622222222222224e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:32:01,317] [INFO] [timer.py:197:stop] 0/8880, RunningAvgSamplesPerSec=5.880158026351529, CurrSamplesPerSec=5.354494222916769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4387/5000 [15:09:36<1:52:15, 10.99s/it][2022-12-21 01:32:11,388] [INFO] [timer.py:197:stop] 0/8882, RunningAvgSamplesPerSec=5.880270546481734, CurrSamplesPerSec=5.348745675590254, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4388/5000 [15:09:51<2:04:22, 12.19s/it][2022-12-21 01:32:22,412] [INFO] [timer.py:197:stop] 0/8884, RunningAvgSamplesPerSec=5.8802683284069275, CurrSamplesPerSec=5.312713544260577, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4389/5000 [15:10:02<2:00:32, 11.84s/it][2022-12-21 01:32:33,409] [INFO] [timer.py:197:stop] 0/8886, RunningAvgSamplesPerSec=5.880269280783198, CurrSamplesPerSec=5.320362559790182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4390/5000 [15:10:13<1:57:53, 11.60s/it][2022-12-21 01:32:44,407] [INFO] [timer.py:197:stop] 0/8888, RunningAvgSamplesPerSec=5.880270301123228, CurrSamplesPerSec=5.351869782840479, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4391/5000 [15:10:24<1:55:46, 11.41s/it][2022-12-21 01:32:55,419] [INFO] [timer.py:197:stop] 0/8890, RunningAvgSamplesPerSec=5.880269597700981, CurrSamplesPerSec=5.31078334391129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4392/5000 [15:10:35<1:54:29, 11.30s/it][2022-12-21 01:33:06,447] [INFO] [timer.py:197:stop] 0/8892, RunningAvgSamplesPerSec=5.88026699943028, CurrSamplesPerSec=5.325827263141906, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4393/5000 [15:10:47<1:53:33, 11.22s/it][2022-12-21 01:33:17,496] [INFO] [timer.py:197:stop] 0/8894, RunningAvgSamplesPerSec=5.880261975719917, CurrSamplesPerSec=5.330734036868159, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4394/5000 [15:10:58<1:52:46, 11.17s/it][2022-12-21 01:33:28,470] [INFO] [timer.py:197:stop] 0/8896, RunningAvgSamplesPerSec=5.880265714245118, CurrSamplesPerSec=5.377370558305674, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4395/5000 [15:11:08<1:51:53, 11.10s/it][2022-12-21 01:33:39,415] [INFO] [timer.py:197:stop] 0/8898, RunningAvgSamplesPerSec=5.880273135573905, CurrSamplesPerSec=5.36779960003577, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4396/5000 [15:11:19<1:51:23, 11.07s/it][2022-12-21 01:33:50,448] [INFO] [logging.py:68:log_dist] [Rank 0] step=4450, skipped=7, lr=[1.2400000000000002e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:33:50,450] [INFO] [timer.py:197:stop] 0/8900, RunningAvgSamplesPerSec=5.880269815857711, CurrSamplesPerSec=5.332768606013655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4397/5000 [15:11:30<1:50:55, 11.04s/it][2022-12-21 01:34:01,412] [INFO] [timer.py:197:stop] 0/8902, RunningAvgSamplesPerSec=5.880275096372063, CurrSamplesPerSec=5.341168559380637, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4398/5000 [15:11:41<1:50:33, 11.02s/it][2022-12-21 01:34:12,405] [INFO] [timer.py:197:stop] 0/8904, RunningAvgSamplesPerSec=5.88027673682472, CurrSamplesPerSec=5.326456471395012, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4399/5000 [15:11:52<1:50:24, 11.02s/it][2022-12-21 01:34:23,396] [INFO] [timer.py:197:stop] 0/8906, RunningAvgSamplesPerSec=5.8802786888204555, CurrSamplesPerSec=5.359688763810507, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4400/5000 [15:12:03<1:50:05, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.2333333333333335e-06, 'epoch': 107.31}
+
 88%|████████▊ | 4400/5000 [15:12:03<1:50:05, 11.01s/it][2022-12-21 01:34:34,393] [INFO] [timer.py:197:stop] 0/8908, RunningAvgSamplesPerSec=5.880281069535819, CurrSamplesPerSec=5.347503061857896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4401/5000 [15:12:14<1:49:47, 11.00s/it][2022-12-21 01:34:45,343] [INFO] [timer.py:197:stop] 0/8910, RunningAvgSamplesPerSec=5.88028800914735, CurrSamplesPerSec=5.361345847612888, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4402/5000 [15:12:25<1:49:33, 10.99s/it][2022-12-21 01:34:56,351] [INFO] [timer.py:197:stop] 0/8912, RunningAvgSamplesPerSec=5.880287755410082, CurrSamplesPerSec=5.3359336809580835, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4403/5000 [15:12:36<1:49:25, 11.00s/it][2022-12-21 01:35:07,384] [INFO] [timer.py:197:stop] 0/8914, RunningAvgSamplesPerSec=5.8802845052551795, CurrSamplesPerSec=5.315763898936041, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4404/5000 [15:12:47<1:49:18, 11.00s/it][2022-12-21 01:35:18,480] [INFO] [timer.py:197:stop] 0/8916, RunningAvgSamplesPerSec=5.880273760100813, CurrSamplesPerSec=5.251975765528925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4405/5000 [15:12:59<1:49:25, 11.03s/it][2022-12-21 01:35:29,469] [INFO] [timer.py:197:stop] 0/8918, RunningAvgSamplesPerSec=5.8802757683286755, CurrSamplesPerSec=5.352019169341559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4406/5000 [15:13:10<1:49:04, 11.02s/it][2022-12-21 01:35:40,446] [INFO] [logging.py:68:log_dist] [Rank 0] step=4460, skipped=7, lr=[1.2177777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:35:40,448] [INFO] [timer.py:197:stop] 0/8920, RunningAvgSamplesPerSec=5.880279087810776, CurrSamplesPerSec=5.35352908121962, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4407/5000 [15:13:20<1:48:39, 10.99s/it][2022-12-21 01:35:51,430] [INFO] [timer.py:197:stop] 0/8922, RunningAvgSamplesPerSec=5.880281937752482, CurrSamplesPerSec=5.314821086044379, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4408/5000 [15:13:31<1:48:37, 11.01s/it][2022-12-21 01:36:02,449] [INFO] [timer.py:197:stop] 0/8924, RunningAvgSamplesPerSec=5.880280437243664, CurrSamplesPerSec=5.336977161629642, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4409/5000 [15:13:42<1:48:24, 11.01s/it][2022-12-21 01:36:13,455] [INFO] [timer.py:197:stop] 0/8926, RunningAvgSamplesPerSec=5.880280549404935, CurrSamplesPerSec=5.331563689251887, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4410/5000 [15:13:54<1:48:18, 11.01s/it][2022-12-21 01:36:24,542] [INFO] [timer.py:197:stop] 0/8928, RunningAvgSamplesPerSec=5.880270746589671, CurrSamplesPerSec=5.2856108602987275, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4411/5000 [15:14:05<1:48:12, 11.02s/it][2022-12-21 01:36:35,510] [INFO] [timer.py:197:stop] 0/8930, RunningAvgSamplesPerSec=5.880275401193019, CurrSamplesPerSec=5.351605389176193, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4412/5000 [15:14:16<1:47:55, 11.01s/it][2022-12-21 01:36:46,491] [INFO] [timer.py:197:stop] 0/8932, RunningAvgSamplesPerSec=5.880278509504864, CurrSamplesPerSec=5.360371169852717, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4413/5000 [15:14:27<1:47:40, 11.01s/it][2022-12-21 01:36:57,476] [INFO] [timer.py:197:stop] 0/8934, RunningAvgSamplesPerSec=5.880281172912061, CurrSamplesPerSec=5.361295734709873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4414/5000 [15:14:38<1:47:24, 11.00s/it][2022-12-21 01:37:08,517] [INFO] [timer.py:197:stop] 0/8936, RunningAvgSamplesPerSec=5.880277071016022, CurrSamplesPerSec=5.306270238108181, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4415/5000 [15:14:49<1:47:26, 11.02s/it][2022-12-21 01:37:19,550] [INFO] [timer.py:197:stop] 0/8938, RunningAvgSamplesPerSec=5.880274093537571, CurrSamplesPerSec=5.339054091516246, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4416/5000 [15:15:00<1:47:15, 11.02s/it][2022-12-21 01:37:30,585] [INFO] [logging.py:68:log_dist] [Rank 0] step=4470, skipped=7, lr=[1.1955555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:37:30,586] [INFO] [timer.py:197:stop] 0/8940, RunningAvgSamplesPerSec=5.880270548479393, CurrSamplesPerSec=5.324862286978595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4417/5000 [15:15:11<1:47:09, 11.03s/it][2022-12-21 01:37:41,627] [INFO] [timer.py:197:stop] 0/8942, RunningAvgSamplesPerSec=5.880268637227993, CurrSamplesPerSec=5.330472785831584, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4418/5000 [15:15:22<1:46:58, 11.03s/it][2022-12-21 01:37:52,677] [INFO] [timer.py:197:stop] 0/8944, RunningAvgSamplesPerSec=5.880263390473467, CurrSamplesPerSec=5.312484335782559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4419/5000 [15:15:33<1:46:51, 11.03s/it][2022-12-21 01:38:03,752] [INFO] [timer.py:197:stop] 0/8946, RunningAvgSamplesPerSec=5.880255192035947, CurrSamplesPerSec=5.289065525072967, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4420/5000 [15:15:44<1:46:48, 11.05s/it][2022-12-21 01:38:14,829] [INFO] [timer.py:197:stop] 0/8948, RunningAvgSamplesPerSec=5.880246775259542, CurrSamplesPerSec=5.294686647431846, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4421/5000 [15:15:55<1:46:42, 11.06s/it][2022-12-21 01:38:25,917] [INFO] [timer.py:197:stop] 0/8950, RunningAvgSamplesPerSec=5.88023701480219, CurrSamplesPerSec=5.288457413367045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4422/5000 [15:16:06<1:46:33, 11.06s/it][2022-12-21 01:38:36,933] [INFO] [timer.py:197:stop] 0/8952, RunningAvgSamplesPerSec=5.880236014410982, CurrSamplesPerSec=5.336610263124945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4423/5000 [15:16:17<1:46:14, 11.05s/it][2022-12-21 01:38:47,969] [INFO] [timer.py:197:stop] 0/8954, RunningAvgSamplesPerSec=5.880232538243196, CurrSamplesPerSec=5.318243035458209, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4424/5000 [15:16:28<1:45:57, 11.04s/it][2022-12-21 01:38:58,947] [INFO] [timer.py:197:stop] 0/8956, RunningAvgSamplesPerSec=5.880235920909027, CurrSamplesPerSec=5.346678665227795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 88%|████████▊ | 4425/5000 [15:16:39<1:45:42, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.1777777777777778e-06, 'epoch': 107.92}
+
 88%|████████▊ | 4425/5000 [15:16:39<1:45:42, 11.03s/it][2022-12-21 01:39:09,985] [INFO] [timer.py:197:stop] 0/8958, RunningAvgSamplesPerSec=5.880232615366944, CurrSamplesPerSec=5.328552297319036, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4426/5000 [15:16:50<1:45:30, 11.03s/it][2022-12-21 01:39:20,979] [INFO] [logging.py:68:log_dist] [Rank 0] step=4480, skipped=7, lr=[1.1733333333333335e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:39:20,980] [INFO] [timer.py:197:stop] 0/8960, RunningAvgSamplesPerSec=5.880234089778217, CurrSamplesPerSec=5.353798363254468, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4427/5000 [15:17:01<1:45:06, 11.01s/it][2022-12-21 01:39:31,992] [INFO] [timer.py:197:stop] 0/8962, RunningAvgSamplesPerSec=5.880233365884771, CurrSamplesPerSec=5.303484114547555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4428/5000 [15:17:12<1:44:58, 11.01s/it][2022-12-21 01:39:42,060] [INFO] [timer.py:197:stop] 0/8964, RunningAvgSamplesPerSec=5.880345141562016, CurrSamplesPerSec=6.283278509372505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 01:39:53,046] [INFO] [timer.py:197:stop] 0/8966, RunningAvgSamplesPerSec=5.880347378546373, CurrSamplesPerSec=5.3268327569644, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4429/5000 [15:17:28<1:59:24, 12.55s/it][2022-12-21 01:40:04,119] [INFO] [timer.py:197:stop] 0/8968, RunningAvgSamplesPerSec=5.880339378429554, CurrSamplesPerSec=5.284863908513213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4430/5000 [15:17:39<1:54:59, 12.10s/it][2022-12-21 01:40:15,166] [INFO] [timer.py:197:stop] 0/8970, RunningAvgSamplesPerSec=5.880334808896696, CurrSamplesPerSec=5.3276499870795755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4431/5000 [15:17:50<1:51:46, 11.79s/it][2022-12-21 01:40:26,167] [INFO] [timer.py:197:stop] 0/8972, RunningAvgSamplesPerSec=5.880335586985099, CurrSamplesPerSec=5.332013139173579, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4432/5000 [15:18:01<1:49:21, 11.55s/it][2022-12-21 01:40:37,214] [INFO] [timer.py:197:stop] 0/8974, RunningAvgSamplesPerSec=5.880331912248216, CurrSamplesPerSec=5.341967019422092, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4433/5000 [15:18:12<1:47:43, 11.40s/it][2022-12-21 01:40:48,288] [INFO] [timer.py:197:stop] 0/8976, RunningAvgSamplesPerSec=5.880323890598916, CurrSamplesPerSec=5.294927064715266, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4434/5000 [15:18:23<1:46:37, 11.30s/it][2022-12-21 01:40:59,325] [INFO] [timer.py:197:stop] 0/8978, RunningAvgSamplesPerSec=5.880320441813125, CurrSamplesPerSec=5.33420174219361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4435/5000 [15:18:34<1:45:40, 11.22s/it][2022-12-21 01:41:10,367] [INFO] [logging.py:68:log_dist] [Rank 0] step=4490, skipped=7, lr=[1.151111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:41:10,368] [INFO] [timer.py:197:stop] 0/8980, RunningAvgSamplesPerSec=5.8803161155711505, CurrSamplesPerSec=5.298192090622961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4436/5000 [15:18:45<1:44:59, 11.17s/it][2022-12-21 01:41:21,384] [INFO] [timer.py:197:stop] 0/8982, RunningAvgSamplesPerSec=5.880315031593987, CurrSamplesPerSec=5.298366940820282, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▊ | 4437/5000 [15:18:56<1:44:22, 11.12s/it][2022-12-21 01:41:32,410] [INFO] [timer.py:197:stop] 0/8984, RunningAvgSamplesPerSec=5.8803127656037955, CurrSamplesPerSec=5.3319085008037925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4438/5000 [15:19:08<1:43:54, 11.09s/it][2022-12-21 01:41:43,410] [INFO] [timer.py:197:stop] 0/8986, RunningAvgSamplesPerSec=5.880313567224994, CurrSamplesPerSec=5.329954382093274, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4439/5000 [15:19:19<1:43:27, 11.07s/it][2022-12-21 01:41:54,431] [INFO] [timer.py:197:stop] 0/8988, RunningAvgSamplesPerSec=5.880311845690864, CurrSamplesPerSec=5.341245078754454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4440/5000 [15:19:30<1:43:09, 11.05s/it][2022-12-21 01:42:05,489] [INFO] [timer.py:197:stop] 0/8990, RunningAvgSamplesPerSec=5.8803058148707885, CurrSamplesPerSec=5.317468083261185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4441/5000 [15:19:41<1:42:59, 11.05s/it][2022-12-21 01:42:16,543] [INFO] [timer.py:197:stop] 0/8992, RunningAvgSamplesPerSec=5.8803002922257015, CurrSamplesPerSec=5.318380856484618, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4442/5000 [15:19:52<1:42:48, 11.05s/it][2022-12-21 01:42:27,568] [INFO] [timer.py:197:stop] 0/8994, RunningAvgSamplesPerSec=5.88029814111771, CurrSamplesPerSec=5.337501177614581, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4443/5000 [15:20:03<1:42:32, 11.05s/it][2022-12-21 01:42:38,582] [INFO] [timer.py:197:stop] 0/8996, RunningAvgSamplesPerSec=5.880300130620294, CurrSamplesPerSec=5.341804161814273, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4444/5000 [15:20:14<1:42:15, 11.04s/it][2022-12-21 01:42:49,621] [INFO] [timer.py:197:stop] 0/8998, RunningAvgSamplesPerSec=5.8802962420489395, CurrSamplesPerSec=5.322498337831215, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4445/5000 [15:20:25<1:42:05, 11.04s/it][2022-12-21 01:43:00,677] [INFO] [logging.py:68:log_dist] [Rank 0] step=4500, skipped=7, lr=[1.128888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:43:00,678] [INFO] [timer.py:197:stop] 0/9000, RunningAvgSamplesPerSec=5.880290152687617, CurrSamplesPerSec=5.29805782354453, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4446/5000 [15:20:36<1:41:57, 11.04s/it][2022-12-21 01:43:11,744] [INFO] [timer.py:197:stop] 0/9002, RunningAvgSamplesPerSec=5.88028292634447, CurrSamplesPerSec=5.310541274469182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4447/5000 [15:20:47<1:41:50, 11.05s/it][2022-12-21 01:43:22,773] [INFO] [timer.py:197:stop] 0/9004, RunningAvgSamplesPerSec=5.880280277839604, CurrSamplesPerSec=5.324671530599606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4448/5000 [15:20:58<1:41:35, 11.04s/it][2022-12-21 01:43:33,778] [INFO] [timer.py:197:stop] 0/9006, RunningAvgSamplesPerSec=5.880280275706951, CurrSamplesPerSec=5.318105010855505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4449/5000 [15:21:09<1:41:18, 11.03s/it][2022-12-21 01:43:44,777] [INFO] [timer.py:197:stop] 0/9008, RunningAvgSamplesPerSec=5.880281269400823, CurrSamplesPerSec=5.363377279379024, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4450/5000 [15:21:20<1:41:02, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.12e-06, 'epoch': 108.53}
+
 89%|████████▉ | 4450/5000 [15:21:20<1:41:02, 11.02s/it][2022-12-21 01:43:55,800] [INFO] [timer.py:197:stop] 0/9010, RunningAvgSamplesPerSec=5.8802794726331555, CurrSamplesPerSec=5.334803880632716, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4451/5000 [15:21:31<1:40:51, 11.02s/it][2022-12-21 01:44:06,779] [INFO] [timer.py:197:stop] 0/9012, RunningAvgSamplesPerSec=5.880282845554919, CurrSamplesPerSec=5.36475979842679, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4452/5000 [15:21:42<1:40:33, 11.01s/it][2022-12-21 01:44:17,726] [INFO] [timer.py:197:stop] 0/9014, RunningAvgSamplesPerSec=5.8802900798659765, CurrSamplesPerSec=5.351650413208872, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4453/5000 [15:21:53<1:40:11, 10.99s/it][2022-12-21 01:44:28,680] [INFO] [timer.py:197:stop] 0/9016, RunningAvgSamplesPerSec=5.880296486377608, CurrSamplesPerSec=5.368660372563817, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4454/5000 [15:22:04<1:39:54, 10.98s/it][2022-12-21 01:44:39,680] [INFO] [timer.py:197:stop] 0/9018, RunningAvgSamplesPerSec=5.880297384226967, CurrSamplesPerSec=5.346908064809192, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4455/5000 [15:22:15<1:39:47, 10.99s/it][2022-12-21 01:44:50,665] [INFO] [logging.py:68:log_dist] [Rank 0] step=4510, skipped=7, lr=[1.1066666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:44:50,667] [INFO] [timer.py:197:stop] 0/9020, RunningAvgSamplesPerSec=5.88029981660445, CurrSamplesPerSec=5.334981367792814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4456/5000 [15:22:26<1:39:36, 10.99s/it][2022-12-21 01:45:01,701] [INFO] [timer.py:197:stop] 0/9022, RunningAvgSamplesPerSec=5.880296565918671, CurrSamplesPerSec=5.314511939818211, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4457/5000 [15:22:37<1:39:33, 11.00s/it][2022-12-21 01:45:12,695] [INFO] [timer.py:197:stop] 0/9024, RunningAvgSamplesPerSec=5.880298121837218, CurrSamplesPerSec=5.303584915814932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4458/5000 [15:22:48<1:39:21, 11.00s/it][2022-12-21 01:45:23,677] [INFO] [timer.py:197:stop] 0/9026, RunningAvgSamplesPerSec=5.880301194614621, CurrSamplesPerSec=5.321048066781988, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4459/5000 [15:22:59<1:39:07, 10.99s/it][2022-12-21 01:45:34,734] [INFO] [timer.py:197:stop] 0/9028, RunningAvgSamplesPerSec=5.88029518625755, CurrSamplesPerSec=5.304463788557064, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4460/5000 [15:23:10<1:39:06, 11.01s/it][2022-12-21 01:45:45,710] [INFO] [timer.py:197:stop] 0/9030, RunningAvgSamplesPerSec=5.880298884692065, CurrSamplesPerSec=5.373287375775656, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4461/5000 [15:23:21<1:38:49, 11.00s/it][2022-12-21 01:45:56,647] [INFO] [timer.py:197:stop] 0/9032, RunningAvgSamplesPerSec=5.8803073328253, CurrSamplesPerSec=5.338945141509502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4462/5000 [15:23:32<1:38:28, 10.98s/it][2022-12-21 01:46:07,639] [INFO] [timer.py:197:stop] 0/9034, RunningAvgSamplesPerSec=5.88030919820418, CurrSamplesPerSec=5.354824702126351, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4463/5000 [15:23:43<1:38:18, 10.99s/it][2022-12-21 01:46:18,671] [INFO] [timer.py:197:stop] 0/9036, RunningAvgSamplesPerSec=5.8803061779804775, CurrSamplesPerSec=5.332678557366441, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4464/5000 [15:23:54<1:38:15, 11.00s/it][2022-12-21 01:46:29,688] [INFO] [timer.py:197:stop] 0/9038, RunningAvgSamplesPerSec=5.880304959770268, CurrSamplesPerSec=5.322764084971344, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4465/5000 [15:24:05<1:38:07, 11.00s/it][2022-12-21 01:46:40,664] [INFO] [logging.py:68:log_dist] [Rank 0] step=4520, skipped=7, lr=[1.0844444444444446e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:46:40,665] [INFO] [timer.py:197:stop] 0/9040, RunningAvgSamplesPerSec=5.880308649653313, CurrSamplesPerSec=5.357850899367342, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4466/5000 [15:24:16<1:37:52, 11.00s/it][2022-12-21 01:46:51,613] [INFO] [timer.py:197:stop] 0/9042, RunningAvgSamplesPerSec=5.880315700622892, CurrSamplesPerSec=5.345496621650281, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4467/5000 [15:24:27<1:37:33, 10.98s/it][2022-12-21 01:47:02,553] [INFO] [timer.py:197:stop] 0/9044, RunningAvgSamplesPerSec=5.880323750764969, CurrSamplesPerSec=5.3944566457921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4468/5000 [15:24:38<1:37:15, 10.97s/it][2022-12-21 01:47:13,494] [INFO] [timer.py:197:stop] 0/9046, RunningAvgSamplesPerSec=5.880331665819761, CurrSamplesPerSec=5.353449220067183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4469/5000 [15:24:49<1:37:00, 10.96s/it][2022-12-21 01:47:23,566] [INFO] [timer.py:197:stop] 0/9048, RunningAvgSamplesPerSec=5.880442142739665, CurrSamplesPerSec=5.335131933662303, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4470/5000 [15:25:04<1:47:34, 12.18s/it][2022-12-21 01:47:34,597] [INFO] [timer.py:197:stop] 0/9050, RunningAvgSamplesPerSec=5.880439138327344, CurrSamplesPerSec=5.31464241239591, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4471/5000 [15:25:15<1:44:19, 11.83s/it][2022-12-21 01:47:45,580] [INFO] [timer.py:197:stop] 0/9052, RunningAvgSamplesPerSec=5.88044180438825, CurrSamplesPerSec=5.3541182909981035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4472/5000 [15:25:26<1:41:54, 11.58s/it][2022-12-21 01:47:56,555] [INFO] [timer.py:197:stop] 0/9054, RunningAvgSamplesPerSec=5.880445610031862, CurrSamplesPerSec=5.372273946099674, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4473/5000 [15:25:37<1:40:04, 11.39s/it][2022-12-21 01:48:07,598] [INFO] [timer.py:197:stop] 0/9056, RunningAvgSamplesPerSec=5.880441239572259, CurrSamplesPerSec=5.293550443830425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 89%|████████▉ | 4474/5000 [15:25:48<1:38:59, 11.29s/it][2022-12-21 01:48:18,642] [INFO] [timer.py:197:stop] 0/9058, RunningAvgSamplesPerSec=5.880436822230419, CurrSamplesPerSec=5.302940775792831, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4475/5000 [15:25:59<1:38:02, 11.20s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.0644444444444445e-06, 'epoch': 109.14}
+
 90%|████████▉ | 4475/5000 [15:25:59<1:38:02, 11.20s/it][2022-12-21 01:48:29,590] [INFO] [logging.py:68:log_dist] [Rank 0] step=4530, skipped=7, lr=[1.0622222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:48:29,591] [INFO] [timer.py:197:stop] 0/9060, RunningAvgSamplesPerSec=5.880443546648543, CurrSamplesPerSec=5.346801563019808, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4476/5000 [15:26:10<1:37:13, 11.13s/it][2022-12-21 01:48:40,603] [INFO] [timer.py:197:stop] 0/9062, RunningAvgSamplesPerSec=5.880443118701863, CurrSamplesPerSec=5.30785834572428, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4477/5000 [15:26:21<1:36:42, 11.10s/it][2022-12-21 01:48:51,574] [INFO] [timer.py:197:stop] 0/9064, RunningAvgSamplesPerSec=5.8804472689290295, CurrSamplesPerSec=5.34007860894287, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4478/5000 [15:26:32<1:36:16, 11.07s/it][2022-12-21 01:49:02,578] [INFO] [timer.py:197:stop] 0/9066, RunningAvgSamplesPerSec=5.880447466225641, CurrSamplesPerSec=5.332307589012845, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4479/5000 [15:26:43<1:35:49, 11.04s/it][2022-12-21 01:49:13,508] [INFO] [timer.py:197:stop] 0/9068, RunningAvgSamplesPerSec=5.88045647482423, CurrSamplesPerSec=5.363792024575232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4480/5000 [15:26:54<1:35:22, 11.01s/it][2022-12-21 01:49:24,498] [INFO] [timer.py:197:stop] 0/9070, RunningAvgSamplesPerSec=5.880458259339314, CurrSamplesPerSec=5.313922578218566, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4481/5000 [15:27:05<1:35:17, 11.02s/it][2022-12-21 01:49:35,542] [INFO] [timer.py:197:stop] 0/9072, RunningAvgSamplesPerSec=5.880453812257237, CurrSamplesPerSec=5.313910375753905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████�� | 4482/5000 [15:27:16<1:35:08, 11.02s/it][2022-12-21 01:49:46,576] [INFO] [timer.py:197:stop] 0/9074, RunningAvgSamplesPerSec=5.88045063929097, CurrSamplesPerSec=5.311650099831585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4483/5000 [15:27:27<1:35:01, 11.03s/it][2022-12-21 01:49:57,608] [INFO] [timer.py:197:stop] 0/9076, RunningAvgSamplesPerSec=5.880447688426041, CurrSamplesPerSec=5.324249506917437, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4484/5000 [15:27:38<1:34:46, 11.02s/it][2022-12-21 01:50:08,624] [INFO] [timer.py:197:stop] 0/9078, RunningAvgSamplesPerSec=5.880446539948908, CurrSamplesPerSec=5.3093709531036, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4485/5000 [15:27:49<1:34:37, 11.02s/it][2022-12-21 01:50:19,661] [INFO] [logging.py:68:log_dist] [Rank 0] step=4540, skipped=7, lr=[1.04e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:50:19,663] [INFO] [timer.py:197:stop] 0/9080, RunningAvgSamplesPerSec=5.880442793430431, CurrSamplesPerSec=5.307689795040298, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4486/5000 [15:28:00<1:34:25, 11.02s/it][2022-12-21 01:50:30,625] [INFO] [timer.py:197:stop] 0/9082, RunningAvgSamplesPerSec=5.880448267816349, CurrSamplesPerSec=5.355936069424194, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4487/5000 [15:28:11<1:34:12, 11.02s/it][2022-12-21 01:50:41,684] [INFO] [timer.py:197:stop] 0/9084, RunningAvgSamplesPerSec=5.880441980253298, CurrSamplesPerSec=5.312055201348783, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4488/5000 [15:28:22<1:34:03, 11.02s/it][2022-12-21 01:50:52,716] [INFO] [timer.py:197:stop] 0/9086, RunningAvgSamplesPerSec=5.880438894218136, CurrSamplesPerSec=5.3120036930567665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4489/5000 [15:28:33<1:33:57, 11.03s/it][2022-12-21 01:51:03,763] [INFO] [timer.py:197:stop] 0/9088, RunningAvgSamplesPerSec=5.880434080770759, CurrSamplesPerSec=5.316222270175449, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4490/5000 [15:28:44<1:33:40, 11.02s/it][2022-12-21 01:51:14,757] [INFO] [timer.py:197:stop] 0/9090, RunningAvgSamplesPerSec=5.880435514913358, CurrSamplesPerSec=5.315166892610625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4491/5000 [15:28:55<1:33:31, 11.02s/it][2022-12-21 01:51:25,788] [INFO] [timer.py:197:stop] 0/9092, RunningAvgSamplesPerSec=5.880432551923307, CurrSamplesPerSec=5.319315446175143, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4492/5000 [15:29:06<1:33:23, 11.03s/it][2022-12-21 01:51:36,845] [INFO] [timer.py:197:stop] 0/9094, RunningAvgSamplesPerSec=5.880426543341033, CurrSamplesPerSec=5.3059515979280905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4493/5000 [15:29:17<1:33:01, 11.01s/it][2022-12-21 01:51:47,809] [INFO] [timer.py:197:stop] 0/9096, RunningAvgSamplesPerSec=5.880431608737186, CurrSamplesPerSec=5.304834666744661, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4494/5000 [15:29:28<1:32:55, 11.02s/it][2022-12-21 01:51:58,806] [INFO] [timer.py:197:stop] 0/9098, RunningAvgSamplesPerSec=5.880432549942161, CurrSamplesPerSec=5.341934489628464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4495/5000 [15:29:39<1:32:48, 11.03s/it][2022-12-21 01:52:09,887] [INFO] [logging.py:68:log_dist] [Rank 0] step=4550, skipped=7, lr=[1.0177777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-21 01:52:09,888] [INFO] [timer.py:197:stop] 0/9100, RunningAvgSamplesPerSec=5.880423577772919, CurrSamplesPerSec=5.308138377624003, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4496/5000 [15:29:50<1:32:41, 11.03s/it][2022-12-21 01:52:20,895] [INFO] [timer.py:197:stop] 0/9102, RunningAvgSamplesPerSec=5.880423563447671, CurrSamplesPerSec=5.350106372135575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4497/5000 [15:30:01<1:32:21, 11.02s/it][2022-12-21 01:52:31,907] [INFO] [timer.py:197:stop] 0/9104, RunningAvgSamplesPerSec=5.880422852134698, CurrSamplesPerSec=5.3170299285873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4498/5000 [15:30:12<1:32:09, 11.01s/it][2022-12-21 01:52:42,883] [INFO] [timer.py:197:stop] 0/9106, RunningAvgSamplesPerSec=5.880426467413257, CurrSamplesPerSec=5.347336457858315, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|████████▉ | 4499/5000 [15:30:23<1:31:49, 11.00s/it][2022-12-21 01:52:53,839] [INFO] [timer.py:197:stop] 0/9108, RunningAvgSamplesPerSec=5.880432417836862, CurrSamplesPerSec=5.345937351420396, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4500/5000 [15:30:34<1:31:39, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 1.008888888888889e-06, 'epoch': 109.75}
+
 90%|█████████ | 4500/5000 [15:30:34<1:31:39, 11.00s/it][2022-12-21 01:53:04,865] [INFO] [timer.py:197:stop] 0/9110, RunningAvgSamplesPerSec=5.880430375024078, CurrSamplesPerSec=5.3315990579172725, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4501/5000 [15:30:45<1:31:24, 10.99s/it][2022-12-21 01:53:15,850] [INFO] [timer.py:197:stop] 0/9112, RunningAvgSamplesPerSec=5.880432702362991, CurrSamplesPerSec=5.315968334955242, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4502/5000 [15:30:56<1:31:19, 11.00s/it][2022-12-21 01:53:26,902] [INFO] [timer.py:197:stop] 0/9114, RunningAvgSamplesPerSec=5.880427067632778, CurrSamplesPerSec=5.297638753258436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4503/5000 [15:31:07<1:31:16, 11.02s/it][2022-12-21 01:53:37,892] [INFO] [timer.py:197:stop] 0/9116, RunningAvgSamplesPerSec=5.880428941391807, CurrSamplesPerSec=5.3543357270010095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4504/5000 [15:31:18<1:30:58, 11.00s/it][2022-12-21 01:53:48,902] [INFO] [timer.py:197:stop] 0/9118, RunningAvgSamplesPerSec=5.880428502210159, CurrSamplesPerSec=5.322274826827844, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4505/5000 [15:31:29<1:30:50, 11.01s/it][2022-12-21 01:53:59,938] [INFO] [logging.py:68:log_dist] [Rank 0] step=4560, skipped=7, lr=[9.955555555555556e-07], mom=[[0.9, 0.999]]
+[2022-12-21 01:53:59,940] [INFO] [timer.py:197:stop] 0/9120, RunningAvgSamplesPerSec=5.880424646723352, CurrSamplesPerSec=5.308895283333363, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4506/5000 [15:31:40<1:30:40, 11.01s/it][2022-12-21 01:54:10,959] [INFO] [timer.py:197:stop] 0/9122, RunningAvgSamplesPerSec=5.880423075584787, CurrSamplesPerSec=5.31188491243612, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4507/5000 [15:31:51<1:30:36, 11.03s/it][2022-12-21 01:54:21,992] [INFO] [timer.py:197:stop] 0/9124, RunningAvgSamplesPerSec=5.880419900996102, CurrSamplesPerSec=5.334929201882353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4508/5000 [15:32:02<1:30:17, 11.01s/it][2022-12-21 01:54:32,995] [INFO] [timer.py:197:stop] 0/9126, RunningAvgSamplesPerSec=5.880420261742048, CurrSamplesPerSec=5.310945366256265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4509/5000 [15:32:13<1:30:07, 11.01s/it][2022-12-21 01:54:43,982] [INFO] [timer.py:197:stop] 0/9128, RunningAvgSamplesPerSec=5.880422626054864, CurrSamplesPerSec=5.339304714647681, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4510/5000 [15:32:24<1:29:56, 11.01s/it][2022-12-21 01:54:54,092] [INFO] [timer.py:197:stop] 0/9130, RunningAvgSamplesPerSec=5.880527447077867, CurrSamplesPerSec=6.2675497047472755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 01:55:05,154] [INFO] [timer.py:197:stop] 0/9132, RunningAvgSamplesPerSec=5.880520633297506, CurrSamplesPerSec=5.3005903848695795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4511/5000 [15:32:40<1:42:29, 12.57s/it][2022-12-21 01:55:16,213] [INFO] [timer.py:197:stop] 0/9134, RunningAvgSamplesPerSec=5.880514647617238, CurrSamplesPerSec=5.313718930628229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4512/5000 [15:32:51<1:38:34, 12.12s/it][2022-12-21 01:55:27,262] [INFO] [timer.py:197:stop] 0/9136, RunningAvgSamplesPerSec=5.880509566493539, CurrSamplesPerSec=5.284806475434704, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4513/5000 [15:33:02<1:35:46, 11.80s/it][2022-12-21 01:55:38,301] [INFO] [timer.py:197:stop] 0/9138, RunningAvgSamplesPerSec=5.880505794490363, CurrSamplesPerSec=5.3278105023473605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4514/5000 [15:33:13<1:33:43, 11.57s/it][2022-12-21 01:55:49,317] [INFO] [logging.py:68:log_dist] [Rank 0] step=4570, skipped=7, lr=[9.733333333333333e-07], mom=[[0.9, 0.999]]
+[2022-12-21 01:55:49,319] [INFO] [timer.py:197:stop] 0/9140, RunningAvgSamplesPerSec=5.880504475731382, CurrSamplesPerSec=5.311544997961143, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4515/5000 [15:33:24<1:32:11, 11.40s/it][2022-12-21 01:56:00,315] [INFO] [timer.py:197:stop] 0/9142, RunningAvgSamplesPerSec=5.880505728686221, CurrSamplesPerSec=5.322365579740216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4516/5000 [15:33:35<1:31:00, 11.28s/it][2022-12-21 01:56:11,371] [INFO] [timer.py:197:stop] 0/9144, RunningAvgSamplesPerSec=5.88049995316461, CurrSamplesPerSec=5.309683072382902, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4517/5000 [15:33:46<1:30:16, 11.21s/it][2022-12-21 01:56:22,485] [INFO] [timer.py:197:stop] 0/9146, RunningAvgSamplesPerSec=5.880487383736593, CurrSamplesPerSec=5.232245869040384, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4518/5000 [15:33:58<1:29:50, 11.18s/it][2022-12-21 01:56:33,486] [INFO] [timer.py:197:stop] 0/9148, RunningAvgSamplesPerSec=5.880488057383731, CurrSamplesPerSec=5.308947991325011, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4519/5000 [15:34:09<1:29:13, 11.13s/it][2022-12-21 01:56:44,459] [INFO] [timer.py:197:stop] 0/9150, RunningAvgSamplesPerSec=5.880492117065759, CurrSamplesPerSec=5.336426090181879, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4520/5000 [15:34:20<1:28:39, 11.08s/it][2022-12-21 01:56:55,478] [INFO] [timer.py:197:stop] 0/9152, RunningAvgSamplesPerSec=5.880490665904338, CurrSamplesPerSec=5.297541523371702, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4521/5000 [15:34:31<1:28:19, 11.06s/it][2022-12-21 01:57:06,523] [INFO] [timer.py:197:stop] 0/9154, RunningAvgSamplesPerSec=5.880486153541542, CurrSamplesPerSec=5.304776798785327, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4522/5000 [15:34:42<1:28:05, 11.06s/it][2022-12-21 01:57:17,556] [INFO] [timer.py:197:stop] 0/9156, RunningAvgSamplesPerSec=5.880483049125236, CurrSamplesPerSec=5.32392743675845, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4523/5000 [15:34:53<1:27:51, 11.05s/it][2022-12-21 01:57:28,641] [INFO] [timer.py:197:stop] 0/9158, RunningAvgSamplesPerSec=5.880473824586998, CurrSamplesPerSec=5.294529792668037, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4524/5000 [15:35:04<1:27:44, 11.06s/it][2022-12-21 01:57:39,697] [INFO] [logging.py:68:log_dist] [Rank 0] step=4580, skipped=7, lr=[9.511111111111111e-07], mom=[[0.9, 0.999]]
+[2022-12-21 01:57:39,698] [INFO] [timer.py:197:stop] 0/9160, RunningAvgSamplesPerSec=5.880467916294156, CurrSamplesPerSec=5.300919477464728, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 90%|█████████ | 4525/5000 [15:35:15<1:27:33, 11.06s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 9.511111111111111e-07, 'epoch': 110.36}
+
 90%|█████████ | 4525/5000 [15:35:15<1:27:33, 11.06s/it][2022-12-21 01:57:50,684] [INFO] [timer.py:197:stop] 0/9162, RunningAvgSamplesPerSec=5.880470719592, CurrSamplesPerSec=5.345484273746796, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4526/5000 [15:35:26<1:27:11, 11.04s/it][2022-12-21 01:58:01,778] [INFO] [timer.py:197:stop] 0/9164, RunningAvgSamplesPerSec=5.8804605196927024, CurrSamplesPerSec=5.283858385788975, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4527/5000 [15:35:37<1:27:08, 11.05s/it][2022-12-21 01:58:12,826] [INFO] [timer.py:197:stop] 0/9166, RunningAvgSamplesPerSec=5.880455690449079, CurrSamplesPerSec=5.320475814456463, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4528/5000 [15:35:48<1:26:56, 11.05s/it][2022-12-21 01:58:23,851] [INFO] [timer.py:197:stop] 0/9168, RunningAvgSamplesPerSec=5.880453431812182, CurrSamplesPerSec=5.3264192685188965, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4529/5000 [15:35:59<1:26:41, 11.04s/it][2022-12-21 01:58:34,885] [INFO] [timer.py:197:stop] 0/9170, RunningAvgSamplesPerSec=5.880450222489306, CurrSamplesPerSec=5.315511270583549, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4530/5000 [15:36:10<1:26:29, 11.04s/it][2022-12-21 01:58:45,872] [INFO] [timer.py:197:stop] 0/9172, RunningAvgSamplesPerSec=5.880452560630593, CurrSamplesPerSec=5.3689595282723985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4531/5000 [15:36:21<1:26:10, 11.02s/it][2022-12-21 01:58:56,936] [INFO] [timer.py:197:stop] 0/9174, RunningAvgSamplesPerSec=5.880445866406771, CurrSamplesPerSec=5.29041041106696, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4532/5000 [15:36:32<1:26:05, 11.04s/it][2022-12-21 01:59:07,996] [INFO] [timer.py:197:stop] 0/9176, RunningAvgSamplesPerSec=5.880439635938803, CurrSamplesPerSec=5.278972499950049, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4533/5000 [15:36:43<1:25:57, 11.04s/it][2022-12-21 01:59:18,972] [INFO] [timer.py:197:stop] 0/9178, RunningAvgSamplesPerSec=5.880443260907056, CurrSamplesPerSec=5.3331896502740035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4534/5000 [15:36:54<1:25:36, 11.02s/it][2022-12-21 01:59:29,962] [INFO] [logging.py:68:log_dist] [Rank 0] step=4590, skipped=7, lr=[9.288888888888889e-07], mom=[[0.9, 0.999]]
+[2022-12-21 01:59:29,964] [INFO] [timer.py:197:stop] 0/9180, RunningAvgSamplesPerSec=5.880445155837406, CurrSamplesPerSec=5.3199323625461705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4535/5000 [15:37:05<1:25:21, 11.01s/it][2022-12-21 01:59:40,976] [INFO] [timer.py:197:stop] 0/9182, RunningAvgSamplesPerSec=5.880444547931888, CurrSamplesPerSec=5.313725452158301, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4536/5000 [15:37:16<1:25:10, 11.01s/it][2022-12-21 01:59:52,011] [INFO] [timer.py:197:stop] 0/9184, RunningAvgSamplesPerSec=5.880441373646404, CurrSamplesPerSec=5.32735435976437, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4537/5000 [15:37:27<1:25:02, 11.02s/it][2022-12-21 02:00:03,011] [INFO] [timer.py:197:stop] 0/9186, RunningAvgSamplesPerSec=5.880442397883138, CurrSamplesPerSec=5.341489955484841, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4538/5000 [15:37:38<1:24:48, 11.01s/it][2022-12-21 02:00:13,940] [INFO] [timer.py:197:stop] 0/9188, RunningAvgSamplesPerSec=5.880451457383316, CurrSamplesPerSec=5.369275256383033, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4539/5000 [15:37:49<1:24:25, 10.99s/it][2022-12-21 02:00:24,926] [INFO] [timer.py:197:stop] 0/9190, RunningAvgSamplesPerSec=5.88045399071944, CurrSamplesPerSec=5.351170764841647, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4540/5000 [15:38:00<1:24:14, 10.99s/it][2022-12-21 02:00:35,933] [INFO] [timer.py:197:stop] 0/9192, RunningAvgSamplesPerSec=5.880453991418233, CurrSamplesPerSec=5.308394924531674, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4541/5000 [15:38:11<1:24:06, 10.99s/it][2022-12-21 02:00:46,974] [INFO] [timer.py:197:stop] 0/9194, RunningAvgSamplesPerSec=5.880450043362405, CurrSamplesPerSec=5.311240015880964, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4542/5000 [15:38:22<1:24:01, 11.01s/it][2022-12-21 02:00:57,965] [INFO] [timer.py:197:stop] 0/9196, RunningAvgSamplesPerSec=5.880451937251019, CurrSamplesPerSec=5.347196705988205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4543/5000 [15:38:33<1:23:48, 11.00s/it][2022-12-21 02:01:08,950] [INFO] [timer.py:197:stop] 0/9198, RunningAvgSamplesPerSec=5.880454453262519, CurrSamplesPerSec=5.363368706511553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4544/5000 [15:38:44<1:23:34, 11.00s/it][2022-12-21 02:01:19,842] [INFO] [logging.py:68:log_dist] [Rank 0] step=4600, skipped=7, lr=[9.066666666666668e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:01:19,843] [INFO] [timer.py:197:stop] 0/9200, RunningAvgSamplesPerSec=5.88046790636436, CurrSamplesPerSec=5.395881267214939, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4545/5000 [15:38:55<1:23:09, 10.97s/it][2022-12-21 02:01:30,809] [INFO] [timer.py:197:stop] 0/9202, RunningAvgSamplesPerSec=5.880472799880131, CurrSamplesPerSec=5.366177572068601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4546/5000 [15:39:06<1:22:58, 10.97s/it][2022-12-21 02:01:41,777] [INFO] [timer.py:197:stop] 0/9204, RunningAvgSamplesPerSec=5.880477303583169, CurrSamplesPerSec=5.329611727917405, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4547/5000 [15:39:17<1:22:47, 10.97s/it][2022-12-21 02:01:52,767] [INFO] [timer.py:197:stop] 0/9206, RunningAvgSamplesPerSec=5.880479329077776, CurrSamplesPerSec=5.307589047563874, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4548/5000 [15:39:28<1:22:40, 10.97s/it][2022-12-21 02:02:03,790] [INFO] [timer.py:197:stop] 0/9208, RunningAvgSamplesPerSec=5.880477537573404, CurrSamplesPerSec=5.329611939549376, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4549/5000 [15:39:39<1:22:35, 10.99s/it][2022-12-21 02:02:14,751] [INFO] [timer.py:197:stop] 0/9210, RunningAvgSamplesPerSec=5.880483038815123, CurrSamplesPerSec=5.353364023229255, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4550/5000 [15:39:50<1:22:21, 10.98s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 8.955555555555557e-07, 'epoch': 110.96}
+
 91%|█████████ | 4550/5000 [15:39:50<1:22:21, 10.98s/it][2022-12-21 02:02:25,729] [INFO] [timer.py:197:stop] 0/9212, RunningAvgSamplesPerSec=5.880486687802154, CurrSamplesPerSec=5.365596001200104, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4551/5000 [15:40:01<1:22:09, 10.98s/it][2022-12-21 02:02:35,855] [INFO] [timer.py:197:stop] 0/9214, RunningAvgSamplesPerSec=5.880588804216937, CurrSamplesPerSec=5.316216374223607, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4552/5000 [15:40:16<1:31:07, 12.20s/it][2022-12-21 02:02:46,861] [INFO] [timer.py:197:stop] 0/9216, RunningAvgSamplesPerSec=5.880588794805165, CurrSamplesPerSec=5.3245530277054804, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4553/5000 [15:40:27<1:28:10, 11.84s/it][2022-12-21 02:02:57,849] [INFO] [timer.py:197:stop] 0/9218, RunningAvgSamplesPerSec=5.880590965366828, CurrSamplesPerSec=5.317940022906913, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4554/5000 [15:40:38<1:26:05, 11.58s/it][2022-12-21 02:03:08,797] [INFO] [logging.py:68:log_dist] [Rank 0] step=4610, skipped=7, lr=[8.844444444444446e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:03:08,799] [INFO] [timer.py:197:stop] 0/9220, RunningAvgSamplesPerSec=5.880597483116103, CurrSamplesPerSec=5.348682156375526, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4555/5000 [15:40:49<1:24:30, 11.40s/it][2022-12-21 02:03:19,799] [INFO] [timer.py:197:stop] 0/9222, RunningAvgSamplesPerSec=5.8805981401574, CurrSamplesPerSec=5.314558025367921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4556/5000 [15:41:00<1:23:20, 11.26s/it][2022-12-21 02:03:30,757] [INFO] [timer.py:197:stop] 0/9224, RunningAvgSamplesPerSec=5.880603738513444, CurrSamplesPerSec=5.309416949534447, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4557/5000 [15:41:11<1:22:33, 11.18s/it][2022-12-21 02:03:41,719] [INFO] [timer.py:197:stop] 0/9226, RunningAvgSamplesPerSec=5.8806088706272215, CurrSamplesPerSec=5.335077644143741, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4558/5000 [15:41:22<1:21:54, 11.12s/it][2022-12-21 02:03:52,701] [INFO] [timer.py:197:stop] 0/9228, RunningAvgSamplesPerSec=5.880611627218014, CurrSamplesPerSec=5.325546207057345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4559/5000 [15:41:33<1:21:25, 11.08s/it][2022-12-21 02:04:03,665] [INFO] [timer.py:197:stop] 0/9230, RunningAvgSamplesPerSec=5.880616570351992, CurrSamplesPerSec=5.347002215924146, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4560/5000 [15:41:44<1:21:02, 11.05s/it][2022-12-21 02:04:14,694] [INFO] [timer.py:197:stop] 0/9232, RunningAvgSamplesPerSec=5.8806139242247335, CurrSamplesPerSec=5.31011182362186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4561/5000 [15:41:55<1:20:52, 11.05s/it][2022-12-21 02:04:25,707] [INFO] [timer.py:197:stop] 0/9234, RunningAvgSamplesPerSec=5.88061308717108, CurrSamplesPerSec=5.347475577874399, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████ | 4562/5000 [15:42:06<1:20:35, 11.04s/it][2022-12-21 02:04:36,745] [INFO] [timer.py:197:stop] 0/9236, RunningAvgSamplesPerSec=5.880609431114901, CurrSamplesPerSec=5.323133515467272, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4563/5000 [15:42:17<1:20:21, 11.03s/it][2022-12-21 02:04:47,737] [INFO] [timer.py:197:stop] 0/9238, RunningAvgSamplesPerSec=5.88061097519919, CurrSamplesPerSec=5.344057197908337, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4564/5000 [15:42:28<1:19:59, 11.01s/it][2022-12-21 02:04:58,706] [INFO] [logging.py:68:log_dist] [Rank 0] step=4620, skipped=7, lr=[8.622222222222224e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:04:58,707] [INFO] [timer.py:197:stop] 0/9240, RunningAvgSamplesPerSec=5.880615181189549, CurrSamplesPerSec=5.328191632676143, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4565/5000 [15:42:39<1:19:44, 11.00s/it][2022-12-21 02:05:09,663] [INFO] [timer.py:197:stop] 0/9242, RunningAvgSamplesPerSec=5.88062103320555, CurrSamplesPerSec=5.347381197043217, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4566/5000 [15:42:50<1:19:31, 11.00s/it][2022-12-21 02:05:20,682] [INFO] [timer.py:197:stop] 0/9244, RunningAvgSamplesPerSec=5.880619488711999, CurrSamplesPerSec=5.317811284632525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4567/5000 [15:43:01<1:19:20, 10.99s/it][2022-12-21 02:05:31,648] [INFO] [timer.py:197:stop] 0/9246, RunningAvgSamplesPerSec=5.880624277501237, CurrSamplesPerSec=5.3393180960358375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4568/5000 [15:43:12<1:19:13, 11.00s/it][2022-12-21 02:05:42,664] [INFO] [timer.py:197:stop] 0/9248, RunningAvgSamplesPerSec=5.88062295969447, CurrSamplesPerSec=5.3458892295635465, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4569/5000 [15:43:23<1:19:02, 11.00s/it][2022-12-21 02:05:53,695] [INFO] [timer.py:197:stop] 0/9250, RunningAvgSamplesPerSec=5.880620046442602, CurrSamplesPerSec=5.321629937137331, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4570/5000 [15:43:34<1:18:53, 11.01s/it][2022-12-21 02:06:04,708] [INFO] [timer.py:197:stop] 0/9252, RunningAvgSamplesPerSec=5.8806191568524655, CurrSamplesPerSec=5.325209612757232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4571/5000 [15:43:45<1:18:40, 11.00s/it][2022-12-21 02:06:15,691] [INFO] [timer.py:197:stop] 0/9254, RunningAvgSamplesPerSec=5.880621792033226, CurrSamplesPerSec=5.335485478762584, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4572/5000 [15:43:56<1:18:27, 11.00s/it][2022-12-21 02:06:26,711] [INFO] [timer.py:197:stop] 0/9256, RunningAvgSamplesPerSec=5.880620106435126, CurrSamplesPerSec=5.31010405044518, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4573/5000 [15:44:07<1:18:15, 11.00s/it][2022-12-21 02:06:37,679] [INFO] [timer.py:197:stop] 0/9258, RunningAvgSamplesPerSec=5.880624626107431, CurrSamplesPerSec=5.330560007803354, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 91%|█████████▏| 4574/5000 [15:44:18<1:18:03, 10.99s/it][2022-12-21 02:06:48,673] [INFO] [logging.py:68:log_dist] [Rank 0] step=4630, skipped=7, lr=[8.400000000000001e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:06:48,675] [INFO] [timer.py:197:stop] 0/9260, RunningAvgSamplesPerSec=5.88062567702227, CurrSamplesPerSec=5.321498699279524, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4575/5000 [15:44:29<1:17:58, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 8.400000000000001e-07, 'epoch': 111.58}
+
 92%|█████████▏| 4575/5000 [15:44:29<1:17:58, 11.01s/it][2022-12-21 02:06:59,724] [INFO] [timer.py:197:stop] 0/9262, RunningAvgSamplesPerSec=5.880620863434008, CurrSamplesPerSec=5.315309606184809, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4576/5000 [15:44:40<1:17:51, 11.02s/it][2022-12-21 02:07:10,732] [INFO] [timer.py:197:stop] 0/9264, RunningAvgSamplesPerSec=5.88062063005357, CurrSamplesPerSec=5.344907817894291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4577/5000 [15:44:51<1:17:40, 11.02s/it][2022-12-21 02:07:21,801] [INFO] [timer.py:197:stop] 0/9266, RunningAvgSamplesPerSec=5.880613316033379, CurrSamplesPerSec=5.299346817402509, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4578/5000 [15:45:02<1:17:34, 11.03s/it][2022-12-21 02:07:32,799] [INFO] [timer.py:197:stop] 0/9268, RunningAvgSamplesPerSec=5.880614300769703, CurrSamplesPerSec=5.350375096663765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4579/5000 [15:45:13<1:17:21, 11.02s/it][2022-12-21 02:07:43,787] [INFO] [timer.py:197:stop] 0/9270, RunningAvgSamplesPerSec=5.880616424227047, CurrSamplesPerSec=5.371789733976641, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4580/5000 [15:45:24<1:17:00, 11.00s/it][2022-12-21 02:07:54,717] [INFO] [timer.py:197:stop] 0/9272, RunningAvgSamplesPerSec=5.880625356419194, CurrSamplesPerSec=5.390039662962207, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4581/5000 [15:45:35<1:16:45, 10.99s/it][2022-12-21 02:08:05,744] [INFO] [timer.py:197:stop] 0/9274, RunningAvgSamplesPerSec=5.880622917200362, CurrSamplesPerSec=5.334644427926808, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4582/5000 [15:45:46<1:16:37, 11.00s/it][2022-12-21 02:08:16,805] [INFO] [timer.py:197:stop] 0/9276, RunningAvgSamplesPerSec=5.880616405448078, CurrSamplesPerSec=5.291830465365266, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4583/5000 [15:45:57<1:16:37, 11.02s/it][2022-12-21 02:08:27,871] [INFO] [timer.py:197:stop] 0/9278, RunningAvgSamplesPerSec=5.880609658836568, CurrSamplesPerSec=5.312225290926635, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4584/5000 [15:46:08<1:16:23, 11.02s/it][2022-12-21 02:08:38,873] [INFO] [logging.py:68:log_dist] [Rank 0] step=4640, skipped=7, lr=[8.177777777777779e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:08:38,874] [INFO] [timer.py:197:stop] 0/9280, RunningAvgSamplesPerSec=5.8806099725725245, CurrSamplesPerSec=5.31215065200807, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4585/5000 [15:46:19<1:16:10, 11.01s/it][2022-12-21 02:08:49,862] [INFO] [timer.py:197:stop] 0/9282, RunningAvgSamplesPerSec=5.880612144744779, CurrSamplesPerSec=5.324734480806193, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4586/5000 [15:46:30<1:15:56, 11.01s/it][2022-12-21 02:09:00,835] [INFO] [timer.py:197:stop] 0/9284, RunningAvgSamplesPerSec=5.88061606481585, CurrSamplesPerSec=5.340500383355103, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4587/5000 [15:46:41<1:15:46, 11.01s/it][2022-12-21 02:09:11,868] [INFO] [timer.py:197:stop] 0/9286, RunningAvgSamplesPerSec=5.880612881389406, CurrSamplesPerSec=5.324371164539262, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4588/5000 [15:46:52<1:15:40, 11.02s/it][2022-12-21 02:09:22,877] [INFO] [timer.py:197:stop] 0/9288, RunningAvgSamplesPerSec=5.880612506910815, CurrSamplesPerSec=5.356191913745254, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4589/5000 [15:47:03<1:15:24, 11.01s/it][2022-12-21 02:09:33,908] [INFO] [timer.py:197:stop] 0/9290, RunningAvgSamplesPerSec=5.880609641505174, CurrSamplesPerSec=5.316292390891075, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4590/5000 [15:47:14<1:15:17, 11.02s/it][2022-12-21 02:09:44,902] [INFO] [timer.py:197:stop] 0/9292, RunningAvgSamplesPerSec=5.8806111918783825, CurrSamplesPerSec=5.358036981939373, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4591/5000 [15:47:25<1:14:58, 11.00s/it][2022-12-21 02:09:55,880] [INFO] [timer.py:197:stop] 0/9294, RunningAvgSamplesPerSec=5.880614464272843, CurrSamplesPerSec=5.334252621874057, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4592/5000 [15:47:36<1:14:54, 11.02s/it][2022-12-21 02:10:06,063] [INFO] [timer.py:197:stop] 0/9296, RunningAvgSamplesPerSec=5.880708945399914, CurrSamplesPerSec=6.227135142718963, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 02:10:17,048] [INFO] [timer.py:197:stop] 0/9298, RunningAvgSamplesPerSec=5.880711170359993, CurrSamplesPerSec=5.3419938089613845, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4593/5000 [15:47:52<1:25:13, 12.56s/it][2022-12-21 02:10:28,065] [INFO] [logging.py:68:log_dist] [Rank 0] step=4650, skipped=7, lr=[7.955555555555557e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:10:28,067] [INFO] [timer.py:197:stop] 0/9300, RunningAvgSamplesPerSec=5.880709766705022, CurrSamplesPerSec=5.338996536475208, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4594/5000 [15:48:03<1:21:52, 12.10s/it][2022-12-21 02:10:39,110] [INFO] [timer.py:197:stop] 0/9302, RunningAvgSamplesPerSec=5.8807055410718725, CurrSamplesPerSec=5.305883008240703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4595/5000 [15:48:14<1:19:32, 11.78s/it][2022-12-21 02:10:50,117] [INFO] [timer.py:197:stop] 0/9304, RunningAvgSamplesPerSec=5.88070564869224, CurrSamplesPerSec=5.312057724229564, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4596/5000 [15:48:25<1:17:46, 11.55s/it][2022-12-21 02:11:01,180] [INFO] [timer.py:197:stop] 0/9306, RunningAvgSamplesPerSec=5.880699168937078, CurrSamplesPerSec=5.311506111323249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4597/5000 [15:48:36<1:16:35, 11.40s/it][2022-12-21 02:11:12,237] [INFO] [timer.py:197:stop] 0/9308, RunningAvgSamplesPerSec=5.8806931509574145, CurrSamplesPerSec=5.2975043051338355, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4598/5000 [15:48:47<1:15:42, 11.30s/it][2022-12-21 02:11:23,220] [INFO] [timer.py:197:stop] 0/9310, RunningAvgSamplesPerSec=5.880696019668126, CurrSamplesPerSec=5.324819191319168, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4599/5000 [15:48:58<1:14:53, 11.20s/it][2022-12-21 02:11:34,231] [INFO] [timer.py:197:stop] 0/9312, RunningAvgSamplesPerSec=5.880695272624321, CurrSamplesPerSec=5.2974313338145125, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4600/5000 [15:49:09<1:14:18, 11.15s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 7.822222222222223e-07, 'epoch': 112.19}
+
 92%|█████████▏| 4600/5000 [15:49:09<1:14:18, 11.15s/it][2022-12-21 02:11:45,229] [INFO] [timer.py:197:stop] 0/9314, RunningAvgSamplesPerSec=5.880696334476508, CurrSamplesPerSec=5.2993564422404, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4601/5000 [15:49:20<1:13:49, 11.10s/it][2022-12-21 02:11:56,256] [INFO] [timer.py:197:stop] 0/9316, RunningAvgSamplesPerSec=5.8806938485761355, CurrSamplesPerSec=5.350169711914274, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4602/5000 [15:49:31<1:13:29, 11.08s/it][2022-12-21 02:12:07,339] [INFO] [timer.py:197:stop] 0/9318, RunningAvgSamplesPerSec=5.880685202537343, CurrSamplesPerSec=5.311841816356127, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4603/5000 [15:49:42<1:13:18, 11.08s/it][2022-12-21 02:12:18,341] [INFO] [logging.py:68:log_dist] [Rank 0] step=4660, skipped=7, lr=[7.733333333333335e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:12:18,343] [INFO] [timer.py:197:stop] 0/9320, RunningAvgSamplesPerSec=5.880685432687263, CurrSamplesPerSec=5.312487279619281, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4604/5000 [15:49:53<1:12:58, 11.06s/it][2022-12-21 02:12:29,369] [INFO] [timer.py:197:stop] 0/9322, RunningAvgSamplesPerSec=5.8806832894527075, CurrSamplesPerSec=5.32899044890735, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4605/5000 [15:50:04<1:12:43, 11.05s/it][2022-12-21 02:12:40,386] [INFO] [timer.py:197:stop] 0/9324, RunningAvgSamplesPerSec=5.88068180333802, CurrSamplesPerSec=5.3088287173653494, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4606/5000 [15:50:15<1:12:29, 11.04s/it][2022-12-21 02:12:51,419] [INFO] [timer.py:197:stop] 0/9326, RunningAvgSamplesPerSec=5.880678782547304, CurrSamplesPerSec=5.329206906724594, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4607/5000 [15:50:27<1:12:17, 11.04s/it][2022-12-21 02:13:02,407] [INFO] [timer.py:197:stop] 0/9328, RunningAvgSamplesPerSec=5.880680749026344, CurrSamplesPerSec=5.319663735438205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4608/5000 [15:50:38<1:12:00, 11.02s/it][2022-12-21 02:13:13,466] [INFO] [timer.py:197:stop] 0/9330, RunningAvgSamplesPerSec=5.880674515019637, CurrSamplesPerSec=5.30754035432129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4609/5000 [15:50:49<1:11:54, 11.03s/it][2022-12-21 02:13:24,495] [INFO] [timer.py:197:stop] 0/9332, RunningAvgSamplesPerSec=5.880671873900263, CurrSamplesPerSec=5.339664973516253, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4610/5000 [15:51:00<1:11:42, 11.03s/it][2022-12-21 02:13:35,448] [INFO] [timer.py:197:stop] 0/9334, RunningAvgSamplesPerSec=5.880678028343625, CurrSamplesPerSec=5.345741462550639, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4611/5000 [15:51:11<1:11:22, 11.01s/it][2022-12-21 02:13:46,470] [INFO] [timer.py:197:stop] 0/9336, RunningAvgSamplesPerSec=5.880676122343861, CurrSamplesPerSec=5.302391264077579, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4612/5000 [15:51:22<1:11:12, 11.01s/it][2022-12-21 02:13:57,429] [INFO] [timer.py:197:stop] 0/9338, RunningAvgSamplesPerSec=5.880681502507283, CurrSamplesPerSec=5.361325716676022, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4613/5000 [15:51:33<1:10:55, 11.00s/it][2022-12-21 02:14:08,411] [INFO] [logging.py:68:log_dist] [Rank 0] step=4670, skipped=7, lr=[7.511111111111112e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:14:08,412] [INFO] [timer.py:197:stop] 0/9340, RunningAvgSamplesPerSec=5.880684103652808, CurrSamplesPerSec=5.323047380801796, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4614/5000 [15:51:44<1:10:43, 10.99s/it][2022-12-21 02:14:19,364] [INFO] [timer.py:197:stop] 0/9342, RunningAvgSamplesPerSec=5.880690577382234, CurrSamplesPerSec=5.350064146449623, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4615/5000 [15:51:54<1:10:27, 10.98s/it][2022-12-21 02:14:30,383] [INFO] [timer.py:197:stop] 0/9344, RunningAvgSamplesPerSec=5.880689073771709, CurrSamplesPerSec=5.331595033913721, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4616/5000 [15:52:05<1:10:20, 10.99s/it][2022-12-21 02:14:41,366] [INFO] [timer.py:197:stop] 0/9346, RunningAvgSamplesPerSec=5.880691773136352, CurrSamplesPerSec=5.335867707996117, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4617/5000 [15:52:16<1:10:08, 10.99s/it][2022-12-21 02:14:52,313] [INFO] [timer.py:197:stop] 0/9348, RunningAvgSamplesPerSec=5.880698594052202, CurrSamplesPerSec=5.3455264271695935, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4618/5000 [15:52:27<1:09:53, 10.98s/it][2022-12-21 02:15:03,327] [INFO] [timer.py:197:stop] 0/9350, RunningAvgSamplesPerSec=5.880697658563655, CurrSamplesPerSec=5.333835861079956, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4619/5000 [15:52:38<1:09:46, 10.99s/it][2022-12-21 02:15:14,347] [INFO] [timer.py:197:stop] 0/9352, RunningAvgSamplesPerSec=5.8806961643112565, CurrSamplesPerSec=5.337276829296905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4620/5000 [15:52:49<1:09:39, 11.00s/it][2022-12-21 02:15:25,339] [INFO] [timer.py:197:stop] 0/9354, RunningAvgSamplesPerSec=5.8806977989082085, CurrSamplesPerSec=5.350207673847223, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4621/5000 [15:53:00<1:09:27, 11.00s/it][2022-12-21 02:15:36,307] [INFO] [timer.py:197:stop] 0/9356, RunningAvgSamplesPerSec=5.880702191766011, CurrSamplesPerSec=5.372426194235268, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4622/5000 [15:53:11<1:09:13, 10.99s/it][2022-12-21 02:15:47,291] [INFO] [timer.py:197:stop] 0/9358, RunningAvgSamplesPerSec=5.880704831476866, CurrSamplesPerSec=5.33752749785861, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4623/5000 [15:53:22<1:09:01, 10.99s/it][2022-12-21 02:15:58,294] [INFO] [logging.py:68:log_dist] [Rank 0] step=4680, skipped=7, lr=[7.28888888888889e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:15:58,295] [INFO] [timer.py:197:stop] 0/9360, RunningAvgSamplesPerSec=5.880704982383556, CurrSamplesPerSec=5.3099899764429, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▏| 4624/5000 [15:53:33<1:08:52, 10.99s/it][2022-12-21 02:16:09,277] [INFO] [timer.py:197:stop] 0/9362, RunningAvgSamplesPerSec=5.880707774198584, CurrSamplesPerSec=5.350771621278556, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 92%|█████████▎| 4625/5000 [15:53:44<1:08:40, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 7.266666666666668e-07, 'epoch': 112.8}
+
 92%|█████████▎| 4625/5000 [15:53:44<1:08:40, 10.99s/it][2022-12-21 02:16:20,337] [INFO] [timer.py:197:stop] 0/9364, RunningAvgSamplesPerSec=5.8807017191884965, CurrSamplesPerSec=5.289321899187964, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4626/5000 [15:53:55<1:08:37, 11.01s/it][2022-12-21 02:16:31,342] [INFO] [timer.py:197:stop] 0/9366, RunningAvgSamplesPerSec=5.880701937197023, CurrSamplesPerSec=5.336895883637636, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4627/5000 [15:54:06<1:08:26, 11.01s/it][2022-12-21 02:16:42,364] [INFO] [timer.py:197:stop] 0/9368, RunningAvgSamplesPerSec=5.880700082216833, CurrSamplesPerSec=5.337319490107559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4628/5000 [15:54:17<1:08:16, 11.01s/it][2022-12-21 02:16:53,368] [INFO] [timer.py:197:stop] 0/9370, RunningAvgSamplesPerSec=5.880700452525664, CurrSamplesPerSec=5.327815366597375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4629/5000 [15:54:28<1:08:04, 11.01s/it][2022-12-21 02:17:04,392] [INFO] [timer.py:197:stop] 0/9372, RunningAvgSamplesPerSec=5.880698355623813, CurrSamplesPerSec=5.3258513550438, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4630/5000 [15:54:39<1:07:55, 11.01s/it][2022-12-21 02:17:15,404] [INFO] [timer.py:197:stop] 0/9374, RunningAvgSamplesPerSec=5.880697516578974, CurrSamplesPerSec=5.323837475269936, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4631/5000 [15:54:50<1:07:44, 11.01s/it][2022-12-21 02:17:26,474] [INFO] [timer.py:197:stop] 0/9376, RunningAvgSamplesPerSec=5.880690092606823, CurrSamplesPerSec=5.282405392001692, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4632/5000 [15:55:02<1:07:39, 11.03s/it][2022-12-21 02:17:37,507] [INFO] [timer.py:197:stop] 0/9378, RunningAvgSamplesPerSec=5.880687165305491, CurrSamplesPerSec=5.306850559413277, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4633/5000 [15:55:13<1:07:28, 11.03s/it][2022-12-21 02:17:47,620] [INFO] [logging.py:68:log_dist] [Rank 0] step=4690, skipped=7, lr=[7.066666666666667e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:17:47,621] [INFO] [timer.py:197:stop] 0/9380, RunningAvgSamplesPerSec=5.88078889299912, CurrSamplesPerSec=5.314390311475187, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4634/5000 [15:55:28<1:14:36, 12.23s/it][2022-12-21 02:17:58,568] [INFO] [timer.py:197:stop] 0/9382, RunningAvgSamplesPerSec=5.880795693869737, CurrSamplesPerSec=5.364394859078534, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4635/5000 [15:55:39<1:12:06, 11.85s/it][2022-12-21 02:18:09,570] [INFO] [timer.py:197:stop] 0/9384, RunningAvgSamplesPerSec=5.880796944054113, CurrSamplesPerSec=5.339768217035852, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4636/5000 [15:55:50<1:10:18, 11.59s/it][2022-12-21 02:18:20,551] [INFO] [timer.py:197:stop] 0/9386, RunningAvgSamplesPerSec=5.880800058239515, CurrSamplesPerSec=5.334349507949397, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4637/5000 [15:56:01<1:08:58, 11.40s/it][2022-12-21 02:18:31,539] [INFO] [timer.py:197:stop] 0/9388, RunningAvgSamplesPerSec=5.880802089305658, CurrSamplesPerSec=5.309938088048904, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4638/5000 [15:56:12<1:08:08, 11.29s/it][2022-12-21 02:18:42,572] [INFO] [timer.py:197:stop] 0/9390, RunningAvgSamplesPerSec=5.88079884255112, CurrSamplesPerSec=5.320466323656873, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4639/5000 [15:56:23<1:07:29, 11.22s/it][2022-12-21 02:18:53,616] [INFO] [timer.py:197:stop] 0/9392, RunningAvgSamplesPerSec=5.88079460272898, CurrSamplesPerSec=5.31288010080947, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4640/5000 [15:56:34<1:06:59, 11.16s/it][2022-12-21 02:19:04,632] [INFO] [timer.py:197:stop] 0/9394, RunningAvgSamplesPerSec=5.880793588239236, CurrSamplesPerSec=5.336838162911671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4641/5000 [15:56:45<1:06:31, 11.12s/it][2022-12-21 02:19:15,662] [INFO] [timer.py:197:stop] 0/9396, RunningAvgSamplesPerSec=5.880790969306624, CurrSamplesPerSec=5.32137527408203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4642/5000 [15:56:56<1:06:12, 11.10s/it][2022-12-21 02:19:26,637] [INFO] [timer.py:197:stop] 0/9398, RunningAvgSamplesPerSec=5.880794466351256, CurrSamplesPerSec=5.384895253548928, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4643/5000 [15:57:07<1:05:42, 11.04s/it][2022-12-21 02:19:37,623] [INFO] [logging.py:68:log_dist] [Rank 0] step=4700, skipped=7, lr=[6.844444444444446e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:19:37,624] [INFO] [timer.py:197:stop] 0/9400, RunningAvgSamplesPerSec=5.8807965547931556, CurrSamplesPerSec=5.322559125940257, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4644/5000 [15:57:18<1:05:29, 11.04s/it][2022-12-21 02:19:48,671] [INFO] [timer.py:197:stop] 0/9402, RunningAvgSamplesPerSec=5.880791747535703, CurrSamplesPerSec=5.304128596194462, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4645/5000 [15:57:29<1:05:18, 11.04s/it][2022-12-21 02:19:59,662] [INFO] [timer.py:197:stop] 0/9404, RunningAvgSamplesPerSec=5.880793389198383, CurrSamplesPerSec=5.3488081305172726, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4646/5000 [15:57:40<1:04:58, 11.01s/it][2022-12-21 02:20:10,653] [INFO] [timer.py:197:stop] 0/9406, RunningAvgSamplesPerSec=5.880795151353624, CurrSamplesPerSec=5.311056539057226, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4647/5000 [15:57:51<1:04:49, 11.02s/it][2022-12-21 02:20:21,681] [INFO] [timer.py:197:stop] 0/9408, RunningAvgSamplesPerSec=5.880792506774223, CurrSamplesPerSec=5.318655466154241, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4648/5000 [15:58:02<1:04:37, 11.01s/it][2022-12-21 02:20:32,666] [INFO] [timer.py:197:stop] 0/9410, RunningAvgSamplesPerSec=5.8807949759984215, CurrSamplesPerSec=5.334540958398183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4649/5000 [15:58:13<1:04:27, 11.02s/it][2022-12-21 02:20:43,712] [INFO] [timer.py:197:stop] 0/9412, RunningAvgSamplesPerSec=5.880790505378182, CurrSamplesPerSec=5.319458171792316, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4650/5000 [15:58:24<1:04:14, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 6.711111111111111e-07, 'epoch': 113.41}
+
 93%|█████████▎| 4650/5000 [15:58:24<1:04:14, 11.01s/it][2022-12-21 02:20:54,678] [INFO] [timer.py:197:stop] 0/9414, RunningAvgSamplesPerSec=5.880795394153776, CurrSamplesPerSec=5.353775512735795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4651/5000 [15:58:35<1:04:02, 11.01s/it][2022-12-21 02:21:05,723] [INFO] [timer.py:197:stop] 0/9416, RunningAvgSamplesPerSec=5.8807909106728795, CurrSamplesPerSec=5.316965686015462, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4652/5000 [15:58:46<1:03:55, 11.02s/it][2022-12-21 02:21:16,746] [INFO] [timer.py:197:stop] 0/9418, RunningAvgSamplesPerSec=5.880788914360871, CurrSamplesPerSec=5.332761190127796, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4653/5000 [15:58:57<1:03:41, 11.01s/it][2022-12-21 02:21:27,765] [INFO] [logging.py:68:log_dist] [Rank 0] step=4710, skipped=7, lr=[6.622222222222223e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:21:27,766] [INFO] [timer.py:197:stop] 0/9420, RunningAvgSamplesPerSec=5.880787360146206, CurrSamplesPerSec=5.309719621724139, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4654/5000 [15:59:08<1:03:31, 11.01s/it][2022-12-21 02:21:38,775] [INFO] [timer.py:197:stop] 0/9422, RunningAvgSamplesPerSec=5.880786969491311, CurrSamplesPerSec=5.319043719459168, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4655/5000 [15:59:19<1:03:17, 11.01s/it][2022-12-21 02:21:49,725] [INFO] [timer.py:197:stop] 0/9424, RunningAvgSamplesPerSec=5.880793473099913, CurrSamplesPerSec=5.356657283910076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4656/5000 [15:59:30<1:03:04, 11.00s/it][2022-12-21 02:22:00,725] [INFO] [timer.py:197:stop] 0/9426, RunningAvgSamplesPerSec=5.880794147833345, CurrSamplesPerSec=5.3453995431203065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4657/5000 [15:59:41<1:02:50, 10.99s/it][2022-12-21 02:22:11,675] [INFO] [timer.py:197:stop] 0/9428, RunningAvgSamplesPerSec=5.880800444833497, CurrSamplesPerSec=5.368231347410081, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4658/5000 [15:59:52<1:02:37, 10.99s/it][2022-12-21 02:22:22,710] [INFO] [timer.py:197:stop] 0/9430, RunningAvgSamplesPerSec=5.880797053989978, CurrSamplesPerSec=5.313987799238561, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4659/5000 [16:00:03<1:02:33, 11.01s/it][2022-12-21 02:22:33,781] [INFO] [timer.py:197:stop] 0/9432, RunningAvgSamplesPerSec=5.880789627469041, CurrSamplesPerSec=5.294869412707697, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4660/5000 [16:00:14<1:02:27, 11.02s/it][2022-12-21 02:22:44,786] [INFO] [timer.py:197:stop] 0/9434, RunningAvgSamplesPerSec=5.880789619836106, CurrSamplesPerSec=5.339970891851903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████��| 4661/5000 [16:00:25<1:02:15, 11.02s/it][2022-12-21 02:22:55,800] [INFO] [timer.py:197:stop] 0/9436, RunningAvgSamplesPerSec=5.880788769089052, CurrSamplesPerSec=5.3382102168041285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4662/5000 [16:00:36<1:02:03, 11.02s/it][2022-12-21 02:23:06,833] [INFO] [timer.py:197:stop] 0/9438, RunningAvgSamplesPerSec=5.880785622079926, CurrSamplesPerSec=5.316435375499112, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4663/5000 [16:00:47<1:01:51, 11.01s/it][2022-12-21 02:23:17,865] [INFO] [logging.py:68:log_dist] [Rank 0] step=4720, skipped=7, lr=[6.4e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:23:17,866] [INFO] [timer.py:197:stop] 0/9440, RunningAvgSamplesPerSec=5.880782435237846, CurrSamplesPerSec=5.2971576570048535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4664/5000 [16:00:58<1:01:45, 11.03s/it][2022-12-21 02:23:28,886] [INFO] [timer.py:197:stop] 0/9442, RunningAvgSamplesPerSec=5.880780786721044, CurrSamplesPerSec=5.331859148504126, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4665/5000 [16:01:09<1:01:33, 11.02s/it][2022-12-21 02:23:39,922] [INFO] [timer.py:197:stop] 0/9444, RunningAvgSamplesPerSec=5.880777362138165, CurrSamplesPerSec=5.313758901548024, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4666/5000 [16:01:20<1:01:21, 11.02s/it][2022-12-21 02:23:50,886] [INFO] [timer.py:197:stop] 0/9446, RunningAvgSamplesPerSec=5.880782115397386, CurrSamplesPerSec=5.361493407712165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4667/5000 [16:01:31<1:01:02, 11.00s/it][2022-12-21 02:24:01,855] [INFO] [timer.py:197:stop] 0/9448, RunningAvgSamplesPerSec=5.880786423819913, CurrSamplesPerSec=5.338266693293489, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4668/5000 [16:01:42<1:00:46, 10.98s/it][2022-12-21 02:24:12,827] [INFO] [timer.py:197:stop] 0/9450, RunningAvgSamplesPerSec=5.880790260114006, CurrSamplesPerSec=5.313325144028703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4669/5000 [16:01:53<1:00:42, 11.00s/it][2022-12-21 02:24:23,842] [INFO] [timer.py:197:stop] 0/9452, RunningAvgSamplesPerSec=5.880789191067208, CurrSamplesPerSec=5.350944625955184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4670/5000 [16:02:04<1:00:28, 11.00s/it][2022-12-21 02:24:34,853] [INFO] [timer.py:197:stop] 0/9454, RunningAvgSamplesPerSec=5.880788568813164, CurrSamplesPerSec=5.317789583049633, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4671/5000 [16:02:15<1:00:20, 11.01s/it][2022-12-21 02:24:45,843] [INFO] [timer.py:197:stop] 0/9456, RunningAvgSamplesPerSec=5.880790376603267, CurrSamplesPerSec=5.353884214609746, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4672/5000 [16:02:26<1:00:09, 11.00s/it][2022-12-21 02:24:56,880] [INFO] [timer.py:197:stop] 0/9458, RunningAvgSamplesPerSec=5.880786852021977, CurrSamplesPerSec=5.322901296026654, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4673/5000 [16:02:37<59:57, 11.00s/it]  [2022-12-21 02:25:07,873] [INFO] [logging.py:68:log_dist] [Rank 0] step=4730, skipped=7, lr=[6.177777777777778e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:25:07,875] [INFO] [timer.py:197:stop] 0/9460, RunningAvgSamplesPerSec=5.880788068581071, CurrSamplesPerSec=5.324713145166745, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 93%|█████████▎| 4674/5000 [16:02:48<59:50, 11.02s/it][2022-12-21 02:25:17,997] [INFO] [timer.py:197:stop] 0/9462, RunningAvgSamplesPerSec=5.880887851594186, CurrSamplesPerSec=6.278946082906602, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 02:25:29,068] [INFO] [timer.py:197:stop] 0/9464, RunningAvgSamplesPerSec=5.880880049918166, CurrSamplesPerSec=5.30392276421961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4675/5000 [16:03:04<1:08:07, 12.58s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 6.133333333333333e-07, 'epoch': 114.02}
+
 94%|█████████▎| 4675/5000 [16:03:04<1:08:07, 12.58s/it][2022-12-21 02:25:40,117] [INFO] [timer.py:197:stop] 0/9466, RunningAvgSamplesPerSec=5.880875437495437, CurrSamplesPerSec=5.313117756390179, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4676/5000 [16:03:15<1:05:26, 12.12s/it][2022-12-21 02:25:51,147] [INFO] [timer.py:197:stop] 0/9468, RunningAvgSamplesPerSec=5.880872643743671, CurrSamplesPerSec=5.30335942805252, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4677/5000 [16:03:26<1:03:28, 11.79s/it][2022-12-21 02:26:02,209] [INFO] [timer.py:197:stop] 0/9470, RunningAvgSamplesPerSec=5.880866206270647, CurrSamplesPerSec=5.29196775543393, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|████████���▎| 4678/5000 [16:03:37<1:02:06, 11.57s/it][2022-12-21 02:26:13,261] [INFO] [timer.py:197:stop] 0/9472, RunningAvgSamplesPerSec=5.8808610317970444, CurrSamplesPerSec=5.316433901391751, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4679/5000 [16:03:48<1:01:04, 11.42s/it][2022-12-21 02:26:24,285] [INFO] [timer.py:197:stop] 0/9474, RunningAvgSamplesPerSec=5.880858933847421, CurrSamplesPerSec=5.322914173119551, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4680/5000 [16:03:59<1:00:15, 11.30s/it][2022-12-21 02:26:35,277] [INFO] [timer.py:197:stop] 0/9476, RunningAvgSamplesPerSec=5.880860279925671, CurrSamplesPerSec=5.365175829805524, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4681/5000 [16:04:10<59:35, 11.21s/it]  [2022-12-21 02:26:46,287] [INFO] [timer.py:197:stop] 0/9478, RunningAvgSamplesPerSec=5.880859778939343, CurrSamplesPerSec=5.335196828052147, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4682/5000 [16:04:21<59:04, 11.15s/it][2022-12-21 02:26:57,288] [INFO] [logging.py:68:log_dist] [Rank 0] step=4740, skipped=7, lr=[5.955555555555556e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:26:57,290] [INFO] [timer.py:197:stop] 0/9480, RunningAvgSamplesPerSec=5.880860078701431, CurrSamplesPerSec=5.338846177131293, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4683/5000 [16:04:32<58:40, 11.10s/it][2022-12-21 02:27:08,307] [INFO] [timer.py:197:stop] 0/9482, RunningAvgSamplesPerSec=5.88085880385782, CurrSamplesPerSec=5.334523148474391, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4684/5000 [16:04:43<58:20, 11.08s/it][2022-12-21 02:27:19,244] [INFO] [timer.py:197:stop] 0/9484, RunningAvgSamplesPerSec=5.880866586789677, CurrSamplesPerSec=5.369438074913051, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4685/5000 [16:04:54<57:56, 11.04s/it][2022-12-21 02:27:30,233] [INFO] [timer.py:197:stop] 0/9486, RunningAvgSamplesPerSec=5.880868364957599, CurrSamplesPerSec=5.334285906190174, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4686/5000 [16:05:05<57:40, 11.02s/it][2022-12-21 02:27:41,264] [INFO] [timer.py:197:stop] 0/9488, RunningAvgSamplesPerSec=5.880865537576707, CurrSamplesPerSec=5.322996081332886, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▎| 4687/5000 [16:05:16<57:30, 11.02s/it][2022-12-21 02:27:52,269] [INFO] [timer.py:197:stop] 0/9490, RunningAvgSamplesPerSec=5.880865503259718, CurrSamplesPerSec=5.339914166846696, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4688/5000 [16:05:27<57:17, 11.02s/it][2022-12-21 02:28:03,211] [INFO] [timer.py:197:stop] 0/9492, RunningAvgSamplesPerSec=5.88087274333504, CurrSamplesPerSec=5.37592856198446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4689/5000 [16:05:38<56:59, 11.00s/it][2022-12-21 02:28:14,206] [INFO] [timer.py:197:stop] 0/9494, RunningAvgSamplesPerSec=5.880874334602242, CurrSamplesPerSec=5.311362971361763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4690/5000 [16:05:49<56:48, 11.00s/it][2022-12-21 02:28:25,251] [INFO] [timer.py:197:stop] 0/9496, RunningAvgSamplesPerSec=5.88086991590282, CurrSamplesPerSec=5.307259336211239, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4691/5000 [16:06:00<56:42, 11.01s/it][2022-12-21 02:28:36,314] [INFO] [timer.py:197:stop] 0/9498, RunningAvgSamplesPerSec=5.880863501356603, CurrSamplesPerSec=5.312874212268554, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4692/5000 [16:06:11<56:35, 11.03s/it][2022-12-21 02:28:47,290] [INFO] [logging.py:68:log_dist] [Rank 0] step=4750, skipped=7, lr=[5.733333333333334e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:28:47,292] [INFO] [timer.py:197:stop] 0/9500, RunningAvgSamplesPerSec=5.880866705535532, CurrSamplesPerSec=5.343637627235034, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4693/5000 [16:06:22<56:20, 11.01s/it][2022-12-21 02:28:58,233] [INFO] [timer.py:197:stop] 0/9502, RunningAvgSamplesPerSec=5.880874005657511, CurrSamplesPerSec=5.372548988212856, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4694/5000 [16:06:33<56:03, 10.99s/it][2022-12-21 02:29:09,222] [INFO] [timer.py:197:stop] 0/9504, RunningAvgSamplesPerSec=5.8808759740417, CurrSamplesPerSec=5.319229856642703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4695/5000 [16:06:44<55:51, 10.99s/it][2022-12-21 02:29:20,217] [INFO] [timer.py:197:stop] 0/9506, RunningAvgSamplesPerSec=5.880877130395758, CurrSamplesPerSec=5.344237642067467, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4696/5000 [16:06:55<55:41, 10.99s/it][2022-12-21 02:29:31,221] [INFO] [timer.py:197:stop] 0/9508, RunningAvgSamplesPerSec=5.880877367509262, CurrSamplesPerSec=5.321058403469919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4697/5000 [16:07:06<55:31, 11.00s/it][2022-12-21 02:29:42,247] [INFO] [timer.py:197:stop] 0/9510, RunningAvgSamplesPerSec=5.880875019658428, CurrSamplesPerSec=5.2961263439907675, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4698/5000 [16:07:17<55:23, 11.00s/it][2022-12-21 02:29:53,202] [INFO] [timer.py:197:stop] 0/9512, RunningAvgSamplesPerSec=5.8808810340777145, CurrSamplesPerSec=5.36053344880522, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4699/5000 [16:07:28<55:07, 10.99s/it][2022-12-21 02:30:04,236] [INFO] [timer.py:197:stop] 0/9514, RunningAvgSamplesPerSec=5.880877955427366, CurrSamplesPerSec=5.307913552073799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4700/5000 [16:07:39<55:00, 11.00s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 5.577777777777779e-07, 'epoch': 114.63}
+
 94%|█████████▍| 4700/5000 [16:07:39<55:00, 11.00s/it][2022-12-21 02:30:15,204] [INFO] [timer.py:197:stop] 0/9516, RunningAvgSamplesPerSec=5.880882361982548, CurrSamplesPerSec=5.362196624955059, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4701/5000 [16:07:50<54:46, 10.99s/it][2022-12-21 02:30:26,223] [INFO] [timer.py:197:stop] 0/9518, RunningAvgSamplesPerSec=5.880880979605738, CurrSamplesPerSec=5.298048203423232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4702/5000 [16:08:01<54:38, 11.00s/it][2022-12-21 02:30:37,169] [INFO] [logging.py:68:log_dist] [Rank 0] step=4760, skipped=7, lr=[5.511111111111111e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:30:37,170] [INFO] [timer.py:197:stop] 0/9520, RunningAvgSamplesPerSec=5.880887499212462, CurrSamplesPerSec=5.359891455368944, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4703/5000 [16:08:12<54:22, 10.98s/it][2022-12-21 02:30:48,166] [INFO] [timer.py:197:stop] 0/9522, RunningAvgSamplesPerSec=5.8808886008077, CurrSamplesPerSec=5.357850685486765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4704/5000 [16:08:23<54:12, 10.99s/it][2022-12-21 02:30:59,214] [INFO] [timer.py:197:stop] 0/9524, RunningAvgSamplesPerSec=5.8808838104937795, CurrSamplesPerSec=5.3304448415317545, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4705/5000 [16:08:34<54:06, 11.01s/it][2022-12-21 02:31:10,238] [INFO] [timer.py:197:stop] 0/9526, RunningAvgSamplesPerSec=5.880881720010783, CurrSamplesPerSec=5.3113793658559985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4706/5000 [16:08:45<53:57, 11.01s/it][2022-12-21 02:31:21,184] [INFO] [timer.py:197:stop] 0/9528, RunningAvgSamplesPerSec=5.880888385630128, CurrSamplesPerSec=5.38144916274789, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4707/5000 [16:08:56<53:40, 10.99s/it][2022-12-21 02:31:32,139] [INFO] [timer.py:197:stop] 0/9530, RunningAvgSamplesPerSec=5.880894165173551, CurrSamplesPerSec=5.3629488838628205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4708/5000 [16:09:07<53:26, 10.98s/it][2022-12-21 02:31:43,121] [INFO] [timer.py:197:stop] 0/9532, RunningAvgSamplesPerSec=5.880896884017917, CurrSamplesPerSec=5.328187825333314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4709/5000 [16:09:18<53:15, 10.98s/it][2022-12-21 02:31:54,148] [INFO] [timer.py:197:stop] 0/9534, RunningAvgSamplesPerSec=5.880894517936693, CurrSamplesPerSec=5.31600897138238, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4710/5000 [16:09:29<53:08, 10.99s/it][2022-12-21 02:32:05,118] [INFO] [timer.py:197:stop] 0/9536, RunningAvgSamplesPerSec=5.880898635147077, CurrSamplesPerSec=5.345557510422361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4711/5000 [16:09:40<52:55, 10.99s/it][2022-12-21 02:32:16,078] [INFO] [timer.py:197:stop] 0/9538, RunningAvgSamplesPerSec=5.880903868224854, CurrSamplesPerSec=5.338669919192283, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4712/5000 [16:09:51<52:41, 10.98s/it][2022-12-21 02:32:27,076] [INFO] [logging.py:68:log_dist] [Rank 0] step=4770, skipped=7, lr=[5.288888888888889e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:32:27,077] [INFO] [timer.py:197:stop] 0/9540, RunningAvgSamplesPerSec=5.880904764747687, CurrSamplesPerSec=5.360762111814961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4713/5000 [16:10:02<52:32, 10.99s/it][2022-12-21 02:32:38,107] [INFO] [timer.py:197:stop] 0/9542, RunningAvgSamplesPerSec=5.8809021717543235, CurrSamplesPerSec=5.3177225831637225, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4714/5000 [16:10:13<52:25, 11.00s/it][2022-12-21 02:32:49,151] [INFO] [timer.py:197:stop] 0/9544, RunningAvgSamplesPerSec=5.880897999404677, CurrSamplesPerSec=5.321691127418676, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4715/5000 [16:10:24<52:18, 11.01s/it][2022-12-21 02:32:59,242] [INFO] [timer.py:197:stop] 0/9546, RunningAvgSamplesPerSec=5.881000593704332, CurrSamplesPerSec=5.34906180203864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4716/5000 [16:10:39<57:49, 12.22s/it][2022-12-21 02:33:10,275] [INFO] [timer.py:197:stop] 0/9548, RunningAvgSamplesPerSec=5.880997517581248, CurrSamplesPerSec=5.3081931699300755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4717/5000 [16:10:50<55:57, 11.86s/it][2022-12-21 02:33:21,271] [INFO] [timer.py:197:stop] 0/9550, RunningAvgSamplesPerSec=5.88099863134607, CurrSamplesPerSec=5.344514077211338, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4718/5000 [16:11:01<54:29, 11.59s/it][2022-12-21 02:33:32,279] [INFO] [timer.py:197:stop] 0/9552, RunningAvgSamplesPerSec=5.880998249494198, CurrSamplesPerSec=5.30742660031388, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4719/5000 [16:11:12<53:31, 11.43s/it][2022-12-21 02:33:43,317] [INFO] [timer.py:197:stop] 0/9554, RunningAvgSamplesPerSec=5.880994565301846, CurrSamplesPerSec=5.309396366558196, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4720/5000 [16:11:23<52:45, 11.31s/it][2022-12-21 02:33:54,324] [INFO] [timer.py:197:stop] 0/9556, RunningAvgSamplesPerSec=5.880994213833595, CurrSamplesPerSec=5.31832964692065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4721/5000 [16:11:34<52:12, 11.23s/it][2022-12-21 02:34:05,387] [INFO] [timer.py:197:stop] 0/9558, RunningAvgSamplesPerSec=5.8809884030759605, CurrSamplesPerSec=5.306666336658286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4722/5000 [16:11:45<51:43, 11.16s/it][2022-12-21 02:34:16,370] [INFO] [logging.py:68:log_dist] [Rank 0] step=4780, skipped=7, lr=[5.066666666666667e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:34:16,371] [INFO] [timer.py:197:stop] 0/9560, RunningAvgSamplesPerSec=5.880990807735305, CurrSamplesPerSec=5.33344798864372, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4723/5000 [16:11:56<51:20, 11.12s/it][2022-12-21 02:34:27,348] [INFO] [timer.py:197:stop] 0/9562, RunningAvgSamplesPerSec=5.880994155099633, CurrSamplesPerSec=5.36987567179094, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4724/5000 [16:12:07<50:56, 11.08s/it][2022-12-21 02:34:38,374] [INFO] [timer.py:197:stop] 0/9564, RunningAvgSamplesPerSec=5.88099197358117, CurrSamplesPerSec=5.320407903258427, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 94%|█████████▍| 4725/5000 [16:12:18<50:37, 11.05s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 5.022222222222222e-07, 'epoch': 115.24}
+
 94%|█████████▍| 4725/5000 [16:12:18<50:37, 11.05s/it][2022-12-21 02:34:49,358] [INFO] [timer.py:197:stop] 0/9566, RunningAvgSamplesPerSec=5.880994530831817, CurrSamplesPerSec=5.31895518765823, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4726/5000 [16:12:29<50:24, 11.04s/it][2022-12-21 02:35:00,331] [INFO] [timer.py:197:stop] 0/9568, RunningAvgSamplesPerSec=5.880998263657485, CurrSamplesPerSec=5.35284862951076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4727/5000 [16:12:40<50:09, 11.02s/it][2022-12-21 02:35:11,355] [INFO] [timer.py:197:stop] 0/9570, RunningAvgSamplesPerSec=5.880996240494401, CurrSamplesPerSec=5.321895386545972, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4728/5000 [16:12:51<49:59, 11.03s/it][2022-12-21 02:35:22,391] [INFO] [timer.py:197:stop] 0/9572, RunningAvgSamplesPerSec=5.880992826698634, CurrSamplesPerSec=5.3294051831268545, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4729/5000 [16:13:02<49:45, 11.02s/it][2022-12-21 02:35:33,402] [INFO] [timer.py:197:stop] 0/9574, RunningAvgSamplesPerSec=5.880992126929935, CurrSamplesPerSec=5.305552040729241, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4730/5000 [16:13:13<49:34, 11.02s/it][2022-12-21 02:35:44,399] [INFO] [timer.py:197:stop] 0/9576, RunningAvgSamplesPerSec=5.880993230611954, CurrSamplesPerSec=5.328104699704492, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4731/5000 [16:13:24<49:20, 11.01s/it][2022-12-21 02:35:55,398] [INFO] [timer.py:197:stop] 0/9578, RunningAvgSamplesPerSec=5.8809940315845735, CurrSamplesPerSec=5.31392825869537, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4732/5000 [16:13:35<49:13, 11.02s/it][2022-12-21 02:36:06,442] [INFO] [logging.py:68:log_dist] [Rank 0] step=4790, skipped=7, lr=[4.844444444444445e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:36:06,443] [INFO] [timer.py:197:stop] 0/9580, RunningAvgSamplesPerSec=5.880989476905555, CurrSamplesPerSec=5.315551689506329, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4733/5000 [16:13:47<49:04, 11.03s/it][2022-12-21 02:36:17,499] [INFO] [timer.py:197:stop] 0/9582, RunningAvgSamplesPerSec=5.880983864531018, CurrSamplesPerSec=5.3123916065958205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4734/5000 [16:13:58<48:55, 11.03s/it][2022-12-21 02:36:28,515] [INFO] [timer.py:197:stop] 0/9584, RunningAvgSamplesPerSec=5.880982646508299, CurrSamplesPerSec=5.340226488003584, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4735/5000 [16:14:09<48:44, 11.03s/it][2022-12-21 02:36:39,537] [INFO] [timer.py:197:stop] 0/9586, RunningAvgSamplesPerSec=5.880980679303313, CurrSamplesPerSec=5.351560365901092, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4736/5000 [16:14:20<48:31, 11.03s/it][2022-12-21 02:36:50,545] [INFO] [timer.py:197:stop] 0/9588, RunningAvgSamplesPerSec=5.880980302633258, CurrSamplesPerSec=5.351210234436573, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4737/5000 [16:14:31<48:15, 11.01s/it][2022-12-21 02:37:01,548] [INFO] [timer.py:197:stop] 0/9590, RunningAvgSamplesPerSec=5.880980703100028, CurrSamplesPerSec=5.317763246415105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4738/5000 [16:14:42<48:04, 11.01s/it][2022-12-21 02:37:12,551] [INFO] [timer.py:197:stop] 0/9592, RunningAvgSamplesPerSec=5.880980879302264, CurrSamplesPerSec=5.325121509411445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4739/5000 [16:14:53<47:55, 11.02s/it][2022-12-21 02:37:23,619] [INFO] [timer.py:197:stop] 0/9594, RunningAvgSamplesPerSec=5.88097407738425, CurrSamplesPerSec=5.298682788010215, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4740/5000 [16:15:04<47:47, 11.03s/it][2022-12-21 02:37:34,652] [INFO] [timer.py:197:stop] 0/9596, RunningAvgSamplesPerSec=5.880970959813388, CurrSamplesPerSec=5.323124015183574, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4741/5000 [16:15:15<47:36, 11.03s/it][2022-12-21 02:37:45,720] [INFO] [timer.py:197:stop] 0/9598, RunningAvgSamplesPerSec=5.880964069559679, CurrSamplesPerSec=5.28422430766218, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4742/5000 [16:15:26<47:30, 11.05s/it][2022-12-21 02:37:56,752] [INFO] [logging.py:68:log_dist] [Rank 0] step=4800, skipped=7, lr=[4.6222222222222225e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:37:56,754] [INFO] [timer.py:197:stop] 0/9600, RunningAvgSamplesPerSec=5.880960865310276, CurrSamplesPerSec=5.332412454901974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4743/5000 [16:15:37<47:15, 11.03s/it][2022-12-21 02:38:07,806] [INFO] [timer.py:197:stop] 0/9602, RunningAvgSamplesPerSec=5.880955429601621, CurrSamplesPerSec=5.282136591548234, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4744/5000 [16:15:48<47:09, 11.05s/it][2022-12-21 02:38:18,873] [INFO] [timer.py:197:stop] 0/9604, RunningAvgSamplesPerSec=5.880948509380528, CurrSamplesPerSec=5.31414665067882, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4745/5000 [16:15:59<46:59, 11.06s/it][2022-12-21 02:38:30,007] [INFO] [timer.py:197:stop] 0/9606, RunningAvgSamplesPerSec=5.880933997160497, CurrSamplesPerSec=5.2495635556628795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4746/5000 [16:16:10<46:53, 11.08s/it][2022-12-21 02:38:41,082] [INFO] [timer.py:197:stop] 0/9608, RunningAvgSamplesPerSec=5.880926249353229, CurrSamplesPerSec=5.297937783663247, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4747/5000 [16:16:21<46:40, 11.07s/it][2022-12-21 02:38:52,096] [INFO] [timer.py:197:stop] 0/9610, RunningAvgSamplesPerSec=5.88092535747982, CurrSamplesPerSec=5.333825898623005, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4748/5000 [16:16:32<46:25, 11.05s/it][2022-12-21 02:39:03,117] [INFO] [timer.py:197:stop] 0/9612, RunningAvgSamplesPerSec=5.8809235396962585, CurrSamplesPerSec=5.330626484717996, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▍| 4749/5000 [16:16:43<46:06, 11.02s/it][2022-12-21 02:39:14,047] [INFO] [timer.py:197:stop] 0/9614, RunningAvgSamplesPerSec=5.880932031233577, CurrSamplesPerSec=5.348564500607492, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4750/5000 [16:16:54<45:52, 11.01s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 4.466666666666667e-07, 'epoch': 115.84}
+
 95%|█████████▌| 4750/5000 [16:16:54<45:52, 11.01s/it][2022-12-21 02:39:25,037] [INFO] [timer.py:197:stop] 0/9616, RunningAvgSamplesPerSec=5.880933872372217, CurrSamplesPerSec=5.342722119610216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4751/5000 [16:17:05<45:40, 11.01s/it][2022-12-21 02:39:36,049] [INFO] [timer.py:197:stop] 0/9618, RunningAvgSamplesPerSec=5.880933187277095, CurrSamplesPerSec=5.32421444689869, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4752/5000 [16:17:16<45:30, 11.01s/it][2022-12-21 02:39:47,021] [INFO] [logging.py:68:log_dist] [Rank 0] step=4810, skipped=7, lr=[4.4e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:39:47,022] [INFO] [timer.py:197:stop] 0/9620, RunningAvgSamplesPerSec=5.880936714529368, CurrSamplesPerSec=5.371673639248028, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4753/5000 [16:17:27<45:18, 11.01s/it][2022-12-21 02:39:58,084] [INFO] [timer.py:197:stop] 0/9622, RunningAvgSamplesPerSec=5.8809304312059325, CurrSamplesPerSec=5.308413610177076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4754/5000 [16:17:38<45:07, 11.01s/it][2022-12-21 02:40:09,090] [INFO] [timer.py:197:stop] 0/9624, RunningAvgSamplesPerSec=5.88093039306304, CurrSamplesPerSec=5.3145104667772936, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4755/5000 [16:17:49<45:02, 11.03s/it][2022-12-21 02:40:20,146] [INFO] [timer.py:197:stop] 0/9626, RunningAvgSamplesPerSec=5.88092492155338, CurrSamplesPerSec=5.341299918987719, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4756/5000 [16:18:00<44:50, 11.03s/it][2022-12-21 02:40:30,303] [INFO] [timer.py:197:stop] 0/9628, RunningAvgSamplesPerSec=5.8810191306122475, CurrSamplesPerSec=6.225931476021153, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 02:40:41,345] [INFO] [timer.py:197:stop] 0/9630, RunningAvgSamplesPerSec=5.881014691462303, CurrSamplesPerSec=5.318506250321466, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4757/5000 [16:18:16<50:59, 12.59s/it][2022-12-21 02:40:52,392] [INFO] [timer.py:197:stop] 0/9632, RunningAvgSamplesPerSec=5.881010105651238, CurrSamplesPerSec=5.307179800267617, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4758/5000 [16:18:27<48:54, 12.13s/it][2022-12-21 02:41:03,404] [INFO] [timer.py:197:stop] 0/9634, RunningAvgSamplesPerSec=5.881009428453584, CurrSamplesPerSec=5.3380386714875545, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4759/5000 [16:18:38<47:22, 11.79s/it][2022-12-21 02:41:14,426] [INFO] [timer.py:197:stop] 0/9636, RunningAvgSamplesPerSec=5.881007540346251, CurrSamplesPerSec=5.307838404549553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4760/5000 [16:18:50<46:14, 11.56s/it][2022-12-21 02:41:25,357] [INFO] [timer.py:197:stop] 0/9638, RunningAvgSamplesPerSec=5.881015902804455, CurrSamplesPerSec=5.390849121708894, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4761/5000 [16:19:00<45:17, 11.37s/it][2022-12-21 02:41:36,369] [INFO] [logging.py:68:log_dist] [Rank 0] step=4820, skipped=7, lr=[4.177777777777778e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:41:36,370] [INFO] [timer.py:197:stop] 0/9640, RunningAvgSamplesPerSec=5.881014969675752, CurrSamplesPerSec=5.3340372378693806, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4762/5000 [16:19:11<44:40, 11.26s/it][2022-12-21 02:41:47,445] [INFO] [timer.py:197:stop] 0/9642, RunningAvgSamplesPerSec=5.881007372975275, CurrSamplesPerSec=5.283105066686653, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4763/5000 [16:19:23<44:16, 11.21s/it][2022-12-21 02:41:58,488] [INFO] [timer.py:197:stop] 0/9644, RunningAvgSamplesPerSec=5.881003312608336, CurrSamplesPerSec=5.291778305352447, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4764/5000 [16:19:34<43:53, 11.16s/it][2022-12-21 02:42:09,546] [INFO] [timer.py:197:stop] 0/9646, RunningAvgSamplesPerSec=5.8809974938904555, CurrSamplesPerSec=5.3024880436194115, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4765/5000 [16:19:45<43:35, 11.13s/it][2022-12-21 02:42:20,572] [INFO] [timer.py:197:stop] 0/9648, RunningAvgSamplesPerSec=5.880995268643816, CurrSamplesPerSec=5.339941998065701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4766/5000 [16:19:56<43:16, 11.10s/it][2022-12-21 02:42:31,609] [INFO] [timer.py:197:stop] 0/9650, RunningAvgSamplesPerSec=5.88099181959029, CurrSamplesPerSec=5.311960805281877, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4767/5000 [16:20:07<43:01, 11.08s/it][2022-12-21 02:42:42,592] [INFO] [timer.py:197:stop] 0/9652, RunningAvgSamplesPerSec=5.880994401294073, CurrSamplesPerSec=5.3276571772799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4768/5000 [16:20:18<42:43, 11.05s/it][2022-12-21 02:42:53,568] [INFO] [timer.py:197:stop] 0/9654, RunningAvgSamplesPerSec=5.8809977978670585, CurrSamplesPerSec=5.329333234807956, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4769/5000 [16:20:29<42:27, 11.03s/it][2022-12-21 02:43:04,583] [INFO] [timer.py:197:stop] 0/9656, RunningAvgSamplesPerSec=5.880996779421266, CurrSamplesPerSec=5.314460173441543, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4770/5000 [16:20:40<42:15, 11.02s/it][2022-12-21 02:43:15,588] [INFO] [timer.py:197:stop] 0/9658, RunningAvgSamplesPerSec=5.880996899718391, CurrSamplesPerSec=5.303342244827158, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4771/5000 [16:20:51<42:03, 11.02s/it][2022-12-21 02:43:26,602] [INFO] [logging.py:68:log_dist] [Rank 0] step=4830, skipped=7, lr=[3.9555555555555557e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:43:26,603] [INFO] [timer.py:197:stop] 0/9660, RunningAvgSamplesPerSec=5.880995855412996, CurrSamplesPerSec=5.327522681452413, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4772/5000 [16:21:02<41:51, 11.02s/it][2022-12-21 02:43:37,640] [INFO] [timer.py:197:stop] 0/9662, RunningAvgSamplesPerSec=5.880992495430277, CurrSamplesPerSec=5.309967078200697, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4773/5000 [16:21:13<41:42, 11.02s/it][2022-12-21 02:43:48,682] [INFO] [timer.py:197:stop] 0/9664, RunningAvgSamplesPerSec=5.880988487053365, CurrSamplesPerSec=5.320686097079864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 95%|█████████▌| 4774/5000 [16:21:24<41:32, 11.03s/it][2022-12-21 02:43:59,717] [INFO] [timer.py:197:stop] 0/9666, RunningAvgSamplesPerSec=5.880985200148974, CurrSamplesPerSec=5.318403827350166, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4775/5000 [16:21:35<41:21, 11.03s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 3.8888888888888895e-07, 'epoch': 116.46}
+
 96%|█████████▌| 4775/5000 [16:21:35<41:21, 11.03s/it][2022-12-21 02:44:10,749] [INFO] [timer.py:197:stop] 0/9668, RunningAvgSamplesPerSec=5.880982451146491, CurrSamplesPerSec=5.33179878319952, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4776/5000 [16:21:46<41:10, 11.03s/it][2022-12-21 02:44:21,728] [INFO] [timer.py:197:stop] 0/9670, RunningAvgSamplesPerSec=5.880985514666687, CurrSamplesPerSec=5.358163183222781, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4777/5000 [16:21:57<40:56, 11.02s/it][2022-12-21 02:44:32,745] [INFO] [timer.py:197:stop] 0/9672, RunningAvgSamplesPerSec=5.880984377710305, CurrSamplesPerSec=5.344094860364544, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4778/5000 [16:22:08<40:45, 11.02s/it][2022-12-21 02:44:43,754] [INFO] [timer.py:197:stop] 0/9674, RunningAvgSamplesPerSec=5.880984125380136, CurrSamplesPerSec=5.309103811508198, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4779/5000 [16:22:19<40:34, 11.01s/it][2022-12-21 02:44:54,779] [INFO] [timer.py:197:stop] 0/9676, RunningAvgSamplesPerSec=5.880982011473916, CurrSamplesPerSec=5.3339658003627575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4780/5000 [16:22:30<40:23, 11.02s/it][2022-12-21 02:45:05,783] [INFO] [timer.py:197:stop] 0/9678, RunningAvgSamplesPerSec=5.8809822052806355, CurrSamplesPerSec=5.342827182847762, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4781/5000 [16:22:41<40:11, 11.01s/it][2022-12-21 02:45:16,787] [INFO] [logging.py:68:log_dist] [Rank 0] step=4840, skipped=7, lr=[3.733333333333334e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:45:16,789] [INFO] [timer.py:197:stop] 0/9680, RunningAvgSamplesPerSec=5.880982075848762, CurrSamplesPerSec=5.311953026691017, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4782/5000 [16:22:52<40:00, 11.01s/it][2022-12-21 02:45:27,777] [INFO] [timer.py:197:stop] 0/9682, RunningAvgSamplesPerSec=5.8809840399254005, CurrSamplesPerSec=5.345632026541421, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4783/5000 [16:23:03<39:47, 11.00s/it][2022-12-21 02:45:38,816] [INFO] [timer.py:197:stop] 0/9684, RunningAvgSamplesPerSec=5.880980450178291, CurrSamplesPerSec=5.307370984354458, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4784/5000 [16:23:14<39:39, 11.02s/it][2022-12-21 02:45:49,780] [INFO] [timer.py:197:stop] 0/9686, RunningAvgSamplesPerSec=5.880985767389646, CurrSamplesPerSec=5.357649217567191, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4785/5000 [16:23:25<39:24, 11.00s/it][2022-12-21 02:46:00,802] [INFO] [timer.py:197:stop] 0/9688, RunningAvgSamplesPerSec=5.880984042902357, CurrSamplesPerSec=5.319238710604539, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4786/5000 [16:23:36<39:15, 11.01s/it][2022-12-21 02:46:11,840] [INFO] [timer.py:197:stop] 0/9690, RunningAvgSamplesPerSec=5.880980420294556, CurrSamplesPerSec=5.282791905564658, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4787/5000 [16:23:47<39:06, 11.02s/it][2022-12-21 02:46:22,768] [INFO] [timer.py:197:stop] 0/9692, RunningAvgSamplesPerSec=5.880989153570312, CurrSamplesPerSec=5.360941116322449, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4788/5000 [16:23:58<38:49, 10.99s/it][2022-12-21 02:46:33,766] [INFO] [timer.py:197:stop] 0/9694, RunningAvgSamplesPerSec=5.8809899894532895, CurrSamplesPerSec=5.374901005958865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4789/5000 [16:24:09<38:39, 10.99s/it][2022-12-21 02:46:44,791] [INFO] [timer.py:197:stop] 0/9696, RunningAvgSamplesPerSec=5.880987934330421, CurrSamplesPerSec=5.337668442936945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4790/5000 [16:24:20<38:30, 11.00s/it][2022-12-21 02:46:55,745] [INFO] [timer.py:197:stop] 0/9698, RunningAvgSamplesPerSec=5.8809936073969125, CurrSamplesPerSec=5.345198158993703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4791/5000 [16:24:31<38:16, 10.99s/it][2022-12-21 02:47:06,722] [INFO] [logging.py:68:log_dist] [Rank 0] step=4850, skipped=7, lr=[3.5111111111111117e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:47:06,723] [INFO] [timer.py:197:stop] 0/9700, RunningAvgSamplesPerSec=5.88099698092743, CurrSamplesPerSec=5.324915523756282, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4792/5000 [16:24:42<38:04, 10.98s/it][2022-12-21 02:47:17,708] [INFO] [timer.py:197:stop] 0/9702, RunningAvgSamplesPerSec=5.880999375427564, CurrSamplesPerSec=5.317700882304791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4793/5000 [16:24:53<37:53, 10.98s/it][2022-12-21 02:47:28,669] [INFO] [timer.py:197:stop] 0/9704, RunningAvgSamplesPerSec=5.881004359227824, CurrSamplesPerSec=5.339721693227279, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4794/5000 [16:25:04<37:41, 10.98s/it][2022-12-21 02:47:39,696] [INFO] [timer.py:197:stop] 0/9706, RunningAvgSamplesPerSec=5.881002035340364, CurrSamplesPerSec=5.314884645409777, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4795/5000 [16:25:15<37:33, 10.99s/it][2022-12-21 02:47:50,639] [INFO] [timer.py:197:stop] 0/9708, RunningAvgSamplesPerSec=5.881008963855953, CurrSamplesPerSec=5.35444466523403, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4796/5000 [16:25:26<37:19, 10.98s/it][2022-12-21 02:48:01,688] [INFO] [timer.py:197:stop] 0/9710, RunningAvgSamplesPerSec=5.881005248041847, CurrSamplesPerSec=5.306625842938296, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4797/5000 [16:25:37<37:12, 11.00s/it][2022-12-21 02:48:11,841] [INFO] [timer.py:197:stop] 0/9712, RunningAvgSamplesPerSec=5.881099024491932, CurrSamplesPerSec=5.2881040309257275, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4798/5000 [16:25:52<41:10, 12.23s/it][2022-12-21 02:48:22,904] [INFO] [timer.py:197:stop] 0/9714, RunningAvgSamplesPerSec=5.881092669966116, CurrSamplesPerSec=5.294900118511985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4799/5000 [16:26:03<39:48, 11.88s/it][2022-12-21 02:48:33,916] [INFO] [timer.py:197:stop] 0/9716, RunningAvgSamplesPerSec=5.88109186185115, CurrSamplesPerSec=5.339365674848051, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4800/5000 [16:26:14<38:42, 11.61s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 3.3333333333333335e-07, 'epoch': 117.07}
+
 96%|█████████▌| 4800/5000 [16:26:14<38:42, 11.61s/it][2022-12-21 02:48:44,927] [INFO] [timer.py:197:stop] 0/9718, RunningAvgSamplesPerSec=5.881091394249807, CurrSamplesPerSec=5.31339602972306, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4801/5000 [16:26:25<37:54, 11.43s/it][2022-12-21 02:48:55,912] [INFO] [logging.py:68:log_dist] [Rank 0] step=4860, skipped=7, lr=[3.2888888888888894e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:48:55,914] [INFO] [timer.py:197:stop] 0/9720, RunningAvgSamplesPerSec=5.881093360889865, CurrSamplesPerSec=5.334486892925552, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4802/5000 [16:26:36<37:18, 11.30s/it][2022-12-21 02:49:06,925] [INFO] [timer.py:197:stop] 0/9722, RunningAvgSamplesPerSec=5.881092555464422, CurrSamplesPerSec=5.332214166512985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4803/5000 [16:26:47<36:48, 11.21s/it][2022-12-21 02:49:17,910] [INFO] [timer.py:197:stop] 0/9724, RunningAvgSamplesPerSec=5.8810947419350965, CurrSamplesPerSec=5.334939168461207, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4804/5000 [16:26:58<36:25, 11.15s/it][2022-12-21 02:49:28,878] [INFO] [timer.py:197:stop] 0/9726, RunningAvgSamplesPerSec=5.881098769008264, CurrSamplesPerSec=5.372962357001803, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4805/5000 [16:27:09<36:04, 11.10s/it][2022-12-21 02:49:39,923] [INFO] [timer.py:197:stop] 0/9728, RunningAvgSamplesPerSec=5.881094302424883, CurrSamplesPerSec=5.316688512914242, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4806/5000 [16:27:20<35:46, 11.07s/it][2022-12-21 02:49:50,932] [INFO] [timer.py:197:stop] 0/9730, RunningAvgSamplesPerSec=5.881093674606915, CurrSamplesPerSec=5.2980216437045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4807/5000 [16:27:31<35:36, 11.07s/it][2022-12-21 02:50:01,925] [INFO] [timer.py:197:stop] 0/9732, RunningAvgSamplesPerSec=5.8810950256856405, CurrSamplesPerSec=5.373086035884832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4808/5000 [16:27:42<35:17, 11.03s/it][2022-12-21 02:50:12,883] [INFO] [timer.py:197:stop] 0/9734, RunningAvgSamplesPerSec=5.881100060525745, CurrSamplesPerSec=5.344942725273218, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4809/5000 [16:27:53<35:06, 11.03s/it][2022-12-21 02:50:23,939] [INFO] [timer.py:197:stop] 0/9736, RunningAvgSamplesPerSec=5.881094274824236, CurrSamplesPerSec=5.3198745864512285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4810/5000 [16:28:04<34:54, 11.03s/it][2022-12-21 02:50:34,944] [INFO] [timer.py:197:stop] 0/9738, RunningAvgSamplesPerSec=5.881094227024425, CurrSamplesPerSec=5.337351539224869, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4811/5000 [16:28:15<34:40, 11.01s/it][2022-12-21 02:50:45,928] [INFO] [logging.py:68:log_dist] [Rank 0] step=4870, skipped=7, lr=[3.0666666666666666e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:50:45,930] [INFO] [timer.py:197:stop] 0/9740, RunningAvgSamplesPerSec=5.8810960783248305, CurrSamplesPerSec=5.32104933249672, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▌| 4812/5000 [16:28:26<34:31, 11.02s/it][2022-12-21 02:50:56,937] [INFO] [timer.py:197:stop] 0/9742, RunningAvgSamplesPerSec=5.881095749778431, CurrSamplesPerSec=5.347296832348291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4813/5000 [16:28:37<34:17, 11.00s/it][2022-12-21 02:51:07,912] [INFO] [timer.py:197:stop] 0/9744, RunningAvgSamplesPerSec=5.881098955435075, CurrSamplesPerSec=5.337720237878165, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4814/5000 [16:28:48<34:04, 10.99s/it][2022-12-21 02:51:18,921] [INFO] [timer.py:197:stop] 0/9746, RunningAvgSamplesPerSec=5.881098458629044, CurrSamplesPerSec=5.3082834432461645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4815/5000 [16:28:59<33:56, 11.01s/it][2022-12-21 02:51:29,949] [INFO] [timer.py:197:stop] 0/9748, RunningAvgSamplesPerSec=5.8810958277765, CurrSamplesPerSec=5.32432807682903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4816/5000 [16:29:10<33:45, 11.01s/it][2022-12-21 02:51:40,990] [INFO] [timer.py:197:stop] 0/9750, RunningAvgSamplesPerSec=5.88109176730643, CurrSamplesPerSec=5.28770299073872, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4817/5000 [16:29:21<33:37, 11.03s/it][2022-12-21 02:51:52,022] [INFO] [timer.py:197:stop] 0/9752, RunningAvgSamplesPerSec=5.881088795477315, CurrSamplesPerSec=5.320243194169213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4818/5000 [16:29:32<33:26, 11.03s/it][2022-12-21 02:52:03,060] [INFO] [timer.py:197:stop] 0/9754, RunningAvgSamplesPerSec=5.881085351390909, CurrSamplesPerSec=5.313563470505324, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4819/5000 [16:29:43<33:15, 11.02s/it][2022-12-21 02:52:14,071] [INFO] [timer.py:197:stop] 0/9756, RunningAvgSamplesPerSec=5.8810846618543025, CurrSamplesPerSec=5.318956241590907, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4820/5000 [16:29:54<33:04, 11.02s/it][2022-12-21 02:52:25,076] [INFO] [timer.py:197:stop] 0/9758, RunningAvgSamplesPerSec=5.8810847963720425, CurrSamplesPerSec=5.339518187648851, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4821/5000 [16:30:05<32:50, 11.01s/it][2022-12-21 02:52:36,043] [INFO] [logging.py:68:log_dist] [Rank 0] step=4880, skipped=7, lr=[2.844444444444445e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:52:36,045] [INFO] [timer.py:197:stop] 0/9760, RunningAvgSamplesPerSec=5.881088717159886, CurrSamplesPerSec=5.337060776559862, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4822/5000 [16:30:16<32:40, 11.01s/it][2022-12-21 02:52:47,083] [INFO] [timer.py:197:stop] 0/9762, RunningAvgSamplesPerSec=5.8810850255846665, CurrSamplesPerSec=5.331853006007156, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4823/5000 [16:30:27<32:27, 11.00s/it][2022-12-21 02:52:58,068] [INFO] [timer.py:197:stop] 0/9764, RunningAvgSamplesPerSec=5.881087247608846, CurrSamplesPerSec=5.324186356975879, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4824/5000 [16:30:38<32:17, 11.01s/it][2022-12-21 02:53:09,090] [INFO] [timer.py:197:stop] 0/9766, RunningAvgSamplesPerSec=5.881085548329467, CurrSamplesPerSec=5.329510146511596, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 96%|█████████▋| 4825/5000 [16:30:49<32:07, 11.01s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 2.7777777777777776e-07, 'epoch': 117.67}
+
 96%|█████████▋| 4825/5000 [16:30:49<32:07, 11.01s/it][2022-12-21 02:53:20,111] [INFO] [timer.py:197:stop] 0/9768, RunningAvgSamplesPerSec=5.88108406596122, CurrSamplesPerSec=5.330138743612372, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4826/5000 [16:31:00<31:57, 11.02s/it][2022-12-21 02:53:31,206] [INFO] [timer.py:197:stop] 0/9770, RunningAvgSamplesPerSec=5.881074169182799, CurrSamplesPerSec=5.276026222553645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4827/5000 [16:31:11<31:50, 11.04s/it][2022-12-21 02:53:42,232] [INFO] [timer.py:197:stop] 0/9772, RunningAvgSamplesPerSec=5.881071894573575, CurrSamplesPerSec=5.33518558806113, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4828/5000 [16:31:22<31:39, 11.04s/it][2022-12-21 02:53:53,310] [INFO] [timer.py:197:stop] 0/9774, RunningAvgSamplesPerSec=5.881063948727733, CurrSamplesPerSec=5.303637308839403, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4829/5000 [16:31:33<31:27, 11.04s/it][2022-12-21 02:54:04,341] [INFO] [timer.py:197:stop] 0/9776, RunningAvgSamplesPerSec=5.881061044887499, CurrSamplesPerSec=5.30129488551071, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4830/5000 [16:31:44<31:18, 11.05s/it][2022-12-21 02:54:15,416] [INFO] [timer.py:197:stop] 0/9778, RunningAvgSamplesPerSec=5.881054459110175, CurrSamplesPerSec=5.313531706430282, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4831/5000 [16:31:55<31:08, 11.06s/it][2022-12-21 02:54:26,445] [INFO] [logging.py:68:log_dist] [Rank 0] step=4890, skipped=7, lr=[2.6222222222222226e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:54:26,447] [INFO] [timer.py:197:stop] 0/9780, RunningAvgSamplesPerSec=5.881053646453292, CurrSamplesPerSec=5.345565600643327, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4832/5000 [16:32:07<30:55, 11.04s/it][2022-12-21 02:54:37,472] [INFO] [timer.py:197:stop] 0/9782, RunningAvgSamplesPerSec=5.881051482091528, CurrSamplesPerSec=5.330822114302117, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4833/5000 [16:32:17<30:41, 11.03s/it][2022-12-21 02:54:48,450] [INFO] [timer.py:197:stop] 0/9784, RunningAvgSamplesPerSec=5.88105439049244, CurrSamplesPerSec=5.336143065761105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4834/5000 [16:32:29<30:30, 11.03s/it][2022-12-21 02:54:59,523] [INFO] [timer.py:197:stop] 0/9786, RunningAvgSamplesPerSec=5.881046988622946, CurrSamplesPerSec=5.29842445985513, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4835/5000 [16:32:40<30:19, 11.03s/it][2022-12-21 02:55:10,520] [INFO] [timer.py:197:stop] 0/9788, RunningAvgSamplesPerSec=5.8810479500291555, CurrSamplesPerSec=5.319647711444614, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4836/5000 [16:32:51<30:09, 11.03s/it][2022-12-21 02:55:21,553] [INFO] [timer.py:197:stop] 0/9790, RunningAvgSamplesPerSec=5.881044794251297, CurrSamplesPerSec=5.32751570308064, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4837/5000 [16:33:02<29:58, 11.03s/it][2022-12-21 02:55:32,596] [INFO] [timer.py:197:stop] 0/9792, RunningAvgSamplesPerSec=5.881040587604242, CurrSamplesPerSec=5.317163895140837, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4838/5000 [16:33:13<29:46, 11.03s/it][2022-12-21 02:55:42,685] [INFO] [timer.py:197:stop] 0/9794, RunningAvgSamplesPerSec=5.881140593095391, CurrSamplesPerSec=6.272706781419294, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 02:55:53,733] [INFO] [timer.py:197:stop] 0/9796, RunningAvgSamplesPerSec=5.881135524406425, CurrSamplesPerSec=5.302798097107958, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4839/5000 [16:33:29<33:45, 12.58s/it][2022-12-21 02:56:04,769] [INFO] [timer.py:197:stop] 0/9798, RunningAvgSamplesPerSec=5.88113211428996, CurrSamplesPerSec=5.319485157703488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4840/5000 [16:33:40<32:18, 12.12s/it][2022-12-21 02:56:15,840] [INFO] [logging.py:68:log_dist] [Rank 0] step=4900, skipped=7, lr=[2.4000000000000003e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:56:15,842] [INFO] [timer.py:197:stop] 0/9800, RunningAvgSamplesPerSec=5.881124765324317, CurrSamplesPerSec=5.302683079792943, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4841/5000 [16:33:51<31:16, 11.80s/it][2022-12-21 02:56:26,815] [INFO] [timer.py:197:stop] 0/9802, RunningAvgSamplesPerSec=5.881128351204375, CurrSamplesPerSec=5.356661559619491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4842/5000 [16:34:02<30:25, 11.55s/it][2022-12-21 02:56:37,834] [INFO] [timer.py:197:stop] 0/9804, RunningAvgSamplesPerSec=5.8811268308221205, CurrSamplesPerSec=5.307711414268036, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4843/5000 [16:34:13<29:48, 11.39s/it][2022-12-21 02:56:48,894] [INFO] [timer.py:197:stop] 0/9806, RunningAvgSamplesPerSec=5.8811208157542385, CurrSamplesPerSec=5.291951897858474, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4844/5000 [16:34:24<29:21, 11.29s/it][2022-12-21 02:56:59,940] [INFO] [timer.py:197:stop] 0/9808, RunningAvgSamplesPerSec=5.881116364077755, CurrSamplesPerSec=5.271964977722606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4845/5000 [16:34:35<28:59, 11.22s/it][2022-12-21 02:57:10,989] [INFO] [timer.py:197:stop] 0/9810, RunningAvgSamplesPerSec=5.8811116763637346, CurrSamplesPerSec=5.318735135662933, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4846/5000 [16:34:46<28:39, 11.17s/it][2022-12-21 02:57:21,928] [INFO] [timer.py:197:stop] 0/9812, RunningAvgSamplesPerSec=5.881118962992291, CurrSamplesPerSec=5.364329681248229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4847/5000 [16:34:57<28:18, 11.10s/it][2022-12-21 02:57:32,912] [INFO] [timer.py:197:stop] 0/9814, RunningAvgSamplesPerSec=5.881121381032901, CurrSamplesPerSec=5.358652001922791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4848/5000 [16:35:08<28:01, 11.06s/it][2022-12-21 02:57:43,980] [INFO] [timer.py:197:stop] 0/9816, RunningAvgSamplesPerSec=5.881114649910023, CurrSamplesPerSec=5.300976842569895, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4849/5000 [16:35:19<27:50, 11.07s/it][2022-12-21 02:57:54,963] [INFO] [timer.py:197:stop] 0/9818, RunningAvgSamplesPerSec=5.881117101706607, CurrSamplesPerSec=5.325304480290763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4850/5000 [16:35:30<27:36, 11.04s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 2.2e-07, 'epoch': 118.29}
+
 97%|█████████▋| 4850/5000 [16:35:30<27:36, 11.04s/it][2022-12-21 02:58:05,988] [INFO] [logging.py:68:log_dist] [Rank 0] step=4910, skipped=7, lr=[2.177777777777778e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:58:05,990] [INFO] [timer.py:197:stop] 0/9820, RunningAvgSamplesPerSec=5.8811148483372495, CurrSamplesPerSec=5.33874105804406, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4851/5000 [16:35:41<27:24, 11.04s/it][2022-12-21 02:58:17,022] [INFO] [timer.py:197:stop] 0/9822, RunningAvgSamplesPerSec=5.8811119471842925, CurrSamplesPerSec=5.316668084125516, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4852/5000 [16:35:52<27:13, 11.04s/it][2022-12-21 02:58:28,067] [INFO] [timer.py:197:stop] 0/9824, RunningAvgSamplesPerSec=5.881107637965968, CurrSamplesPerSec=5.317996914146575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4853/5000 [16:36:03<27:02, 11.04s/it][2022-12-21 02:58:39,105] [INFO] [timer.py:197:stop] 0/9826, RunningAvgSamplesPerSec=5.881104086336548, CurrSamplesPerSec=5.303822159698251, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4854/5000 [16:36:14<26:51, 11.04s/it][2022-12-21 02:58:50,113] [INFO] [timer.py:197:stop] 0/9828, RunningAvgSamplesPerSec=5.881103804143445, CurrSamplesPerSec=5.344964010484144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4855/5000 [16:36:25<26:39, 11.03s/it][2022-12-21 02:59:01,121] [INFO] [timer.py:197:stop] 0/9830, RunningAvgSamplesPerSec=5.8811033900707255, CurrSamplesPerSec=5.336411662423746, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4856/5000 [16:36:36<26:27, 11.02s/it][2022-12-21 02:59:12,185] [INFO] [timer.py:197:stop] 0/9832, RunningAvgSamplesPerSec=5.8810968217213055, CurrSamplesPerSec=5.309499493073067, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4857/5000 [16:36:47<26:18, 11.04s/it][2022-12-21 02:59:23,190] [INFO] [timer.py:197:stop] 0/9834, RunningAvgSamplesPerSec=5.881096721686233, CurrSamplesPerSec=5.33305551068884, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4858/5000 [16:36:58<26:05, 11.03s/it][2022-12-21 02:59:34,316] [INFO] [timer.py:197:stop] 0/9836, RunningAvgSamplesPerSec=5.881083337539888, CurrSamplesPerSec=5.248332115010413, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4859/5000 [16:37:09<25:58, 11.06s/it][2022-12-21 02:59:45,315] [INFO] [timer.py:197:stop] 0/9838, RunningAvgSamplesPerSec=5.881083996184068, CurrSamplesPerSec=5.336562521222161, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4860/5000 [16:37:20<25:45, 11.04s/it][2022-12-21 02:59:56,283] [INFO] [logging.py:68:log_dist] [Rank 0] step=4920, skipped=7, lr=[1.9555555555555558e-07], mom=[[0.9, 0.999]]
+[2022-12-21 02:59:56,284] [INFO] [timer.py:197:stop] 0/9840, RunningAvgSamplesPerSec=5.881088101585368, CurrSamplesPerSec=5.344239982814685, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4861/5000 [16:37:31<25:31, 11.02s/it][2022-12-21 03:00:07,294] [INFO] [timer.py:197:stop] 0/9842, RunningAvgSamplesPerSec=5.88108773628177, CurrSamplesPerSec=5.3468501273095335, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4862/5000 [16:37:42<25:20, 11.02s/it][2022-12-21 03:00:18,285] [INFO] [timer.py:197:stop] 0/9844, RunningAvgSamplesPerSec=5.881089347327349, CurrSamplesPerSec=5.300731270017787, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4863/5000 [16:37:53<25:08, 11.01s/it][2022-12-21 03:00:29,279] [INFO] [timer.py:197:stop] 0/9846, RunningAvgSamplesPerSec=5.881090795756704, CurrSamplesPerSec=5.361119062076674, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4864/5000 [16:38:04<24:56, 11.00s/it][2022-12-21 03:00:40,260] [INFO] [timer.py:197:stop] 0/9848, RunningAvgSamplesPerSec=5.881093604153213, CurrSamplesPerSec=5.319455431051029, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4865/5000 [16:38:15<24:44, 11.00s/it][2022-12-21 03:00:51,279] [INFO] [timer.py:197:stop] 0/9850, RunningAvgSamplesPerSec=5.881092144237029, CurrSamplesPerSec=5.327926823802811, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4866/5000 [16:38:26<24:34, 11.00s/it][2022-12-21 03:01:02,244] [INFO] [timer.py:197:stop] 0/9852, RunningAvgSamplesPerSec=5.881096722359585, CurrSamplesPerSec=5.333900935916359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4867/5000 [16:38:37<24:21, 10.99s/it][2022-12-21 03:01:13,203] [INFO] [timer.py:197:stop] 0/9854, RunningAvgSamplesPerSec=5.8811017963926435, CurrSamplesPerSec=5.379798822099924, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4868/5000 [16:38:48<24:09, 10.98s/it][2022-12-21 03:01:24,132] [INFO] [timer.py:197:stop] 0/9856, RunningAvgSamplesPerSec=5.88111031772632, CurrSamplesPerSec=5.344157632304582, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4869/5000 [16:38:59<23:56, 10.97s/it][2022-12-21 03:01:35,117] [INFO] [timer.py:197:stop] 0/9858, RunningAvgSamplesPerSec=5.881112518386495, CurrSamplesPerSec=5.344831193539486, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4870/5000 [16:39:10<23:46, 10.97s/it][2022-12-21 03:01:46,108] [INFO] [logging.py:68:log_dist] [Rank 0] step=4930, skipped=7, lr=[1.7333333333333335e-07], mom=[[0.9, 0.999]]
+[2022-12-21 03:01:46,110] [INFO] [timer.py:197:stop] 0/9860, RunningAvgSamplesPerSec=5.881113800008539, CurrSamplesPerSec=5.319881333959638, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4871/5000 [16:39:21<23:36, 10.98s/it][2022-12-21 03:01:57,108] [INFO] [timer.py:197:stop] 0/9862, RunningAvgSamplesPerSec=5.88111467715933, CurrSamplesPerSec=5.350249475250512, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4872/5000 [16:39:32<23:25, 10.98s/it][2022-12-21 03:02:08,071] [INFO] [timer.py:197:stop] 0/9864, RunningAvgSamplesPerSec=5.8811193704716205, CurrSamplesPerSec=5.333700841158969, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4873/5000 [16:39:43<23:14, 10.98s/it][2022-12-21 03:02:19,064] [INFO] [timer.py:197:stop] 0/9866, RunningAvgSamplesPerSec=5.881120730001057, CurrSamplesPerSec=5.346225248942059, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 97%|█████████▋| 4874/5000 [16:39:54<23:03, 10.98s/it][2022-12-21 03:02:29,995] [INFO] [timer.py:197:stop] 0/9868, RunningAvgSamplesPerSec=5.881128856960425, CurrSamplesPerSec=5.335659193251529, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4875/5000 [16:40:05<22:50, 10.97s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 1.6444444444444447e-07, 'epoch': 118.89}
+
 98%|█████████▊| 4875/5000 [16:40:05<22:50, 10.97s/it][2022-12-21 03:02:40,974] [INFO] [timer.py:197:stop] 0/9870, RunningAvgSamplesPerSec=5.881131866184139, CurrSamplesPerSec=5.365476313211763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4876/5000 [16:40:16<22:40, 10.97s/it][2022-12-21 03:02:51,918] [INFO] [timer.py:197:stop] 0/9872, RunningAvgSamplesPerSec=5.881138587510723, CurrSamplesPerSec=5.3696369932724375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4877/5000 [16:40:27<22:28, 10.96s/it][2022-12-21 03:03:02,872] [INFO] [timer.py:197:stop] 0/9874, RunningAvgSamplesPerSec=5.881144273738245, CurrSamplesPerSec=5.356830028003102, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4878/5000 [16:40:38<22:17, 10.96s/it][2022-12-21 03:03:13,842] [INFO] [timer.py:197:stop] 0/9876, RunningAvgSamplesPerSec=5.881148240551594, CurrSamplesPerSec=5.328577894729863, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4879/5000 [16:40:49<22:06, 10.96s/it][2022-12-21 03:03:23,928] [INFO] [timer.py:197:stop] 0/9878, RunningAvgSamplesPerSec=5.881247832402207, CurrSamplesPerSec=5.3351493235073555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4880/5000 [16:41:04<24:20, 12.17s/it][2022-12-21 03:03:34,871] [INFO] [logging.py:68:log_dist] [Rank 0] step=4940, skipped=7, lr=[1.5111111111111112e-07], mom=[[0.9, 0.999]]
+[2022-12-21 03:03:34,872] [INFO] [timer.py:197:stop] 0/9880, RunningAvgSamplesPerSec=5.881254335323087, CurrSamplesPerSec=5.347510518801368, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4881/5000 [16:41:15<23:24, 11.80s/it][2022-12-21 03:03:45,841] [INFO] [timer.py:197:stop] 0/9882, RunningAvgSamplesPerSec=5.881258195066024, CurrSamplesPerSec=5.330498189994769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4882/5000 [16:41:26<22:45, 11.57s/it][2022-12-21 03:03:56,838] [INFO] [timer.py:197:stop] 0/9884, RunningAvgSamplesPerSec=5.881259028773293, CurrSamplesPerSec=5.361381398359978, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4883/5000 [16:41:37<22:11, 11.38s/it][2022-12-21 03:04:07,823] [INFO] [timer.py:197:stop] 0/9886, RunningAvgSamplesPerSec=5.881261110617577, CurrSamplesPerSec=5.3142110355566015, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4884/5000 [16:41:48<21:47, 11.27s/it][2022-12-21 03:04:18,798] [INFO] [timer.py:197:stop] 0/9888, RunningAvgSamplesPerSec=5.881264327620437, CurrSamplesPerSec=5.341422782202486, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4885/5000 [16:41:59<21:26, 11.19s/it][2022-12-21 03:04:29,759] [INFO] [timer.py:197:stop] 0/9890, RunningAvgSamplesPerSec=5.881269080466366, CurrSamplesPerSec=5.384018468835765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4886/5000 [16:42:10<21:07, 11.12s/it][2022-12-21 03:04:40,756] [INFO] [timer.py:197:stop] 0/9892, RunningAvgSamplesPerSec=5.8812699399123884, CurrSamplesPerSec=5.356439445548449, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4887/5000 [16:42:21<20:50, 11.07s/it][2022-12-21 03:04:51,737] [INFO] [timer.py:197:stop] 0/9894, RunningAvgSamplesPerSec=5.881272418274777, CurrSamplesPerSec=5.321790300863083, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4888/5000 [16:42:32<20:38, 11.05s/it][2022-12-21 03:05:02,759] [INFO] [timer.py:197:stop] 0/9896, RunningAvgSamplesPerSec=5.881270562505185, CurrSamplesPerSec=5.3222288183940485, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4889/5000 [16:42:43<20:26, 11.05s/it][2022-12-21 03:05:13,798] [INFO] [timer.py:197:stop] 0/9898, RunningAvgSamplesPerSec=5.881266913513807, CurrSamplesPerSec=5.326018102339611, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4890/5000 [16:42:54<20:14, 11.04s/it][2022-12-21 03:05:24,774] [INFO] [logging.py:68:log_dist] [Rank 0] step=4950, skipped=7, lr=[1.288888888888889e-07], mom=[[0.9, 0.999]]
+[2022-12-21 03:05:24,775] [INFO] [timer.py:197:stop] 0/9900, RunningAvgSamplesPerSec=5.881270029192826, CurrSamplesPerSec=5.361373260195539, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4891/5000 [16:43:05<20:01, 11.02s/it][2022-12-21 03:05:35,763] [INFO] [timer.py:197:stop] 0/9902, RunningAvgSamplesPerSec=5.881272010755754, CurrSamplesPerSec=5.353279469650216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4892/5000 [16:43:16<19:48, 11.01s/it][2022-12-21 03:05:46,730] [INFO] [timer.py:197:stop] 0/9904, RunningAvgSamplesPerSec=5.881276049252223, CurrSamplesPerSec=5.352018742511104, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4893/5000 [16:43:27<19:36, 11.00s/it][2022-12-21 03:05:57,774] [INFO] [timer.py:197:stop] 0/9906, RunningAvgSamplesPerSec=5.881271752456181, CurrSamplesPerSec=5.288471999742152, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4894/5000 [16:43:38<19:26, 11.01s/it][2022-12-21 03:06:08,789] [INFO] [timer.py:197:stop] 0/9908, RunningAvgSamplesPerSec=5.881270703082028, CurrSamplesPerSec=5.310281998867976, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4895/5000 [16:43:49<19:17, 11.03s/it][2022-12-21 03:06:19,834] [INFO] [timer.py:197:stop] 0/9910, RunningAvgSamplesPerSec=5.8812664413084175, CurrSamplesPerSec=5.328548701008293, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4896/5000 [16:44:00<19:05, 11.02s/it][2022-12-21 03:06:30,841] [INFO] [timer.py:197:stop] 0/9912, RunningAvgSamplesPerSec=5.881266277604089, CurrSamplesPerSec=5.316582579765042, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4897/5000 [16:44:11<18:54, 11.02s/it][2022-12-21 03:06:41,851] [INFO] [timer.py:197:stop] 0/9914, RunningAvgSamplesPerSec=5.881265718482466, CurrSamplesPerSec=5.322109580054575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4898/5000 [16:44:22<18:44, 11.03s/it][2022-12-21 03:06:52,895] [INFO] [timer.py:197:stop] 0/9916, RunningAvgSamplesPerSec=5.881261520192311, CurrSamplesPerSec=5.3238000977990145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4899/5000 [16:44:33<18:34, 11.03s/it][2022-12-21 03:07:03,937] [INFO] [timer.py:197:stop] 0/9918, RunningAvgSamplesPerSec=5.881257429248042, CurrSamplesPerSec=5.32185191686462, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4900/5000 [16:44:44<18:22, 11.03s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 1.088888888888889e-07, 'epoch': 119.51}
+
 98%|█████████▊| 4900/5000 [16:44:44<18:22, 11.03s/it][2022-12-21 03:07:14,992] [INFO] [logging.py:68:log_dist] [Rank 0] step=4960, skipped=7, lr=[1.0666666666666667e-07], mom=[[0.9, 0.999]]
+[2022-12-21 03:07:14,993] [INFO] [timer.py:197:stop] 0/9920, RunningAvgSamplesPerSec=5.881252001212702, CurrSamplesPerSec=5.296224357890981, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4901/5000 [16:44:55<18:12, 11.04s/it][2022-12-21 03:07:26,028] [INFO] [timer.py:197:stop] 0/9922, RunningAvgSamplesPerSec=5.881248696874494, CurrSamplesPerSec=5.324292382146269, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4902/5000 [16:45:06<18:01, 11.04s/it][2022-12-21 03:07:37,064] [INFO] [timer.py:197:stop] 0/9924, RunningAvgSamplesPerSec=5.881245278861728, CurrSamplesPerSec=5.314292465724569, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4903/5000 [16:45:17<17:49, 11.02s/it][2022-12-21 03:07:48,010] [INFO] [timer.py:197:stop] 0/9926, RunningAvgSamplesPerSec=5.881251677516267, CurrSamplesPerSec=5.3551759477589, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4904/5000 [16:45:28<17:37, 11.01s/it][2022-12-21 03:07:59,039] [INFO] [timer.py:197:stop] 0/9928, RunningAvgSamplesPerSec=5.881249024705565, CurrSamplesPerSec=5.317215714203256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4905/5000 [16:45:39<17:26, 11.02s/it][2022-12-21 03:08:10,081] [INFO] [timer.py:197:stop] 0/9930, RunningAvgSamplesPerSec=5.88124487811138, CurrSamplesPerSec=5.310954823118706, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4906/5000 [16:45:50<17:16, 11.03s/it][2022-12-21 03:08:21,123] [INFO] [timer.py:197:stop] 0/9932, RunningAvgSamplesPerSec=5.881240674099582, CurrSamplesPerSec=5.320687573546213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4907/5000 [16:46:01<17:05, 11.03s/it][2022-12-21 03:08:32,129] [INFO] [timer.py:197:stop] 0/9934, RunningAvgSamplesPerSec=5.881240539041054, CurrSamplesPerSec=5.343051998027551, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4908/5000 [16:46:12<16:54, 11.02s/it][2022-12-21 03:08:43,162] [INFO] [timer.py:197:stop] 0/9936, RunningAvgSamplesPerSec=5.881237427243262, CurrSamplesPerSec=5.324150875391776, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4909/5000 [16:46:23<16:42, 11.02s/it][2022-12-21 03:08:54,223] [INFO] [timer.py:197:stop] 0/9938, RunningAvgSamplesPerSec=5.88123135254784, CurrSamplesPerSec=5.279598992288359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4910/5000 [16:46:34<16:32, 11.03s/it][2022-12-21 03:09:05,271] [INFO] [logging.py:68:log_dist] [Rank 0] step=4970, skipped=7, lr=[8.444444444444444e-08], mom=[[0.9, 0.999]]
+[2022-12-21 03:09:05,273] [INFO] [timer.py:197:stop] 0/9940, RunningAvgSamplesPerSec=5.881226305924845, CurrSamplesPerSec=5.28592498026212, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4911/5000 [16:46:45<16:23, 11.05s/it][2022-12-21 03:09:16,333] [INFO] [timer.py:197:stop] 0/9942, RunningAvgSamplesPerSec=5.881220200700927, CurrSamplesPerSec=5.319063112532863, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|██��██████▊| 4912/5000 [16:46:56<16:10, 11.03s/it][2022-12-21 03:09:27,297] [INFO] [timer.py:197:stop] 0/9944, RunningAvgSamplesPerSec=5.881224499405726, CurrSamplesPerSec=5.344868228369991, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4913/5000 [16:47:07<15:58, 11.02s/it][2022-12-21 03:09:38,345] [INFO] [timer.py:197:stop] 0/9946, RunningAvgSamplesPerSec=5.881219741902373, CurrSamplesPerSec=5.2998856528932325, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4914/5000 [16:47:18<15:48, 11.03s/it][2022-12-21 03:09:49,392] [INFO] [timer.py:197:stop] 0/9948, RunningAvgSamplesPerSec=5.881215129864212, CurrSamplesPerSec=5.308986840514734, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4915/5000 [16:47:29<15:37, 11.03s/it][2022-12-21 03:10:00,339] [INFO] [timer.py:197:stop] 0/9950, RunningAvgSamplesPerSec=5.88122133864681, CurrSamplesPerSec=5.373435378821797, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4916/5000 [16:47:40<15:24, 11.01s/it][2022-12-21 03:10:11,321] [INFO] [timer.py:197:stop] 0/9952, RunningAvgSamplesPerSec=5.88122377502043, CurrSamplesPerSec=5.362875598439042, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4917/5000 [16:47:51<15:12, 11.00s/it][2022-12-21 03:10:22,346] [INFO] [timer.py:197:stop] 0/9954, RunningAvgSamplesPerSec=5.88122158669117, CurrSamplesPerSec=5.306522408417547, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4918/5000 [16:48:02<15:02, 11.01s/it][2022-12-21 03:10:33,386] [INFO] [timer.py:197:stop] 0/9956, RunningAvgSamplesPerSec=5.881217656529153, CurrSamplesPerSec=5.311343003845229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4919/5000 [16:48:13<14:52, 11.02s/it][2022-12-21 03:10:44,404] [INFO] [timer.py:197:stop] 0/9958, RunningAvgSamplesPerSec=5.8812160776662035, CurrSamplesPerSec=5.3289809277062385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4920/5000 [16:48:24<14:41, 11.02s/it][2022-12-21 03:10:54,527] [INFO] [logging.py:68:log_dist] [Rank 0] step=4980, skipped=7, lr=[6.222222222222223e-08], mom=[[0.9, 0.999]]
+[2022-12-21 03:10:54,528] [INFO] [timer.py:197:stop] 0/9960, RunningAvgSamplesPerSec=5.881310406137595, CurrSamplesPerSec=6.264844214324032, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-21 03:11:05,525] [INFO] [timer.py:197:stop] 0/9962, RunningAvgSamplesPerSec=5.881310809623745, CurrSamplesPerSec=5.313677908467399, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4921/5000 [16:48:41<16:32, 12.56s/it][2022-12-21 03:11:16,549] [INFO] [timer.py:197:stop] 0/9964, RunningAvgSamplesPerSec=5.881308695669039, CurrSamplesPerSec=5.334383005485661, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4922/5000 [16:48:52<15:43, 12.10s/it][2022-12-21 03:11:27,575] [INFO] [timer.py:197:stop] 0/9966, RunningAvgSamplesPerSec=5.881306452931631, CurrSamplesPerSec=5.307873249226387, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4923/5000 [16:49:03<15:06, 11.78s/it][2022-12-21 03:11:38,617] [INFO] [timer.py:197:stop] 0/9968, RunningAvgSamplesPerSec=5.881302473662925, CurrSamplesPerSec=5.298105506400605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4924/5000 [16:49:14<14:38, 11.56s/it][2022-12-21 03:11:49,641] [INFO] [timer.py:197:stop] 0/9970, RunningAvgSamplesPerSec=5.8813003621421105, CurrSamplesPerSec=5.307426390440144, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 98%|█████████▊| 4925/5000 [16:49:25<14:14, 11.40s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 5.111111111111112e-08, 'epoch': 120.12}
+
 98%|█████████▊| 4925/5000 [16:49:25<14:14, 11.40s/it][2022-12-21 03:12:00,672] [INFO] [timer.py:197:stop] 0/9972, RunningAvgSamplesPerSec=5.881297828453425, CurrSamplesPerSec=5.326023174664336, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4926/5000 [16:49:36<13:55, 11.29s/it][2022-12-21 03:12:11,679] [INFO] [timer.py:197:stop] 0/9974, RunningAvgSamplesPerSec=5.881297694126449, CurrSamplesPerSec=5.321831237362568, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4927/5000 [16:49:47<13:37, 11.20s/it][2022-12-21 03:12:22,736] [INFO] [timer.py:197:stop] 0/9976, RunningAvgSamplesPerSec=5.881292137187019, CurrSamplesPerSec=5.320043279296575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4928/5000 [16:49:58<13:23, 11.16s/it][2022-12-21 03:12:33,790] [INFO] [timer.py:197:stop] 0/9978, RunningAvgSamplesPerSec=5.881286867633184, CurrSamplesPerSec=5.318340605262924, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4929/5000 [16:50:09<13:10, 11.13s/it][2022-12-21 03:12:44,850] [INFO] [logging.py:68:log_dist] [Rank 0] step=4990, skipped=7, lr=[4e-08], mom=[[0.9, 0.999]]
+[2022-12-21 03:12:44,851] [INFO] [timer.py:197:stop] 0/9980, RunningAvgSamplesPerSec=5.881280658512985, CurrSamplesPerSec=5.317547506526408, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4930/5000 [16:50:20<12:57, 11.11s/it][2022-12-21 03:12:55,857] [INFO] [timer.py:197:stop] 0/9982, RunningAvgSamplesPerSec=5.881280511208736, CurrSamplesPerSec=5.339307051075701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4931/5000 [16:50:31<12:44, 11.08s/it][2022-12-21 03:13:06,894] [INFO] [timer.py:197:stop] 0/9984, RunningAvgSamplesPerSec=5.881276949274559, CurrSamplesPerSec=5.319455009398775, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4932/5000 [16:50:42<12:32, 11.07s/it][2022-12-21 03:13:17,951] [INFO] [timer.py:197:stop] 0/9986, RunningAvgSamplesPerSec=5.881271301530712, CurrSamplesPerSec=5.315437170821566, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4933/5000 [16:50:53<12:21, 11.06s/it][2022-12-21 03:13:28,989] [INFO] [timer.py:197:stop] 0/9988, RunningAvgSamplesPerSec=5.881267827397466, CurrSamplesPerSec=5.316848368341264, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4934/5000 [16:51:04<12:09, 11.06s/it][2022-12-21 03:13:39,953] [INFO] [timer.py:197:stop] 0/9990, RunningAvgSamplesPerSec=5.88127229282922, CurrSamplesPerSec=5.340523120694523, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4935/5000 [16:51:15<11:56, 11.03s/it][2022-12-21 03:13:50,984] [INFO] [timer.py:197:stop] 0/9992, RunningAvgSamplesPerSec=5.8812693415513495, CurrSamplesPerSec=5.325216162522858, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4936/5000 [16:51:26<11:45, 11.03s/it][2022-12-21 03:14:02,014] [INFO] [timer.py:197:stop] 0/9994, RunningAvgSamplesPerSec=5.8812666873656925, CurrSamplesPerSec=5.30069149484874, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▊| 4937/5000 [16:51:37<11:34, 11.03s/it][2022-12-21 03:14:13,010] [INFO] [timer.py:197:stop] 0/9996, RunningAvgSamplesPerSec=5.8812675051848675, CurrSamplesPerSec=5.309296184350492, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4938/5000 [16:51:48<11:23, 11.02s/it][2022-12-21 03:14:23,956] [INFO] [timer.py:197:stop] 0/9998, RunningAvgSamplesPerSec=5.881273955265471, CurrSamplesPerSec=5.350657499779842, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4939/5000 [16:51:59<11:10, 11.00s/it][2022-12-21 03:14:34,929] [INFO] [logging.py:68:log_dist] [Rank 0] step=5000, skipped=7, lr=[1.777777777777778e-08], mom=[[0.9, 0.999]]
+[2022-12-21 03:14:34,931] [INFO] [timer.py:197:stop] 0/10000, RunningAvgSamplesPerSec=5.8812771123839065, CurrSamplesPerSec=5.330708418737848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4940/5000 [16:52:10<10:59, 10.99s/it][2022-12-21 03:14:45,963] [INFO] [timer.py:197:stop] 0/10002, RunningAvgSamplesPerSec=5.88127412422575, CurrSamplesPerSec=5.329034458456789, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4941/5000 [16:52:21<10:49, 11.00s/it][2022-12-21 03:14:56,917] [INFO] [timer.py:197:stop] 0/10004, RunningAvgSamplesPerSec=5.8812795926481956, CurrSamplesPerSec=5.323810867524946, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4942/5000 [16:52:32<10:37, 10.99s/it][2022-12-21 03:15:07,892] [INFO] [timer.py:197:stop] 0/10006, RunningAvgSamplesPerSec=5.881282877952258, CurrSamplesPerSec=5.3429847854519625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4943/5000 [16:52:43<10:26, 10.98s/it][2022-12-21 03:15:18,928] [INFO] [timer.py:197:stop] 0/10008, RunningAvgSamplesPerSec=5.881279497163925, CurrSamplesPerSec=5.344004854359268, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4944/5000 [16:52:54<10:15, 11.00s/it][2022-12-21 03:15:29,923] [INFO] [timer.py:197:stop] 0/10010, RunningAvgSamplesPerSec=5.881280660743498, CurrSamplesPerSec=5.324802291250986, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4945/5000 [16:53:05<10:04, 11.00s/it][2022-12-21 03:15:40,924] [INFO] [timer.py:197:stop] 0/10012, RunningAvgSamplesPerSec=5.881281027707231, CurrSamplesPerSec=5.353477192552662, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4946/5000 [16:53:16<09:53, 11.00s/it][2022-12-21 03:15:51,923] [INFO] [timer.py:197:stop] 0/10014, RunningAvgSamplesPerSec=5.881281732983303, CurrSamplesPerSec=5.3478328916973386, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4947/5000 [16:53:27<09:42, 11.00s/it][2022-12-21 03:16:02,904] [INFO] [timer.py:197:stop] 0/10016, RunningAvgSamplesPerSec=5.881284430933788, CurrSamplesPerSec=5.348701766173452, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4948/5000 [16:53:38<09:31, 10.99s/it][2022-12-21 03:16:13,889] [INFO] [timer.py:197:stop] 0/10018, RunningAvgSamplesPerSec=5.881286523101808, CurrSamplesPerSec=5.326754535924154, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4949/5000 [16:53:49<09:20, 10.99s/it][2022-12-21 03:16:24,837] [INFO] [logging.py:68:log_dist] [Rank 0] step=5010, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:16:24,838] [INFO] [timer.py:197:stop] 0/10020, RunningAvgSamplesPerSec=5.881292564011672, CurrSamplesPerSec=5.346546614971619, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4950/5000 [16:54:00<09:08, 10.98s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 0.0, 'epoch': 120.72}
+
 99%|█████████▉| 4950/5000 [16:54:00<09:08, 10.98s/it][2022-12-21 03:16:35,760] [INFO] [timer.py:197:stop] 0/10022, RunningAvgSamplesPerSec=5.881301597989175, CurrSamplesPerSec=5.372602537606953, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4951/5000 [16:54:11<08:57, 10.96s/it][2022-12-21 03:16:46,806] [INFO] [timer.py:197:stop] 0/10024, RunningAvgSamplesPerSec=5.881297158673873, CurrSamplesPerSec=5.307300049463672, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4952/5000 [16:54:22<08:47, 10.99s/it][2022-12-21 03:16:57,805] [INFO] [timer.py:197:stop] 0/10026, RunningAvgSamplesPerSec=5.881297853823559, CurrSamplesPerSec=5.322416444935216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4953/5000 [16:54:33<08:36, 10.99s/it][2022-12-21 03:17:08,782] [INFO] [timer.py:197:stop] 0/10028, RunningAvgSamplesPerSec=5.8813008304838865, CurrSamplesPerSec=5.338586466179769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4954/5000 [16:54:44<08:25, 10.99s/it][2022-12-21 03:17:19,821] [INFO] [timer.py:197:stop] 0/10030, RunningAvgSamplesPerSec=5.88129711425438, CurrSamplesPerSec=5.302242749962984, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4955/5000 [16:54:55<08:15, 11.00s/it][2022-12-21 03:17:30,871] [INFO] [timer.py:197:stop] 0/10032, RunningAvgSamplesPerSec=5.881292074051597, CurrSamplesPerSec=5.2975229141873985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4956/5000 [16:55:06<08:04, 11.02s/it][2022-12-21 03:17:41,877] [INFO] [timer.py:197:stop] 0/10034, RunningAvgSamplesPerSec=5.8812919482005, CurrSamplesPerSec=5.335591742326389, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4957/5000 [16:55:17<07:53, 11.01s/it][2022-12-21 03:17:52,832] [INFO] [timer.py:197:stop] 0/10036, RunningAvgSamplesPerSec=5.88129724258707, CurrSamplesPerSec=5.34237399424286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4958/5000 [16:55:28<07:41, 11.00s/it][2022-12-21 03:18:03,877] [INFO] [timer.py:197:stop] 0/10038, RunningAvgSamplesPerSec=5.881292815474794, CurrSamplesPerSec=5.311328291034376, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4959/5000 [16:55:39<07:31, 11.01s/it][2022-12-21 03:18:14,888] [INFO] [logging.py:68:log_dist] [Rank 0] step=5020, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:18:14,889] [INFO] [timer.py:197:stop] 0/10040, RunningAvgSamplesPerSec=5.881291976412807, CurrSamplesPerSec=5.338066270856126, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4960/5000 [16:55:50<07:20, 11.01s/it][2022-12-21 03:18:25,892] [INFO] [timer.py:197:stop] 0/10042, RunningAvgSamplesPerSec=5.8812922555520055, CurrSamplesPerSec=5.331409300992028, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4961/5000 [16:56:01<07:09, 11.01s/it][2022-12-21 03:18:35,941] [INFO] [timer.py:197:stop] 0/10044, RunningAvgSamplesPerSec=5.881394103001211, CurrSamplesPerSec=5.365999933953345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4962/5000 [16:56:16<07:43, 12.21s/it][2022-12-21 03:18:46,935] [INFO] [timer.py:197:stop] 0/10046, RunningAvgSamplesPerSec=5.881395138690692, CurrSamplesPerSec=5.351609016676832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4963/5000 [16:56:27<07:18, 11.85s/it][2022-12-21 03:18:57,982] [INFO] [timer.py:197:stop] 0/10048, RunningAvgSamplesPerSec=5.881390553685634, CurrSamplesPerSec=5.315996759333798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4964/5000 [16:56:38<06:57, 11.60s/it][2022-12-21 03:19:08,986] [INFO] [timer.py:197:stop] 0/10050, RunningAvgSamplesPerSec=5.88139055519217, CurrSamplesPerSec=5.330042857097472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4965/5000 [16:56:49<06:39, 11.41s/it][2022-12-21 03:19:19,931] [INFO] [timer.py:197:stop] 0/10052, RunningAvgSamplesPerSec=5.881396854770055, CurrSamplesPerSec=5.350455505995745, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4966/5000 [16:57:00<06:23, 11.28s/it][2022-12-21 03:19:30,932] [INFO] [timer.py:197:stop] 0/10054, RunningAvgSamplesPerSec=5.881397241758793, CurrSamplesPerSec=5.332520926585494, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4967/5000 [16:57:11<06:09, 11.19s/it][2022-12-21 03:19:41,928] [INFO] [timer.py:197:stop] 0/10056, RunningAvgSamplesPerSec=5.881398091923873, CurrSamplesPerSec=5.325531204145525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4968/5000 [16:57:22<05:56, 11.14s/it][2022-12-21 03:19:52,969] [INFO] [timer.py:197:stop] 0/10058, RunningAvgSamplesPerSec=5.88139417474471, CurrSamplesPerSec=5.3184107818715285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4969/5000 [16:57:33<05:44, 11.11s/it][2022-12-21 03:20:04,002] [INFO] [logging.py:68:log_dist] [Rank 0] step=5030, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:20:04,004] [INFO] [timer.py:197:stop] 0/10060, RunningAvgSamplesPerSec=5.881390781540597, CurrSamplesPerSec=5.316073822511502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4970/5000 [16:57:44<05:32, 11.09s/it][2022-12-21 03:20:15,014] [INFO] [timer.py:197:stop] 0/10062, RunningAvgSamplesPerSec=5.88139012696381, CurrSamplesPerSec=5.337560398528682, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4971/5000 [16:57:55<05:20, 11.07s/it][2022-12-21 03:20:26,047] [INFO] [timer.py:197:stop] 0/10064, RunningAvgSamplesPerSec=5.881387001527627, CurrSamplesPerSec=5.326416943356395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4972/5000 [16:58:06<05:09, 11.05s/it][2022-12-21 03:20:37,124] [INFO] [timer.py:197:stop] 0/10066, RunningAvgSamplesPerSec=5.881379125770512, CurrSamplesPerSec=5.271700551656549, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4973/5000 [16:58:17<04:58, 11.07s/it][2022-12-21 03:20:48,159] [INFO] [timer.py:197:stop] 0/10068, RunningAvgSamplesPerSec=5.881375895165819, CurrSamplesPerSec=5.328140022491944, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 99%|█████████▉| 4974/5000 [16:58:28<04:47, 11.05s/it][2022-12-21 03:20:59,157] [INFO] [timer.py:197:stop] 0/10070, RunningAvgSamplesPerSec=5.881376712110741, CurrSamplesPerSec=5.354996900736099, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4975/5000 [16:58:39<04:35, 11.03s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 0.0, 'epoch': 121.34}
+
100%|█████████▉| 4975/5000 [16:58:39<04:35, 11.03s/it][2022-12-21 03:21:10,127] [INFO] [timer.py:197:stop] 0/10072, RunningAvgSamplesPerSec=5.881380590559962, CurrSamplesPerSec=5.353564101299987, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4976/5000 [16:58:50<04:24, 11.02s/it][2022-12-21 03:21:21,154] [INFO] [timer.py:197:stop] 0/10074, RunningAvgSamplesPerSec=5.881378389499504, CurrSamplesPerSec=5.3343020184924015, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4977/5000 [16:59:01<04:13, 11.02s/it][2022-12-21 03:21:32,189] [INFO] [timer.py:197:stop] 0/10076, RunningAvgSamplesPerSec=5.8813751060673045, CurrSamplesPerSec=5.327441056792944, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4978/5000 [16:59:12<04:02, 11.02s/it][2022-12-21 03:21:43,167] [INFO] [timer.py:197:stop] 0/10078, RunningAvgSamplesPerSec=5.881378003258725, CurrSamplesPerSec=5.348466458088087, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4979/5000 [16:59:23<03:51, 11.00s/it][2022-12-21 03:21:54,120] [INFO] [logging.py:68:log_dist] [Rank 0] step=5040, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:21:54,122] [INFO] [timer.py:197:stop] 0/10080, RunningAvgSamplesPerSec=5.881383155009212, CurrSamplesPerSec=5.364761942753, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4980/5000 [16:59:34<03:39, 10.99s/it][2022-12-21 03:22:05,171] [INFO] [timer.py:197:stop] 0/10082, RunningAvgSamplesPerSec=5.881378246182036, CurrSamplesPerSec=5.280441887068784, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4981/5000 [16:59:45<03:29, 11.00s/it][2022-12-21 03:22:16,174] [INFO] [timer.py:197:stop] 0/10084, RunningAvgSamplesPerSec=5.881378418990187, CurrSamplesPerSec=5.3117329230357155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4982/5000 [16:59:56<03:17, 10.99s/it][2022-12-21 03:22:27,120] [INFO] [timer.py:197:stop] 0/10086, RunningAvgSamplesPerSec=5.881384629528429, CurrSamplesPerSec=5.320820669807597, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4983/5000 [17:00:07<03:07, 11.00s/it][2022-12-21 03:22:38,160] [INFO] [timer.py:197:stop] 0/10088, RunningAvgSamplesPerSec=5.881380819315018, CurrSamplesPerSec=5.3151060627528945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|███���█████▉| 4984/5000 [17:00:18<02:56, 11.01s/it][2022-12-21 03:22:49,214] [INFO] [timer.py:197:stop] 0/10090, RunningAvgSamplesPerSec=5.881375611525982, CurrSamplesPerSec=5.301347023906035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4985/5000 [17:00:29<02:45, 11.03s/it][2022-12-21 03:23:00,239] [INFO] [timer.py:197:stop] 0/10092, RunningAvgSamplesPerSec=5.881373397196765, CurrSamplesPerSec=5.340788758161633, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4986/5000 [17:00:40<02:34, 11.03s/it][2022-12-21 03:23:11,310] [INFO] [timer.py:197:stop] 0/10094, RunningAvgSamplesPerSec=5.881366258632091, CurrSamplesPerSec=5.2960341849759835, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4987/5000 [17:00:51<02:23, 11.02s/it][2022-12-21 03:23:22,308] [INFO] [timer.py:197:stop] 0/10096, RunningAvgSamplesPerSec=5.881366958902828, CurrSamplesPerSec=5.312095357485689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4988/5000 [17:01:02<02:12, 11.02s/it][2022-12-21 03:23:33,347] [INFO] [timer.py:197:stop] 0/10098, RunningAvgSamplesPerSec=5.881363276141535, CurrSamplesPerSec=5.297693329016797, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4989/5000 [17:01:13<02:01, 11.04s/it][2022-12-21 03:23:44,385] [INFO] [logging.py:68:log_dist] [Rank 0] step=5050, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:23:44,386] [INFO] [timer.py:197:stop] 0/10100, RunningAvgSamplesPerSec=5.881359542013666, CurrSamplesPerSec=5.325262222620043, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4990/5000 [17:01:24<01:50, 11.02s/it][2022-12-21 03:23:55,394] [INFO] [timer.py:197:stop] 0/10102, RunningAvgSamplesPerSec=5.8813591452139375, CurrSamplesPerSec=5.307864013243388, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4991/5000 [17:01:35<01:39, 11.02s/it][2022-12-21 03:24:06,354] [INFO] [timer.py:197:stop] 0/10104, RunningAvgSamplesPerSec=5.881364061779938, CurrSamplesPerSec=5.35067200467225, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4992/5000 [17:01:46<01:28, 11.02s/it][2022-12-21 03:24:17,429] [INFO] [timer.py:197:stop] 0/10106, RunningAvgSamplesPerSec=5.881356451745012, CurrSamplesPerSec=5.299697518849052, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4993/5000 [17:01:57<01:17, 11.03s/it][2022-12-21 03:24:28,444] [INFO] [timer.py:197:stop] 0/10108, RunningAvgSamplesPerSec=5.881355443884368, CurrSamplesPerSec=5.3328732781455175, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4994/5000 [17:02:09<01:06, 11.03s/it][2022-12-21 03:24:39,488] [INFO] [timer.py:197:stop] 0/10110, RunningAvgSamplesPerSec=5.881351212272025, CurrSamplesPerSec=5.315564320545756, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4995/5000 [17:02:19<00:55, 11.02s/it][2022-12-21 03:24:50,495] [INFO] [timer.py:197:stop] 0/10112, RunningAvgSamplesPerSec=5.881350984915537, CurrSamplesPerSec=5.308833127046617, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4996/5000 [17:02:31<00:44, 11.02s/it][2022-12-21 03:25:01,524] [INFO] [timer.py:197:stop] 0/10114, RunningAvgSamplesPerSec=5.8813482856803985, CurrSamplesPerSec=5.305472346229712, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4997/5000 [17:02:42<00:33, 11.04s/it][2022-12-21 03:25:12,609] [INFO] [timer.py:197:stop] 0/10116, RunningAvgSamplesPerSec=5.881339695009787, CurrSamplesPerSec=5.292267606688984, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4998/5000 [17:02:53<00:22, 11.05s/it][2022-12-21 03:25:23,644] [INFO] [timer.py:197:stop] 0/10118, RunningAvgSamplesPerSec=5.881336428470138, CurrSamplesPerSec=5.324769758921748, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|█████████▉| 4999/5000 [17:03:04<00:11, 11.04s/it][2022-12-21 03:25:34,660] [INFO] [logging.py:68:log_dist] [Rank 0] step=5060, skipped=7, lr=[0.0], mom=[[0.9, 0.999]]
+[2022-12-21 03:25:34,661] [INFO] [timer.py:197:stop] 0/10120, RunningAvgSamplesPerSec=5.881335021976626, CurrSamplesPerSec=5.317251735366098, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
100%|██████████| 5000/5000 [17:03:15<00:00, 11.02s/it]
                                                      
{'loss': 0.0001, 'learning_rate': 0.0, 'epoch': 121.94}
+
100%|██████████| 5000/5000 [17:03:15<00:00, 11.02s/it][INFO|trainer.py:2955] 2022-12-21 03:25:39,586 >> ***** Running Evaluation *****
+[INFO|trainer.py:2957] 2022-12-21 03:25:39,586 >>   Num examples = 1325
+[INFO|trainer.py:2960] 2022-12-21 03:25:39,586 >>   Batch size = 8
+
+
  0%|          | 0/166 [00:00<?, ?it/s]
+
  1%|          | 2/166 [00:05<07:53,  2.89s/it]
+
  2%|▏         | 3/166 [00:11<11:10,  4.11s/it]
+
  2%|▏         | 4/166 [00:17<12:56,  4.79s/it]
+
  3%|���         | 5/166 [00:24<15:07,  5.64s/it]
+
  4%|▎         | 6/166 [00:32<16:32,  6.20s/it]
+
  4%|▍         | 7/166 [00:38<16:32,  6.24s/it]
+
  5%|▍         | 8/166 [00:44<16:08,  6.13s/it]
+
  5%|▌         | 9/166 [00:50<15:53,  6.07s/it]
+
  6%|▌         | 10/166 [00:56<15:33,  5.98s/it]
+
  7%|▋         | 11/166 [01:01<15:11,  5.88s/it]
+
  7%|▋         | 12/166 [01:07<15:10,  5.92s/it]
+
  8%|▊         | 13/166 [01:14<15:30,  6.08s/it]
+
  8%|▊         | 14/166 [01:21<16:03,  6.34s/it]
+
  9%|▉         | 15/166 [01:26<15:26,  6.13s/it]
+
 10%|▉         | 16/166 [01:34<16:13,  6.49s/it]
+
 10%|█         | 17/166 [01:40<16:16,  6.55s/it]
+
 11%|█         | 18/166 [01:47<15:56,  6.46s/it]
+
 11%|█▏        | 19/166 [01:52<15:24,  6.29s/it]
+
 12%|█▏        | 20/166 [02:03<18:33,  7.62s/it]
+
 13%|█▎        | 21/166 [02:10<18:07,  7.50s/it]
+
 13%|█▎        | 22/166 [02:18<17:52,  7.45s/it]
+
 14%|█▍        | 23/166 [02:25<17:50,  7.48s/it]
+
 14%|█▍        | 24/166 [02:32<16:59,  7.18s/it]
+
 15%|█▌        | 25/166 [02:39<16:53,  7.19s/it]
+
 16%|█▌        | 26/166 [02:46<16:40,  7.14s/it]
+
 16%|█▋        | 27/166 [02:53<16:42,  7.21s/it]
+
 17%|█▋        | 28/166 [02:59<15:46,  6.86s/it]
+
 17%|█▋        | 29/166 [03:08<16:34,  7.26s/it]
+
 18%|█▊        | 30/166 [03:14<16:11,  7.14s/it]
+
 19%|█▊        | 31/166 [03:22<16:09,  7.18s/it]
+
 19%|█▉        | 32/166 [03:29<16:22,  7.33s/it]
+
 20%|█▉        | 33/166 [03:37<16:16,  7.35s/it]
+
 20%|██        | 34/166 [03:43<15:32,  7.07s/it]
+
 21%|██        | 35/166 [03:51<16:00,  7.33s/it]
+
 22%|██▏       | 36/166 [03:59<15:59,  7.38s/it]
+
 22%|██▏       | 37/166 [04:05<15:18,  7.12s/it]
+
 23%|██▎       | 38/166 [04:14<16:05,  7.54s/it]
+
 23%|██▎       | 39/166 [04:21<15:31,  7.34s/it]
+
 24%|██▍       | 40/166 [04:28<15:40,  7.46s/it]
+
 25%|██▍       | 41/166 [04:35<15:04,  7.23s/it]
+
 25%|██▌       | 42/166 [04:41<14:28,  7.00s/it]
+
 26%|██▌       | 43/166 [04:48<13:57,  6.81s/it]
+
 27%|██▋       | 44/166 [04:56<14:53,  7.33s/it]
+
 27%|██▋       | 45/166 [05:04<15:13,  7.55s/it]
+
 28%|██▊       | 46/166 [05:14<16:37,  8.31s/it]
+
 28%|██▊       | 47/166 [05:23<16:25,  8.28s/it]
+
 29%|██▉       | 48/166 [05:30<15:34,  7.92s/it]
+
 30%|██▉       | 49/166 [05:36<14:14,  7.31s/it]
+
 30%|███       | 50/166 [05:44<14:50,  7.67s/it]
+
 31%|███       | 51/166 [05:52<14:34,  7.60s/it]
+
 31%|███▏      | 52/166 [05:59<14:27,  7.61s/it]
+
 32%|███▏      | 53/166 [06:06<14:04,  7.47s/it]
+
 33%|███▎      | 54/166 [06:14<14:12,  7.61s/it]
+
 33%|███▎      | 55/166 [06:20<13:08,  7.11s/it]
+
 34%|███▎      | 56/166 [06:26<12:29,  6.82s/it]
+
 34%|███▍      | 57/166 [06:35<13:06,  7.21s/it]
+
 35%|███▍      | 58/166 [06:40<12:05,  6.71s/it]
+
 36%|███▌      | 59/166 [06:47<12:10,  6.83s/it]
+
 36%|███▌      | 60/166 [06:54<12:13,  6.92s/it]
+
 37%|███▋      | 61/166 [07:03<13:15,  7.58s/it]
+
 37%|███▋      | 62/166 [07:11<13:00,  7.50s/it]
+
 38%|███▊      | 63/166 [07:18<12:45,  7.43s/it]
+
 39%|███▊      | 64/166 [07:27<13:27,  7.92s/it]
+
 39%|███▉      | 65/166 [07:36<13:35,  8.08s/it]
+
 40%|███▉      | 66/166 [07:42<12:51,  7.71s/it]
+
 40%|████      | 67/166 [07:49<12:20,  7.48s/it]
+
 41%|████      | 68/166 [07:54<11:03,  6.78s/it]
+
 42%|████▏     | 69/166 [08:01<10:59,  6.80s/it]
+
 42%|████▏     | 70/166 [08:09<11:29,  7.18s/it]
+
 43%|████▎     | 71/166 [08:17<11:42,  7.39s/it]
+
 43%|████▎     | 72/166 [08:24<11:10,  7.13s/it]
+
 44%|████▍     | 73/166 [08:30<10:43,  6.92s/it]
+
 45%|████▍     | 74/166 [08:37<10:19,  6.73s/it]
+
 45%|████▌     | 75/166 [08:43<10:01,  6.61s/it]
+
 46%|████▌     | 76/166 [08:50<10:13,  6.82s/it]
+
 46%|████▋     | 77/166 [08:56<09:38,  6.50s/it]
+
 47%|████▋     | 78/166 [09:03<09:47,  6.68s/it]
+
 48%|████▊     | 79/166 [09:11<10:14,  7.06s/it]
+
 48%|████▊     | 80/166 [09:19<10:35,  7.38s/it]
+
 49%|████▉     | 81/166 [09:27<10:40,  7.53s/it]
+
 49%|████▉     | 82/166 [09:34<10:17,  7.35s/it]
+
 50%|█████     | 83/166 [09:39<09:20,  6.75s/it]
+
 51%|█████     | 84/166 [09:46<09:05,  6.65s/it]
+
 51%|█████     | 85/166 [09:53<09:13,  6.84s/it]
+
 52%|█████▏    | 86/166 [10:01<09:41,  7.27s/it]
+
 52%|█████▏    | 87/166 [10:09<09:41,  7.36s/it]
+
 53%|█████▎    | 88/166 [10:15<09:18,  7.16s/it]
+
 54%|█████▎    | 89/166 [10:23<09:15,  7.21s/it]
+
 54%|█████▍    | 90/166 [10:29<08:52,  7.01s/it]
+
 55%|█████▍    | 91/166 [10:36<08:38,  6.92s/it]
+
 55%|█████▌    | 92/166 [10:42<08:12,  6.66s/it]
+
 56%|█████▌    | 93/166 [10:50<08:37,  7.08s/it]
+
 57%|█████▋    | 94/166 [11:00<09:36,  8.01s/it]
+
 57%|█████▋    | 95/166 [11:06<08:30,  7.18s/it]
+
 58%|█████▊    | 96/166 [11:12<08:09,  6.99s/it]
+
 58%|█████▊    | 97/166 [11:22<09:09,  7.97s/it]
+
 59%|█████▉    | 98/166 [11:32<09:41,  8.55s/it]
+
 60%|█████▉    | 99/166 [11:40<09:10,  8.22s/it]
+
 60%|██████    | 100/166 [11:46<08:22,  7.62s/it]
+
 61%|██████    | 101/166 [11:53<07:55,  7.31s/it]
+
 61%|██████▏   | 102/166 [12:01<08:17,  7.78s/it]
+
 62%|██████▏   | 103/166 [12:10<08:31,  8.12s/it]
+
 63%|██████▎   | 104/166 [12:18<08:07,  7.87s/it]
+
 63%|██████▎   | 105/166 [12:26<08:06,  7.97s/it]
+
 64%|██████▍   | 106/166 [12:36<08:31,  8.52s/it]
+
 64%|██████▍   | 107/166 [12:45<08:41,  8.84s/it]
+
 65%|██████▌   | 108/166 [12:51<07:32,  7.81s/it]
+
 66%|██████▌   | 109/166 [12:57<07:00,  7.37s/it]
+
 66%|██████▋   | 110/166 [13:04<06:54,  7.40s/it]
+
 67%|██████▋   | 111/166 [13:10<06:23,  6.97s/it]
+
 67%|██████▋   | 112/166 [13:18<06:22,  7.08s/it]
+
 68%|██████▊   | 113/166 [13:26<06:38,  7.52s/it]
+
 69%|██████▊   | 114/166 [13:34<06:29,  7.50s/it]
+
 69%|██████▉   | 115/166 [13:40<06:04,  7.14s/it]
+
 70%|██████▉   | 116/166 [13:46<05:43,  6.88s/it]
+
 70%|███████   | 117/166 [13:54<05:46,  7.07s/it]
+
 71%|███████   | 118/166 [14:02<05:50,  7.30s/it]
+
 72%|███████▏  | 119/166 [14:09<05:48,  7.42s/it]
+
 72%|███████▏  | 120/166 [14:18<05:52,  7.65s/it]
+
 73%|███████▎  | 121/166 [14:26<05:53,  7.86s/it]
+
 73%|███████▎  | 122/166 [14:34<05:42,  7.79s/it]
+
 74%|███████▍  | 123/166 [14:41<05:29,  7.65s/it]
+
 75%|███████▍  | 124/166 [14:48<05:10,  7.39s/it]
+
 75%|███████▌  | 125/166 [14:55<05:03,  7.40s/it]
+
 76%|███████▌  | 126/166 [15:04<05:08,  7.72s/it]
+
 77%|███████▋  | 127/166 [15:09<04:29,  6.91s/it]
+
 77%|███████▋  | 128/166 [15:19<04:58,  7.86s/it]
+
 78%|███████▊  | 129/166 [15:27<04:51,  7.87s/it]
+
 78%|███████▊  | 130/166 [15:33<04:32,  7.57s/it]
+
 79%|███████▉  | 131/166 [15:40<04:19,  7.41s/it]
+
 80%|███████▉  | 132/166 [15:49<04:18,  7.61s/it]
+
 80%|████████  | 133/166 [15:56<04:12,  7.65s/it]
+
 81%|████████  | 134/166 [16:03<03:54,  7.34s/it]
+
 81%|████████▏ | 135/166 [16:09<03:36,  6.98s/it]
+
 82%|████████▏ | 136/166 [16:19<04:00,  8.02s/it]
+
 83%|████████▎ | 137/166 [16:28<04:00,  8.31s/it]
+
 83%|████████▎ | 138/166 [16:39<04:07,  8.84s/it]
+
 84%|████████▎ | 139/166 [16:47<03:54,  8.67s/it]
+
 84%|████████▍ | 140/166 [16:53<03:26,  7.96s/it]
+
 85%|████████▍ | 141/166 [16:59<03:03,  7.35s/it]
+
 86%|████████▌ | 142/166 [17:06<02:54,  7.29s/it]
+
 86%|████████▌ | 143/166 [17:12<02:35,  6.75s/it]
+
 87%|████████▋ | 144/166 [17:20<02:41,  7.36s/it]
+
 87%|████████▋ | 145/166 [17:30<02:49,  8.09s/it]
+
 88%|████████▊ | 146/166 [17:40<02:49,  8.46s/it]
+
 89%|████████▊ | 147/166 [17:48<02:40,  8.46s/it]
+
 89%|████████▉ | 148/166 [17:56<02:31,  8.44s/it]
+
 90%|████████▉ | 149/166 [18:06<02:27,  8.66s/it]
+
 90%|█████████ | 150/166 [18:15<02:23,  8.96s/it]
+
 91%|█████████ | 151/166 [18:23<02:10,  8.68s/it]
+
 92%|█████████▏| 152/166 [18:29<01:49,  7.80s/it]
+
 92%|█████████▏| 153/166 [18:36<01:36,  7.40s/it]
+
 93%|█████████▎| 154/166 [18:43<01:28,  7.38s/it]
+
 93%|█████████▎| 155/166 [18:51<01:22,  7.49s/it]
+
 94%|█████████▍| 156/166 [18:56<01:10,  7.00s/it]
+
 95%|█████████▍| 157/166 [19:03<01:02,  6.91s/it]
+
 95%|█████████▌| 158/166 [19:09<00:52,  6.52s/it]
+
 96%|█████████▌| 159/166 [19:14<00:41,  5.98s/it]
+
 96%|█████████▋| 160/166 [19:19<00:34,  5.70s/it]
+
 97%|█████████▋| 161/166 [19:25<00:29,  5.82s/it]
+
 98%|█████���███▊| 162/166 [19:32<00:24,  6.15s/it]
+
 98%|█████████▊| 163/166 [19:39<00:19,  6.46s/it]
+
 99%|█████████▉| 164/166 [19:45<00:12,  6.37s/it]
+
 99%|█████████▉| 165/166 [19:51<00:06,  6.24s/it]
+
100%|██████████| 166/166 [19:55<00:00,  5.63s/it]
                                                      
+
                                                 
{'eval_loss': 0.291259765625, 'eval_wer': 8.473140457818456, 'eval_runtime': 1274.9287, 'eval_samples_per_second': 1.039, 'eval_steps_per_second': 0.13, 'epoch': 121.94}
+
100%|██████████| 5000/5000 [17:24:30<00:00, 11.02s/it]
+
100%|██████████| 166/166 [21:09<00:00,  5.63s/it]
+
                                                 [INFO|trainer.py:2700] 2022-12-21 03:46:54,517 >> Saving model checkpoint to ./checkpoint-5000
+[INFO|configuration_utils.py:447] 2022-12-21 03:46:54,518 >> Configuration saved in ./checkpoint-5000/config.json
+[INFO|modeling_utils.py:1680] 2022-12-21 03:46:58,246 >> Model weights saved in ./checkpoint-5000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:368] 2022-12-21 03:46:58,261 >> Feature extractor saved in ./checkpoint-5000/preprocessor_config.json
+[2022-12-21 03:46:58,261] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step5060 is begin to save!
+/home/milan/hf_env/lib/python3.8/site-packages/torch/nn/modules/module.py:1428: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2022-12-21 03:46:58,272] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-5000/global_step5060/mp_rank_00_model_states.pt
+[2022-12-21 03:46:58,272] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-5000/global_step5060/mp_rank_00_model_states.pt...
+[2022-12-21 03:47:01,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-5000/global_step5060/mp_rank_00_model_states.pt.
+[2022-12-21 03:47:01,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-5000/global_step5060/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2022-12-21 03:47:16,832] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-5000/global_step5060/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2022-12-21 03:47:16,833] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-5000/global_step5060/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2022-12-21 03:47:16,833] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5060 is ready now!
+[INFO|feature_extraction_utils.py:368] 2022-12-21 03:47:22,152 >> Feature extractor saved in ./preprocessor_config.json