diff --git "a/checkpoints/wandb/debug-internal.log" "b/checkpoints/wandb/debug-internal.log" --- "a/checkpoints/wandb/debug-internal.log" +++ "b/checkpoints/wandb/debug-internal.log" @@ -38036,3 +38036,11441 @@ 2024-08-31 17:12:17,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report 2024-08-31 17:12:23,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages 2024-08-31 17:12:23,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:28,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:31,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:12:31,162 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:12:31,434 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:12:33,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:12:34,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:39,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:43,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:12:45,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:46,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:12:46,163 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:12:50,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:12:53,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:12:56,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:01,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:01,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:13:01,162 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:13:01,436 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:13:03,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:07,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:12,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:13,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:16,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:13:16,163 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:13:17,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:23,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:23,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:28,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:31,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:13:31,163 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:13:31,442 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:13:33,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:34,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:39,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:43,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:45,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:46,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:13:46,163 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:13:50,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:13:53,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:13:56,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:01,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:01,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:14:01,164 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:14:01,441 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:14:03,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:05,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:14:05,480 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:14:05,480 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:14:05,482 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:14:06,319 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:14:06,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:07,319 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:14:12,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:13,052 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:16,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:14:16,164 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:14:17,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:23,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:23,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:29,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:31,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:14:31,164 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:14:31,443 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:14:33,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:35,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:40,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:43,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:46,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:46,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:14:46,164 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:14:51,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:14:53,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:14:57,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:01,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:15:01,165 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:15:01,448 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:15:02,452 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:03,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:08,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:13,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:14,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:16,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:15:16,166 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:15:19,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:23,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:24,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:29,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:31,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:15:31,166 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:15:31,448 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:15:33,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:35,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:41,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:43,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:46,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:15:46,167 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:15:46,316 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:52,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:15:53,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:15:58,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:01,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:16:01,167 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:16:01,451 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:16:03,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:03,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:07,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:16:07,220 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:16:07,221 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:16:07,221 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:16:07,378 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:16:09,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:09,379 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:16:13,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:14,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:16,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:16:16,167 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:16:19,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:23,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:24,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:29,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:31,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:16:31,168 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:16:31,453 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:16:33,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:35,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:40,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:43,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:46,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:46,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:16:46,167 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:16:51,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:16:53,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:16:57,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:01,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:17:01,168 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:17:01,456 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:17:02,457 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:03,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:08,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:13,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:14,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:16,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:17:16,168 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:17:19,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:23,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:25,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:30,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:31,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:17:31,169 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:17:31,458 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:17:33,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:36,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:41,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:43,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:46,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:17:46,170 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:17:46,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:52,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:17:53,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:17:58,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:01,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:18:01,170 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:18:01,465 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:18:03,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:04,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:10,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:10,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:18:10,559 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:18:10,559 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:18:10,560 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:18:11,435 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:18:11,435 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:18:13,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:15,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:16,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:18:16,170 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:18:20,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:23,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:26,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:31,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:31,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:18:31,170 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:18:31,463 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:18:33,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:36,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:42,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:43,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:46,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:18:46,170 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:18:47,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:53,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:18:53,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:18:58,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:01,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:19:01,171 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:19:01,467 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:19:03,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:03,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:08,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:13,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:14,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:16,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:19:16,171 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:19:19,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:23,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:25,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:30,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:31,171 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:19:31,172 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:19:31,469 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:19:33,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:36,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:41,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:43,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:46,171 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:19:46,172 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:19:46,387 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:52,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:19:53,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:19:58,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:01,172 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:20:01,172 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:20:01,471 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:20:03,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:03,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:09,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:12,172 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:20:12,174 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:20:12,174 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:20:12,175 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:20:12,489 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:20:13,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:13,490 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:20:15,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:16,171 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:20:16,172 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:20:21,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:23,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:27,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:31,172 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:20:31,172 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:20:31,473 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:20:32,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:33,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:38,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:43,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:44,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:46,172 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:20:46,173 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:20:49,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:20:53,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:20:55,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:00,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:01,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:21:01,173 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:21:01,476 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:21:03,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:06,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:11,093 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:13,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:16,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:21:16,173 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:21:16,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:22,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:23,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:28,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:31,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:21:31,174 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:21:31,478 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:21:33,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:34,093 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:40,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:43,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:46,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:46,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:21:46,174 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:21:52,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:21:53,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:21:57,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:01,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:22:01,174 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:22:01,480 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:22:02,482 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:03,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:08,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:13,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:13,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:13,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:22:13,880 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:22:13,880 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:22:13,881 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:22:14,548 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:22:15,549 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:22:16,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:22:16,175 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:22:18,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:23,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:24,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:29,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:31,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:22:31,174 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:22:31,483 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:22:33,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:35,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:40,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:43,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:46,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:46,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:22:46,175 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:22:51,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:22:53,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:22:57,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:01,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:23:01,175 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:23:01,488 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:23:02,490 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:03,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:08,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:13,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:13,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:16,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:23:16,175 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:23:18,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:23,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:24,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:29,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:31,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:23:31,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:23:31,488 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:23:34,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:34,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:40,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:44,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:45,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:46,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:23:46,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:23:51,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:23:54,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:23:56,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:01,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:24:01,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:24:01,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:01,492 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:24:04,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:07,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:12,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:14,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:16,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:24:16,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:24:17,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:24:17,281 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:24:17,281 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:24:17,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:17,282 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:24:17,599 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:24:19,600 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:24:23,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:24,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:28,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:31,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:24:31,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:24:31,493 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:24:33,494 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:34,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:39,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:44,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:45,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:46,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:24:46,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:24:50,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:24:54,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:24:55,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:01,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:25:01,177 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:25:01,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:01,495 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:25:04,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:07,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:13,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:14,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:16,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:25:16,176 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:25:18,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:24,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:24,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:29,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:31,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:25:31,177 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:25:31,498 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:25:34,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:34,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:40,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:44,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:46,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:25:46,177 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:25:46,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:52,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:25:54,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:25:58,265 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:01,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:26:01,177 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:26:01,500 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:26:03,501 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:04,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:09,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:14,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:15,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:16,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:26:16,179 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:26:19,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:26:19,128 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:26:19,128 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:26:19,129 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:26:19,648 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:26:21,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:21,650 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:26:24,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:26,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:31,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:26:31,179 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:26:31,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:31,503 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:26:34,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:37,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:43,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:44,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:46,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:26:46,179 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:26:48,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:54,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:26:54,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:26:59,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:01,179 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:27:01,179 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:27:01,506 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:27:04,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:04,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:10,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:14,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:15,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:16,179 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:27:16,180 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:27:20,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:24,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:26,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:31,180 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:27:31,180 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:27:31,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:31,512 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:27:34,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:37,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:42,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:44,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:46,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:27:46,191 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:27:47,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:53,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:27:54,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:27:59,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:01,180 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:28:01,180 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:28:01,510 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:28:04,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:05,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:10,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:14,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:16,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:28:16,182 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:28:16,370 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:20,908 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:28:20,909 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:28:20,909 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:28:20,911 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:28:21,700 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:28:21,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:23,701 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:28:24,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:27,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:31,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:28:31,182 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:28:31,512 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:28:32,514 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:34,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:38,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:43,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:44,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:46,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:28:46,183 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:28:48,317 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:54,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:28:54,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:28:59,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:01,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:29:01,183 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:29:01,516 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:29:04,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:05,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:11,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:14,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:16,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:29:16,183 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:29:16,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:22,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:24,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:28,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:31,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:29:31,183 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:29:31,517 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:29:33,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:34,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:39,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:44,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:45,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:46,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:29:46,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:29:50,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:29:54,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:29:56,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:01,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:30:01,184 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:30:01,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:01,520 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:30:04,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:07,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:13,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:14,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:16,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:30:16,184 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:30:18,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:24,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:30:24,239 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:30:24,240 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:30:24,240 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:24,241 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:30:24,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:24,753 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:30:25,754 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:30:29,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:31,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:30:31,184 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:30:31,522 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:30:34,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:34,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:40,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:44,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:45,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:46,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:30:46,184 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:30:51,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:30:54,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:30:57,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:01,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:31:01,184 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:31:01,524 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:31:02,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:04,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:08,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:13,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:14,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:16,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:31:16,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:31:18,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:24,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:24,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:29,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:31,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:31:31,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:31:31,526 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:31:34,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:35,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:40,326 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:44,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:46,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:31:46,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:31:46,319 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:51,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:31:54,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:31:56,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:01,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:32:01,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:32:01,532 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:32:01,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:04,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:07,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:13,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:14,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:16,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:32:16,185 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:32:19,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:24,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:24,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:26,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:32:26,170 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:32:26,171 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:32:26,171 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:32:26,807 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:32:27,808 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:32:29,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:31,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:32:31,186 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:32:31,531 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:32:34,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:34,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:39,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:44,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:45,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:46,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:32:46,186 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:32:51,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:32:54,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:32:57,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:01,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:33:01,186 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:33:01,535 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:33:02,536 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:04,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:08,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:13,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:14,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:16,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:33:16,186 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:33:18,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:24,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:24,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:29,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:31,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:33:31,187 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:33:31,536 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:33:34,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:35,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:41,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:44,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:46,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:33:46,187 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:33:46,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:52,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:33:54,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:33:58,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:01,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:34:01,188 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:34:01,538 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:34:03,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:04,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:09,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:14,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:15,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:16,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:34:16,187 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:34:20,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:24,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:25,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:28,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:34:28,679 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:34:28,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:34:28,681 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:34:28,861 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:34:29,863 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:34:31,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:34:31,188 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:34:31,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:31,541 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:34:34,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:36,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:41,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:44,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:46,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:34:46,189 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:34:46,382 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:52,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:34:54,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:34:58,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:01,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:35:01,202 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:35:01,543 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:35:03,545 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:04,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:09,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:14,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:15,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:16,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:35:16,201 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:35:20,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:24,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:25,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:30,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:31,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:35:31,202 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:35:31,546 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:35:34,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:36,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:41,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:44,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:46,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:35:46,214 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:35:46,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:52,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:35:54,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:35:58,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:01,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:36:01,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:36:01,549 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:36:03,551 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:04,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:09,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:14,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:15,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:16,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:36:16,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:36:20,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:24,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:26,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:31,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:36:31,214 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:36:31,428 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:31,555 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:36:32,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:36:32,094 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:36:32,094 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:36:32,094 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:36:32,924 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:36:33,925 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:36:34,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:37,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:43,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:44,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:46,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:36:46,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:36:48,434 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:54,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:36:54,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:36:59,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:01,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:37:01,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:37:01,554 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:37:04,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:05,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:10,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:14,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:15,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:16,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:37:16,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:37:21,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:24,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:27,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:31,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:37:31,215 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:37:31,558 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:37:32,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:34,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:38,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:44,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:44,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:46,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:37:46,216 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:37:50,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:37:54,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:37:56,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:01,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:38:01,216 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:38:01,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:01,559 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:38:04,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:07,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:13,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:14,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:16,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:38:16,216 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:38:18,390 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:24,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:24,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:29,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:31,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:38:31,216 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:38:31,562 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:38:33,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:38:33,748 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:38:33,748 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:38:33,749 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:38:34,000 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:38:34,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:35,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:36,001 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:38:41,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:44,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:46,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:38:46,216 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:38:46,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:52,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:38:54,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:38:58,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:01,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:39:01,217 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:39:01,565 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:39:03,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:04,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:09,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:14,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:15,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:16,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:39:16,217 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:39:20,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:24,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:26,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:31,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:39:31,217 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:39:31,431 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:31,567 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:39:34,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:37,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:43,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:44,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:46,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:39:46,217 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:39:48,382 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:54,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:39:54,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:39:59,350 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:01,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:40:01,217 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:40:01,572 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:40:04,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:04,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:09,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:14,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:15,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:16,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:40:16,218 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:40:21,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:24,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:27,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:31,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:40:31,219 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:40:31,572 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:40:32,573 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:34,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:35,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:40:35,416 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:40:35,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:40:35,419 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:40:36,067 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:40:38,067 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:40:38,459 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:44,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:44,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:46,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:40:46,218 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:40:49,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:54,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:40:54,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:40:59,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:01,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:41:01,219 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:41:01,579 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:41:04,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:05,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:10,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:14,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:15,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:16,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:41:16,219 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:41:21,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:24,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:26,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:31,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:41:31,219 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:41:31,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:31,577 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:41:34,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:37,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:43,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:44,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:46,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:41:46,220 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:41:48,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:54,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:41:54,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:41:59,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:01,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:42:01,220 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:42:01,580 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:42:04,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:04,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:09,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:14,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:15,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:16,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:42:16,221 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:42:20,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:24,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:26,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:31,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:42:31,221 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:42:31,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:31,582 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:42:34,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:36,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:42:36,855 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:42:36,855 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:42:36,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:36,856 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:42:37,132 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:42:38,132 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:42:42,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:44,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:46,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:42:46,221 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:42:47,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:53,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:42:54,300 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:42:59,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:01,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:43:01,221 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:43:01,585 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:43:04,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:05,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:10,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:14,300 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:15,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:16,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:43:16,222 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:43:21,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:24,300 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:27,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:31,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:43:31,222 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:43:31,587 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:43:32,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:34,300 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:38,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:43,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:44,300 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:46,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:43:46,222 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:43:49,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:54,301 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:43:54,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:43:59,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:01,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:44:01,223 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:44:01,590 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:44:04,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:04,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:10,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:14,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:16,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:44:16,223 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:44:16,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:22,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:24,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:28,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:31,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:44:31,224 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:44:31,593 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:44:33,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:34,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:39,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:40,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:44:40,127 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:44:40,127 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:44:40,128 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:44:40,201 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:44:42,203 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:44:44,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:44,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:46,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:44:46,224 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:44:49,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:44:54,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:44:55,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:00,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:01,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:45:01,224 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:45:01,596 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:45:04,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:06,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:12,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:14,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:16,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:45:16,225 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:45:17,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:23,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:24,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:29,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:31,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:45:31,224 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:45:31,598 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:45:34,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:34,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:39,382 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:44,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:45,382 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:46,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:45:46,224 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:45:50,383 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:45:54,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:45:56,383 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:01,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:46:01,225 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:46:04,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:05,158 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:46:05,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:10,384 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:14,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:15,384 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:16,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:46:16,225 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:46:21,384 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:24,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:27,384 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:31,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:46:31,226 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:46:31,603 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:46:32,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:34,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:38,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:42,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:46:42,053 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:46:42,054 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:46:42,056 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:46:42,268 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:46:44,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:44,269 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:46:44,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:46,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:46:46,225 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:46:49,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:46:54,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:46:55,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:00,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:01,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:47:01,226 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:47:01,606 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:47:04,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:06,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:11,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:14,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:16,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:47:16,227 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:47:16,439 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:22,388 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:24,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:28,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:31,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:47:31,227 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:47:31,611 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:47:33,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:34,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:39,388 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:44,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:45,388 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:46,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:47:46,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:47:50,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:47:54,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:47:56,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:01,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:48:01,227 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:48:01,612 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:48:01,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:04,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:07,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:12,390 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:14,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:16,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:48:16,227 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:48:17,441 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:23,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:24,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:29,390 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:31,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:48:31,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:48:31,614 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:48:34,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:35,390 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:40,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:43,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:48:43,833 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:48:43,833 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:48:43,834 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:48:44,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:44,339 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:48:45,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:46,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:48:46,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:48:46,340 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:48:50,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:48:54,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:48:55,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:00,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:01,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:49:01,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:49:01,616 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:49:04,316 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:06,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:11,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:14,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:16,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:49:16,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:49:16,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:22,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:24,316 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:27,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:31,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:49:31,229 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:49:31,619 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:49:32,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:34,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:38,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:44,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:44,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:46,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:49:46,229 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:49:49,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:49:54,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:49:55,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:00,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:01,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:50:01,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:50:01,621 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:50:04,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:06,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:12,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:14,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:16,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:50:16,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:50:18,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:23,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:24,324 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:29,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:31,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:50:31,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:50:31,627 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:50:34,324 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:34,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:40,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:44,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:45,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:46,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:50:46,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:50:47,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:50:47,290 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:50:47,291 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:50:47,291 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:50:47,424 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:50:48,425 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:50:50,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:50:54,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:50:56,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:01,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:51:01,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:51:01,435 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:01,626 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:51:04,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:07,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:12,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:14,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:16,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:51:16,231 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:51:18,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:23,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:24,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:29,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:31,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:51:31,231 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:51:31,630 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:51:34,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:35,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:41,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:44,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:46,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:51:46,231 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:51:46,454 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:52,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:51:54,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:51:57,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:01,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:52:01,231 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:52:01,631 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:52:02,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:04,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:08,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:14,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:14,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:16,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:52:16,232 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:52:19,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:24,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:25,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:30,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:31,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:52:31,232 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:52:31,634 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:52:34,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:35,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:41,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:44,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:46,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:52:46,233 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:52:46,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:48,992 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:52:48,994 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:52:48,994 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:52:48,996 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:52:49,501 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:52:50,501 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:52:52,414 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:52:54,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:52:58,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:01,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:53:01,232 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:53:01,636 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:53:03,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:04,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:09,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:14,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:15,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:16,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:53:16,232 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:53:20,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:24,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:26,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:31,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:53:31,233 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:53:31,639 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:53:31,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:34,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:37,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:43,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:44,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:46,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:53:46,234 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:53:49,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:54,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:53:54,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:53:59,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:01,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:54:01,234 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:54:01,642 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:54:04,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:04,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:10,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:14,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:16,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:54:16,234 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:54:16,452 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:22,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:24,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:28,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:31,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:54:31,234 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:54:31,644 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:54:33,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:34,345 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:39,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:44,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:45,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:46,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:54:46,235 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:54:50,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:54:50,724 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:54:50,725 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:54:50,726 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:54:50,726 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:54:51,578 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:54:52,579 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:54:54,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:54:56,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:01,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:55:01,236 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:55:01,650 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:55:01,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:04,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:07,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:13,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:14,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:16,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:55:16,235 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:55:19,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:24,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:24,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:29,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:31,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:55:31,235 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:55:31,649 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:55:34,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:35,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:40,422 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:44,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:46,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:55:46,236 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:55:46,460 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:52,428 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:55:54,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:55:58,428 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:01,237 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:56:01,238 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:56:01,654 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:56:03,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:04,347 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:09,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:14,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:15,428 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:16,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:56:16,239 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:56:20,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:24,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:26,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:31,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:56:31,239 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:56:31,655 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:56:31,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:34,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:37,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:42,430 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:44,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:46,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:56:46,239 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:56:48,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:53,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:56:54,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:56:54,018 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:56:54,019 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:56:54,019 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:56:54,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:56:54,655 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:56:55,656 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:56:59,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:01,243 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:57:01,243 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:57:01,657 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:57:04,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:04,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:10,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:14,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:16,239 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:57:16,240 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:57:16,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:21,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:24,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:27,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:31,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:57:31,247 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:57:31,659 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:57:32,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:34,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:38,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:44,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:44,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:46,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:57:46,247 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:57:49,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:57:54,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:57:55,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:00,439 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:01,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:58:01,248 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:58:01,673 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:58:04,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:05,440 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:11,440 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:14,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:16,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:58:16,248 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:58:16,463 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:22,442 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:24,365 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:28,442 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:31,248 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:58:31,248 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:58:34,366 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:34,951 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:58:34,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:40,442 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:44,367 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:46,248 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:58:46,248 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:58:46,464 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:52,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:58:54,367 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:58:55,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 17:58:55,943 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 17:58:55,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 17:58:55,944 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 17:58:56,718 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 17:58:57,719 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 17:58:57,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:01,248 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:59:01,249 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:59:01,680 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:59:03,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:04,368 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:09,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:14,368 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:15,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:16,248 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:59:16,249 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:59:20,444 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:24,368 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:26,444 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:31,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:59:31,249 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:59:31,688 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 17:59:31,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:34,369 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:37,446 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:42,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:44,368 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:46,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 17:59:46,249 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 17:59:48,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:53,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 17:59:54,369 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 17:59:59,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:01,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:00:01,250 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:00:01,685 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:00:04,369 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:05,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:11,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:14,369 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:16,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:00:16,268 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:00:16,480 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:22,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:24,370 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:28,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:31,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:00:31,269 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:00:31,688 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:00:33,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:34,370 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:39,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:44,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:45,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:46,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:00:46,269 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:00:51,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:54,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:00:57,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:00:57,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:00:57,480 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:00:57,481 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:00:57,482 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:00:57,786 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:00:59,788 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:01:01,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:01:01,269 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:01:04,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:04,592 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:01:04,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:10,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:14,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:15,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:16,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:01:16,270 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:01:21,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:24,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:27,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:31,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:01:31,270 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:01:31,692 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:01:32,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:34,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:38,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:43,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:44,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:46,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:01:46,270 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:01:48,456 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:54,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:01:54,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:01:59,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:01,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:02:01,271 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:02:01,697 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:02:04,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:05,452 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:10,452 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:14,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:16,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:02:16,271 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:02:16,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:21,454 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:24,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:27,452 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:31,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:02:31,270 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:02:31,697 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:02:32,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:34,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:38,454 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:44,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:44,453 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:46,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:02:46,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:02:50,454 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:02:54,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:02:56,454 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:00,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:03:00,737 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:03:00,737 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:03:00,738 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:03:00,851 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:03:01,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:03:01,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:03:01,502 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:01,701 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:03:01,852 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:03:04,373 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:07,462 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:12,463 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:14,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:16,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:03:16,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:03:18,446 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:23,465 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:24,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:29,464 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:31,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:03:31,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:03:31,703 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:03:34,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:34,464 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:40,464 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:44,374 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:46,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:03:46,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:03:46,512 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:52,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:03:54,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:03:57,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:01,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:04:01,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:04:01,705 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:04:02,707 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:04,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:08,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:14,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:14,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:16,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:04:16,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:04:19,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:24,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:25,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:30,467 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:31,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:04:31,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:04:31,712 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:04:34,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:35,467 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:40,468 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:44,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:46,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:04:46,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:04:46,464 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:51,468 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:04:54,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:04:57,468 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:01,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:05:01,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:05:01,711 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:05:02,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:05:02,588 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:05:02,588 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:05:02,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:02,591 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:05:02,915 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:05:03,917 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:05:04,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:08,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:14,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:14,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:16,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:05:16,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:05:20,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:24,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:25,470 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:31,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:05:31,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:05:31,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:31,713 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:05:34,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:36,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:42,470 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:44,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:46,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:05:46,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:05:47,508 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:53,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:05:54,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:05:59,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:01,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:06:01,291 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:06:01,717 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:06:04,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:04,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:09,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:14,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:15,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:16,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:06:16,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:06:20,472 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:24,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:26,472 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:31,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:06:31,291 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:06:31,473 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:31,718 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:06:34,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:37,473 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:43,474 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:44,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:46,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:06:46,291 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:06:48,489 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:54,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:06:54,473 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:06:59,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:01,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:07:01,292 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:07:01,720 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:07:04,387 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:04,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:07:04,440 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:07:04,440 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:07:04,440 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:07:04,483 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:04,980 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:07:05,981 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:07:10,474 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:14,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:15,474 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:16,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:07:16,292 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:07:20,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:24,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:26,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:31,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:07:31,292 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:07:31,723 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:07:31,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:34,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:37,476 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:43,476 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:44,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:46,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:07:46,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:07:49,457 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:54,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:07:54,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:07:59,476 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:01,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:08:01,292 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:08:01,725 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:08:04,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:05,476 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:10,476 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:14,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:16,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:08:16,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:08:16,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:21,477 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:24,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:27,477 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:31,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:08:31,294 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:08:31,728 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:08:32,729 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:34,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:38,479 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:44,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:44,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:46,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:08:46,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:08:49,479 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:08:54,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:08:55,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:00,488 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:01,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:09:01,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:09:01,733 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:09:04,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:05,488 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:07,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:09:07,903 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:09:07,903 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:09:07,904 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:09:08,052 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:09:10,053 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:09:10,488 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:14,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:16,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:09:16,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:09:16,431 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:21,489 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:24,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:27,488 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:31,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:09:31,293 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:09:31,732 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:09:32,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:34,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:38,490 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:44,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:44,489 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:46,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:09:46,294 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:09:49,489 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:09:54,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:09:55,489 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:00,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:01,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:10:01,294 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:10:01,736 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:10:04,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:06,490 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:11,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:14,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:16,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:10:16,294 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:10:17,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:22,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:24,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:28,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:31,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:10:31,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:10:31,737 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:10:33,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:34,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:39,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:44,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:45,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:46,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:10:46,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:10:50,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:10:54,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:10:55,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:01,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:11:01,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:11:01,506 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:01,739 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:11:04,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:07,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:09,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:11:09,651 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:11:09,652 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:11:09,652 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:11:10,110 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:11:10,111 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:11:12,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:14,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:16,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:11:16,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:11:18,497 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:24,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:24,493 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:29,493 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:31,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:11:31,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:11:31,741 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:11:34,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:35,493 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:40,494 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:44,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:46,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:11:46,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:11:46,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:51,495 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:11:54,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:11:57,495 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:01,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:12:01,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:12:01,745 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:12:02,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:04,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:08,497 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:14,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:14,495 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:16,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:12:16,297 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:12:19,496 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:24,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:25,496 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:31,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:12:31,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:12:31,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:31,748 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:12:34,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:36,496 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:41,497 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:44,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:46,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:12:46,297 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:12:47,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:52,498 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:12:54,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:12:57,498 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:01,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:13:01,297 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:13:01,751 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:13:02,753 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:04,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:08,499 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:11,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:13:11,769 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:13:11,769 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:13:11,770 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:13:12,184 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:13:13,811 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:14,185 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:13:14,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:16,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:13:16,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:13:19,499 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:24,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:25,499 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:30,499 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:31,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:13:31,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:13:31,756 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:13:34,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:35,515 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:40,516 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:44,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:46,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:13:46,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:13:46,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:51,515 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:13:54,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:13:57,515 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:01,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:14:01,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:14:01,756 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:14:02,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:04,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:08,517 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:14,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:14,515 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:16,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:14:16,298 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:14:19,516 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:24,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:24,516 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:29,517 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:31,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:14:31,309 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:14:31,760 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:14:34,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:35,517 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:40,518 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:44,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:46,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:14:46,309 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:14:46,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:52,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:14:54,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:14:58,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:01,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:15:01,310 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:15:01,760 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:15:03,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:04,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:09,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:13,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:15:13,821 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:15:13,821 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:15:13,822 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:15:14,266 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:15:14,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:15,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:16,267 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:15:16,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:15:16,310 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:15:20,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:25,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:25,935 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:30,935 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:31,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:15:31,310 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:15:31,763 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:15:35,934 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:35,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:41,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:45,935 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:46,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:15:46,310 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:15:47,480 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:52,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:15:55,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:15:58,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:01,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:16:01,310 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:16:01,765 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:16:04,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:05,935 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:09,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:14,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:15,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:16,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:16:16,327 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:16:20,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:25,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:25,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:30,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:31,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:16:31,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:16:31,768 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:16:35,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:35,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:41,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:45,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:46,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:16:46,327 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:16:47,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:52,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:16:55,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:16:58,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:01,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:17:01,327 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:17:01,770 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:17:04,772 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:05,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:09,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:14,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:15,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:16,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:17:16,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:17:17,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:17:17,099 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:17:17,100 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:17:17,101 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:17:17,327 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:17:18,327 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:17:20,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:25,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:25,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:30,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:31,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:17:31,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:17:31,774 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:17:35,938 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:36,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:41,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:45,938 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:46,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:17:46,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:17:47,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:52,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:17:55,938 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:17:58,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:01,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:18:01,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:18:01,781 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:18:04,783 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:05,938 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:09,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:14,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:15,938 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:16,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:18:16,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:18:20,510 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:25,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:25,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:30,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:31,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:18:31,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:18:31,779 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:18:35,940 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:36,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:41,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:45,940 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:46,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:18:46,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:18:47,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:52,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:18:55,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:18:57,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:01,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:19:01,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:19:01,784 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:19:03,786 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:05,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:08,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:13,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:15,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:16,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:19:16,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:19:18,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:19:18,883 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:19:18,883 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:19:18,884 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:19:19,384 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:19:19,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:20,385 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:19:24,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:25,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:30,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:31,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:19:31,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:19:31,784 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:19:35,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:36,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:41,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:45,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:46,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:19:46,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:19:47,481 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:52,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:19:55,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:19:57,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:01,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:20:01,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:20:01,787 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:20:03,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:05,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:08,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:13,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:15,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:16,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:20:16,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:20:19,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:24,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:25,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:30,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:31,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:20:31,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:20:31,789 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:20:35,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:36,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:41,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:45,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:46,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:20:46,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:20:47,512 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:52,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:20:55,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:20:58,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:01,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:21:01,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:21:05,402 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:21:05,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:05,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:10,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:15,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:16,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:21:16,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:21:16,484 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:20,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:21:20,568 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:21:20,568 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:21:20,569 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:21:21,444 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:21:21,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:22,445 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:21:25,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:27,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:31,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:21:31,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:21:31,857 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:21:33,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:35,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:38,962 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:44,960 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:45,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:46,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:21:46,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:21:50,546 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:21:55,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:21:55,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:00,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:01,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:22:01,336 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:22:01,861 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:22:05,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:06,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:11,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:15,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:16,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:22:16,335 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:22:17,463 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:22,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:25,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:28,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:31,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:22:31,335 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:22:31,876 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:22:34,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:35,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:39,971 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:45,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:45,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:46,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:22:46,335 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:22:50,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:22:55,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:22:56,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:01,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:23:01,335 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:23:01,864 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:23:02,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:05,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:07,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:13,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:15,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:16,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:23:16,338 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:23:19,486 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:24,550 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:23:24,552 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:23:24,553 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:23:24,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:24,556 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:23:25,505 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:23:25,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:26,506 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:23:29,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:31,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:23:31,336 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:23:31,867 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:23:35,871 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:35,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:40,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:45,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:46,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:23:46,336 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:23:46,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:51,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:23:55,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:23:56,973 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:01,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:24:01,337 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:24:01,876 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:24:02,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:05,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:07,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:12,973 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:15,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:16,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:24:16,336 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:24:18,547 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:23,972 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:25,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:29,972 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:31,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:24:31,337 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:24:31,872 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:24:35,876 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:35,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:40,972 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:45,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:46,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:24:46,337 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:24:46,472 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:51,973 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:24:55,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:24:57,973 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:01,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:25:01,337 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:25:01,875 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:25:03,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:05,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:08,974 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:13,977 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:15,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:16,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:25:16,338 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:25:19,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:24,979 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:25,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:26,461 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:25:26,463 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:25:26,463 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:25:26,464 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:25:26,565 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:25:28,566 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:25:30,503 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:31,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:25:31,337 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:25:31,877 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:25:35,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:35,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:40,979 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:45,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:46,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:25:46,338 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:25:46,562 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:51,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:25:55,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:25:57,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:01,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:26:01,338 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:26:01,880 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:26:03,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:05,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:08,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:13,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:15,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:16,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:26:16,339 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:26:19,516 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:24,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:25,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:29,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:31,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:26:31,339 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:26:31,882 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:26:35,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:35,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:41,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:45,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:46,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:26:46,339 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:26:46,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:52,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:26:55,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:26:58,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:01,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:27:01,340 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:27:01,884 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:27:03,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:05,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:09,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:14,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:15,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:16,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:27:16,340 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:27:19,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:25,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:25,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:28,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:27:28,285 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:27:28,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:27:28,286 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:27:28,628 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:27:30,320 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:30,630 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:27:31,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:27:31,341 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:27:31,887 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:27:35,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:35,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:41,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:45,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:46,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:27:46,341 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:27:46,509 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:52,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:27:55,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:27:57,009 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:01,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:28:01,341 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:28:01,889 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:28:02,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:05,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:08,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:14,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:15,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:16,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:28:16,342 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:28:19,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:25,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:25,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:31,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:31,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:28:31,342 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:28:31,894 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:28:35,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:37,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:42,017 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:45,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:46,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:28:46,342 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:28:47,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:53,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:28:55,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:28:59,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:01,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:29:01,342 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:29:01,897 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:29:04,900 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:05,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:10,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:15,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:15,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:16,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:29:16,342 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:29:20,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:25,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:26,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:31,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:31,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:29:31,343 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:29:31,899 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:29:32,921 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:29:32,923 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:29:32,924 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:29:32,925 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:29:33,701 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:29:34,702 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:29:35,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:37,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:42,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:45,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:46,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:29:46,343 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:29:47,562 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:53,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:29:55,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:29:59,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:01,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:30:01,343 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:30:01,902 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:30:04,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:05,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:10,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:15,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:15,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:16,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:30:16,344 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:30:20,593 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:25,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:26,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:31,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:31,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:30:31,344 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:30:31,905 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:30:35,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:37,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:42,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:45,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:46,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:30:46,344 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:30:47,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:53,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:30:55,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:30:58,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:01,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:31:01,344 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:31:01,907 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:31:03,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:05,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:09,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:15,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:15,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:16,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:31:16,394 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:31:20,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:25,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:26,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:31,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:31,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:31:31,400 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:31:31,910 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:31:35,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:31:35,151 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:31:35,151 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:31:35,152 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:31:35,770 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:31:35,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:36,771 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:31:37,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:43,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:45,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:46,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:31:46,400 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:31:48,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:54,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:31:55,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:31:59,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:01,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:32:01,400 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:32:05,035 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:32:05,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:05,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:11,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:15,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:16,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:16,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:32:16,401 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:32:21,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:25,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:26,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:31,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:31,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:32:31,401 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:32:31,915 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:32:35,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:37,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:42,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:45,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:46,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:32:46,401 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:32:47,606 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:53,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:32:55,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:32:59,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:01,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:33:01,402 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:33:01,919 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:33:04,921 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:05,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:10,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:15,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:15,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:16,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:33:16,402 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:33:20,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:25,992 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:26,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:31,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:31,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:33:31,402 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:33:31,920 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:33:35,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:36,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:37,365 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:33:37,367 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:33:37,368 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:33:37,368 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:33:37,842 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:33:38,843 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:33:41,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:45,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:46,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:33:46,402 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:33:46,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:52,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:33:55,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:33:57,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:01,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:34:01,403 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:34:01,923 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:34:02,925 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:05,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:08,052 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:14,052 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:15,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:16,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:34:16,404 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:34:19,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:25,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:25,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:30,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:31,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:34:31,404 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:34:31,925 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:34:35,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:35,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:41,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:45,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:46,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:34:46,404 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:34:46,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:52,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:34:55,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:34:58,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:01,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:35:01,405 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:35:01,927 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:35:03,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:05,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:35:09,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:15,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:15,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:35:16,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:35:16,405 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:35:20,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:25,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:35:26,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:31,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:31,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:35:31,405 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:35:31,930 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:35:35,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:35:37,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:40,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:35:40,917 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:35:40,918 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:35:40,919 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:35:41,914 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:35:42,915 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:35:43,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:46,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:35:46,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:35:46,406 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:35:48,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:54,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:35:56,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:00,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:01,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:36:01,406 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:36:01,932 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:36:05,934 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:06,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:11,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:16,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:16,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:36:16,406 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:36:16,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:22,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:26,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:28,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:31,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:36:31,406 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:36:31,939 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:36:33,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:36,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:39,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:44,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:46,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:36:46,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:36:46,407 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:36:49,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:55,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:36:56,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:01,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:01,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:37:01,407 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:37:01,938 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:37:06,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:06,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:11,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:16,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:16,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:37:16,407 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:37:16,629 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:22,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:26,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:27,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:31,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:37:31,408 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:37:32,224 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:37:32,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:36,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:38,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:43,466 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:37:43,468 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:37:43,469 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:37:43,469 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:43,470 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:37:43,985 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:37:44,986 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:37:46,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:46,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:37:46,408 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:37:48,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:54,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:37:56,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:37:59,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:01,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:38:01,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:38:01,943 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:38:04,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:06,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:10,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:15,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:16,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:16,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:38:16,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:38:20,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:26,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:26,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:31,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:31,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:38:31,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:38:31,947 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:38:36,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:36,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:42,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:46,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:46,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:38:46,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:38:47,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:53,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:38:56,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:38:59,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:01,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:39:01,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:39:01,949 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:39:04,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:06,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:10,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:15,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:16,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:16,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:39:16,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:39:20,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:26,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:26,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:31,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:39:31,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:39:31,575 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:31,951 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:39:36,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:37,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:43,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:45,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:39:45,639 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:39:45,639 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:39:45,640 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:39:46,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:46,070 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:39:46,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:39:46,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:39:47,070 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:39:48,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:54,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:39:56,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:39:59,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:01,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:40:01,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:40:01,953 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:40:04,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:06,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:40:10,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:16,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:40:16,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:16,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:40:16,420 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:40:21,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:26,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:40:27,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:31,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:40:31,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:40:31,955 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:40:32,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:36,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:40:38,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:43,105 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:46,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:40:46,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:40:46,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:40:48,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:54,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:40:56,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:00,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:01,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:41:01,417 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:41:01,965 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:41:05,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:06,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:11,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:16,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:16,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:16,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:41:16,418 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:41:22,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:27,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:27,557 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:31,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:41:31,418 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:41:31,960 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:41:32,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:37,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:37,560 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:42,561 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:46,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:41:46,419 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:41:47,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:48,560 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:49,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:41:49,407 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:41:49,407 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:41:49,409 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:41:50,149 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:41:51,150 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:41:53,561 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:41:57,561 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:41:58,564 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:01,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:42:01,419 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:42:01,963 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:42:04,565 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:07,560 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:42:10,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:16,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:42:16,420 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:42:16,598 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:17,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:42:22,574 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:27,575 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:27,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:42:31,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:42:31,420 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:42:31,967 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:42:33,575 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:37,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:42:38,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:44,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:46,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:42:46,420 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:42:47,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:42:50,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:55,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:42:57,577 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:00,577 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:01,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:43:01,420 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:43:01,968 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:43:06,577 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:07,578 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:11,577 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:16,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:43:16,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:43:16,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:17,577 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:22,578 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:27,578 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:28,578 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:31,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:43:31,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:43:31,971 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:43:34,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:37,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:40,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:45,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:46,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:43:46,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:43:47,578 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:43:51,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:43:51,105 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:43:51,106 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:43:51,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:51,107 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:43:51,214 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:43:53,215 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:43:56,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:43:57,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:01,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:44:01,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:44:01,972 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:44:01,975 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:07,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:07,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:12,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:16,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:44:16,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:44:17,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:17,597 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:22,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:27,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:28,598 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:31,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:44:31,421 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:44:31,975 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:44:33,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:37,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:39,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:44,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:46,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:44:46,422 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:44:47,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:44:50,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:55,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:44:57,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:00,602 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:01,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:45:01,422 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:45:01,978 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:45:06,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:07,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:12,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:16,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:45:16,422 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:45:17,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:18,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:23,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:27,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:29,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:31,422 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:45:31,422 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:45:31,983 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:45:35,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:37,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:40,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:45,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:46,422 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:45:46,422 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:45:47,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:45:51,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:52,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:45:52,945 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:45:52,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:45:52,946 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:45:53,268 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:45:55,269 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:45:56,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:45:57,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:01,422 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:46:01,423 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:46:01,982 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:46:02,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:07,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:08,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:13,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:16,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:46:16,424 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:46:17,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:19,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:24,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:27,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:30,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:31,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:46:31,424 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:46:31,986 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:46:35,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:37,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:41,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:46,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:46:46,424 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:46:46,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:47,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:52,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:46:57,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:46:58,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:01,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:47:01,424 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:47:01,987 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:47:03,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:07,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:09,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:14,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:16,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:47:16,425 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:47:17,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:20,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:25,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:27,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:31,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:47:31,425 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:47:31,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:31,989 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:47:36,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:37,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:42,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:46,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:47:46,425 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:47:47,593 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:48,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:53,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:47:56,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:47:56,100 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:47:56,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:47:56,102 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:47:56,317 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:47:57,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:47:59,318 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:47:59,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:01,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:48:01,425 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:48:01,992 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:48:04,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:07,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:10,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:15,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:16,425 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:48:16,425 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:48:17,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:21,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:26,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:27,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:31,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:48:31,443 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:48:31,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:31,994 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:48:37,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:37,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:42,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:46,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:48:46,444 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:48:47,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:48,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:53,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:48:57,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:48:59,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:01,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:49:01,444 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:49:01,997 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:49:04,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:07,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:09,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:15,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:16,443 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:49:16,444 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:49:17,597 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:21,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:26,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:27,597 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:31,444 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:49:31,444 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:49:31,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:31,999 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:49:37,597 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:37,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:43,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:46,444 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:49:46,445 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:49:47,598 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:49,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:54,629 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:49:57,598 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:49:57,801 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:49:57,803 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:49:57,803 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:49:57,804 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:49:58,375 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:49:59,376 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:49:59,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:01,446 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:50:01,447 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:50:02,007 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:50:05,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:07,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:50:11,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:16,445 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:50:16,446 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:50:16,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:17,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:50:22,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:27,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:50:28,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:31,445 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:50:31,445 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:50:32,004 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:50:33,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:37,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:50:39,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:44,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:46,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:50:46,447 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:50:47,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:50:49,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:55,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:50:57,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:01,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:51:01,447 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:51:01,671 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:02,010 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:51:07,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:07,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:12,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:16,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:51:16,448 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:51:17,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:18,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:23,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:27,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:29,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:31,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:51:31,448 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:51:32,011 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:51:34,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:37,602 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:39,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:44,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:46,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:51:46,448 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:51:47,602 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:50,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:55,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:51:57,602 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:51:59,437 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:51:59,439 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:51:59,440 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:51:59,441 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:52:00,433 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:52:01,433 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:52:01,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:52:01,449 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:52:01,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:02,014 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:52:06,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:07,603 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:12,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:16,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:52:16,450 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:52:17,603 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:18,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:23,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:27,603 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:28,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:31,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:52:31,449 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:52:32,016 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:52:34,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:37,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:40,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:45,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:46,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:52:46,449 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:52:47,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:51,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:52:57,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:52:57,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:01,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:53:01,450 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:53:02,020 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:53:02,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:07,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:08,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:13,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:16,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:53:16,450 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:53:17,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:19,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:24,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:27,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:30,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:31,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:53:31,450 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:53:32,022 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:53:35,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:37,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:41,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:46,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:53:46,451 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:53:47,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:47,623 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:52,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:53:57,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:53:58,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:00,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:54:00,857 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:54:00,858 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:54:00,858 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:54:01,450 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:54:01,451 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:54:01,493 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:54:02,025 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:54:03,494 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:54:03,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:07,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:09,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:14,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:17,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:17,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:54:17,818 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:54:20,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:25,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:27,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:30,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:32,027 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:54:32,813 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:54:32,813 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:54:35,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:37,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:41,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:46,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:47,813 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:54:47,813 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:54:47,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:51,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:54:57,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:54:57,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:02,033 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:55:02,813 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:55:02,814 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:55:03,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:07,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:08,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:13,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:17,813 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:55:17,814 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:55:17,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:19,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:24,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:27,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:30,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:32,033 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:55:32,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:55:32,815 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:55:36,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:37,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:41,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:46,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:47,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:55:47,815 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:55:47,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:51,969 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:55:57,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:55:57,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:02,036 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:56:02,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:56:02,815 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:56:02,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:03,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:56:03,855 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:56:03,855 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:56:03,856 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:56:04,551 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:56:05,551 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:56:07,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:08,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:13,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:17,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:56:17,815 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:56:17,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:19,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:24,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:27,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:30,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:32,037 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:56:32,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:56:32,816 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:56:36,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:37,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:41,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:46,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:47,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:56:47,816 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:56:47,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:51,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:56:57,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:56:57,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:02,040 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:57:02,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:57:02,829 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:57:02,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:07,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:08,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:14,844 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:17,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:17,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:57:17,820 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:57:20,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:25,844 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:27,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:31,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:32,042 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:57:32,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:57:32,819 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:57:36,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:37,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:42,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:47,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:57:47,820 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:57:47,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:48,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:53,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:57:57,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:57:59,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:02,044 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:58:02,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:58:02,820 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:58:05,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:05,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 18:58:05,203 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 18:58:05,204 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 18:58:05,205 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 18:58:05,601 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 18:58:07,602 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 18:58:07,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:10,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:15,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:17,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:58:17,820 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:58:17,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:21,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:26,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:27,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:32,047 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:58:32,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:32,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:58:32,821 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:58:37,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:37,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:42,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:47,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:47,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:58:47,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:58:48,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:53,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:58:57,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:58:59,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:02,050 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:59:02,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:59:02,822 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:59:05,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:07,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:59:10,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:16,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:17,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:59:17,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:59:17,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:59:21,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:27,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:59:27,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:32,055 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 18:59:32,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:59:32,822 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:59:33,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:37,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:59:38,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:43,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:47,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 18:59:47,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 18:59:47,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 18:59:48,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:54,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 18:59:57,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:00,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:02,054 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:00:02,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:00:02,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:00:05,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:06,563 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:00:06,565 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:00:06,566 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:00:06,569 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:00:06,660 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:00:07,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:09,662 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:00:11,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:16,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:17,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:00:17,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:00:17,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:22,009 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:27,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:27,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:32,057 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:00:32,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:00:32,823 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:00:32,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:37,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:38,876 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:43,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:47,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:00:47,824 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:00:47,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:48,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:54,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:00:57,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:00:59,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:02,060 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:01:02,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:01:02,824 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:01:05,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:07,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:01:11,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:16,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:17,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:01:17,825 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:01:17,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:01:22,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:27,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:01:28,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:32,062 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:01:32,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:01:32,825 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:01:34,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:37,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:01:39,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:45,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:47,825 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:01:47,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:01:47,826 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:01:51,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:56,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:01:57,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:01,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:02,064 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:02:02,825 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:02:02,826 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:02:07,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:07,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:09,317 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:02:09,319 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:02:09,319 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:02:09,320 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:02:09,716 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:02:10,717 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:02:13,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:17,825 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:02:17,826 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:02:17,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:18,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:23,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:27,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:28,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:32,068 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:02:32,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:02:32,827 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:02:34,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:37,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:39,992 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:45,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:47,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:02:47,828 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:02:47,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:02:51,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:56,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:02:57,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:01,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:02,070 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:03:02,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:03:02,829 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:03:07,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:07,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:13,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:17,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:03:17,828 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:03:17,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:18,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:24,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:27,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:29,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:32,072 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:03:32,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:03:32,829 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:03:35,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:37,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:41,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:46,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:47,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:03:47,828 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:03:47,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:52,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:03:57,836 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:03:57,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:02,078 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:04:02,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:04:02,829 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:04:03,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:07,837 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:08,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:10,472 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:04:10,474 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:04:10,474 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:04:10,475 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:04:10,768 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:04:12,769 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:04:13,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:17,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:17,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:04:17,852 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:04:19,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:24,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:27,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:30,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:32,078 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:04:32,839 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:04:32,840 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:04:35,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:37,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:40,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:46,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:47,839 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:04:47,840 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:04:47,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:52,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:04:57,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:04:58,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:02,082 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:05:02,839 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:05:02,840 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:05:04,931 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:07,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:05:10,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:15,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:17,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:05:17,841 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:05:17,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:05:21,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:27,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:27,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:05:32,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:32,083 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:05:32,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:05:32,840 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:05:37,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:37,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:05:43,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:47,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:05:47,841 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:05:47,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:05:49,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:54,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:05:57,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:00,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:02,085 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:06:02,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:06:02,841 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:06:05,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:07,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:10,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:11,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:06:11,585 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:06:11,586 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:06:11,587 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:06:11,817 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:06:12,818 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:06:16,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:17,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:06:17,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:06:17,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:22,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:27,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:27,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:32,088 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:06:32,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:32,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:06:32,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:06:37,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:38,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:44,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:47,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:06:47,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:06:47,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:49,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:54,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:06:57,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:06:59,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:02,090 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:07:02,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:07:02,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:07:05,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:07,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:07:11,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:17,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:17,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:07:17,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:07:17,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:07:23,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:27,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:07:29,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:32,093 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:07:32,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:07:32,842 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:07:34,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:37,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:07:40,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:45,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:47,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:07:47,845 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:07:47,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:07:50,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:56,017 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:07:57,866 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:01,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:02,095 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:08:02,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:08:02,845 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:08:06,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:07,867 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:12,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:14,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:08:14,364 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:08:14,365 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:08:14,365 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:08:14,889 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:08:14,890 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:08:17,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:17,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:08:17,845 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:08:17,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:23,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:27,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:29,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:32,100 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:08:32,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:08:32,846 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:08:34,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:37,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:39,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:44,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:47,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:08:47,846 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:08:47,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:08:50,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:55,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:08:57,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:00,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:02,099 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:09:02,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:09:02,846 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:09:06,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:07,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:11,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:16,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:17,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:09:17,847 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:09:17,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:21,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:26,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:27,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:32,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:32,103 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:09:32,847 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:09:32,848 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:09:37,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:38,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:44,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:47,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:09:47,847 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:09:47,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:09:50,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:55,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:09:57,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:01,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:02,104 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:10:02,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:10:02,847 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:10:07,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:07,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:12,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:15,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:10:15,179 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:10:15,179 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:10:15,180 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:10:15,951 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:10:16,951 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:10:17,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:17,847 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:10:17,848 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:10:17,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:23,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:27,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:29,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:32,106 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:10:32,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:10:32,849 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:10:34,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:37,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:40,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:45,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:47,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:10:47,849 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:10:47,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:10:51,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:56,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:10:57,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:01,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:02,108 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:11:02,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:11:02,850 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:11:06,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:07,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:12,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:17,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:11:17,850 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:11:17,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:18,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:23,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:27,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:29,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:32,110 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:11:32,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:11:32,850 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:11:34,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:37,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:39,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:44,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:47,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:11:47,851 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:11:47,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:11:50,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:55,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:11:57,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:00,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:02,113 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:12:02,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:12:02,851 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:12:05,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:07,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:11,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:16,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:16,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:12:16,112 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:12:16,112 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:12:16,113 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:12:17,018 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:12:17,019 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:12:17,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:12:17,851 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:12:17,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:22,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:27,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:27,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:32,115 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:12:32,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:32,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:12:32,851 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:12:37,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:38,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:44,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:47,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:12:47,851 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:12:47,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:49,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:54,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:12:57,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:12:59,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:02,117 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:13:02,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:13:02,852 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:13:04,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:07,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:10,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:16,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:17,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:13:17,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:13:17,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:21,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:26,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:27,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:32,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:32,122 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:13:32,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:13:32,861 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:13:37,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:38,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:43,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:47,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:13:47,861 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:13:48,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:48,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:54,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:13:58,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:13:59,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:02,122 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:14:02,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:14:02,861 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:14:04,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:08,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:10,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:16,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:17,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:14:17,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:14:18,572 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:14:18,574 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:14:18,574 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:14:18,576 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:14:18,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:19,074 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:14:19,074 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:14:22,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:27,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:28,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:32,124 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:14:32,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:32,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:14:32,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:14:38,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:38,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:43,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:47,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:14:47,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:14:48,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:49,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:54,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:14:58,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:14:59,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:02,128 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:15:02,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:15:02,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:15:05,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:08,847 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:10,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:15,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:17,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:15:17,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:15:18,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:21,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:27,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:28,847 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:32,128 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:15:32,129 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:32,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:15:32,863 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:15:38,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:38,848 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:44,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:47,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:15:47,863 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:15:48,848 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:49,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:54,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:15:58,848 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:15:59,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:02,131 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:16:02,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:16:02,863 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:16:04,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:08,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:16:09,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:15,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:17,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:16:17,864 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:16:18,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:16:19,446 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:16:19,447 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:16:19,447 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:16:19,448 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:16:20,126 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:16:20,487 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:21,127 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:16:26,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:28,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:16:32,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:32,133 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:16:32,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:16:32,864 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:16:37,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:38,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:16:43,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:47,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:16:47,865 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:16:48,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:16:49,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:54,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:16:58,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:00,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:02,136 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:17:02,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:17:02,865 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:17:05,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:08,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:11,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:16,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:17,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:17:17,865 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:17:18,868 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:22,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:27,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:28,868 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:32,138 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:17:32,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:32,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:17:32,865 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:17:38,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:38,868 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:43,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:47,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:17:47,874 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:17:48,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:17:49,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:54,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:17:58,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:00,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:02,145 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:18:02,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:18:02,874 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:18:05,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:08,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:11,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:17,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:17,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:18:17,874 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:18:18,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:20,449 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:18:20,450 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:18:20,451 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:18:20,451 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:18:21,185 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:18:21,185 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:18:22,491 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:28,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:28,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:32,143 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:18:32,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:18:32,875 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:18:34,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:38,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:39,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:44,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:47,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:18:47,875 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:18:48,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:18:49,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:54,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:18:58,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:00,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:02,147 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:19:02,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:19:02,875 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:19:06,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:08,871 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:12,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:17,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:17,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:19:17,875 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:19:18,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:23,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:28,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:28,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:32,148 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:19:32,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:19:32,875 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:19:33,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:38,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:39,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:44,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:47,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:19:47,876 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:19:48,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:19:50,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:55,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:19:58,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:01,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:02,150 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:20:02,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:20:02,878 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:20:07,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:08,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:12,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:17,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:20:17,878 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:20:18,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:18,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:23,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:20:23,027 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:20:23,027 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:20:23,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:23,028 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:20:23,252 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:20:25,252 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:20:28,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:28,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:32,152 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:20:32,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:20:32,885 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:20:33,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:38,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:39,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:45,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:47,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:20:47,885 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:20:48,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:20:50,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:56,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:20:58,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:02,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:02,154 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:21:02,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:21:02,885 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:21:07,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:08,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:13,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:17,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:21:17,886 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:21:18,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:18,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:24,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:28,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:30,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:32,156 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:21:32,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:21:32,886 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:21:36,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:38,876 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:41,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:46,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:47,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:21:47,886 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:21:48,876 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:21:52,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:57,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:21:58,877 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:02,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:02,158 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:22:02,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:22:02,887 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:22:08,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:08,877 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:13,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:17,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:22:17,887 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:22:18,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:18,877 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:23,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:22:23,948 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:22:23,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:22:23,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:23,950 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:22:24,300 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:22:25,301 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:22:28,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:29,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:32,164 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:22:32,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:22:32,887 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:22:35,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:38,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:40,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:46,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:47,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:22:47,887 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:22:48,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:22:51,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:56,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:22:58,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:01,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:02,163 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:23:02,899 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:23:02,900 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:23:07,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:08,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:12,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:17,900 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:23:17,900 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:23:18,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:18,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:24,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:28,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:30,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:32,168 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:23:32,902 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:23:32,903 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:23:35,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:38,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:41,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:46,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:47,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:23:47,902 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:23:48,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:23:52,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:58,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:23:58,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:02,170 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:24:02,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:24:02,902 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:24:03,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:08,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:09,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:15,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:17,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:24:17,902 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:24:18,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:20,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:25,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:25,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:24:25,363 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:24:25,364 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:24:25,366 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:24:26,357 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:24:27,357 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:24:28,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:31,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:32,173 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:24:32,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:24:32,902 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:24:36,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:38,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:41,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:47,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:47,901 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:24:47,902 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:24:48,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:24:53,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:58,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:24:58,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:02,177 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:25:02,902 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:25:02,903 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:25:03,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:08,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:09,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:15,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:17,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:25:17,903 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:25:18,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:21,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:26,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:28,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:32,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:32,178 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:25:32,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:25:32,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:25:37,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:38,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:42,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:47,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:25:47,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:25:48,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:48,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:25:54,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:25:58,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:00,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:02,179 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:26:02,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:26:02,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:26:05,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:08,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:10,105 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:16,105 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:17,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:26:17,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:26:18,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:21,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:26,446 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:26:26,447 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:26:26,448 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:26:26,448 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:26,449 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:26:27,415 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:26:27,415 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:26:28,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:32,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:32,181 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:26:32,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:26:32,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:26:37,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:38,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:42,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:47,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:26:47,905 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:26:48,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:48,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:26:53,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:58,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:26:58,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:02,188 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:27:02,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:27:02,905 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:27:04,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:08,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:09,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:15,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:17,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:27:17,905 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:27:18,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:20,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:25,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:28,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:31,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:32,186 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:27:32,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:27:32,905 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:27:36,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:38,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:42,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:47,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:27:47,906 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:27:48,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:48,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:27:53,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:58,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:27:58,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:02,190 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:28:02,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:28:02,906 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:28:03,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:08,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:09,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:15,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:17,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:28:17,906 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:28:18,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:21,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:26,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:28,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:29,312 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:28:29,314 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:28:29,315 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:28:29,315 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:28:29,474 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:28:31,475 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:28:32,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:32,190 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:28:32,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:28:32,907 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:28:37,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:38,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:43,114 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:47,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:28:47,907 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:28:48,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:48,890 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:28:54,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:28:58,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:00,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:02,194 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:29:02,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:29:02,907 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:29:06,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:08,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:11,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:16,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:17,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:29:17,907 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:29:18,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:22,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:28,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:28,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:32,195 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:29:32,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:29:32,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:29:34,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:38,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:39,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:44,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:47,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:29:47,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:29:48,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:29:50,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:55,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:29:58,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:01,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:02,198 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:30:02,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:30:02,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:30:07,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:08,895 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:13,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:17,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:30:17,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:30:18,895 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:19,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:24,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:28,895 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:29,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:30,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:30:30,862 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:30:30,862 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:30:30,863 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:30:31,533 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:30:32,201 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:30:32,908 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:30:32,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:30:33,534 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:30:35,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:38,895 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:40,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:46,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:47,908 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:30:47,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:30:48,896 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:30:52,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:57,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:30:58,896 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:02,203 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:31:02,206 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:02,908 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:31:02,909 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:31:08,127 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:08,897 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:13,127 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:17,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:31:17,910 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:31:18,129 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:18,897 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:24,127 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:28,897 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:29,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:32,209 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:31:32,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:31:32,910 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:31:35,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:38,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:40,129 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:45,129 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:47,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:31:47,909 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:31:48,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:31:50,131 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:56,132 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:31:58,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:02,132 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:02,208 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:32:02,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:32:02,910 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:32:08,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:08,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:14,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:17,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:32:17,910 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:32:18,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:20,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:25,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:28,899 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:30,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:32,212 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:32:32,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:32:32,560 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:32:32,560 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:32:32,560 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:32:32,593 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:32:32,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:32:32,910 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:32:33,594 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:32:35,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:38,899 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:41,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:47,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:47,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:32:47,911 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:32:48,899 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:32:53,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:59,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:32:59,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:02,212 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:33:02,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:33:02,911 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:33:05,132 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:09,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:10,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:15,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:17,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:33:17,912 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:33:19,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:21,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:26,139 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:29,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:31,139 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:32,215 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:33:32,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:33:32,911 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:33:36,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:39,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:42,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:47,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:47,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:33:47,911 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:33:49,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:33:53,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:58,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:33:59,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:02,219 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:34:02,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:34:02,912 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:34:05,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:09,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:11,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:16,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:17,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:34:17,912 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:34:19,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:22,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:27,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:29,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:32,219 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:34:32,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:32,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:34:32,912 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:34:35,555 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:34:35,558 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:34:35,558 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:34:35,559 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:34:35,644 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:34:37,599 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:37,645 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:34:39,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:43,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:47,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:34:47,912 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:34:49,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:49,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:34:54,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:59,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:34:59,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:02,222 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:35:02,914 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:35:02,915 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:35:05,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:09,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:10,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:15,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:17,914 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:35:17,915 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:35:19,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:21,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:26,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:29,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:32,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:32,224 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:35:32,914 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:35:32,915 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:35:38,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:39,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:43,146 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:47,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:35:47,915 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:35:49,130 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:49,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:35:54,146 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:59,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:35:59,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:02,230 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:36:02,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:36:02,915 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:36:05,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:09,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:10,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:15,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:17,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:36:17,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:36:19,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:21,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:26,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:29,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:32,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:32,229 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:36:32,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:36:32,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:36:37,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:36:37,054 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:36:37,054 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:36:37,057 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:36:37,701 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:36:38,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:39,702 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:36:39,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:43,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:47,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:36:47,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:36:49,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:49,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:36:54,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:59,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:36:59,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:02,232 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:37:02,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:37:02,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:37:05,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:09,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:10,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:15,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:17,916 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:37:17,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:37:19,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:21,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:26,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:29,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:32,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:32,235 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:37:32,916 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:37:32,917 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:37:37,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:39,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:43,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:47,916 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:37:47,917 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:37:49,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:49,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:37:54,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:59,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:37:59,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:02,236 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:38:02,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:38:02,918 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:38:05,136 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:09,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:10,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:15,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:17,917 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:38:17,917 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:38:19,960 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:20,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:26,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:29,960 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:32,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:32,238 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:38:32,917 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:38:32,918 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:38:37,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:38,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:38:38,165 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:38:38,166 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:38:38,167 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:38:38,756 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:38:39,756 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:38:39,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:42,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:47,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:47,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:38:47,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:38:49,960 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:38:53,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:58,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:38:59,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:02,243 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:39:02,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:39:02,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:39:04,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:09,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:09,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:15,158 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:17,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:39:17,918 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:39:19,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:21,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:26,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:29,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:32,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:32,245 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:39:32,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:39:32,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:39:38,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:39,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:44,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:47,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:39:47,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:39:49,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:39:50,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:55,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:39:59,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:01,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:02,249 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:40:02,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:40:02,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:40:07,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:09,961 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:13,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:17,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:40:17,920 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:40:19,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:19,962 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:24,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:29,161 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:29,962 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:32,256 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:40:32,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:40:32,920 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:40:34,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:39,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:40,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:41,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:40:41,122 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:40:41,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:40:41,123 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:40:41,823 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:40:43,824 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:40:46,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:47,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:40:47,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:40:49,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:40:52,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:57,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:40:59,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:02,256 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:41:02,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:02,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:41:02,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:41:08,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:09,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:13,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:17,928 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:41:17,929 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:41:19,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:19,963 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:24,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:29,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:30,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:32,260 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:41:32,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:41:32,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:41:35,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:39,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:40,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:46,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:47,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:41:47,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:41:49,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:41:51,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:57,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:41:59,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:02,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:02,262 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:42:02,929 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:42:02,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:42:07,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:09,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:13,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:17,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:42:17,930 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:42:19,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:19,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:24,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:29,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:29,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:32,264 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:42:32,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:42:32,931 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:42:34,913 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:39,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:40,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:42,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:42:42,656 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:42:42,656 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:42:42,657 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:42:42,874 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:42:43,874 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:42:45,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:47,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:42:47,931 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:42:49,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:42:51,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:56,174 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:42:59,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:01,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:02,267 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:43:02,931 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:43:02,931 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:43:07,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:09,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:13,175 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:17,931 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:43:17,932 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:43:19,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:19,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:24,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:29,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:30,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:32,269 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:43:32,931 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:43:32,932 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:43:35,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:39,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:41,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:46,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:47,931 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:43:47,932 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:43:49,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:43:52,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:57,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:43:59,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:02,271 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:44:02,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:02,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:44:02,940 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:44:08,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:09,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:14,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:17,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:44:17,940 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:44:19,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:20,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:25,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:29,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:30,179 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:32,274 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:44:32,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:44:32,940 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:44:36,179 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:39,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:42,180 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:44,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:44:44,128 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:44:44,129 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:44:44,130 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:44:44,924 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:44:45,925 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:44:47,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:44:47,940 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:44:48,128 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:49,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:44:53,180 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:58,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:44:59,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:02,280 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:45:02,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:45:02,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:45:04,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:09,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:09,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:14,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:17,940 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:45:17,941 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:45:19,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:20,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:25,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:29,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:31,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:32,278 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:45:32,940 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:45:32,942 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:45:37,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:39,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:42,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:47,940 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:45:47,941 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:45:48,094 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:49,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:45:53,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:58,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:45:59,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:02,282 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:46:02,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:46:02,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:46:04,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:09,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:09,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:15,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:17,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:46:17,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:46:19,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:21,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:26,192 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:29,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:32,192 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:32,283 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:46:32,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:46:32,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:46:37,192 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:39,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:42,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:47,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:46:47,268 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:46:47,269 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:46:47,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:47,270 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:46:47,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:46:47,942 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:46:47,984 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:46:49,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:46:49,985 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:46:53,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:59,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:46:59,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:02,285 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:47:02,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:47:02,942 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:47:05,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:09,983 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:10,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:15,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:17,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:47:17,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:47:19,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:21,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:26,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:29,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:32,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:32,287 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:47:32,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:47:32,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:47:37,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:39,985 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:43,195 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:47,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:47:47,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:47:49,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:49,985 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:47:54,196 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:59,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:47:59,985 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:02,290 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:48:02,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:48:02,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:48:04,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:09,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:10,196 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:15,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:17,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:48:17,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:48:19,986 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:21,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:27,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:29,986 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:32,292 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:48:32,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:32,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:48:32,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:48:38,210 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:39,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:44,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:47,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:48:47,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:48:48,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:48:48,695 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:48:48,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:48:48,696 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:48:49,047 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:48:49,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:49,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:48:50,048 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:48:55,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:48:59,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:01,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:02,294 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:49:02,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:49:02,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:49:06,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:09,989 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:12,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:17,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:17,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:49:17,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:49:19,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:23,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:28,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:30,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:32,300 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:49:32,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:49:32,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:49:34,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:39,207 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:39,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:45,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:47,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:49:47,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:49:49,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:49:50,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:56,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:49:59,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:02,204 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:02,299 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:50:02,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:50:02,946 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:50:08,204 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:09,992 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:13,210 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:17,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:50:17,946 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:50:19,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:19,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:24,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:29,207 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:29,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:32,310 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:50:32,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:50:32,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:50:35,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:39,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:40,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:45,208 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:47,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:50:47,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:50:49,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:50:50,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:50:50,524 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:50:50,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:50:50,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:50,526 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:50:51,112 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:50:52,114 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:50:55,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:50:58,118 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:50:59,993 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:01,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:02,304 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:51:02,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:51:02,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:51:06,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:08,127 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:51:09,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:12,177 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:17,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:17,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:51:17,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:51:19,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:22,181 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:28,181 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:29,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:32,306 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:51:32,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:51:32,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:51:33,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:39,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:39,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:44,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:47,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:51:47,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:51:49,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:51:50,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:55,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:51:59,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:00,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:02,308 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:52:02,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:52:02,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:52:06,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:09,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:11,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:16,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:17,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:52:17,953 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:52:19,995 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:21,210 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:27,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:29,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:32,310 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:52:32,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:34,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:52:34,520 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:52:38,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:39,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:44,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:49,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:49,518 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:52:49,519 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:52:49,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:52:55,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:52:59,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:01,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:02,313 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:53:04,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:53:04,520 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:53:06,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:08,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:53:08,759 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:53:08,760 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:53:08,761 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:53:09,197 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:53:09,996 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:10,198 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:53:11,800 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:17,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:19,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:53:19,520 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:53:19,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:22,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:28,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:29,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:32,316 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:53:33,318 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:34,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:53:34,520 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:53:39,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:39,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:44,192 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:49,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:49,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:53:49,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:53:49,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:53:55,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:53:59,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:01,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:02,322 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:54:04,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:54:04,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:54:06,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:09,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:12,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:17,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:19,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:54:19,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:54:19,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:22,197 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:27,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:29,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:32,321 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:54:32,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:34,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:54:34,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:54:38,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:39,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:44,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:49,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:54:49,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:54:49,794 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:49,998 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:54:55,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:54:59,999 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:00,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:02,325 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:55:04,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:55:04,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:55:05,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:09,923 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:55:09,925 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:55:09,925 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:55:09,926 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:55:09,999 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:10,260 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:55:10,971 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:12,261 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:55:16,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:19,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:55:19,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:55:19,999 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:21,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:27,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:30,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:32,327 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:55:32,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:34,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:55:34,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:55:38,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:40,000 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:43,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:49,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:49,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:55:49,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:55:50,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:55:54,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:55:59,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:00,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:02,328 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:56:04,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:04,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:56:04,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:56:10,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:10,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:15,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:19,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:56:19,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:56:20,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:20,781 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:26,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:30,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:32,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:32,330 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:56:34,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:56:34,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:56:37,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:40,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:42,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:47,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:49,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:56:49,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:56:50,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:56:52,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:56:57,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:00,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:02,333 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:57:02,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:04,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:57:04,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:57:08,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:10,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:11,243 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:57:11,245 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:57:11,245 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:57:11,246 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:57:11,330 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:57:12,330 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:57:13,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:19,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:19,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:57:19,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:57:20,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:25,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:30,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:30,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:32,335 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:57:34,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:57:34,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:57:35,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:40,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:41,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:47,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:49,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:57:49,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:57:50,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:57:52,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:57:57,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:00,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:02,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:02,338 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:58:04,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:58:04,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:58:08,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:10,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:14,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:19,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:19,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:58:19,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:58:20,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:24,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:30,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:30,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:32,344 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:58:34,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:58:34,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:58:35,719 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:40,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:41,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:46,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:49,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:58:49,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:58:50,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:58:51,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:58:57,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:00,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:02,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:02,342 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:59:04,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:59:04,529 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:59:08,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:10,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:14,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 19:59:14,100 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 19:59:14,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 19:59:14,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:14,103 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 19:59:14,396 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 19:59:16,396 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 19:59:19,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:19,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:59:19,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:59:20,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:24,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:29,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:30,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:32,344 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 19:59:34,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:34,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:59:34,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:59:40,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:40,366 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:45,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:49,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 19:59:49,539 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 19:59:50,366 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 19:59:50,719 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 19:59:56,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:00,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:01,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:02,357 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:00:04,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:00:04,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:00:06,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:10,371 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:12,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:17,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:19,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:00:19,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:00:20,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:22,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:28,236 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:30,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:32,351 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:00:33,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:34,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:00:34,543 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:00:39,236 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:40,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:45,237 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:49,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:00:49,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:00:50,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:00:50,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:00:56,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:00,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:01,239 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:02,353 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:01:04,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:01:04,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:01:06,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:10,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:12,240 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:15,562 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:01:15,563 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:01:15,564 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:01:15,564 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:01:16,457 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:01:16,458 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:01:17,241 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:19,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:01:19,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:01:20,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:22,241 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:28,241 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:30,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:32,355 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:01:33,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:34,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:01:34,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:01:39,242 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:40,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:45,242 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:49,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:01:49,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:01:50,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:01:50,705 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:01:56,244 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:00,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:02,243 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:02,358 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:02:04,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:02:04,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:02:08,243 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:10,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:13,244 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:18,244 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:19,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:02:19,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:02:20,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:24,244 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:29,245 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:30,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:32,360 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:02:34,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:34,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:02:34,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:02:40,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:40,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:46,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:49,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:02:49,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:02:50,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:02:51,708 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:02:57,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:00,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:02,250 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:02,365 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:03:04,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:03:04,543 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:03:08,250 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:10,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:14,250 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:16,470 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:03:16,472 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:03:16,472 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:03:16,473 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:03:16,516 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:03:18,517 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:03:19,252 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:19,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:03:19,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:03:20,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:25,251 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:30,252 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:30,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:32,365 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:03:34,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:03:34,543 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:03:35,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:40,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:41,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:46,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:49,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:03:49,544 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:03:50,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:03:51,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:03:57,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:00,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:02,369 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:04:02,370 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:04,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:04:04,544 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:04:08,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:10,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:13,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:19,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:19,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:04:19,544 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:04:20,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:24,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:30,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:30,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:32,369 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:04:34,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:04:34,544 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:04:35,732 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:40,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:41,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:47,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:49,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:04:49,544 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:04:50,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:04:52,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:04:57,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:00,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:02,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:02,371 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:05:04,544 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:05:04,545 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:05:07,272 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:10,385 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:12,272 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:17,916 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:05:17,918 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:05:17,919 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:05:17,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:17,920 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:05:18,586 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:05:19,544 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:05:19,545 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:05:20,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:20,587 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:05:23,272 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:29,272 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:30,386 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:32,374 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:05:34,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:34,544 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:05:34,545 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:05:40,273 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:40,387 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:45,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:49,544 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:05:49,545 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:05:50,387 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:05:50,730 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:05:56,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:00,388 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:01,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:02,376 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:06:04,546 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:06:04,547 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:06:06,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:10,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:12,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:18,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:19,547 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:06:19,547 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:06:20,388 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:23,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:29,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:30,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:32,378 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:06:34,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:34,547 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:06:34,547 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:06:40,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:40,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:46,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:49,547 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:06:49,548 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:06:50,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:06:51,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:06:57,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:00,390 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:02,380 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:07:02,382 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:04,548 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:07:04,549 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:07:08,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:10,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:14,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:19,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:19,548 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:07:19,549 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:07:20,391 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:20,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:07:20,685 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:07:20,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:07:20,687 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:07:21,656 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:07:22,657 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:07:24,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:29,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:30,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:32,388 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:07:34,389 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:34,548 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:07:34,549 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:07:40,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:40,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:45,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:49,548 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:07:49,549 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:07:50,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:07:50,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:07:56,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:00,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:01,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:02,385 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:08:04,548 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:08:04,549 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:08:06,790 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:10,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:12,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:17,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:19,549 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:08:19,550 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:08:20,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:22,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:28,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:30,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:32,389 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:08:33,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:34,549 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:08:34,550 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:08:39,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:40,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:45,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:49,551 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:08:49,552 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:08:50,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:08:50,792 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:08:56,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:00,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:02,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:02,390 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:09:04,551 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:09:04,552 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:09:07,301 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:10,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:12,301 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:18,301 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:19,551 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:09:19,551 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:09:20,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:22,414 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:09:22,415 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:09:22,416 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:09:22,416 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:09:22,720 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:09:23,459 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:24,722 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:09:29,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:30,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:32,393 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:09:34,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:34,551 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:09:34,552 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:09:40,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:40,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:46,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:49,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:09:49,552 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:09:50,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:09:52,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:09:57,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:00,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:02,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:02,395 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:10:04,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:10:04,553 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:10:07,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:10,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:12,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:18,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:19,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:10:19,553 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:10:20,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:23,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:28,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:30,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:32,397 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:10:33,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:34,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:10:34,554 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:10:39,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:40,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:45,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:49,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:10:49,554 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:10:50,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:10:50,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:10:56,310 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:00,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:02,310 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:02,403 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:11:04,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:11:04,554 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:11:08,310 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:10,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:13,311 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:19,311 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:19,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:11:19,554 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:11:20,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:24,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:11:24,035 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:11:24,035 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:11:24,036 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:11:24,782 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:11:25,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:26,782 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:11:30,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:30,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:32,403 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:11:34,556 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:11:34,557 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:11:35,709 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:40,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:41,317 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:46,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:50,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:11:50,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:11:50,522 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:11:51,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:11:57,324 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:00,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:02,324 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:02,413 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:12:05,520 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:12:05,522 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:12:07,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:10,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:12,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:17,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:20,520 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:20,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:12:20,521 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:12:23,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:29,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:30,520 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:32,407 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:12:34,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:35,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:12:35,522 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:12:40,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:40,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:45,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:50,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:12:50,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:12:50,523 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:12:50,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:12:56,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:00,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:01,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:02,409 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:13:05,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:13:05,523 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:13:06,666 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:10,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:12,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:17,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:20,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:20,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:13:20,523 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:13:23,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:26,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:13:26,603 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:13:26,604 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:13:26,605 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:13:26,842 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:13:28,843 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:13:29,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:30,522 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:32,412 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:13:34,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:35,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:13:35,524 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:13:40,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:40,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:45,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:50,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:13:50,524 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:13:50,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:13:50,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:13:56,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:00,531 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:01,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:02,414 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:14:05,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:14:05,524 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:14:06,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:10,531 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:12,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:17,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:20,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:14:20,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:14:20,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:22,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:28,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:30,531 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:32,416 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:14:33,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:35,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:14:35,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:14:39,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:40,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:45,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:50,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:14:50,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:14:50,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:14:50,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:14:56,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:00,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:01,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:02,418 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:15:05,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:15:05,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:15:06,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:10,552 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:12,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:17,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:20,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:15:20,525 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:15:20,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:22,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:27,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:27,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:15:27,609 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:15:27,609 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:15:27,610 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:15:27,892 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:15:29,893 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:15:30,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:32,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:32,421 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:15:35,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:15:35,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:15:38,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:40,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:44,359 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:50,359 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:15:50,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:15:50,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:15:50,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:15:56,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:00,553 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:02,359 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:02,423 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:16:05,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:16:05,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:16:08,359 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:10,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:14,359 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:19,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:20,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:16:20,526 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:16:20,566 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:25,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:30,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:31,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:32,425 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:16:35,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:16:35,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:16:36,727 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:40,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:42,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:48,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:50,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:16:50,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:16:50,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:16:54,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:16:59,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:00,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:02,431 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:17:04,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:05,526 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:17:05,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:17:10,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:10,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:16,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:20,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:17:20,527 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:17:20,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:21,667 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:27,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:28,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:17:28,618 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:17:28,618 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:17:28,619 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:17:28,945 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:17:29,946 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:17:30,555 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:32,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:32,430 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:17:35,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:17:35,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:17:37,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:40,556 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:42,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:48,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:50,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:17:50,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:17:50,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:17:54,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:17:59,365 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:00,556 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:02,434 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:18:04,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:05,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:18:05,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:18:10,367 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:10,557 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:15,845 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:20,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:18:20,528 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:18:20,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:21,726 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:26,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:30,557 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:32,435 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:18:32,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:35,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:18:35,529 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:18:37,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:40,557 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:42,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:47,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:50,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:18:50,529 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:18:50,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:18:53,728 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:18:58,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:00,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:02,437 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:19:03,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:05,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:19:05,529 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:19:09,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:10,558 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:14,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:19,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:20,528 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:19:20,529 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:19:20,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:24,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:30,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:30,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:31,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:19:31,034 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:19:31,035 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:19:31,035 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:19:32,023 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:19:32,023 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:19:32,439 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:19:35,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:19:35,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:19:36,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:40,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:41,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:47,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:50,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:19:50,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:19:50,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:19:53,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:19:58,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:00,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:02,442 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:20:03,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:05,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:20:05,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:20:09,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:10,579 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:15,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:20,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:20:20,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:20:20,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:21,726 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:26,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:30,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:32,444 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:20:32,447 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:35,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:20:35,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:20:37,714 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:40,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:42,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:48,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:50,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:20:50,530 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:20:50,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:20:54,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:20:59,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:00,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:02,447 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:21:05,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:21:05,531 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:21:05,726 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:10,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:10,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:15,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:20,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:21:20,531 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:21:20,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:21,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:26,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:30,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:32,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:21:32,135 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:21:32,136 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:21:32,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:32,137 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:21:32,454 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:21:33,094 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:21:34,094 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:21:35,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:21:35,532 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:21:37,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:40,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:42,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:48,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:50,531 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:21:50,532 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:21:50,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:21:54,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:21:59,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:00,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:02,452 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:22:05,531 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:22:05,532 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:22:05,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:10,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:10,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:15,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:20,532 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:22:20,533 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:22:20,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:21,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:26,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:30,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:32,456 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:22:32,457 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:35,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:22:35,534 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:22:37,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:40,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:42,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:48,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:50,532 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:22:50,533 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:22:50,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:22:54,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:22:59,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:00,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:02,457 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:23:05,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:23:05,533 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:23:05,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:10,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:10,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:16,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:20,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:23:20,533 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:23:20,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:22,728 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:27,860 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:30,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:32,460 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:23:33,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:23:33,153 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:23:33,153 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:23:33,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:33,155 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:23:33,156 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:23:34,156 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:23:35,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:23:35,534 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:23:38,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:40,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:43,861 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:49,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:50,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:23:50,534 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:23:50,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:23:54,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:23:59,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:00,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:02,462 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:24:05,534 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:24:05,535 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:24:05,770 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:10,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:10,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:16,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:20,534 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:24:20,535 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:24:20,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:22,711 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:27,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:30,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:32,464 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:24:33,465 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:35,534 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:24:35,535 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:24:38,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:40,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:44,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:49,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:24:50,535 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:24:50,535 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:24:50,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:24:55,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:00,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:01,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:02,466 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:25:05,535 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:25:05,536 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:25:06,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:10,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:12,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:18,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:20,535 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:25:20,536 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:25:20,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:23,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:29,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:30,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:32,468 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:25:35,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:35,535 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:25:35,536 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:25:35,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:25:35,830 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:25:35,830 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:25:35,831 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:25:36,210 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:25:38,211 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:25:40,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:41,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:46,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:50,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:25:50,543 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:25:50,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:25:51,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:25:57,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:00,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:02,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:02,475 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:26:05,536 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:26:05,537 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:26:07,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:10,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:13,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:18,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:20,536 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:26:20,537 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:26:20,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:23,729 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:29,006 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:30,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:32,474 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:26:35,007 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:35,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:26:35,537 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:26:40,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:41,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:47,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:50,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:26:50,538 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:26:50,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:26:52,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:26:58,009 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:00,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:02,477 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:27:03,479 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:05,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:27:05,538 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:27:08,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:10,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:14,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:19,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:20,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:27:20,538 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:27:20,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:24,009 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:29,009 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:30,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:32,478 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:27:35,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:35,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:27:35,539 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:27:36,892 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:27:36,894 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:27:36,895 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:27:36,895 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:27:37,272 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:27:38,272 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:27:40,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:41,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:46,011 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:50,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:27:50,539 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:27:50,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:27:51,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:27:57,011 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:00,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:02,481 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:28:02,483 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:05,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:28:05,539 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:28:07,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:10,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:13,014 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:19,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:20,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:28:20,539 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:28:20,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:25,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:30,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:31,013 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:32,483 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:28:35,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:28:35,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:28:36,667 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:40,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:42,015 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:48,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:50,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:28:50,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:28:50,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:28:53,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:28:59,014 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:00,608 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:02,486 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:29:05,014 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:05,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:29:05,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:29:10,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:11,015 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:17,015 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:20,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:29:20,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:29:20,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:22,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:28,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:30,609 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:32,488 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:29:33,490 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:35,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:29:35,540 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:29:38,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:29:38,069 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:29:38,069 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:29:38,070 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:29:38,327 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:29:39,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:40,328 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:29:40,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:44,017 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:50,017 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:29:50,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:29:50,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:29:50,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:29:56,017 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:00,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:01,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:02,491 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:30:05,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:30:05,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:30:06,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:10,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:12,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:18,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:20,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:30:20,541 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:30:20,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:23,710 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:29,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:30,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:32,498 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:30:34,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:35,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:30:35,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:30:40,019 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:40,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:46,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:50,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:30:50,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:30:50,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:30:51,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:30:57,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:00,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:02,495 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:31:02,497 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:06,942 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:31:06,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:31:08,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:10,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:14,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:20,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:20,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:21,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:31:21,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:31:26,020 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:30,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:32,021 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:32,499 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:31:36,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:31:36,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:31:37,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:40,570 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:31:40,571 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:31:40,572 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:31:40,573 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:31:40,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:41,378 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:31:42,378 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:31:42,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:48,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:50,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:31:51,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:31:51,943 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:31:53,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:31:59,022 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:00,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:02,500 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:32:04,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:06,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:32:06,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:32:10,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:10,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:16,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:20,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:21,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:21,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:32:21,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:32:27,024 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:30,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:32,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:32,503 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:32:36,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:32:36,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:32:37,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:40,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:43,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:48,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:50,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:32:51,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:32:51,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:32:53,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:32:59,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:00,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:02,504 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:33:04,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:06,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:33:06,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:33:09,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:10,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:15,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:20,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:21,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:21,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:33:21,944 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:33:26,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:30,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:32,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:32,507 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:33:36,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:33:36,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:33:37,126 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:40,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:41,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:33:41,410 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:33:41,410 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:33:41,411 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:33:41,421 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:33:42,421 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:33:42,451 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:48,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:50,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:33:51,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:33:51,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:33:53,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:33:59,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:00,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:02,510 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:34:05,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:06,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:34:06,945 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:34:10,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:10,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:16,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:20,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:21,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:21,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:34:21,946 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:34:27,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:30,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:32,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:32,512 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:34:36,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:34:36,946 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:34:37,158 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:40,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:43,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:48,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:50,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:34:51,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:34:51,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:34:53,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:34:59,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:00,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:02,516 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:35:05,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:06,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:35:06,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:35:10,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:11,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:16,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:20,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:21,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:35:21,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:35:22,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:28,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:30,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:32,522 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:35:33,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:36,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:35:36,947 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:35:39,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:40,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:42,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:35:42,354 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:35:42,354 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:35:42,355 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:35:42,472 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:35:44,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:44,473 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:35:49,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:35:50,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:35:51,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:35:51,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:35:55,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:00,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:00,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:02,521 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:36:06,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:06,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:36:06,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:36:10,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:12,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:17,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:20,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:21,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:36:21,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:36:22,171 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:28,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:30,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:32,525 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:36:33,527 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:36,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:36:36,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:36:39,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:40,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:44,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:50,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:36:50,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:36:51,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:36:51,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:36:55,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:00,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:01,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:02,525 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:37:06,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:06,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:37:06,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:37:10,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:12,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:17,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:20,621 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:21,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:37:21,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:37:22,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:28,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:30,621 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:32,528 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:37:33,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:36,948 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:37:36,949 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:37:39,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:40,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:44,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:45,322 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:37:45,324 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:37:45,324 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:37:45,325 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:37:45,526 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:37:46,526 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:37:50,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:37:50,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:37:51,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:37:51,950 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:37:56,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:00,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:01,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:02,530 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:38:06,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:06,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:38:06,953 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:38:10,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:12,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:18,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:20,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:21,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:38:21,951 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:38:23,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:29,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:30,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:32,532 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:38:35,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:36,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:38:36,951 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:38:40,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:41,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:46,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:50,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:38:51,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:38:51,951 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:38:52,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:38:58,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:00,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:02,535 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:39:03,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:06,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:39:06,951 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:39:09,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:10,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:14,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:20,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:20,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:21,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:39:21,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:39:26,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:30,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:31,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:32,537 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:39:36,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:39:36,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:39:37,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:40,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:43,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:46,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:39:46,983 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:39:46,984 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:39:46,985 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:39:47,588 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:39:48,588 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:39:49,027 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:50,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:39:51,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:39:51,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:39:54,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:39:59,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:00,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:02,543 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:40:04,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:06,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:40:06,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:40:10,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:10,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:16,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:20,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:21,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:40:21,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:40:22,171 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:28,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:30,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:32,541 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:40:33,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:36,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:40:36,953 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:40:39,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:40,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:45,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:50,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:40:50,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:40:51,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:40:51,953 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:40:56,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:00,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:01,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:02,546 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:41:06,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:06,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:41:06,953 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:41:10,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:12,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:18,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:20,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:21,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:41:21,955 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:41:23,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:29,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:30,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:32,546 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:41:34,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:36,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:41:36,955 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:41:40,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:40,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:45,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:48,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:41:48,586 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:41:48,586 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:41:48,587 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:41:48,635 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:41:50,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:41:50,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:41:50,636 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:41:51,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:41:51,955 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:41:56,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:00,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:01,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:02,548 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:42:06,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:42:06,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:42:07,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:10,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:13,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:18,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:20,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:21,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:42:21,955 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:42:23,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:29,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:30,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:32,551 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:42:34,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:36,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:42:36,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:42:40,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:40,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:46,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:50,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:42:51,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:42:51,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:42:52,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:42:58,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:00,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:02,553 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:43:03,554 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:06,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:43:06,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:43:09,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:10,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:14,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:20,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:20,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:21,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:43:21,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:43:25,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:30,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:30,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:32,556 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:43:36,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:36,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:43:36,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:43:40,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:42,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:47,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:49,907 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:43:49,908 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:43:49,908 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:43:49,909 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:43:50,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:43:50,681 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:43:51,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:43:51,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:43:52,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:43:52,682 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:43:58,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:00,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:02,558 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:44:03,559 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:06,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:44:06,957 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:44:09,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:10,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:14,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:20,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:20,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:21,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:44:21,957 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:44:26,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:30,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:31,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:32,564 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:44:36,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:36,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:44:36,957 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:44:40,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:42,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:47,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:50,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:44:51,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:44:51,958 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:44:52,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:44:58,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:00,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:02,563 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:45:03,564 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:06,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:45:06,958 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:45:09,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:10,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:14,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:19,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:20,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:21,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:45:21,958 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:45:25,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:30,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:30,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:32,567 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:45:35,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:36,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:45:36,958 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:45:40,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:41,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:46,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:50,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:45:51,958 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:45:51,958 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:45:52,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:45:52,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:45:52,971 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:45:52,971 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:45:52,972 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:45:53,728 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:45:54,728 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:45:58,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:00,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:02,568 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:46:03,569 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:06,975 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:46:06,975 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:46:09,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:10,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:15,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:20,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:21,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:21,975 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:46:21,975 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:46:26,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:30,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:31,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:32,570 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:46:36,975 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:46:36,975 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:46:37,194 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:40,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:43,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:48,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:50,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:46:51,976 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:46:51,976 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:46:53,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:46:59,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:00,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:02,573 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:47:05,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:06,979 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:47:06,980 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:47:10,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:11,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:17,084 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:20,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:21,979 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:47:21,980 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:47:22,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:28,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:30,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:32,575 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:47:33,576 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:36,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:47:36,980 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:47:39,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:40,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:45,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:50,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:47:51,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:51,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:47:51,980 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:47:54,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:47:54,326 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:47:54,326 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:47:54,327 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:47:54,775 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:47:56,367 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:47:56,776 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:48:00,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:02,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:02,578 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:48:06,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:48:06,981 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:48:07,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:10,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:13,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:18,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:20,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:21,980 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:48:21,981 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:48:23,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:29,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:30,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:32,580 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:48:34,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:36,981 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:48:36,981 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:48:40,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:40,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:45,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:50,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:48:50,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:48:51,986 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:48:51,987 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:48:55,090 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:00,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:00,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:02,586 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:49:06,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:06,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:49:06,987 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:49:10,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:12,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:17,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:20,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:21,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:49:21,988 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:49:22,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:28,093 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:30,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:32,584 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:49:33,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:36,987 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:49:36,988 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:49:39,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:40,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:44,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:50,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:50,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:49:51,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:49:51,988 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:49:55,092 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:49:55,663 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:49:55,665 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:49:55,665 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:49:55,667 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:49:55,833 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:49:56,833 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:50:00,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:01,093 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:02,587 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:50:06,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:50:06,988 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:50:07,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:10,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:13,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:19,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:20,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:22,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:50:22,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:50:25,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:30,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:31,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:32,590 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:50:36,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:37,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:50:37,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:50:40,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:42,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:47,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:50,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:50:52,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:50:52,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:50:52,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:50:58,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:00,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:02,592 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:51:03,593 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:07,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:51:07,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:51:09,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:10,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:14,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:20,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:20,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:22,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:51:22,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:51:26,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:30,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:32,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:32,594 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:51:37,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:37,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:51:37,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:51:40,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:43,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:48,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:50,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:51:52,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:51:52,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:51:53,496 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:58,657 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:51:58,658 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:51:58,659 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:51:58,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:51:58,660 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:51:58,896 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:52:00,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:00,897 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:52:02,597 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:52:04,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:07,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:52:07,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:52:09,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:10,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:15,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:20,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:20,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:22,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:52:22,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:52:26,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:30,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:32,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:32,599 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:52:37,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:37,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:52:37,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:52:40,662 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:43,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:48,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:50,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:52:52,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:52:52,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:52:53,461 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:52:59,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:00,664 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:02,602 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:53:04,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:07,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:53:07,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:53:10,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:10,664 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:16,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:20,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:22,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:22,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:53:22,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:53:28,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:30,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:32,604 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:53:33,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:37,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:53:37,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:53:39,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:40,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:45,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:50,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:53:51,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:53:52,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:53:52,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:53:57,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:00,083 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:54:00,090 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:54:00,090 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:54:00,090 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:54:00,666 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:00,965 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:54:02,127 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:02,607 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:54:02,966 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:54:07,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:54:07,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:54:07,525 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:10,666 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:13,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:19,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:20,666 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:22,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:54:22,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:54:25,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:30,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:30,666 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:32,608 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:54:35,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:37,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:54:37,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:54:40,667 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:41,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:46,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:50,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:54:51,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:54:52,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:54:52,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:54:56,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:00,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:02,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:02,611 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:55:07,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:55:07,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:55:07,482 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:10,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:13,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:18,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:20,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:22,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:55:22,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:55:23,484 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:29,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:30,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:32,614 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:55:34,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:37,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:55:37,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:55:40,124 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:40,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:45,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:50,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:55:51,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:55:52,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:55:52,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:55:57,125 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:00,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:01,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:56:01,852 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:56:01,852 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:56:01,853 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:56:02,040 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:56:02,616 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:56:02,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:03,041 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:56:07,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:56:07,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:56:08,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:10,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:14,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:19,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:20,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:22,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:56:22,332 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:56:25,133 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:30,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:31,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:32,618 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:56:36,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:37,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:56:37,333 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:56:40,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:41,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:47,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:50,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:56:53,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:56:53,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:56:53,257 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:56:59,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:00,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:02,620 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:57:04,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:08,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:57:08,257 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:57:10,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:10,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:15,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:20,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:21,135 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:23,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:57:23,258 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:57:26,136 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:30,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:32,136 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:32,623 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:57:37,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:38,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:57:38,258 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:57:40,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:43,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:49,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:57:50,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:57:53,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:57:53,258 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:57:55,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:00,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:00,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:02,625 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:58:04,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 20:58:04,691 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 20:58:04,691 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 20:58:04,692 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 20:58:05,107 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 20:58:05,731 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:07,109 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 20:58:08,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:58:08,258 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:58:10,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:11,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:17,138 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:20,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:23,138 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:23,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:58:23,258 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:58:29,138 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:30,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:32,631 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:58:34,138 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:38,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:58:38,259 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:58:40,139 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:40,691 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:45,139 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:50,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:58:50,691 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:58:53,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:58:53,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:58:56,139 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:00,691 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:01,140 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:02,630 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:59:06,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:08,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:59:08,259 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:59:10,691 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:12,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:17,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:20,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:23,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:23,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:59:23,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:59:29,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:30,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:32,634 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 20:59:34,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:38,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:59:38,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:59:40,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:40,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:46,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:50,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 20:59:52,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 20:59:53,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 20:59:53,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 20:59:58,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:00,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:02,635 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:00:03,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:06,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:00:06,227 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:00:06,228 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:00:06,228 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:00:07,172 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:00:08,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:00:08,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:00:09,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:09,173 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:00:10,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:15,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:20,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:20,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:23,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:00:23,260 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:00:26,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:30,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:32,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:32,637 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:00:37,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:38,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:00:38,261 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:00:40,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:43,146 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:48,146 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:50,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:00:53,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:00:53,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:00:53,261 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:00:58,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:00,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:02,640 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:01:03,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:08,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:01:08,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:01:09,146 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:10,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:14,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:19,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:20,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:23,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:01:23,261 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:01:25,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:30,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:31,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:32,642 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:01:36,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:38,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:01:38,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:01:40,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:42,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:47,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:50,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:01:53,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:01:53,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:01:53,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:01:59,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:00,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:02,645 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:02:05,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:07,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:02:07,972 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:02:07,972 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:02:07,973 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:02:08,220 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:02:08,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:02:08,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:02:09,221 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:02:10,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:10,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:16,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:20,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:22,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:23,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:02:23,263 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:02:27,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:30,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:32,647 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:02:32,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:38,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:38,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:02:38,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:02:40,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:44,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:50,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:02:50,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:02:53,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:02:53,262 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:02:55,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:00,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:00,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:02,652 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:03:06,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:08,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:03:08,263 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:03:10,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:12,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:18,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:20,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:23,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:23,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:03:23,263 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:03:29,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:30,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:32,651 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:03:35,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:38,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:03:38,263 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:03:40,700 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:41,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:47,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:50,700 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:03:52,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:03:53,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:03:53,264 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:03:57,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:00,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:02,655 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:04:02,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:08,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:08,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:04:08,264 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:04:10,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:11,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:04:11,037 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:04:11,037 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:04:11,038 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:04:11,269 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:04:13,270 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:04:14,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:19,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:20,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:23,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:04:23,264 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:04:25,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:30,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:30,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:32,656 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:04:35,161 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:38,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:04:38,264 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:04:40,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:40,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:45,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:50,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:04:51,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:04:53,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:04:53,264 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:04:57,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:00,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:02,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:02,659 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:05:07,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:08,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:05:08,282 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:05:10,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:13,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:19,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:20,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:23,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:05:23,283 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:05:25,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:30,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:31,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:32,661 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:05:36,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:38,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:05:38,283 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:05:40,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:42,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:48,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:50,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:05:53,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:05:53,282 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:05:53,494 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:05:59,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:00,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:02,664 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:06:04,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:08,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:06:08,283 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:06:09,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:10,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:12,731 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:06:12,732 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:06:12,732 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:06:12,733 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:06:13,324 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:06:14,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:15,325 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:06:20,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:20,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:23,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:06:23,283 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:06:25,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:30,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:31,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:32,666 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:06:36,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:38,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:06:38,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:06:40,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:42,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:48,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:50,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:06:53,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:06:53,283 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:06:53,438 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:06:59,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:00,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:02,668 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:07:05,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:08,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:07:08,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:07:10,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:11,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:16,181 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:20,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:22,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:23,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:07:23,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:07:28,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:30,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:32,674 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:07:33,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:38,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:07:38,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:07:39,182 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:40,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:44,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:50,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:07:50,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:07:53,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:07:53,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:07:56,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:00,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:01,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:02,673 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:08:06,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:08,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:08:08,284 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:08:10,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:12,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:14,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:08:14,294 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:08:14,294 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:08:14,296 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:08:14,381 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:08:17,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:17,382 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:08:20,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:23,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:23,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:08:23,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:08:28,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:30,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:32,675 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:08:33,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:38,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:08:38,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:08:39,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:40,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:44,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:49,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:08:50,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:08:53,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:08:53,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:08:55,197 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:00,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:00,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:02,680 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:09:05,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:08,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:09:08,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:09:10,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:11,198 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:16,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:20,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:22,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:23,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:09:23,286 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:09:28,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:30,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:32,680 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:09:33,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:38,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:09:38,286 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:09:39,200 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:40,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:44,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:50,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:09:50,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:09:53,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:09:53,286 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:09:56,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:00,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:02,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:02,682 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:10:07,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:08,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:10:08,286 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:10:10,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:13,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:17,481 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:10:17,482 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:10:17,482 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:10:17,483 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:10:18,434 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:10:18,523 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:19,434 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:10:20,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:23,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:10:23,286 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:10:24,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:29,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:30,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:32,685 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:10:34,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:38,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:10:38,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:10:40,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:40,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:46,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:50,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:10:52,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:10:53,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:10:53,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:10:57,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:00,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:02,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:02,689 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:11:07,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:08,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:11:08,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:11:10,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:13,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:19,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:20,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:23,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:11:23,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:11:24,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:29,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:30,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:32,751 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:11:35,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:38,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:11:38,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:11:40,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:40,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:45,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:50,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:11:50,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:11:53,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:11:53,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:11:56,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:00,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:01,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:02,757 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:12:06,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:08,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:12:08,287 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:12:10,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:12,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:18,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:19,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:12:19,116 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:12:19,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:12:19,117 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:12:19,477 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:12:20,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:21,478 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:12:23,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:12:23,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:12:23,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:29,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:30,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:32,756 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:12:34,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:38,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:12:38,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:12:40,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:40,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:45,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:50,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:12:51,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:12:53,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:12:53,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:12:57,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:00,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:02,759 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:13:02,760 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:08,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:08,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:13:08,288 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:13:10,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:14,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:19,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:20,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:23,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:13:23,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:13:24,246 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:30,246 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:30,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:32,760 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:13:35,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:38,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:13:38,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:13:40,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:41,246 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:46,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:50,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:13:52,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:13:53,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:13:53,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:13:57,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:00,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:02,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:02,763 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:14:07,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:08,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:14:08,289 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:14:10,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:13,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:19,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:20,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:14:20,605 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:14:20,606 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:14:20,606 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:14:20,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:21,523 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:14:23,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:14:23,290 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:14:23,524 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:14:25,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:30,750 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:31,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:32,765 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:14:36,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:38,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:14:38,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:14:40,750 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:42,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:48,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:50,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:14:53,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:14:53,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:14:53,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:14:59,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:00,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:02,767 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:15:04,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:08,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:15:08,295 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:15:10,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:10,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:15,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:20,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:21,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:23,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:15:23,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:15:27,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:30,752 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:32,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:32,770 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:15:37,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:38,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:15:38,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:15:40,752 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:42,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:48,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:50,752 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:15:53,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:15:53,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:15:53,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:15:59,280 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:00,753 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:02,772 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:16:04,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:08,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:16:08,296 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:16:10,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:10,753 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:15,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:20,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:20,753 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:23,668 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:16:23,668 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:16:24,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:16:24,833 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:16:24,833 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:16:24,834 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:16:25,573 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:16:25,875 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:27,574 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:16:30,754 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:31,281 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:32,778 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:16:36,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:38,668 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:16:38,668 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:16:40,754 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:42,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:47,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:50,755 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:16:52,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:16:53,668 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:16:53,668 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:16:58,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:00,755 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:02,777 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:17:03,778 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:08,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:17:08,669 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:17:08,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:10,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:14,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:20,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:20,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:23,668 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:17:23,669 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:17:26,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:30,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:31,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:32,781 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:17:37,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:38,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:17:38,669 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:17:40,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:42,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:48,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:50,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:17:53,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:17:53,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:17:53,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:17:59,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:00,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:02,782 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:18:04,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:08,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:18:08,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:18:09,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:10,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:15,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:20,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:21,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:23,669 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:18:23,669 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:18:26,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:18:26,218 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:18:26,218 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:18:26,219 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:18:26,621 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:18:27,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:27,621 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:18:30,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:32,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:32,784 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:18:37,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:38,670 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:18:38,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:18:40,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:43,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:49,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:50,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:18:53,670 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:18:53,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:18:54,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:18:59,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:00,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:02,787 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:19:04,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:08,670 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:19:08,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:19:09,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:10,760 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:14,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:19,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:20,760 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:23,670 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:19:23,670 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:19:25,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:30,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:30,760 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:32,789 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:19:35,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:38,670 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:19:38,671 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:19:40,761 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:41,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:46,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:50,761 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:19:51,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:19:53,671 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:19:53,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:19:56,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:00,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:02,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:02,792 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:20:07,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:08,671 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:20:08,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:20:10,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:13,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:18,309 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:20,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:23,671 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:20:23,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:20:23,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:27,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:20:27,686 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:20:27,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:20:27,687 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:20:28,679 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:20:29,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:29,680 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:20:30,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:32,794 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:20:34,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:38,671 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:20:38,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:20:39,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:40,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:45,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:50,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:20:51,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:20:53,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:20:53,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:20:57,308 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:00,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:02,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:02,796 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:21:07,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:08,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:21:08,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:21:10,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:13,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:18,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:20,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:23,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:21:23,672 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:21:23,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:29,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:30,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:32,802 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:21:34,314 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:38,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:21:38,673 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:21:40,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:40,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:46,315 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:50,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:21:51,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:21:53,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:21:53,674 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:21:56,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:00,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:01,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:02,801 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:22:07,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:08,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:22:08,674 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:22:10,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:13,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:18,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:20,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:23,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:23,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:22:23,674 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:22:28,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:29,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:22:29,532 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:22:29,533 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:22:29,535 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:22:29,750 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:22:30,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:31,751 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:22:32,803 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:22:33,805 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:38,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:22:38,674 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:22:38,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:40,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:44,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:49,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:22:50,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:22:53,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:22:53,675 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:22:55,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:00,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:00,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:02,807 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:23:06,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:08,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:23:08,675 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:23:10,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:11,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:16,338 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:20,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:21,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:23,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:23:23,675 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:23:27,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:30,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:32,340 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:32,808 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:23:37,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:38,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:23:38,675 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:23:40,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:43,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:49,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:23:50,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:23:53,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:23:53,676 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:23:55,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:00,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:00,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:02,810 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:24:05,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:08,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:24:08,676 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:24:10,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:11,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:16,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:20,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:22,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:23,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:24:23,676 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:24:28,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:30,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:32,812 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:24:33,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:24:33,117 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:24:33,118 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:24:33,119 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:24:33,822 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:24:34,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:35,823 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:24:38,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:24:38,676 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:24:39,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:40,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:44,348 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:50,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:24:50,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:24:53,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:24:53,677 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:24:56,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:00,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:02,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:02,815 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:25:07,350 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:08,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:25:08,677 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:25:10,769 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:13,350 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:19,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:20,769 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:23,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:25:23,677 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:25:25,351 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:30,769 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:31,351 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:32,815 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:25:37,351 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:38,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:25:38,677 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:25:40,770 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:42,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:48,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:50,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:25:53,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:25:53,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:25:53,677 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:25:59,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:00,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:02,820 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:26:04,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:08,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:26:08,678 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:26:09,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:10,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:15,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:20,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:20,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:23,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:26:23,678 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:26:25,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:30,772 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:31,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:32,824 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:26:35,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:26:35,122 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:26:35,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:26:35,124 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:26:35,896 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:26:37,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:37,895 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:26:38,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:26:38,678 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:26:40,772 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:42,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:48,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:50,772 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:26:53,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:26:53,679 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:26:53,896 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:26:59,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:00,772 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:02,823 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:27:04,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:08,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:27:08,679 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:27:09,356 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:10,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:15,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:20,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:21,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:23,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:27:23,679 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:27:26,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:30,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:32,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:32,827 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:27:37,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:38,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:27:38,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:27:40,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:42,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:47,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:50,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:27:53,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:27:53,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:27:53,679 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:27:59,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:00,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:02,828 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:28:05,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:08,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:28:08,679 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:28:10,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:10,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:16,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:20,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:22,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:23,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:28:23,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:28:28,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:30,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:32,830 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:28:33,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:37,128 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:28:37,130 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:28:37,130 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:28:37,131 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:28:37,964 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:28:38,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:28:38,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:28:39,091 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:39,965 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:28:40,775 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:44,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:49,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:28:50,775 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:28:53,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:28:53,681 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:28:55,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:00,363 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:00,775 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:02,833 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:29:06,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:08,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:29:08,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:29:10,776 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:11,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:17,364 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:20,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:23,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:23,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:29:23,680 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:29:29,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:30,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:32,835 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:29:34,836 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:38,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:29:38,681 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:29:39,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:40,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:45,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:50,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:29:50,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:29:53,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:29:53,681 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:29:55,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:00,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:00,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:02,837 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:30:05,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:08,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:30:08,681 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:30:10,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:11,253 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:17,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:20,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:23,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:23,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:30:23,682 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:30:28,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:30,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:32,839 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:30:33,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:38,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:30:38,682 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:30:38,941 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:40,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:30:40,397 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:30:40,398 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:30:40,398 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:30:40,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:41,039 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:30:42,039 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:30:44,440 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:50,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:30:50,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:30:53,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:30:53,682 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:30:55,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:00,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:00,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:02,890 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:31:05,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:08,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:31:08,682 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:31:10,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:10,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:16,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:20,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:21,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:23,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:31:23,683 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:31:27,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:30,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:32,845 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:31:32,847 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:38,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:38,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:31:38,683 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:31:40,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:44,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:49,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:31:50,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:31:53,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:31:53,683 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:31:54,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:00,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:00,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:02,848 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:32:05,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:08,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:32:08,684 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:32:10,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:11,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:16,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:20,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:22,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:23,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:32:23,684 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:32:28,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:30,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:32,850 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:32:33,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:38,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:32:38,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:32:38,944 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:40,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:44,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:49,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:32:50,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:32:53,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:32:53,684 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:32:54,918 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:00,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:00,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:02,853 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:33:03,172 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:33:03,173 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:33:03,173 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:33:03,174 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:33:04,112 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:33:05,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:06,113 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:33:08,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:33:08,684 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:33:10,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:10,800 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:16,263 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:20,801 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:21,276 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:23,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:33:23,685 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:33:27,265 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:30,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:32,856 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:33:32,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:38,265 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:38,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:33:38,685 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:33:40,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:44,265 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:49,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:33:50,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:33:53,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:33:53,685 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:33:54,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:00,266 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:00,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:02,859 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:34:05,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:08,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:34:08,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:34:10,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:11,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:16,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:20,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:22,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:23,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:34:23,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:34:27,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:30,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:32,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:32,861 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:34:38,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:38,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:34:38,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:34:40,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:44,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:49,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:34:50,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:34:53,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:34:53,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:34:54,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:00,284 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:00,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:02,864 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:35:04,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:35:04,723 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:35:04,723 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:35:04,724 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:35:05,171 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:35:06,172 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:35:06,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:08,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:35:08,687 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:35:10,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:12,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:17,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:20,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:23,285 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:23,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:35:23,686 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:35:28,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:30,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:32,870 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:35:33,871 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:38,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:35:38,687 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:35:38,934 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:40,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:44,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:49,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:35:50,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:35:53,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:35:53,688 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:35:54,899 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:00,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:00,805 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:02,869 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:36:05,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:08,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:36:08,688 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:36:10,805 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:11,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:17,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:20,805 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:23,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:23,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:36:23,689 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:36:28,289 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:30,805 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:32,873 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:36:33,876 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:38,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:36:38,689 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:36:38,933 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:40,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:44,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:50,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:36:50,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:36:53,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:36:53,690 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:36:55,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:00,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:00,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:02,873 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:37:05,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:08,128 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:37:08,130 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:37:08,130 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:37:08,131 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:37:08,244 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:37:08,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:37:08,689 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:37:09,244 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:37:10,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:11,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:16,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:20,807 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:21,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:23,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:37:23,690 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:37:27,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:30,807 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:32,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:32,875 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:37:38,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:38,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:37:38,690 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:37:40,807 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:43,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:48,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:50,807 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:37:53,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:37:53,694 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:37:53,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:37:59,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:00,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:02,879 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:38:04,880 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:08,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:38:08,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:38:10,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:10,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:16,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:20,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:21,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:23,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:38:23,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:38:27,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:30,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:32,881 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:38:32,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:38,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:38,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:38:38,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:38:40,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:43,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:49,298 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:38:50,808 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:38:53,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:38:53,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:38:54,889 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:00,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:00,809 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:02,884 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:39:05,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:08,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:39:08,695 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:39:09,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:39:09,860 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:39:09,860 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:39:09,861 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:39:10,309 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:39:10,809 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:11,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:11,309 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:39:16,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:20,810 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:22,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:23,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:39:23,696 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:39:28,301 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:30,810 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:32,886 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:39:33,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:38,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:39:38,696 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:39:38,894 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:40,811 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:44,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:49,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:39:50,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:39:53,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:39:53,696 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:39:54,867 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:00,302 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:00,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:02,895 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:40:05,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:08,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:40:08,696 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:40:10,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:11,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:16,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:20,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:22,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:23,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:40:23,697 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:40:27,304 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:30,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:32,325 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:32,891 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:40:37,326 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:38,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:40:38,697 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:40:40,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:43,326 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:48,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:50,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:40:53,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:40:53,697 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:40:53,950 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:40:59,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:00,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:02,895 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:41:04,898 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:08,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:41:08,697 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:41:10,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:10,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:11,700 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:41:11,701 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:41:11,702 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:41:11,703 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:41:12,375 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:41:13,376 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:41:15,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:20,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:20,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:23,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:41:23,698 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:41:26,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:30,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:31,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:32,896 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:41:37,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:38,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:41:38,698 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:41:40,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:42,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:48,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:50,838 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:41:53,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:41:53,698 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:41:54,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:41:59,334 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:00,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:02,898 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:42:04,900 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:08,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:42:08,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:42:09,926 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:10,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:15,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:20,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:20,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:23,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:42:23,698 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:42:26,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:30,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:31,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:32,902 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:42:36,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:38,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:42:38,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:42:40,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:42,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:47,350 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:50,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:42:53,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:42:53,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:42:53,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:42:59,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:00,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:02,904 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:43:04,906 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:08,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:43:08,700 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:43:09,939 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:10,826 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:15,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:43:15,118 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:43:15,119 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:43:15,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:15,120 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:43:15,438 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:43:17,439 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:43:20,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:20,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:23,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:43:23,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:43:25,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:30,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:31,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:32,906 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:43:37,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:38,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:43:38,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:43:40,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:43,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:48,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:50,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:43:53,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:43:53,699 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:43:54,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:43:59,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:00,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:02,909 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:44:04,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:08,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:44:08,702 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:44:09,922 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:10,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:15,354 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:20,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:20,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:23,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:44:23,710 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:44:25,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:30,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:31,375 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:32,912 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:44:36,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:38,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:44:38,703 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:44:40,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:42,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:48,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:50,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:44:53,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:44:53,703 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:44:53,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:44:59,376 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:00,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:02,918 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:45:04,919 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:08,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:45:08,703 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:45:10,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:10,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:15,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:19,432 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:45:19,434 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:45:19,434 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:45:19,435 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:45:19,505 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:45:20,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:21,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:21,506 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:45:23,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:45:23,704 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:45:27,377 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:30,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:32,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:32,916 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:45:37,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:38,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:45:38,704 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:45:40,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:43,378 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:49,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:45:50,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:45:53,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:45:53,704 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:45:54,900 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:00,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:00,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:02,920 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:46:05,379 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:08,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:46:08,704 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:46:10,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:10,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:16,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:20,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:22,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:23,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:46:23,705 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:46:28,380 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:30,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:32,921 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:46:33,922 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:38,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:46:38,707 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:46:39,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:40,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:44,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:49,384 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:46:50,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:46:53,708 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:46:53,708 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:46:54,952 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:00,381 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:00,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:02,923 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:47:05,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:08,708 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:47:08,709 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:47:10,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:11,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:17,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:20,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:22,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:23,709 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:47:23,709 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:47:27,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:29,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:47:29,034 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:47:29,034 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:47:29,035 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:47:29,576 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:47:29,576 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:47:30,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:32,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:32,926 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:47:37,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:38,709 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:47:38,709 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:47:40,841 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:43,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:48,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:50,844 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:47:53,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:47:53,709 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:47:53,709 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:47:59,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:00,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:02,929 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:48:04,932 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:08,709 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:48:08,710 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:48:10,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:10,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:15,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:20,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:21,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:23,710 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:48:23,713 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:48:27,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:30,844 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:32,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:32,932 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:48:38,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:38,710 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:48:38,711 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:48:40,848 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:43,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:49,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:48:50,848 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:48:53,710 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:48:53,712 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:48:54,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:00,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:00,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:02,937 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:49:06,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:08,712 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:49:08,712 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:49:10,849 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:11,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:17,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:20,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:22,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:23,713 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:49:23,715 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:49:28,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:30,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:32,942 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:49:33,512 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:49:33,514 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:49:33,514 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:49:33,514 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:33,515 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:49:33,672 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:49:35,673 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:49:38,712 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:49:38,713 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:49:38,900 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:40,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:44,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:50,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:49:50,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:49:53,712 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:49:53,713 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:49:56,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:00,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:01,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:02,939 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:50:07,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:08,713 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:50:08,713 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:50:10,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:13,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:18,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:20,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:23,713 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:50:23,713 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:50:23,893 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:29,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:30,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:32,943 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:50:34,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:38,713 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:50:38,714 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:50:40,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:40,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:46,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:50,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:50:52,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:50:53,713 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:50:53,714 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:50:58,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:00,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:02,944 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:51:03,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:08,714 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:51:08,714 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:51:09,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:10,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:15,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:20,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:20,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:23,732 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:51:23,733 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:51:26,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:30,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:31,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:32,947 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:51:35,196 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:51:35,197 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:51:35,198 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:51:35,198 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:51:35,718 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:51:37,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:37,719 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:51:38,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:51:38,734 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:51:40,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:42,413 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:47,414 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:50,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:51:52,414 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:51:53,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:51:53,734 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:51:58,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:00,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:02,950 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:52:03,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:08,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:52:08,734 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:52:09,894 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:10,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:15,415 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:20,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:20,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:23,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:52:23,734 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:52:25,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:30,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:31,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:32,952 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:52:36,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:38,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:52:38,734 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:52:40,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:42,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:48,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:50,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:52:53,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:52:53,735 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:52:53,937 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:52:59,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:00,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:02,954 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:53:04,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:08,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:53:08,735 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:53:10,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:10,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:15,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:20,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:21,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:23,734 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:53:23,735 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:53:26,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:30,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:31,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:32,956 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:53:36,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:53:36,790 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:53:36,790 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:53:36,791 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:36,791 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:53:37,764 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:53:37,764 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:53:38,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:53:38,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:53:40,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:42,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:48,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:50,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:53:53,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:53:53,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:53:53,735 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:53:59,420 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:00,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:02,963 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:54:04,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:08,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:54:08,735 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:54:10,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:10,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:15,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:20,424 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:20,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:23,735 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:54:23,736 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:54:25,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:30,429 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:30,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:32,961 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:54:35,430 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:38,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:54:38,736 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:54:40,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:41,430 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:46,430 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:50,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:54:51,431 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:54:53,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:54:53,737 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:54:56,431 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:00,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:02,432 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:02,966 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:55:07,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:08,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:55:08,737 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:55:10,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:12,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:18,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:20,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:23,736 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:55:23,736 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:55:23,936 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:29,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:30,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:32,966 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:55:34,967 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:38,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:55:38,314 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:55:38,315 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:55:38,315 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:55:38,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:55:38,737 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:55:38,809 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:55:39,810 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:55:40,433 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:40,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:45,434 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:50,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:55:51,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:55:53,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:55:53,738 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:55:55,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:01,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:01,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:02,969 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:56:06,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:08,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:56:08,738 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:56:11,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:12,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:17,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:21,028 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:23,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:23,737 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:56:23,738 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:56:28,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:31,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:32,972 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:56:33,974 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:38,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:56:38,738 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:56:39,930 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:41,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:45,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:50,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:56:51,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:56:53,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:56:53,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:56:55,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:01,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:01,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:02,974 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:57:06,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:08,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:57:08,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:57:11,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:12,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:17,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:21,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:23,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:23,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:57:23,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:57:28,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:31,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:32,977 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:57:33,978 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:38,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:57:38,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:57:39,945 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:41,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:41,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:57:41,331 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:57:41,331 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:57:41,332 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:57:41,872 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:57:43,874 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:57:45,372 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:51,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:57:51,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:57:53,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:57:53,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:57:57,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:01,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:02,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:02,979 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:58:08,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:08,738 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:58:08,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:58:11,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:14,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:19,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:21,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:23,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:58:23,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:58:24,922 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:30,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:31,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:32,986 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:58:35,988 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:38,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:58:38,739 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:58:41,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:41,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:47,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:51,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:58:53,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:58:53,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:58:53,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:58:58,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:01,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:02,984 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:59:03,986 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:08,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:59:08,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:59:09,947 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:11,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:15,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:20,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:21,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:23,739 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:59:23,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:59:25,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:31,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:31,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:32,989 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 21:59:36,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:38,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:59:38,741 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:59:41,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:42,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:42,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 21:59:42,855 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 21:59:42,856 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 21:59:42,857 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 21:59:42,922 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 21:59:43,923 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 21:59:48,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:51,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 21:59:53,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 21:59:53,741 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 21:59:53,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 21:59:59,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:01,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:02,989 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:00:04,991 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:08,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:00:08,740 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:00:10,912 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:11,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:16,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:21,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:22,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:23,740 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:00:23,741 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:00:27,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:31,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:32,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:32,992 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:00:37,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:38,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:00:38,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:00:41,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:43,052 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:48,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:51,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:00:53,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:00:53,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:00:53,954 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:00:59,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:01,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:02,995 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:01:04,997 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:08,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:01:08,741 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:01:10,962 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:11,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:16,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:21,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:21,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:23,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:01:23,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:01:26,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:31,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:31,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:32,997 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:01:36,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:38,741 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:01:38,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:01:41,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:42,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:44,492 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:01:44,494 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:01:44,495 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:01:44,496 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:01:44,973 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:01:45,974 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:01:47,054 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:51,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:01:53,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:01:53,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:01:53,742 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:01:59,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:01,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:02,999 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:02:05,001 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:08,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:02:08,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:02:10,903 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:11,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:16,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:21,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:22,055 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:23,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:02:23,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:02:27,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:31,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:33,002 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:02:33,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:38,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:38,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:02:38,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:02:41,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:44,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:49,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:02:51,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:02:53,742 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:02:53,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:02:54,891 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:00,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:01,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:03,008 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:03:06,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:08,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:03:08,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:03:11,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:11,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:16,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:21,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:22,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:23,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:03:23,744 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:03:27,058 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:31,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:33,007 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:03:33,010 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:38,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:38,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:03:38,743 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:03:41,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:44,059 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:47,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:03:47,663 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:03:47,663 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:03:47,664 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:03:48,048 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:03:49,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:03:50,050 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:03:51,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:03:53,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:03:53,744 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:03:54,957 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:00,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:01,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:03,010 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:04:06,012 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:08,743 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:04:08,744 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:04:11,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:11,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:16,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:21,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:22,060 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:23,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:04:23,744 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:04:27,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:31,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:33,012 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:04:33,015 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:38,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:38,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:04:38,744 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:04:41,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:44,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:49,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:04:51,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:04:53,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:04:53,745 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:04:54,932 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:00,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:01,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:03,015 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:05:06,016 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:08,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:05:08,745 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:05:11,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:11,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:16,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:21,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:22,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:23,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:05:23,745 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:05:28,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:31,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:33,017 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:05:34,018 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:38,744 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:05:38,745 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:05:39,951 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:41,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:45,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:49,134 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:05:49,136 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:05:49,136 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:05:49,137 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:05:50,108 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:05:50,108 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:05:50,179 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:05:51,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:05:53,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:05:53,746 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:05:55,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:01,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:01,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:03,019 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:06:07,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:08,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:06:08,746 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:06:11,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:12,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:18,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:21,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:23,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:23,745 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:06:23,746 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:06:29,066 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:31,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:33,022 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:06:35,023 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:38,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:06:38,746 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:06:40,886 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:41,086 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:46,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:51,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:06:52,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:06:53,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:06:53,746 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:06:58,067 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:01,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:03,024 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:07:04,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:08,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:07:08,747 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:07:09,964 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:11,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:15,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:21,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:21,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:23,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:07:23,747 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:07:27,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:31,087 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:32,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:33,027 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:07:38,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:38,746 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:07:38,747 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:07:41,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:44,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:49,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:07:50,623 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:07:50,625 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:07:50,625 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:07:50,626 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:07:51,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:07:51,176 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:07:52,177 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:07:53,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:07:53,747 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:07:54,956 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:00,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:01,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:03,030 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:08:06,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:08,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:08:08,747 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:08:11,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:11,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:16,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:21,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:21,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:23,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:08:23,748 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:08:26,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:31,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:32,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:33,032 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:08:37,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:38,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:08:38,748 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:08:41,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:43,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:48,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:51,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:08:53,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:08:53,747 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:08:53,748 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:08:59,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:01,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:03,035 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:09:05,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:08,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:09:08,749 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:09:10,917 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:11,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:16,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:21,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:21,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:23,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:09:23,749 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:09:26,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:31,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:32,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:33,037 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:09:37,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:38,748 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:09:38,750 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:09:41,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:42,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:48,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:51,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:09:53,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:09:53,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:09:53,749 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:09:53,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:09:53,947 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:09:53,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:09:53,949 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:09:54,244 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:09:56,246 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:09:59,397 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:01,396 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:03,040 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:10:05,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:08,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:10:08,750 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:10:10,968 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:11,400 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:16,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:21,399 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:21,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:23,749 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:10:23,750 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:10:27,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:31,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:33,043 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:10:33,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:38,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:38,750 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:10:38,751 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:10:41,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:43,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:49,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:10:51,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:10:53,750 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:10:53,751 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:10:54,922 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:00,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:01,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:03,046 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:11:06,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:08,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:11:08,751 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:11:11,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:11,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:16,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:21,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:22,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:23,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:11:23,751 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:11:27,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:31,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:33,049 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:11:33,052 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:38,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:38,751 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:11:38,751 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:11:41,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:44,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:50,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:51,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:11:53,754 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:11:53,755 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:11:55,529 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:11:55,532 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:11:55,532 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:11:55,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:11:55,534 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:11:56,296 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:11:56,297 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:12:01,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:01,405 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:03,051 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:12:06,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:08,754 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:12:08,755 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:12:11,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:12,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:17,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:21,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:23,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:23,755 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:12:23,755 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:12:28,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:31,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:33,056 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:12:34,057 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:38,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:12:38,757 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:12:39,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:41,403 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:45,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:50,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:12:51,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:12:53,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:12:53,756 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:12:55,943 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:01,404 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:01,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:03,055 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:13:06,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:08,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:13:08,757 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:13:11,406 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:11,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:17,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:21,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:22,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:23,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:13:23,759 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:13:27,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:31,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:32,411 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:33,059 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:13:37,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:38,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:13:38,758 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:13:41,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:43,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:48,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:51,407 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:13:53,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:13:53,758 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:13:54,008 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:13:57,272 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:13:57,273 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:13:57,274 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:13:57,274 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:13:57,356 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:13:58,358 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:13:59,316 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:01,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:03,060 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:14:05,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:08,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:14:08,758 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:14:11,011 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:11,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:16,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:21,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:21,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:23,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:14:23,758 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:14:27,418 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:31,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:33,063 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:14:33,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:38,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:38,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:14:38,758 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:14:41,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:44,419 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:49,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:14:51,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:14:53,760 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:14:53,763 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:14:54,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:00,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:01,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:03,065 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:15:06,068 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:08,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:15:08,759 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:15:11,410 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:11,421 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:16,422 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:21,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:21,580 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:23,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:15:23,760 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:15:26,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:31,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:32,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:33,067 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:15:38,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:38,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:15:38,767 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:15:41,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:43,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:49,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:15:51,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:15:53,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:15:53,764 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:15:55,241 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:00,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:01,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:16:01,230 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:16:01,230 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:16:01,231 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:16:01,431 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:16:01,581 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:02,433 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:16:03,070 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:16:06,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:08,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:16:08,764 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:16:11,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:11,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:17,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:21,582 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:23,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:23,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:16:23,765 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:16:28,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:31,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:33,072 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:16:34,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:38,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:16:38,764 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:16:39,915 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:41,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:45,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:50,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:16:51,583 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:16:53,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:16:53,764 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:16:55,921 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:01,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:01,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:03,081 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:17:06,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:08,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:17:08,765 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:17:11,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:12,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:17,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:21,584 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:23,591 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:23,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:17:23,765 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:17:28,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:31,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:33,077 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:17:34,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:38,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:17:38,765 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:17:39,985 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:41,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:45,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:50,608 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:17:51,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:17:53,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:17:53,765 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:17:56,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:01,585 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:01,623 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:03,081 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:18:03,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:18:03,768 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:18:03,768 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:18:03,769 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:18:04,503 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:18:04,504 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:18:06,623 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:08,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:18:08,766 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:18:11,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:11,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:16,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:21,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:22,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:23,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:18:23,766 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:18:27,625 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:31,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:33,083 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:18:33,085 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:38,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:38,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:18:38,767 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:18:41,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:44,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:50,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:18:51,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:18:53,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:18:53,767 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:18:55,971 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:01,587 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:01,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:03,085 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:19:06,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:08,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:19:08,767 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:19:11,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:12,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:17,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:21,588 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:23,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:23,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:19:23,767 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:19:29,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:31,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:33,087 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:19:35,088 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:38,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:19:38,768 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:19:40,971 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:41,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:46,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:51,589 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:19:52,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:19:53,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:19:53,768 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:19:57,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:01,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:02,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:03,089 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:20:06,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:20:06,055 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:20:06,055 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:20:06,056 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:20:06,577 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:20:08,578 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:20:08,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:08,768 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:20:08,768 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:20:11,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:14,630 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:20,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:21,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:23,782 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:20:23,782 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:20:25,966 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:31,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:31,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:33,092 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:20:36,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:38,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:20:38,774 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:20:41,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:42,631 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:47,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:51,592 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:20:53,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:20:53,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:20:53,774 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:20:58,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:01,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:03,095 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:21:04,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:08,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:21:08,775 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:21:09,994 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:11,593 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:15,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:21,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:21,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:23,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:21:23,775 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:21:26,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:31,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:32,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:33,102 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:21:37,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:38,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:21:38,775 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:21:41,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:43,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:48,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:51,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:21:54,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:21:54,041 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:21:54,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:21:59,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:01,596 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:03,102 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:22:05,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:09,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:22:09,041 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:22:09,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:22:09,835 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:22:09,835 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:22:09,839 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:22:10,654 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:22:10,654 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:22:10,839 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:11,601 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:16,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:21,603 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:22,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:24,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:22:24,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:22:27,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:31,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:32,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:33,102 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:22:37,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:39,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:22:39,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:22:41,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:43,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:49,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:22:51,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:22:54,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:22:54,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:22:55,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:00,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:01,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:03,106 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:23:06,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:09,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:23:09,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:23:11,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:11,604 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:16,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:21,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:22,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:24,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:23:24,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:23:28,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:31,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:33,106 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:23:34,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:39,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:23:39,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:23:39,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:41,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:44,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:49,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:23:51,606 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:23:54,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:23:54,043 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:23:55,213 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:00,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:01,606 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:03,109 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:24:06,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:09,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:24:09,044 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:24:11,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:11,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:24:11,541 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:24:11,542 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:24:11,543 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:24:11,605 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:11,736 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:24:12,737 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:24:16,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:21,606 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:22,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:24,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:24:24,044 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:24:27,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:31,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:33,111 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:24:33,114 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:38,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:39,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:24:39,044 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:24:41,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:44,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:49,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:24:51,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:24:54,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:24:54,044 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:24:55,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:00,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:01,607 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:03,113 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:25:06,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:09,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:25:09,044 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:25:11,196 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:11,608 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:16,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:21,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:22,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:24,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:25:24,045 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:25:27,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:31,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:33,115 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:25:33,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:38,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:39,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:25:39,045 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:25:41,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:43,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:49,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:25:51,611 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:25:54,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:25:54,045 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:25:55,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:00,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:01,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:03,121 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:26:06,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:09,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:26:09,045 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:26:11,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:11,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:13,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:26:13,256 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:26:13,256 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:26:13,257 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:26:13,817 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:26:14,818 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:26:16,299 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:21,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:21,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:24,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:26:24,046 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:26:26,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:31,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:32,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:33,120 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:26:37,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:39,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:26:39,046 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:26:41,612 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:43,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:48,653 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:51,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:26:54,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:26:54,047 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:26:54,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:26:59,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:01,613 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:03,125 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:27:05,127 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:09,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:27:09,047 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:27:10,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:11,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:15,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:21,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:21,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:24,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:27:24,047 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:27:26,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:31,614 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:32,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:33,126 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:27:37,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:39,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:27:39,048 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:27:41,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:43,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:48,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:51,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:27:54,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:27:54,048 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:27:54,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:27:59,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:01,615 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:03,129 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:28:05,131 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:09,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:28:09,048 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:28:10,268 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:11,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:15,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:28:15,092 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:28:15,092 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:28:15,093 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:28:15,922 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:28:16,131 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:16,923 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:28:21,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:21,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:24,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:28:24,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:28:26,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:31,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:31,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:33,132 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:28:37,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:39,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:28:39,048 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:28:41,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:43,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:48,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:51,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:28:54,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:28:54,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:28:54,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:28:59,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:01,616 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:03,135 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:29:05,137 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:09,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:29:09,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:29:10,255 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:11,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:15,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:20,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:21,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:24,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:29:24,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:29:26,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:31,618 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:31,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:33,137 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:29:36,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:39,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:29:39,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:29:41,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:42,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:47,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:51,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:29:53,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:29:54,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:29:54,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:29:58,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:01,619 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:03,139 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:30:04,141 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:09,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:30:09,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:30:09,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:11,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:14,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:18,617 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:30:18,619 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:30:18,619 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:30:18,620 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:30:19,016 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:30:20,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:21,018 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:30:21,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:24,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:30:24,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:30:26,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:31,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:31,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:33,142 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:30:36,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:39,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:30:39,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:30:41,620 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:42,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:47,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:51,621 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:30:53,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:30:54,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:30:54,062 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:30:58,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:01,621 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:03,148 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:31:04,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:09,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:31:09,061 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:31:09,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:11,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:14,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:20,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:21,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:24,061 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:31:24,062 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:31:26,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:31,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:31,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:33,147 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:31:36,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:39,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:31:39,062 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:31:41,624 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:42,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:47,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:51,625 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:31:53,690 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:31:54,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:31:54,062 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:31:58,691 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:01,625 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:03,151 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:32:04,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:09,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:32:09,062 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:32:09,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:11,625 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:14,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:19,693 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:20,459 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:32:20,461 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:32:20,462 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:32:20,462 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:32:21,100 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:32:21,625 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:23,101 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:32:24,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:32:24,063 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:32:25,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:30,692 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:31,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:33,152 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:32:36,154 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:39,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:32:39,063 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:32:41,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:41,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:46,694 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:51,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:32:51,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:32:54,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:32:54,063 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:32:56,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:01,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:02,695 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:03,154 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:33:07,696 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:09,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:33:09,063 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:33:11,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:12,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:18,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:21,626 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:23,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:24,062 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:33:24,063 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:33:29,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:31,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:33,157 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:33:35,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:39,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:33:39,064 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:33:40,261 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:41,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:45,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:50,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:33:51,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:33:54,063 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:33:54,064 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:33:56,195 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:01,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:01,702 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:03,159 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:34:06,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:09,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:34:09,064 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:34:11,627 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:12,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:17,703 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:21,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:21,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:34:21,992 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:34:21,992 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:34:21,993 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:34:22,154 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:34:23,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:23,155 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:34:24,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:34:24,065 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:34:28,704 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:31,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:33,162 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:34:34,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:39,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:34:39,065 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:34:39,287 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:41,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:44,705 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:49,705 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:34:51,628 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:34:54,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:34:54,065 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:34:55,234 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:00,705 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:01,629 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:03,164 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:35:06,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:09,064 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:35:09,066 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:35:11,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:11,629 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:16,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:21,629 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:21,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:24,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:35:24,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:35:26,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:31,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:32,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:33,170 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:35:37,707 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:39,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:35:39,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:35:41,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:43,706 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:48,707 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:51,633 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:35:54,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:35:54,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:35:54,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:35:59,707 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:01,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:03,169 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:36:05,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:09,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:36:09,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:36:10,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:11,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:15,720 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:20,721 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:21,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:24,069 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:36:24,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:36:24,959 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:36:24,961 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:36:24,961 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:36:24,964 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:36:25,212 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:36:25,965 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:27,213 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:36:31,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:31,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:33,171 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:36:36,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:39,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:36:39,070 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:36:41,634 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:42,722 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:47,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:51,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:36:53,723 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:36:54,070 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:36:54,071 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:36:58,724 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:01,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:03,175 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:37:04,176 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:09,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:37:09,071 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:37:09,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:11,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:14,724 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:19,725 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:21,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:24,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:37:24,071 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:37:25,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:30,726 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:31,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:33,176 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:37:36,178 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:39,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:37:39,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:37:41,297 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:41,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:46,730 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:51,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:37:52,730 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:37:54,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:37:54,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:37:57,731 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:01,636 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:03,178 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:38:03,181 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:08,731 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:09,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:38:09,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:38:11,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:14,731 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:19,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:21,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:24,071 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:38:24,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:38:25,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:26,479 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:38:26,482 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:38:26,482 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:38:26,483 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:38:27,269 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:38:29,270 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:38:30,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:31,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:33,181 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:38:36,183 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:39,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:38:39,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:38:41,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:41,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:46,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:51,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:38:51,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:38:54,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:38:54,072 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:38:57,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:01,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:02,733 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:03,184 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:39:07,755 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:09,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:39:09,073 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:39:11,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:13,755 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:18,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:21,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:24,072 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:39:24,073 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:39:24,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:29,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:31,638 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:33,187 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:39:35,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:39,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:39:39,073 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:39:40,313 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:41,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:45,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:51,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:39:51,756 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:39:54,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:39:54,074 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:39:56,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:01,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:02,757 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:03,192 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:40:07,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:09,073 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:40:09,074 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:40:11,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:13,758 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:18,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:21,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:24,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:40:24,074 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:40:24,269 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:28,432 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:40:28,434 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:40:28,434 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:40:28,435 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:40:29,337 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:40:29,475 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:31,338 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:40:31,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:33,192 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:40:35,193 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:39,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:40:39,074 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:40:40,236 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:41,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:45,759 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:50,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:40:51,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:40:54,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:40:54,075 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:40:56,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:01,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:01,762 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:03,193 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:41:06,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:09,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:41:09,075 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:41:11,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:11,763 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:16,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:21,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:21,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:24,074 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:41:24,075 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:41:26,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:31,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:32,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:33,196 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:41:37,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:39,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:41:39,075 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:41:41,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:43,765 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:48,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:51,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:41:53,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:41:54,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:41:54,075 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:41:59,767 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:01,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:03,200 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:42:05,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:09,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:42:09,076 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:42:10,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:11,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:15,766 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:20,769 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:21,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:24,075 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:42:24,076 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:42:25,774 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:31,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:31,770 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:31,962 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:42:31,964 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:42:31,964 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:42:31,965 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:42:32,400 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:42:32,400 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:42:33,201 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:42:36,770 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:39,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:42:39,077 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:42:41,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:42,770 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:47,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:51,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:42:52,777 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:42:54,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:42:54,077 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:42:57,778 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:01,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:03,206 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:43:03,207 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:08,779 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:09,076 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:43:09,077 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:43:11,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:13,779 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:18,781 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:21,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:24,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:43:24,078 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:43:24,271 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:29,780 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:31,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:33,207 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:43:35,209 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:39,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:43:39,078 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:43:40,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:41,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:45,781 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:51,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:43:51,780 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:43:54,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:43:54,078 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:43:56,781 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:01,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:01,781 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:03,209 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:44:06,783 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:09,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:44:09,078 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:44:11,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:12,782 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:17,782 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:21,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:23,782 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:24,077 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:44:24,078 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:44:28,783 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:31,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:33,217 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:44:33,970 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:44:33,973 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:44:33,973 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:44:33,974 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:33,974 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:44:34,459 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:44:34,459 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:44:39,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:44:39,081 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:44:39,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:41,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:44,785 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:49,786 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:44:51,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:44:54,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:44:54,079 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:44:55,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:00,786 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:01,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:03,214 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:45:06,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:09,078 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:45:09,079 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:45:11,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:11,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:16,786 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:21,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:22,786 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:24,079 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:45:24,079 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:45:27,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:31,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:33,218 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:45:33,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:38,787 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:39,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:45:39,082 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:45:41,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:44,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:50,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:45:51,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:45:54,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:45:54,081 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:45:56,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:01,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:01,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:03,220 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:46:06,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:09,080 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:46:09,081 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:46:11,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:12,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:18,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:21,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:24,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:46:24,082 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:46:24,546 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:29,790 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:31,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:33,222 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:46:35,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:38,990 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:46:38,992 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:46:38,992 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:46:38,993 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:46:39,081 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:46:39,081 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:46:39,519 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:46:40,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:40,520 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:46:41,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:45,791 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:50,793 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:46:51,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:46:54,082 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:46:54,083 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:46:56,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:01,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:01,792 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:03,225 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:47:06,794 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:09,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:47:09,095 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:47:11,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:12,794 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:18,794 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:21,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:24,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:47:24,095 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:47:24,352 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:29,795 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:31,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:33,227 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:47:35,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:39,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:47:39,096 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:47:40,341 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:41,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:45,795 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:51,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:47:51,796 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:47:54,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:47:54,096 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:47:57,797 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:01,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:03,230 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:48:03,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:08,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:09,095 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:48:09,095 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:48:11,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:14,797 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:20,797 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:21,653 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:24,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:48:24,098 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:48:26,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:31,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:31,797 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:33,232 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:48:36,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:39,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:48:39,097 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:48:41,652 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:42,541 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:48:42,542 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:48:42,543 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:48:42,543 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:48:42,544 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:42,577 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:48:44,579 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:48:47,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:51,653 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:48:53,798 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:48:54,096 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:48:54,096 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:48:58,799 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:01,653 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:03,239 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:49:04,241 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:09,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:49:09,097 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:49:09,343 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:11,653 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:14,800 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:19,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:21,657 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:24,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:49:24,099 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:49:25,357 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:30,801 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:31,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:33,237 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:49:36,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:39,097 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:49:39,098 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:49:41,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:41,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:46,801 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:51,654 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:49:51,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:49:54,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:49:54,098 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:49:56,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:01,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:02,802 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:03,242 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:50:07,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:09,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:50:09,098 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:50:11,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:13,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:18,804 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:21,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:23,811 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:24,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:50:24,098 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:50:29,809 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:31,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:33,242 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:50:35,243 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:39,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:50:39,099 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:50:40,471 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:41,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:45,810 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:48,499 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:50:48,501 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:50:48,501 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:50:48,502 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:50:48,646 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:50:50,647 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:50:51,543 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:50:51,655 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:50:54,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:50:54,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:50:56,810 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:01,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:02,810 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:03,245 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:51:07,827 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:09,098 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:51:09,099 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:51:11,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:13,811 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:18,812 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:21,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:23,812 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:24,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:51:24,099 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:51:28,813 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:31,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:33,248 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:51:34,249 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:39,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:51:39,099 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:51:39,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:41,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:44,814 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:50,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:51:51,656 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:51:54,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:51:54,100 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:51:56,353 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:01,657 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:01,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:03,252 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:52:06,815 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:09,099 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:52:09,100 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:52:11,657 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:11,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:17,816 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:21,657 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:23,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:24,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:52:24,100 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:52:28,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:31,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:33,253 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:52:34,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:39,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:52:39,100 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:52:39,484 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:41,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:44,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:49,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:52:50,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:52:50,153 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:52:50,153 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:52:50,154 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:52:50,706 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:52:50,706 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:52:51,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:52:54,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:52:54,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:52:55,349 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:00,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:01,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:03,256 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:53:06,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:09,100 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:53:09,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:53:11,463 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:11,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:16,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:21,658 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:22,819 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:24,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:53:24,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:53:28,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:31,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:33,258 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:53:34,260 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:39,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:53:39,101 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:53:39,361 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:41,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:44,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:49,821 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:53:51,659 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:53:54,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:53:54,102 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:53:55,312 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:00,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:01,660 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:03,265 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:54:06,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:09,101 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:54:09,102 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:54:11,310 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:11,661 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:16,822 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:21,662 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:22,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:24,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:54:24,102 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:54:27,823 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:31,662 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:33,263 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:54:33,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:38,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:39,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:54:39,102 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:54:41,662 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:43,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:48,824 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:51,663 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:54:53,252 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:54:53,253 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:54:53,254 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:54:53,256 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:54:53,769 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:54:54,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:54:54,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:54:54,358 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:54:54,770 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:54:59,825 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:01,663 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:03,266 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:55:05,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:09,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:55:09,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:55:11,247 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:11,663 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:16,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:21,663 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:21,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:24,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:55:24,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:55:26,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:31,664 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:31,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:33,270 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:55:36,829 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:39,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:55:39,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:55:41,664 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:42,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:48,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:51,664 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:55:53,838 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:55:54,102 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:55:54,103 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:55:59,831 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:01,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:03,271 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:56:05,273 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:09,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:56:09,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:56:10,355 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:11,665 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:15,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:20,833 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:21,672 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:24,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:56:24,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:56:26,288 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:31,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:31,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:33,273 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:56:37,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:39,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:56:39,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:56:41,673 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:43,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:48,832 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:51,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:56:54,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:56:54,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:56:54,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:56:55,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:56:55,120 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:56:55,121 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:56:55,121 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:56:55,841 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:56:56,842 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:56:59,833 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:01,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:03,275 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:57:05,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:09,103 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:57:09,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:57:11,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:11,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:16,834 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:21,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:22,833 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:24,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:57:24,104 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:57:27,834 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:31,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:32,834 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:33,277 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:57:37,835 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:39,104 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:57:39,105 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:57:41,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:43,835 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:48,839 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:51,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:57:54,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:57:54,107 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:57:54,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:57:59,837 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:01,675 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:03,280 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:58:05,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:09,106 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:58:09,107 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:58:11,401 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:11,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:16,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:21,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:22,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:24,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:58:24,107 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:58:28,850 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:31,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:33,288 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:58:34,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:39,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:58:39,108 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:58:39,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:41,676 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:44,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:50,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:51,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:58:54,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:58:54,108 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:58:56,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:58:56,863 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 22:58:56,864 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 22:58:56,865 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 22:58:56,866 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 22:58:56,901 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 22:58:58,902 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 22:59:01,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:01,851 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:03,284 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:59:06,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:09,107 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:59:09,108 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:59:11,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:12,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:17,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:21,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:23,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:24,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:59:24,108 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:59:28,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:31,677 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:33,288 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 22:59:34,290 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:39,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:59:39,108 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:59:39,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:41,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:44,853 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:49,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 22:59:51,678 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 22:59:54,108 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 22:59:54,109 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 22:59:55,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:00,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:01,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:03,290 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:00:06,291 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:09,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:00:09,110 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:00:11,516 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:11,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:16,855 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:21,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:21,858 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:24,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:00:24,110 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:00:27,856 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:31,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:33,292 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:00:33,293 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:38,857 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:39,109 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:00:39,110 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:00:41,679 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:43,862 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:48,867 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:51,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:00:53,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:00:54,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:00:54,110 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:00:58,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:01,680 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:01,949 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:01:01,951 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:01:01,952 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:01:01,952 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:01:01,976 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:01:02,976 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:01:03,294 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:01:04,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:09,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:01:09,111 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:01:10,273 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:11,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:15,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:21,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:21,869 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:24,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:01:24,111 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:01:26,870 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:31,681 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:32,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:33,300 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:01:38,871 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:39,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:01:39,111 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:01:41,682 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:43,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:49,872 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:01:51,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:01:54,110 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:01:54,111 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:01:55,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:00,877 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:01,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:03,302 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:02:06,306 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:09,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:02:09,111 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:02:11,336 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:11,683 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:16,873 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:21,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:21,874 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:24,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:02:24,112 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:02:26,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:31,684 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:32,878 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:33,304 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:02:37,879 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:39,111 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:02:39,112 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:02:41,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:42,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:48,882 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:51,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:02:54,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:02:54,112 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:02:54,322 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:02:59,881 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:01,685 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:03,361 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:03:04,913 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:03:04,915 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:03:04,916 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:03:04,916 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:04,917 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:03:05,061 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:03:07,062 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:03:09,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:03:09,112 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:03:10,326 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:11,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:15,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:21,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:21,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:24,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:03:24,113 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:03:27,883 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:31,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:32,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:33,310 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:03:37,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:39,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:03:39,113 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:03:41,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:43,884 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:48,885 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:51,686 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:03:54,112 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:03:54,113 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:03:54,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:03:59,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:01,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:03,314 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:04:05,316 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:09,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:04:09,113 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:04:11,303 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:11,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:16,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:21,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:21,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:24,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:04:24,113 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:04:26,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:31,687 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:32,887 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:33,315 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:04:37,888 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:39,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:04:39,114 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:04:41,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:42,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:48,904 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:51,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:04:53,905 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:04:54,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:04:54,114 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:04:58,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:01,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:03,326 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:05:04,330 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:06,946 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:05:06,947 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:05:06,948 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:05:06,949 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:05:07,115 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:05:09,113 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:05:09,114 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:05:09,116 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:05:10,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:11,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:15,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:20,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:21,688 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:24,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:05:24,125 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:05:26,264 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:31,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:31,909 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:33,322 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:05:36,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:39,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:05:39,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:05:41,689 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:42,910 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:48,911 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:51,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:05:54,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:05:54,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:05:54,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:05:59,913 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:01,697 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:03,327 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:06:05,331 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:09,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:06:09,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:06:11,292 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:11,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:16,913 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:21,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:22,913 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:24,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:06:24,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:06:27,914 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:31,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:32,924 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:33,327 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:06:37,924 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:39,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:06:39,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:06:41,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:43,924 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:48,925 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:51,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:06:54,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:06:54,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:06:54,283 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:06:59,926 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:01,698 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:03,329 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:07:05,333 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:08,902 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:07:08,904 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:07:08,904 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:07:08,905 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:07:09,115 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:07:09,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:07:09,184 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:07:11,185 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:07:11,267 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:11,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:16,926 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:21,927 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:23,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:24,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:07:24,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:07:27,265 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:33,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:33,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:33,335 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:07:38,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:39,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:07:39,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:07:43,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:43,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:49,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:07:53,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:07:54,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:07:54,116 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:07:54,254 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:00,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:03,215 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:03,334 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:08:05,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:09,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:08:09,117 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:08:11,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:13,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:17,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:22,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:23,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:24,116 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:08:24,117 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:08:27,337 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:33,216 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:33,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:33,338 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:08:38,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:39,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:08:39,117 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:08:43,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:44,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:49,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:08:53,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:08:54,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:08:54,117 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:08:54,274 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:00,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:03,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:03,339 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:09:05,342 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:09,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:09:09,118 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:09:11,256 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:12,065 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:09:12,068 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:09:12,068 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:09:12,069 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:09:12,241 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:09:13,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:13,241 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:09:17,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:22,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:23,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:24,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:09:24,118 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:09:27,310 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:33,219 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:33,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:33,341 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:09:38,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:39,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:09:39,118 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:09:43,221 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:44,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:49,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:09:53,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:09:54,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:09:54,118 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:09:54,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:00,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:03,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:03,343 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:10:05,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:09,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:10:09,119 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:10:11,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:13,222 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:17,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:22,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:23,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:24,118 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:10:24,119 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:10:27,307 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:33,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:33,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:33,346 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:10:38,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:39,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:10:39,120 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:10:43,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:43,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:48,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:10:53,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:10:54,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:10:54,120 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:10:54,296 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:00,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:03,224 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:03,348 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:11:05,351 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:09,119 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:11:09,120 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:11:11,295 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:13,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:13,586 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:11:13,588 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:11:13,588 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:11:13,589 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:11:14,295 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:11:15,296 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:11:16,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:22,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:23,225 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:24,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:11:24,121 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:11:27,240 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:33,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:33,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:33,351 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:11:38,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:39,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:11:39,120 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:11:43,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:44,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:49,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:11:53,226 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:11:54,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:11:54,121 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:11:54,273 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:00,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:03,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:03,356 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:12:05,360 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:09,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:12:09,121 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:12:11,279 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:13,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:17,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:22,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:23,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:24,120 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:12:24,121 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:12:27,252 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:33,227 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:33,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:33,355 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:12:38,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:39,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:12:39,122 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:12:43,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:44,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:49,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:12:53,228 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:12:54,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:12:54,122 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:12:54,277 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:00,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:03,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:03,360 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:13:05,362 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:09,121 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:13:09,122 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:13:11,294 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:13,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:15,436 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:13:15,437 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:13:15,438 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:13:15,439 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:13:16,353 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:13:16,479 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:17,354 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:13:22,235 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:23,229 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:24,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:13:24,122 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:13:27,275 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:33,230 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:33,237 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:33,361 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:13:39,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:13:39,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:13:39,305 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:43,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:45,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:50,238 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:13:53,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:13:54,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:13:54,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:13:55,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:01,239 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:03,231 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:03,363 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:14:06,365 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:09,122 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:14:09,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:14:12,318 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:13,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:18,240 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:23,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:24,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:14:24,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:14:24,339 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:30,239 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:33,232 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:33,365 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:14:35,368 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:39,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:14:39,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:14:41,270 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:43,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:47,240 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:52,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:14:53,233 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:14:54,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:14:54,124 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:14:57,346 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:03,237 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:03,257 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:03,368 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:15:08,259 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:09,123 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:15:09,123 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:15:13,237 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:14,258 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:18,590 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:15:18,591 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:15:18,592 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:15:18,593 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:15:19,405 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:15:19,635 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:21,406 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:15:24,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:26,327 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:26,328 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:15:26,328 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:15:29,409 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:15:30,524 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:33,373 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:15:36,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:36,344 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:37,418 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:15:41,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:15:41,329 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:15:41,519 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:46,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:47,395 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:53,394 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:15:56,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:15:56,392 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:15:56,402 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:15:58,594 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:03,398 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:16:04,402 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:06,393 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:10,398 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:11,329 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:16:11,330 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:16:15,533 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:16,542 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:20,555 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:26,335 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:16:26,336 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:16:26,478 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:26,530 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:31,521 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:33,404 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:16:36,537 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:37,204 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:41,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:16:41,601 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:16:42,213 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:46,540 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:47,214 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:52,217 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:16:56,538 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:16:56,600 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:16:56,600 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:16:57,218 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:02,220 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:03,404 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:17:06,539 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:07,223 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:11,699 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:17:11,700 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:17:12,332 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:16,568 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:18,262 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:23,278 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:26,632 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:17:26,633 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:17:26,674 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:28,323 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:33,324 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:33,407 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:17:36,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:39,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:41,637 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:17:41,638 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:17:44,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:46,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:49,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:54,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:17:56,595 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:17:56,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:17:56,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:17:59,089 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:03,409 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:18:04,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:06,597 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:10,117 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:11,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:18:11,639 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:18:15,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:16,621 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:20,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:26,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:26,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:26,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:18:26,639 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:18:31,142 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:33,411 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:18:36,143 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:36,610 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:41,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:18:41,639 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:18:41,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:47,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:48,025 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:53,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:18:56,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:18:56,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:18:58,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:18:58,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:03,414 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:19:04,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:08,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:19:10,144 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:11,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:19:11,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:19:15,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:18,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:19:21,145 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:26,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:26,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:19:26,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:19:28,026 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:19:32,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:33,416 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:19:38,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:19:38,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:41,639 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:19:41,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:19:43,803 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:48,029 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:19:49,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:55,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:19:56,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:19:56,640 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:19:58,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:00,147 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:03,418 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:20:05,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:08,030 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:11,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:11,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:20:11,641 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:20:17,148 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:18,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:22,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:26,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:20:26,641 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:20:27,846 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:28,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:33,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:33,420 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:20:38,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:38,149 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:41,640 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:20:41,641 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:20:43,807 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:48,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:20:49,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:54,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:20:56,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:20:56,641 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:20:58,031 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:00,150 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:03,423 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:21:05,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:08,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:10,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:11,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:21:11,641 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:21:16,151 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:18,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:22,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:26,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:21:26,642 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:21:27,830 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:28,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:33,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:33,430 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:21:38,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:38,152 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:41,641 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:21:41,642 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:21:43,854 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:48,032 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:21:49,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:54,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:21:56,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:21:56,642 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:21:58,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:00,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:03,429 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:22:06,153 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:08,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:11,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:22:11,642 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:22:11,773 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:17,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:18,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:22,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:26,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:22:26,642 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:22:27,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:28,033 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:33,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:33,432 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:22:38,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:38,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:41,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:22:41,643 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:22:43,820 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:48,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:49,155 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:54,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:22:56,642 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:22:56,643 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:22:58,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:22:59,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:03,433 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:23:05,156 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:08,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:23:11,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:11,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:23:11,643 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:23:16,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:18,034 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:23:22,157 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:26,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:23:26,643 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:23:27,837 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:28,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:23:33,158 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:33,435 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:23:38,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:23:39,158 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:41,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:23:41,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:23:44,158 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:48,035 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:23:49,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:55,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:23:56,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:23:56,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:23:58,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:01,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:03,438 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:24:07,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:08,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:11,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:24:11,643 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:24:12,840 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:18,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:18,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:24,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:26,643 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:24:26,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:24:28,036 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:30,159 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:33,441 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:24:35,161 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:38,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:41,160 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:41,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:24:41,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:24:46,161 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:48,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:24:52,161 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:56,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:24:56,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:24:57,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:24:58,037 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:03,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:03,443 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:25:08,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:08,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:11,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:25:11,644 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:25:13,764 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:18,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:19,162 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:24,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:26,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:25:26,645 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:25:28,038 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:30,163 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:33,445 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:25:35,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:38,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:41,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:41,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:25:41,645 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:25:46,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:48,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:25:52,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:56,644 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:25:56,645 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:25:57,771 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:25:58,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:03,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:03,456 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:26:08,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:09,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:11,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:26:11,646 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:26:14,164 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:18,039 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:20,165 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:25,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:26,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:26:26,646 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:26:28,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:31,166 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:33,450 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:26:36,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:38,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:41,645 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:26:41,646 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:26:41,864 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:47,167 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:48,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:53,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:26:56,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:26:56,647 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:26:58,040 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:26:58,859 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:03,454 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:27:04,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:08,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:27:10,168 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:11,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:27:11,647 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:27:16,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:18,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:27:22,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:26,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:27:26,646 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:27:27,843 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:28,041 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:27:33,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:33,455 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:27:38,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:27:38,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:41,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:27:41,647 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:27:43,865 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:48,042 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:27:49,169 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:55,170 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:27:56,646 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:27:56,647 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:27:58,046 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:00,173 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:03,457 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:28:05,184 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:08,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:11,185 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:11,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:28:11,648 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:28:16,186 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:18,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:21,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:26,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:28:26,648 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:28:26,788 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:28,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:32,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:33,459 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:28:38,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:38,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:41,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:28:41,648 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:28:43,818 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:48,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:28:49,187 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:54,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:28:56,647 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:28:56,648 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:28:58,043 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:00,188 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:03,462 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:29:05,189 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:08,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:10,190 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:11,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:29:11,649 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:29:15,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:18,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:21,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:26,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:29:26,648 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:29:26,806 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:28,044 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:32,191 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:33,465 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:29:37,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:38,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:41,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:29:41,651 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:29:42,779 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:48,045 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:48,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:53,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:29:56,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:29:56,649 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:29:58,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:29:58,795 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:03,468 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:30:04,199 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:08,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:09,203 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:11,648 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:30:11,649 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:30:15,201 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:18,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:21,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:26,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:26,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:30:26,649 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:30:28,048 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:32,202 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:33,470 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:30:37,204 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:38,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:41,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:30:41,650 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:30:42,852 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:48,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:48,204 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:53,212 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:30:56,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:30:56,650 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:30:58,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:30:58,817 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:03,477 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:31:04,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:08,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:31:10,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:11,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:31:11,650 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:31:16,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:18,049 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:31:21,205 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:26,649 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:31:26,650 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:31:26,789 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:28,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:31:32,206 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:33,475 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:31:37,207 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:38,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:31:41,650 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:31:41,651 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:31:42,282 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: partial_history +2024-08-31 23:31:42,284 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:31:42,285 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:31:42,286 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:42,288 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:31:42,957 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:31:43,958 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:31:48,207 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:50,977 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:31:53,963 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:31:53,982 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:31:56,651 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: stop_status +2024-08-31 23:31:56,652 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: stop_status +2024-08-31 23:31:59,842 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:32:00,796 DEBUG SenderThread:29126 [sender.py:send():391] send: telemetry +2024-08-31 23:32:00,801 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: check_version +2024-08-31 23:32:00,801 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: check_version +2024-08-31 23:32:00,828 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: server_info +2024-08-31 23:32:00,829 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: server_info +2024-08-31 23:32:01,002 DEBUG SenderThread:29126 [sender.py:send():391] send: exit +2024-08-31 23:32:01,002 INFO SenderThread:29126 [sender.py:send_exit():598] handling exit code: 0 +2024-08-31 23:32:01,002 INFO SenderThread:29126 [sender.py:send_exit():600] handling runtime: 99156 +2024-08-31 23:32:01,003 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:32:01,003 INFO SenderThread:29126 [sender.py:send_exit():606] send defer +2024-08-31 23:32:01,003 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,003 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 0 +2024-08-31 23:32:01,004 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,004 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 0 +2024-08-31 23:32:01,004 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 1 +2024-08-31 23:32:01,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,004 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 1 +2024-08-31 23:32:01,004 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,004 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 1 +2024-08-31 23:32:01,004 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 2 +2024-08-31 23:32:01,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,005 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 2 +2024-08-31 23:32:01,005 INFO HandlerThread:29126 [system_monitor.py:finish():203] Stopping system monitor +2024-08-31 23:32:01,006 DEBUG SystemMonitor:29126 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-31 23:32:01,006 DEBUG SystemMonitor:29126 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-31 23:32:01,008 INFO HandlerThread:29126 [interfaces.py:finish():200] Joined cpu monitor +2024-08-31 23:32:01,009 INFO HandlerThread:29126 [interfaces.py:finish():200] Joined disk monitor +2024-08-31 23:32:01,044 INFO HandlerThread:29126 [interfaces.py:finish():200] Joined gpu monitor +2024-08-31 23:32:01,045 INFO HandlerThread:29126 [interfaces.py:finish():200] Joined memory monitor +2024-08-31 23:32:01,045 INFO HandlerThread:29126 [interfaces.py:finish():200] Joined network monitor +2024-08-31 23:32:01,046 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,046 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 2 +2024-08-31 23:32:01,046 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 3 +2024-08-31 23:32:01,046 DEBUG SenderThread:29126 [sender.py:send():391] send: stats +2024-08-31 23:32:01,047 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,047 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 3 +2024-08-31 23:32:01,049 DEBUG SenderThread:29126 [sender.py:send():391] send: history +2024-08-31 23:32:01,049 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: summary_record +2024-08-31 23:32:01,049 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:32:01,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,050 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 3 +2024-08-31 23:32:01,050 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 4 +2024-08-31 23:32:01,050 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,050 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 4 +2024-08-31 23:32:01,050 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,051 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 4 +2024-08-31 23:32:01,051 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 5 +2024-08-31 23:32:01,051 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,051 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 5 +2024-08-31 23:32:01,052 DEBUG SenderThread:29126 [sender.py:send():391] send: summary +2024-08-31 23:32:01,052 INFO SenderThread:29126 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-08-31 23:32:01,052 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,052 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 5 +2024-08-31 23:32:01,052 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 6 +2024-08-31 23:32:01,053 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,053 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 6 +2024-08-31 23:32:01,053 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,053 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 6 +2024-08-31 23:32:01,056 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: status_report +2024-08-31 23:32:01,239 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 7 +2024-08-31 23:32:01,239 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:01,240 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 7 +2024-08-31 23:32:01,240 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:01,240 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 7 +2024-08-31 23:32:01,969 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/config.yaml +2024-08-31 23:32:01,969 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:32:01,970 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:32:02,002 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: poll_exit +2024-08-31 23:32:03,208 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 8 +2024-08-31 23:32:03,209 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: poll_exit +2024-08-31 23:32:03,209 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:03,210 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 8 +2024-08-31 23:32:03,210 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:03,211 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 8 +2024-08-31 23:32:03,211 INFO SenderThread:29126 [job_builder.py:build():443] Attempting to build job artifact +2024-08-31 23:32:03,212 INFO SenderThread:29126 [job_builder.py:_get_source_type():572] is repo sourced job +2024-08-31 23:32:03,212 WARNING SenderThread:29126 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job +2024-08-31 23:32:03,212 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 9 +2024-08-31 23:32:03,213 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:03,213 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 9 +2024-08-31 23:32:03,213 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:03,213 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 9 +2024-08-31 23:32:03,213 INFO SenderThread:29126 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-31 23:32:03,970 INFO Thread-12 :29126 [dir_watcher.py:_on_file_modified():288] file/dir modified: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:32:03,971 INFO SenderThread:29126 [dir_watcher.py:finish():388] scan: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files +2024-08-31 23:32:03,971 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/config.yaml config.yaml +2024-08-31 23:32:03,972 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/diff.patch diff.patch +2024-08-31 23:32:03,972 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-metadata.json wandb-metadata.json +2024-08-31 23:32:03,972 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/requirements.txt requirements.txt +2024-08-31 23:32:03,972 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log output.log +2024-08-31 23:32:03,972 INFO SenderThread:29126 [dir_watcher.py:finish():402] scan save: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json wandb-summary.json +2024-08-31 23:32:03,972 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 10 +2024-08-31 23:32:03,984 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:03,984 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 10 +2024-08-31 23:32:03,985 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:03,985 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 10 +2024-08-31 23:32:03,985 INFO SenderThread:29126 [file_pusher.py:finish():169] shutting down file pusher +2024-08-31 23:32:04,004 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: poll_exit +2024-08-31 23:32:04,004 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: poll_exit +2024-08-31 23:32:04,389 INFO wandb-upload_0:29126 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json +2024-08-31 23:32:04,396 INFO wandb-upload_3:29126 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/requirements.txt +2024-08-31 23:32:04,726 INFO wandb-upload_1:29126 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/output.log +2024-08-31 23:32:04,751 INFO wandb-upload_2:29126 [upload_job.py:push():130] Uploaded file /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/files/config.yaml +2024-08-31 23:32:04,952 INFO Thread-11 (_thread_body):29126 [sender.py:transition_state():626] send defer: 11 +2024-08-31 23:32:04,953 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:04,953 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 11 +2024-08-31 23:32:04,954 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:04,954 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 11 +2024-08-31 23:32:04,954 INFO SenderThread:29126 [file_pusher.py:join():175] waiting for file pusher +2024-08-31 23:32:04,955 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 12 +2024-08-31 23:32:04,955 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:04,955 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 12 +2024-08-31 23:32:04,956 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:04,956 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 12 +2024-08-31 23:32:04,956 INFO SenderThread:29126 [file_stream.py:finish():601] file stream finish called +2024-08-31 23:32:05,005 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: poll_exit +2024-08-31 23:32:05,407 INFO SenderThread:29126 [file_stream.py:finish():605] file stream finish is done +2024-08-31 23:32:05,407 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 13 +2024-08-31 23:32:05,407 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: poll_exit +2024-08-31 23:32:05,408 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:05,408 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 13 +2024-08-31 23:32:05,409 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:05,409 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 13 +2024-08-31 23:32:05,409 INFO SenderThread:29126 [sender.py:transition_state():626] send defer: 14 +2024-08-31 23:32:05,409 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: defer +2024-08-31 23:32:05,410 INFO HandlerThread:29126 [handler.py:handle_request_defer():184] handle defer: 14 +2024-08-31 23:32:05,410 DEBUG SenderThread:29126 [sender.py:send():391] send: final +2024-08-31 23:32:05,411 DEBUG SenderThread:29126 [sender.py:send():391] send: footer +2024-08-31 23:32:05,411 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: defer +2024-08-31 23:32:05,411 INFO SenderThread:29126 [sender.py:send_request_defer():622] handle sender defer: 14 +2024-08-31 23:32:05,412 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: poll_exit +2024-08-31 23:32:05,413 DEBUG SenderThread:29126 [sender.py:send_request():418] send_request: poll_exit +2024-08-31 23:32:05,414 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: internal_messages +2024-08-31 23:32:05,416 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: get_summary +2024-08-31 23:32:05,417 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: sampled_history +2024-08-31 23:32:05,423 DEBUG HandlerThread:29126 [handler.py:handle_request():158] handle_request: shutdown +2024-08-31 23:32:05,423 INFO HandlerThread:29126 [handler.py:finish():884] shutting down handler +2024-08-31 23:32:06,413 INFO WriterThread:29126 [datastore.py:close():296] close: /workspace/nanoT5/outputs/2024-08-30/19-59-22/wandb/run-20240830_195924-mao0tqjy/run-mao0tqjy.wandb +2024-08-31 23:32:06,414 INFO SenderThread:29126 [sender.py:finish():1631] shutting down sender +2024-08-31 23:32:06,414 INFO SenderThread:29126 [file_pusher.py:finish():169] shutting down file pusher +2024-08-31 23:32:06,414 INFO SenderThread:29126 [file_pusher.py:join():175] waiting for file pusher