diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log"
--- "a/wandb/debug-internal.log"
+++ "b/wandb/debug-internal.log"
@@ -1,32 +1,32 @@
-2024-04-30 12:03:37,860 INFO StreamThr :139 [internal.py:wandb_internal():86] W&B internal server running at pid: 139, started at: 2024-04-30 12:03:37.860272
-2024-04-30 12:03:37,862 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status
-2024-04-30 12:03:38,253 INFO WriterThread:139 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/run-od3lqfzi.wandb
-2024-04-30 12:03:38,254 DEBUG SenderThread:139 [sender.py:send():379] send: header
-2024-04-30 12:03:38,257 DEBUG SenderThread:139 [sender.py:send():379] send: run
-2024-04-30 12:03:38,464 INFO SenderThread:139 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files
-2024-04-30 12:03:38,465 INFO SenderThread:139 [sender.py:_start_run_threads():1124] run started: od3lqfzi with start time 1714478617.861226
-2024-04-30 12:03:38,472 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: check_version
-2024-04-30 12:03:38,472 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: check_version
-2024-04-30 12:03:38,562 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: run_start
-2024-04-30 12:03:38,573 DEBUG HandlerThread:139 [system_info.py:__init__():26] System info init
-2024-04-30 12:03:38,573 DEBUG HandlerThread:139 [system_info.py:__init__():41] System info init done
-2024-04-30 12:03:38,573 INFO HandlerThread:139 [system_monitor.py:start():194] Starting system monitor
-2024-04-30 12:03:38,573 INFO SystemMonitor:139 [system_monitor.py:_start():158] Starting system asset monitoring threads
-2024-04-30 12:03:38,574 INFO SystemMonitor:139 [interfaces.py:start():190] Started cpu monitoring
-2024-04-30 12:03:38,574 INFO HandlerThread:139 [system_monitor.py:probe():214] Collecting system info
-2024-04-30 12:03:38,575 INFO SystemMonitor:139 [interfaces.py:start():190] Started disk monitoring
-2024-04-30 12:03:38,576 INFO SystemMonitor:139 [interfaces.py:start():190] Started gpu monitoring
-2024-04-30 12:03:38,577 INFO SystemMonitor:139 [interfaces.py:start():190] Started memory monitoring
-2024-04-30 12:03:38,578 INFO SystemMonitor:139 [interfaces.py:start():190] Started network monitoring
-2024-04-30 12:03:38,593 DEBUG HandlerThread:139 [system_info.py:probe():150] Probing system
-2024-04-30 12:03:38,595 DEBUG HandlerThread:139 [gitlib.py:_init_repo():56] git repository is invalid
-2024-04-30 12:03:38,595 DEBUG HandlerThread:139 [system_info.py:probe():198] Probing system done
-2024-04-30 12:03:38,595 DEBUG HandlerThread:139 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-30T12:03:38.593415', 'startedAt': '2024-04-30T12:03:37.854192', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '2f7b60d19abc', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.194, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.194, 'min': 0.0, 'max': 0.0}, {'current': 2000.194, 'min': 0.0, 'max': 0.0}, {'current': 2000.194, 'min': 0.0, 'max': 0.0}, {'current': 2000.194, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5605.298015594482}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}}
-2024-04-30 12:03:38,595 INFO HandlerThread:139 [system_monitor.py:probe():224] Finished collecting system info
-2024-04-30 12:03:38,595 INFO HandlerThread:139 [system_monitor.py:probe():227] Publishing system info
-2024-04-30 12:03:38,595 DEBUG HandlerThread:139 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment
-2024-04-30 12:03:39,466 INFO Thread-12 :139 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/conda-environment.yaml
-2024-04-30 12:03:53,609 ERROR HandlerThread:139 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds
+2024-04-30 13:43:30,429 INFO StreamThr :161 [internal.py:wandb_internal():86] W&B internal server running at pid: 161, started at: 2024-04-30 13:43:30.429000
+2024-04-30 13:43:30,431 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status
+2024-04-30 13:43:30,931 INFO WriterThread:161 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/run-p3yujlfg.wandb
+2024-04-30 13:43:30,932 DEBUG SenderThread:161 [sender.py:send():379] send: header
+2024-04-30 13:43:30,935 DEBUG SenderThread:161 [sender.py:send():379] send: run
+2024-04-30 13:43:31,062 INFO SenderThread:161 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files
+2024-04-30 13:43:31,062 INFO SenderThread:161 [sender.py:_start_run_threads():1124] run started: p3yujlfg with start time 1714484610.430593
+2024-04-30 13:43:31,070 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: check_version
+2024-04-30 13:43:31,070 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: check_version
+2024-04-30 13:43:31,164 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: run_start
+2024-04-30 13:43:31,176 DEBUG HandlerThread:161 [system_info.py:__init__():26] System info init
+2024-04-30 13:43:31,176 DEBUG HandlerThread:161 [system_info.py:__init__():41] System info init done
+2024-04-30 13:43:31,176 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor
+2024-04-30 13:43:31,176 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-04-30 13:43:31,176 INFO HandlerThread:161 [system_monitor.py:probe():214] Collecting system info
+2024-04-30 13:43:31,177 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring
+2024-04-30 13:43:31,178 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring
+2024-04-30 13:43:31,178 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring
+2024-04-30 13:43:31,180 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring
+2024-04-30 13:43:31,180 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring
+2024-04-30 13:43:31,191 DEBUG HandlerThread:161 [system_info.py:probe():150] Probing system
+2024-04-30 13:43:31,193 DEBUG HandlerThread:161 [gitlib.py:_init_repo():56] git repository is invalid
+2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_info.py:probe():198] Probing system done
+2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-30T13:43:31.191665', 'startedAt': '2024-04-30T13:43:30.422591', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'fde755c1ca53', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.144, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5605.322723388672}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}}
+2024-04-30 13:43:31,194 INFO HandlerThread:161 [system_monitor.py:probe():224] Finished collecting system info
+2024-04-30 13:43:31,194 INFO HandlerThread:161 [system_monitor.py:probe():227] Publishing system info
+2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment
+2024-04-30 13:43:32,064 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/conda-environment.yaml
+2024-04-30 13:43:46,209 ERROR HandlerThread:161 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds
Traceback (most recent call last):
File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda
subprocess.call(
@@ -37,629 +37,841 @@ Traceback (most recent call last):
File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait
raise TimeoutExpired(self.args, timeout)
subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds
-2024-04-30 12:03:53,611 DEBUG HandlerThread:139 [system_info.py:_save_conda():222] Saving conda packages done
-2024-04-30 12:03:53,612 INFO HandlerThread:139 [system_monitor.py:probe():229] Finished publishing system info
-2024-04-30 12:03:53,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:03:53,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: keepalive
-2024-04-30 12:03:53,620 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:03:53,620 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: keepalive
-2024-04-30 12:03:53,620 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:03:53,620 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: keepalive
-2024-04-30 12:03:53,621 DEBUG SenderThread:139 [sender.py:send():379] send: files
-2024-04-30 12:03:53,621 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now
-2024-04-30 12:03:53,866 INFO wandb-upload_0:139 [upload_job.py:push():131] Uploaded file /tmp/tmpblqkyruvwandb/bhay6br5-wandb-metadata.json
-2024-04-30 12:03:54,469 INFO Thread-12 :139 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-metadata.json
-2024-04-30 12:03:54,573 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: python_packages
-2024-04-30 12:03:54,573 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: python_packages
-2024-04-30 12:03:54,576 DEBUG SenderThread:139 [sender.py:send():379] send: telemetry
-2024-04-30 12:03:54,586 DEBUG SenderThread:139 [sender.py:send():379] send: config
-2024-04-30 12:03:54,588 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:03:54,589 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:03:54,589 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:03:54,590 DEBUG SenderThread:139 [sender.py:send():379] send: telemetry
-2024-04-30 12:03:54,590 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:03:54,590 WARNING SenderThread:139 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen)
-2024-04-30 12:03:54,590 DEBUG SenderThread:139 [sender.py:send():379] send: telemetry
-2024-04-30 12:03:54,591 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:03:55,470 INFO Thread-12 :139 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/requirements.txt
-2024-04-30 12:03:58,638 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:03,639 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:08,649 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:09,474 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/config.yaml
-2024-04-30 12:04:09,574 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:04:09,574 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:04:09,615 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:04:13,791 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:13,794 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:13,795 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:13,797 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:14,476 INFO Thread-12 :139 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:04:15,433 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send():379] send: metric
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:15,436 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:15,437 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:15,477 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:04:19,438 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:24,439 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:24,574 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:04:24,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:04:24,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:04:29,622 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:34,185 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:34,187 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:34,187 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:34,189 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:34,483 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:04:35,190 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:35,862 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:35,863 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:35,863 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:35,863 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:36,484 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:04:38,579 DEBUG SystemMonitor:139 [system_monitor.py:_start():172] Starting system metrics aggregation loop
-2024-04-30 12:04:38,580 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:04:39,574 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:04:39,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:04:39,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:04:40,683 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:41,486 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/config.yaml
-2024-04-30 12:04:45,801 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:50,802 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:54,574 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:04:54,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:04:54,615 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:04:55,090 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:55,091 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:55,091 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:55,092 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:55,491 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:04:56,092 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:04:56,807 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:04:56,808 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:04:56,808 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:04:56,808 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:04:57,492 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:05:01,810 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:06,811 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:08,581 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:05:09,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:05:09,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:05:09,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:05:12,649 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:16,943 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:05:16,944 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:05:16,944 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:05:16,946 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:05:17,499 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:05:17,947 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:18,719 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:05:18,721 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:05:18,721 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:05:18,721 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:05:19,500 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:05:23,723 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:24,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:05:24,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:05:24,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:05:29,707 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:34,707 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:38,581 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:05:39,199 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:05:39,200 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:05:39,201 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:05:39,202 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:05:39,507 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:05:39,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:05:39,575 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:05:39,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:05:40,698 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:41,007 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:05:41,008 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:05:41,009 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:05:41,009 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:05:41,507 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:05:46,010 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:51,011 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:05:54,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:05:54,576 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:05:54,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:05:56,642 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:01,278 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:01,279 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:01,280 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:01,280 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:01,514 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:02,281 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:03,019 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:03,020 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:03,020 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:03,021 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:03,515 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:08,022 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:08,582 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:06:09,575 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:06:09,576 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:06:09,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:06:13,641 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:18,642 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:22,803 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:22,804 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:22,804 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:22,804 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:23,522 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:23,805 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:24,547 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:24,548 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:24,548 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:24,549 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:24,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:06:24,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:06:24,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:06:25,523 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:29,642 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:34,643 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:38,583 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:06:39,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:06:39,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:06:39,617 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:06:39,696 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:44,072 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:44,073 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:44,074 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:44,074 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:44,529 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:45,075 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:45,839 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:06:45,840 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:06:45,840 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:06:45,843 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:06:46,530 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:06:50,844 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:06:54,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:06:54,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:06:54,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:06:56,655 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:01,656 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:05,728 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:05,729 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:05,729 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:05,731 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:06,538 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:06,732 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:07,512 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:07,513 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:07,513 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:07,513 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:07,538 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:08,584 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:07:09,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:07:09,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:07:09,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:07:12,668 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:17,669 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:22,670 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:24,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:07:24,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:07:24,617 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:07:27,502 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:27,503 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:27,504 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:27,504 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:27,545 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:28,505 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:29,261 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:29,263 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:29,263 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:29,265 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:29,546 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:34,266 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:38,585 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:07:39,576 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:07:39,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:07:39,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:07:39,671 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:44,672 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:48,871 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:48,872 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:48,872 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:48,873 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:49,552 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:49,873 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:07:50,628 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:07:50,629 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:07:50,629 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:07:50,630 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:07:51,553 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:07:54,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:07:54,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:07:54,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:07:55,714 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:00,715 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:05,716 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:08,586 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:08:09,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:08:09,577 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:08:09,617 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:08:10,698 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:10,699 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:10,699 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:10,702 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:11,560 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:11,702 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:12,462 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:12,464 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:12,464 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:12,464 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:12,560 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:17,466 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:22,467 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:24,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:08:24,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:08:24,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:08:27,660 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:32,197 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:32,198 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:32,199 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:32,199 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:32,567 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:33,200 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:33,959 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:33,960 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:33,960 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:33,962 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:34,568 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:38,587 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:08:38,588 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:39,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:08:39,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:08:39,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:08:43,663 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:48,664 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:53,664 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:08:54,132 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:54,133 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:54,134 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:54,134 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:54,575 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:54,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:08:54,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:08:54,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:08:55,889 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:08:55,890 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:08:55,890 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:08:55,891 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:08:56,576 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:08:58,892 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:03,893 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:08,588 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:09:09,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:09:09,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:09:09,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:09:09,661 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:14,662 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:15,876 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:09:15,877 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:09:15,878 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:09:15,878 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:09:16,584 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:09:17,636 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:09:17,637 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:09:17,637 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:09:17,638 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:09:18,585 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:09:20,639 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:24,577 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:09:24,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:09:24,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:09:25,655 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:30,656 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:35,656 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:37,454 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:09:37,456 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:09:37,456 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:09:37,456 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:09:37,591 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:09:38,588 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:09:39,216 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:09:39,218 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:09:39,218 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:09:39,219 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:09:39,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:09:39,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:09:39,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:09:39,592 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:09:41,618 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:46,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:51,620 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:54,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:09:54,578 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:09:54,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:09:56,649 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:09:59,146 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:09:59,147 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:09:59,147 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:09:59,147 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:09:59,599 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:00,906 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:10:00,907 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:10:00,907 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:10:00,908 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:10:01,600 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:01,909 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:06,909 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:08,590 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:10:09,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:10:09,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:10:09,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:10:12,706 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:17,706 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:20,662 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:10:20,663 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:10:20,663 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:10:20,664 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:10:21,607 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:22,414 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:10:22,415 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:10:22,416 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:10:22,416 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:10:22,607 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:23,417 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:24,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:10:24,579 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:10:24,581 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:10:28,621 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:33,622 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:38,590 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:10:39,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:10:39,579 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:10:39,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:10:39,674 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:41,929 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:10:41,930 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:10:41,931 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:10:41,931 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:10:42,614 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:43,681 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:10:43,682 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:10:43,683 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:10:43,683 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:10:44,615 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:10:44,684 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:49,685 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:10:54,578 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:10:54,579 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:10:54,582 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:10:55,666 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:00,667 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:03,584 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:03,585 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:03,585 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:03,586 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:03,622 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:05,339 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:05,340 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:05,341 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:05,341 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:05,623 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:06,342 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:08,591 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:11:09,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:11:09,579 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:11:09,582 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:11:11,664 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:16,665 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:21,666 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:24,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:11:24,579 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:11:24,619 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:11:25,029 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:25,030 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:25,031 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:25,031 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:25,630 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:26,780 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:26,781 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:26,781 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:26,782 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:26,782 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:27,631 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:31,783 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:36,784 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:38,592 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:11:39,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:11:39,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:11:39,583 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:11:42,612 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:46,629 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:46,635 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:46,635 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:46,635 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:46,637 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:47,636 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:48,390 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:11:48,391 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:11:48,392 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:11:48,392 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:11:48,638 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:11:53,393 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:11:54,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:11:54,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:11:54,582 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:11:58,608 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:03,609 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:08,181 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:08,182 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:08,182 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:08,183 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:08,593 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:12:08,645 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:09,579 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:12:09,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:12:09,583 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:12:09,688 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:09,935 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:09,936 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:09,936 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:09,936 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:10,646 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:14,938 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:19,938 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:24,583 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:12:24,585 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:12:24,585 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:12:25,680 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:29,663 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:29,665 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:29,665 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:29,665 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:30,653 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:31,423 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:31,424 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:31,425 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:31,425 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:31,425 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:31,653 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:36,426 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:38,594 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:12:39,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:12:39,580 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:12:39,583 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:12:41,631 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:46,632 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:50,996 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:50,998 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:50,998 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:50,998 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:51,660 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:51,999 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:12:52,751 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:12:52,752 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:12:52,753 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:12:52,753 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:12:53,660 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:12:54,580 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:12:54,581 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:12:54,584 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:12:57,677 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:02,678 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:07,554 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:13:07,555 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:13:07,555 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:07,556 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:07,665 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:13:08,557 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:08,595 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:13:09,311 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:13:09,312 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:13:09,312 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,313 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,587 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:13:09,587 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:13:09,587 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:13:09,630 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: partial_history
-2024-04-30 12:13:09,630 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: pause
-2024-04-30 12:13:09,630 INFO HandlerThread:139 [handler.py:handle_request_pause():708] stopping system metrics thread
-2024-04-30 12:13:09,631 INFO HandlerThread:139 [system_monitor.py:finish():203] Stopping system monitor
-2024-04-30 12:13:09,631 DEBUG SystemMonitor:139 [system_monitor.py:_start():179] Finished system metrics aggregation loop
-2024-04-30 12:13:09,632 DEBUG SystemMonitor:139 [system_monitor.py:_start():183] Publishing last batch of metrics
-2024-04-30 12:13:09,632 INFO HandlerThread:139 [interfaces.py:finish():202] Joined cpu monitor
-2024-04-30 12:13:09,633 INFO HandlerThread:139 [interfaces.py:finish():202] Joined disk monitor
-2024-04-30 12:13:09,643 INFO HandlerThread:139 [interfaces.py:finish():202] Joined gpu monitor
-2024-04-30 12:13:09,643 INFO HandlerThread:139 [interfaces.py:finish():202] Joined memory monitor
-2024-04-30 12:13:09,643 INFO HandlerThread:139 [interfaces.py:finish():202] Joined network monitor
-2024-04-30 12:13:09,666 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:13:09,722 DEBUG SenderThread:139 [sender.py:send():379] send: telemetry
-2024-04-30 12:13:09,722 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,723 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,723 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,723 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,724 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,724 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,724 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,724 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,725 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,725 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,725 DEBUG SenderThread:139 [sender.py:send():379] send: history
-2024-04-30 12:13:09,725 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: summary_record
-2024-04-30 12:13:09,725 INFO SenderThread:139 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
-2024-04-30 12:13:09,726 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:13:10,666 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/wandb-summary.json
-2024-04-30 12:13:13,732 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:14,668 INFO Thread-12 :139 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_120337-od3lqfzi/files/config.yaml
-2024-04-30 12:13:18,562 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: resume
-2024-04-30 12:13:18,562 INFO HandlerThread:139 [handler.py:handle_request_resume():699] starting system metrics thread
-2024-04-30 12:13:18,562 INFO HandlerThread:139 [system_monitor.py:start():194] Starting system monitor
-2024-04-30 12:13:18,562 INFO SystemMonitor:139 [system_monitor.py:_start():158] Starting system asset monitoring threads
-2024-04-30 12:13:18,562 INFO SystemMonitor:139 [interfaces.py:start():190] Started cpu monitoring
-2024-04-30 12:13:18,563 INFO SystemMonitor:139 [interfaces.py:start():190] Started disk monitoring
-2024-04-30 12:13:18,564 INFO SystemMonitor:139 [interfaces.py:start():190] Started gpu monitoring
-2024-04-30 12:13:18,565 INFO SystemMonitor:139 [interfaces.py:start():190] Started memory monitoring
-2024-04-30 12:13:18,565 INFO SystemMonitor:139 [interfaces.py:start():190] Started network monitoring
-2024-04-30 12:13:18,587 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: pause
-2024-04-30 12:13:18,587 INFO HandlerThread:139 [handler.py:handle_request_pause():708] stopping system metrics thread
-2024-04-30 12:13:18,587 INFO HandlerThread:139 [system_monitor.py:finish():203] Stopping system monitor
-2024-04-30 12:13:18,587 DEBUG SystemMonitor:139 [system_monitor.py:_start():172] Starting system metrics aggregation loop
-2024-04-30 12:13:18,588 INFO HandlerThread:139 [interfaces.py:finish():202] Joined cpu monitor
-2024-04-30 12:13:18,588 DEBUG SystemMonitor:139 [system_monitor.py:_start():179] Finished system metrics aggregation loop
-2024-04-30 12:13:18,588 INFO HandlerThread:139 [interfaces.py:finish():202] Joined disk monitor
-2024-04-30 12:13:18,589 DEBUG SystemMonitor:139 [system_monitor.py:_start():183] Publishing last batch of metrics
-2024-04-30 12:13:18,599 INFO HandlerThread:139 [interfaces.py:finish():202] Joined gpu monitor
-2024-04-30 12:13:18,599 INFO HandlerThread:139 [interfaces.py:finish():202] Joined memory monitor
-2024-04-30 12:13:18,600 INFO HandlerThread:139 [interfaces.py:finish():202] Joined network monitor
-2024-04-30 12:13:18,600 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:13:19,601 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:24,585 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:13:24,586 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:13:24,588 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:13:24,694 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:29,695 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:34,696 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:39,586 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:13:39,586 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:13:39,586 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:13:39,731 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:44,732 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:49,733 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
-2024-04-30 12:13:54,174 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: resume
-2024-04-30 12:13:54,174 INFO HandlerThread:139 [handler.py:handle_request_resume():699] starting system metrics thread
-2024-04-30 12:13:54,174 INFO HandlerThread:139 [system_monitor.py:start():194] Starting system monitor
-2024-04-30 12:13:54,174 INFO SystemMonitor:139 [system_monitor.py:_start():158] Starting system asset monitoring threads
-2024-04-30 12:13:54,175 INFO SystemMonitor:139 [interfaces.py:start():190] Started cpu monitoring
-2024-04-30 12:13:54,175 INFO SystemMonitor:139 [interfaces.py:start():190] Started disk monitoring
-2024-04-30 12:13:54,176 INFO SystemMonitor:139 [interfaces.py:start():190] Started gpu monitoring
-2024-04-30 12:13:54,177 INFO SystemMonitor:139 [interfaces.py:start():190] Started memory monitoring
-2024-04-30 12:13:54,177 INFO SystemMonitor:139 [interfaces.py:start():190] Started network monitoring
-2024-04-30 12:13:54,215 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: pause
-2024-04-30 12:13:54,215 INFO HandlerThread:139 [handler.py:handle_request_pause():708] stopping system metrics thread
-2024-04-30 12:13:54,215 INFO HandlerThread:139 [system_monitor.py:finish():203] Stopping system monitor
-2024-04-30 12:13:54,215 DEBUG SystemMonitor:139 [system_monitor.py:_start():172] Starting system metrics aggregation loop
-2024-04-30 12:13:54,215 DEBUG SystemMonitor:139 [system_monitor.py:_start():179] Finished system metrics aggregation loop
-2024-04-30 12:13:54,215 DEBUG SystemMonitor:139 [system_monitor.py:_start():183] Publishing last batch of metrics
-2024-04-30 12:13:54,216 INFO HandlerThread:139 [interfaces.py:finish():202] Joined cpu monitor
-2024-04-30 12:13:54,216 INFO HandlerThread:139 [interfaces.py:finish():202] Joined disk monitor
-2024-04-30 12:13:54,226 INFO HandlerThread:139 [interfaces.py:finish():202] Joined gpu monitor
-2024-04-30 12:13:54,226 INFO HandlerThread:139 [interfaces.py:finish():202] Joined memory monitor
-2024-04-30 12:13:54,226 INFO HandlerThread:139 [interfaces.py:finish():202] Joined network monitor
-2024-04-30 12:13:54,227 DEBUG SenderThread:139 [sender.py:send():379] send: stats
-2024-04-30 12:13:54,389 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: resume
-2024-04-30 12:13:54,389 INFO HandlerThread:139 [handler.py:handle_request_resume():699] starting system metrics thread
-2024-04-30 12:13:54,389 INFO HandlerThread:139 [system_monitor.py:start():194] Starting system monitor
-2024-04-30 12:13:54,389 INFO SystemMonitor:139 [system_monitor.py:_start():158] Starting system asset monitoring threads
-2024-04-30 12:13:54,390 INFO SystemMonitor:139 [interfaces.py:start():190] Started cpu monitoring
-2024-04-30 12:13:54,390 INFO SystemMonitor:139 [interfaces.py:start():190] Started disk monitoring
-2024-04-30 12:13:54,391 INFO SystemMonitor:139 [interfaces.py:start():190] Started gpu monitoring
-2024-04-30 12:13:54,393 INFO SystemMonitor:139 [interfaces.py:start():190] Started memory monitoring
-2024-04-30 12:13:54,394 INFO SystemMonitor:139 [interfaces.py:start():190] Started network monitoring
-2024-04-30 12:13:54,585 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: stop_status
-2024-04-30 12:13:54,585 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: internal_messages
-2024-04-30 12:13:54,586 DEBUG SenderThread:139 [sender.py:send_request():406] send_request: stop_status
-2024-04-30 12:13:55,621 DEBUG HandlerThread:139 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:43:46,214 DEBUG HandlerThread:161 [system_info.py:_save_conda():222] Saving conda packages done
+2024-04-30 13:43:46,215 INFO HandlerThread:161 [system_monitor.py:probe():229] Finished publishing system info
+2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive
+2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive
+2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:43:46,225 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive
+2024-04-30 13:43:46,225 DEBUG SenderThread:161 [sender.py:send():379] send: files
+2024-04-30 13:43:46,225 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now
+2024-04-30 13:43:46,421 INFO wandb-upload_0:161 [upload_job.py:push():131] Uploaded file /tmp/tmp1w88py37wandb/09spuzjy-wandb-metadata.json
+2024-04-30 13:43:47,068 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-metadata.json
+2024-04-30 13:43:47,242 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: python_packages
+2024-04-30 13:43:47,243 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: python_packages
+2024-04-30 13:43:47,246 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry
+2024-04-30 13:43:47,258 DEBUG SenderThread:161 [sender.py:send():379] send: config
+2024-04-30 13:43:47,260 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:43:47,260 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry
+2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:43:47,262 WARNING SenderThread:161 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen)
+2024-04-30 13:43:47,262 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:43:47,382 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry
+2024-04-30 13:43:48,068 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/requirements.txt
+2024-04-30 13:43:51,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:43:56,384 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:01,390 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:02,073 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml
+2024-04-30 13:44:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:44:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:44:02,285 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:44:03,133 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:03,137 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:04,073 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:04,443 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric
+2024-04-30 13:44:04,446 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:04,446 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:04,446 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:05,074 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:07,448 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:12,449 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:17,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:44:17,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:44:17,247 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:44:18,281 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:19,008 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:19,009 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:19,009 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:19,011 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:19,079 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:20,329 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:20,330 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:20,331 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:20,331 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:21,080 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:23,332 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:28,333 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:31,180 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-04-30 13:44:31,182 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:44:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:44:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:44:32,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:44:34,335 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:34,874 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:34,875 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:34,875 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:34,878 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:35,085 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml
+2024-04-30 13:44:35,085 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:36,200 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:36,201 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:36,201 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:36,202 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:37,086 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:40,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:45,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:47,243 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:44:47,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:44:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:44:50,304 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:44:50,897 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:50,899 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:50,899 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:50,899 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:51,091 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:52,231 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:44:52,232 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:44:52,233 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:44:52,233 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:44:53,092 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:44:56,234 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:01,183 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:45:02,184 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:02,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:45:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:45:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:45:06,865 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:06,866 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:06,866 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:06,867 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:07,097 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:07,867 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:08,202 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:08,203 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:08,204 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:08,204 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:09,098 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:13,205 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:17,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:45:17,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:45:17,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:45:18,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:23,079 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:23,080 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:23,080 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:23,081 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:23,104 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:24,081 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:24,427 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:24,428 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:24,429 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:24,429 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:25,105 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:29,430 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:31,184 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:45:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:45:32,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:45:32,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:45:35,373 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:39,293 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:39,294 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:39,294 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:39,295 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:40,111 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:40,643 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:40,645 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:40,645 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:40,645 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:40,646 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:41,111 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:45,647 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:47,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:45:47,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:45:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:45:51,390 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:55,508 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:55,509 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:55,510 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:55,510 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:56,117 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:45:56,511 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:45:56,859 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:45:56,861 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:45:56,861 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:45:56,861 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:45:57,118 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:01,185 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:46:02,186 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:02,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:46:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:46:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:46:07,361 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:11,758 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:11,759 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:11,760 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:11,762 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:12,123 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:12,763 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:13,117 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:13,118 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:13,119 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:13,119 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:13,124 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:17,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:46:17,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:46:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:46:18,378 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:23,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:28,029 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:28,030 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:28,030 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:28,030 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:28,129 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:29,031 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:29,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:29,385 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:29,385 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:29,385 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:30,130 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:31,186 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:46:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:46:32,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:46:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:46:34,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:39,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:44,384 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:44,396 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:44,398 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:44,398 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:44,398 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:45,136 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:45,756 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:46:45,758 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:46:45,758 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:46:45,758 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:46:46,136 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:46:47,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:46:47,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:46:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:46:50,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:46:55,366 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:00,367 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:00,620 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:00,621 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:00,621 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:00,622 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:01,142 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:01,187 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:47:01,988 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:01,989 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:01,990 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:01,990 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:02,142 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:47:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:47:02,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:47:06,275 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:11,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:16,277 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:16,906 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:16,907 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:16,908 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:16,908 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:17,148 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:17,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:47:17,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:47:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:47:18,272 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:18,274 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:18,274 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:18,275 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:19,149 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:22,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:27,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:31,188 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:47:32,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:47:32,246 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:47:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:47:32,320 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:33,205 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:33,206 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:33,207 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:33,207 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:34,155 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:34,574 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:34,575 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:34,575 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:34,576 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:35,155 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:37,577 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:42,578 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:47,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:47:47,246 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:47:47,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:47:48,289 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:49,483 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:49,484 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:49,484 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:49,485 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:50,161 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:50,855 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:47:50,857 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:47:50,857 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:47:50,857 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:47:51,161 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:47:53,858 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:47:58,859 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:01,189 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:48:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:48:02,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:48:02,247 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:48:04,367 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:05,685 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:05,686 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:05,686 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:05,686 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:06,167 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:07,053 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:07,054 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:07,054 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:07,055 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:07,168 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:10,056 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:15,057 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:48:17,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:48:17,257 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:48:20,322 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:22,092 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:22,094 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:22,094 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:22,095 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:22,174 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:23,467 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:23,468 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:23,469 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:23,469 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:24,174 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:25,470 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:30,471 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:31,190 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:48:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:48:32,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:48:32,251 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:48:36,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:38,399 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:38,401 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:38,401 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:38,401 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:39,181 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:39,775 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:39,776 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:39,777 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:39,777 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:40,181 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:41,778 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:46,779 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:48:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:48:47,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:48:52,291 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:48:54,832 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:54,833 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:54,833 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:54,835 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:55,186 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:56,209 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:48:56,210 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:48:56,211 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:48:56,211 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:48:57,187 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:48:58,212 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:01,191 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:49:02,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:49:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:49:02,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:49:03,371 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:08,372 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:11,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:11,290 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:11,290 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:11,291 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:12,193 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:12,663 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:12,664 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:12,664 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:12,665 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:13,193 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:13,666 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:17,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:49:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:49:17,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:49:19,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:24,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:27,712 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:27,713 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:27,713 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:27,715 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:28,199 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:29,092 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:29,093 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:29,093 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:29,094 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:29,199 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:30,094 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:31,192 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:49:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:49:32,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:49:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:49:35,296 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:40,297 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:44,173 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:44,174 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:44,174 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:44,175 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:44,204 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:45,558 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:49:45,559 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:49:45,559 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:49:45,559 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:45,560 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:49:46,205 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:49:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:49:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:49:47,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:49:51,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:49:56,366 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:00,624 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:00,626 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:00,626 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:00,628 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:01,193 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:50:01,211 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:02,006 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:02,008 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:02,008 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:02,008 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:02,009 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:02,211 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:02,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:50:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:50:02,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:50:07,315 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:12,316 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:17,096 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:17,098 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:17,098 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:17,098 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:17,217 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:50:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:50:17,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:50:17,346 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:18,482 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:18,483 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:18,484 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:18,484 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:19,217 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:22,485 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:27,486 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:31,194 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:50:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:50:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:50:32,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:50:33,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:33,634 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:33,635 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:33,635 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:33,636 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:34,223 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:35,021 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:35,027 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:35,027 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:35,028 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:35,223 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:39,029 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:44,030 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:50:47,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:50:47,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:50:49,286 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:50,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:50,205 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:50,205 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:50,206 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:50,228 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:51,592 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:50:51,594 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:50:51,594 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:50:51,594 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:50:52,229 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:50:54,595 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:50:59,596 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:01,195 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:51:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:51:02,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:51:02,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:51:05,356 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:06,693 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:06,694 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:06,694 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:06,695 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:07,234 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:08,073 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:08,075 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:08,075 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:08,075 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:08,235 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:11,076 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:16,077 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:51:17,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:51:17,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:51:21,358 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:23,153 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:23,154 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:23,154 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:23,155 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:23,240 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:24,537 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:24,539 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:24,539 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:24,539 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:25,241 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:26,540 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:31,195 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:51:32,196 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:51:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:51:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:51:37,290 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:39,679 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:39,680 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:39,680 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:39,681 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:40,246 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:41,065 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:41,066 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:41,066 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:41,067 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:41,247 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:43,068 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:51:47,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:51:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:51:48,302 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:53,303 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:51:56,290 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:56,291 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:56,292 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:56,292 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,253 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:57,671 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:57,672 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:57,672 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,673 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,855 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-30 13:51:57,856 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry
+2024-04-30 13:51:57,856 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,856 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,857 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,857 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,858 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,858 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause
+2024-04-30 13:51:57,858 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread
+2024-04-30 13:51:57,858 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor
+2024-04-30 13:51:57,859 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-04-30 13:51:57,859 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-04-30 13:51:57,860 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor
+2024-04-30 13:51:57,860 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor
+2024-04-30 13:51:57,860 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,861 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,862 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,862 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,863 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,863 DEBUG SenderThread:161 [sender.py:send():379] send: history
+2024-04-30 13:51:57,863 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record
+2024-04-30 13:51:57,864 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor
+2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor
+2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor
+2024-04-30 13:51:57,875 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:51:58,253 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json
+2024-04-30 13:51:58,876 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:02,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:52:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:52:02,251 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:52:04,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:05,256 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml
+2024-04-30 13:52:09,404 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:14,405 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:17,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:52:17,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:52:17,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:52:19,435 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:24,436 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:29,438 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:52:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:52:32,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:52:35,309 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:40,310 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:45,311 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:47,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:52:47,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:52:47,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:52:50,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:52:55,350 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:00,350 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:53:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:53:02,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:53:05,357 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:10,358 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:15,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:17,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:53:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:53:17,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:53:20,489 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:25,490 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:30,491 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:53:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:53:32,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:53:36,333 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:41,334 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:46,335 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:53:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:53:47,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:53:51,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:53:56,380 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:01,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:54:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:54:02,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:54:07,378 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:12,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:54:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:54:17,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:54:18,320 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:23,321 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:28,321 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:54:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:54:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:54:33,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:38,328 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:43,329 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:54:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:54:47,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:54:49,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:54,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:54:59,328 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:55:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:55:02,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:55:04,387 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:09,388 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:14,389 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:55:17,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:55:17,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:55:20,316 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:25,317 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:30,318 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:55:32,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:55:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:55:36,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:41,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:46,308 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:55:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:55:47,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:55:51,362 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:55:56,363 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:01,364 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:56:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:56:02,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:56:07,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:12,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:17,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:56:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:56:17,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:56:18,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:23,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:28,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:32,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:56:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:56:32,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:56:33,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:38,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:43,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:56:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:56:47,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:56:48,715 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:53,716 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:56:58,717 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:57:02,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:57:02,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:57:04,299 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:09,300 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:14,301 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:57:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:57:17,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:57:19,324 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:24,325 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:29,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:57:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:57:32,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:57:34,392 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:39,393 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:44,394 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:47,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:57:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:57:47,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:57:50,284 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:57:55,286 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:00,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:02,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:58:02,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:58:02,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:58:05,348 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:10,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:15,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:58:17,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:58:17,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:58:20,363 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:25,364 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:30,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:58:32,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:58:32,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:58:36,357 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:41,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:46,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:58:47,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:58:47,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:58:52,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:58:57,289 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:02,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:59:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:59:02,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:59:02,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:07,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:12,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:17,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:59:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:59:17,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:59:17,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:22,380 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:27,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:32,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:59:32,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:59:32,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:59:32,414 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume
+2024-04-30 13:59:32,414 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread
+2024-04-30 13:59:32,414 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor
+2024-04-30 13:59:32,414 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-04-30 13:59:32,415 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring
+2024-04-30 13:59:32,415 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring
+2024-04-30 13:59:32,416 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring
+2024-04-30 13:59:32,418 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring
+2024-04-30 13:59:32,420 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring
+2024-04-30 13:59:32,454 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause
+2024-04-30 13:59:32,454 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread
+2024-04-30 13:59:32,454 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor
+2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-04-30 13:59:32,455 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor
+2024-04-30 13:59:32,456 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor
+2024-04-30 13:59:32,466 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor
+2024-04-30 13:59:32,466 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor
+2024-04-30 13:59:32,467 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor
+2024-04-30 13:59:32,467 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:59:32,467 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:33,550 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume
+2024-04-30 13:59:33,550 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread
+2024-04-30 13:59:33,550 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor
+2024-04-30 13:59:33,550 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-04-30 13:59:33,551 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring
+2024-04-30 13:59:33,551 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring
+2024-04-30 13:59:33,552 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring
+2024-04-30 13:59:33,553 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring
+2024-04-30 13:59:33,554 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring
+2024-04-30 13:59:35,391 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause
+2024-04-30 13:59:35,392 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread
+2024-04-30 13:59:35,392 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor
+2024-04-30 13:59:35,392 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor
+2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-04-30 13:59:35,393 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor
+2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-04-30 13:59:35,405 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor
+2024-04-30 13:59:35,406 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor
+2024-04-30 13:59:35,406 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor
+2024-04-30 13:59:35,406 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 13:59:38,407 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:43,408 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 13:59:47,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 13:59:47,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 13:59:49,351 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:54,351 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 13:59:59,352 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 14:00:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 14:00:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 14:00:02,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 14:00:05,312 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume
+2024-04-30 14:00:05,312 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread
+2024-04-30 14:00:05,312 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor
+2024-04-30 14:00:05,313 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-04-30 14:00:05,313 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring
+2024-04-30 14:00:05,314 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring
+2024-04-30 14:00:05,314 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring
+2024-04-30 14:00:05,315 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring
+2024-04-30 14:00:05,316 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring
+2024-04-30 14:00:05,334 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 14:00:05,340 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause
+2024-04-30 14:00:05,340 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread
+2024-04-30 14:00:05,340 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor
+2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-04-30 14:00:05,341 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor
+2024-04-30 14:00:05,344 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor
+2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor
+2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor
+2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor
+2024-04-30 14:00:05,354 DEBUG SenderThread:161 [sender.py:send():379] send: stats
+2024-04-30 14:00:10,355 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 14:00:15,356 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 14:00:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-30 14:00:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-30 14:00:17,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status
+2024-04-30 14:00:20,361 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report
+2024-04-30 14:00:22,024 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume
+2024-04-30 14:00:22,024 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread
+2024-04-30 14:00:22,024 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor
+2024-04-30 14:00:22,024 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-04-30 14:00:22,025 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring
+2024-04-30 14:00:22,025 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring
+2024-04-30 14:00:22,027 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring
+2024-04-30 14:00:22,027 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring
+2024-04-30 14:00:22,028 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring
+2024-04-30 14:00:25,362 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report