diff --git "a/wandb/run-20240416_172306-uygw9yfk/logs/debug-internal.log" "b/wandb/run-20240416_172306-uygw9yfk/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240416_172306-uygw9yfk/logs/debug-internal.log" @@ -0,0 +1,3622 @@ +2024-04-16 17:23:06,350 INFO StreamThr :940772 [internal.py:wandb_internal():86] W&B internal server running at pid: 940772, started at: 2024-04-16 17:23:06.348497 +2024-04-16 17:23:06,351 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status +2024-04-16 17:23:06,354 INFO WriterThread:940772 [datastore.py:open_for_write():85] open: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/run-uygw9yfk.wandb +2024-04-16 17:23:06,357 DEBUG SenderThread:940772 [sender.py:send():382] send: header +2024-04-16 17:23:06,357 DEBUG SenderThread:940772 [sender.py:send():382] send: run +2024-04-16 17:23:06,536 INFO SenderThread:940772 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files +2024-04-16 17:23:06,536 INFO SenderThread:940772 [sender.py:_start_run_threads():1136] run started: uygw9yfk with start time 1713288186.348831 +2024-04-16 17:23:06,544 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: check_version +2024-04-16 17:23:06,544 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: check_version +2024-04-16 17:23:06,605 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: run_start +2024-04-16 17:23:06,664 DEBUG HandlerThread:940772 [system_info.py:__init__():32] System info init +2024-04-16 17:23:06,664 DEBUG HandlerThread:940772 [system_info.py:__init__():47] System info init done +2024-04-16 17:23:06,664 INFO HandlerThread:940772 [system_monitor.py:start():194] Starting system monitor +2024-04-16 17:23:06,665 INFO SystemMonitor:940772 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-16 17:23:06,665 INFO HandlerThread:940772 [system_monitor.py:probe():214] Collecting system info +2024-04-16 17:23:06,666 INFO SystemMonitor:940772 [interfaces.py:start():190] Started cpu monitoring +2024-04-16 17:23:06,666 INFO SystemMonitor:940772 [interfaces.py:start():190] Started disk monitoring +2024-04-16 17:23:06,667 INFO SystemMonitor:940772 [interfaces.py:start():190] Started gpu monitoring +2024-04-16 17:23:06,667 INFO SystemMonitor:940772 [interfaces.py:start():190] Started memory monitoring +2024-04-16 17:23:06,668 INFO SystemMonitor:940772 [interfaces.py:start():190] Started network monitoring +2024-04-16 17:23:06,717 DEBUG HandlerThread:940772 [system_info.py:probe():196] Probing system +2024-04-16 17:23:06,719 DEBUG HandlerThread:940772 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-16 17:23:06,719 DEBUG HandlerThread:940772 [system_info.py:probe():244] Probing system done +2024-04-16 17:23:06,719 DEBUG HandlerThread:940772 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.5', 'heartbeatAt': '2024-04-16T17:23:06.717620', 'startedAt': '2024-04-16T17:23:06.329363', 'docker': None, 'cuda': None, 'args': ('config_mistral.yaml',), 'state': 'running', 'program': '/fsx/sanchit/mistral-debug-4bit/run_distillation.py', 'codePathLocal': 'run_distillation.py', 'codePath': 'run_distillation.py', 'host': 'ip-26-0-164-187', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2659.601562499998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3571.94, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 58.41170883178711}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855461120605}} +2024-04-16 17:23:06,720 INFO HandlerThread:940772 [system_monitor.py:probe():224] Finished collecting system info +2024-04-16 17:23:06,720 INFO HandlerThread:940772 [system_monitor.py:probe():227] Publishing system info +2024-04-16 17:23:06,720 DEBUG HandlerThread:940772 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2024-04-16 17:23:06,722 DEBUG HandlerThread:940772 [system_info.py:_save_pip():68] Saving pip packages done +2024-04-16 17:23:06,722 DEBUG HandlerThread:940772 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2024-04-16 17:23:07,538 INFO Thread-12 :940772 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/conda-environment.yaml +2024-04-16 17:23:07,539 INFO Thread-12 :940772 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/requirements.txt +2024-04-16 17:23:10,859 DEBUG HandlerThread:940772 [system_info.py:_save_conda():87] Saving conda packages done +2024-04-16 17:23:10,862 INFO HandlerThread:940772 [system_monitor.py:probe():229] Finished publishing system info +2024-04-16 17:23:10,882 DEBUG SenderThread:940772 [sender.py:send():382] send: files +2024-04-16 17:23:10,883 INFO SenderThread:940772 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2024-04-16 17:23:10,889 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:23:10,889 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:23:10,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:23:11,039 DEBUG SenderThread:940772 [sender.py:send():382] send: telemetry +2024-04-16 17:23:11,146 INFO wandb-upload_0:940772 [upload_job.py:push():131] Uploaded file /tmp/tmp6psxmfvkwandb/zwsbh3r7-wandb-metadata.json +2024-04-16 17:23:11,548 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/conda-environment.yaml +2024-04-16 17:23:11,548 INFO Thread-12 :940772 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:11,548 INFO Thread-12 :940772 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-metadata.json +2024-04-16 17:23:12,041 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:13,550 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:17,046 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:22,047 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:24,416 DEBUG SenderThread:940772 [sender.py:send():382] send: telemetry +2024-04-16 17:23:24,416 DEBUG SenderThread:940772 [sender.py:send():382] send: telemetry +2024-04-16 17:23:25,562 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:25,889 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:23:25,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:23:25,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:23:27,167 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:27,564 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:29,567 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:31,569 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:32,559 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:35,573 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:37,925 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:38,578 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/config.yaml +2024-04-16 17:23:40,889 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:23:40,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:23:40,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:23:41,581 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:43,227 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:43,583 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:45,585 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:48,303 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:49,590 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:51,592 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:53,816 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:23:55,596 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:55,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:23:55,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:23:55,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:23:57,598 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:23:59,013 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:03,604 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:04,450 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:06,668 DEBUG SystemMonitor:940772 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-16 17:24:06,671 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:24:09,611 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:10,290 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:10,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:24:10,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:24:10,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:24:15,617 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:16,243 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:17,620 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:21,321 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:23,628 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:25,631 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:25,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:24:25,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:24:25,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:24:27,012 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:29,635 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:31,637 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:32,097 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:33,639 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:35,642 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:36,673 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:24:37,180 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:37,644 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:39,646 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:40,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:24:40,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:24:40,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:24:41,649 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:42,248 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:43,651 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:45,654 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:47,297 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:47,656 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:49,658 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:51,660 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:52,357 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:53,663 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:55,665 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:55,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:24:55,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:24:55,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:24:57,411 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:24:57,668 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:24:59,671 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:01,673 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:02,471 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:03,676 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:05,678 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:06,676 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:25:07,529 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:07,680 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:09,683 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:10,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:25:10,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:25:10,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:25:11,685 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:12,565 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:13,687 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:15,691 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:17,632 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:17,694 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:19,698 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:21,700 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:22,683 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:23,703 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:25,706 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:25,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:25:25,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:25:25,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:25:27,708 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:27,730 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:29,710 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:31,712 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:32,778 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:33,714 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:35,717 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:36,678 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:25:37,719 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:37,815 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:39,721 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:40,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:25:40,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:25:40,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:25:41,723 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:42,887 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:43,725 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:45,728 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:47,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:47,939 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:49,732 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:51,734 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:52,958 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:53,736 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:55,739 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:55,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:25:55,890 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:25:55,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:25:57,741 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:25:57,987 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:25:59,743 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:01,746 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:03,022 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:03,748 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:05,750 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:06,679 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:26:07,753 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:08,058 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:09,755 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:10,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:26:10,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:26:10,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:26:11,758 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:13,091 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:13,760 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:15,763 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:17,765 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:18,156 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:19,768 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:21,770 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:23,198 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:23,773 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:25,775 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:25,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:26:25,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:26:25,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:26:27,777 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:28,250 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:29,779 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:31,781 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:33,262 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:33,783 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:35,786 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:36,681 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:26:37,792 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:38,302 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:39,795 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:40,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:26:40,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:26:40,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:26:41,797 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:43,359 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:43,799 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:45,801 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:47,804 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:48,410 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:49,806 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:51,808 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:53,431 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:53,811 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:55,813 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:55,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:26:55,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:26:55,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:26:57,815 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:26:58,456 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:26:59,817 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:01,820 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:03,517 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:03,822 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:05,824 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:06,684 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:27:07,826 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:08,566 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:09,829 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:10,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:27:10,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:27:10,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:27:11,831 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:13,608 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:13,834 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:15,836 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:17,838 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:18,636 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:19,842 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:21,846 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:23,680 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:23,848 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:25,851 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:25,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:27:25,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:27:25,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:27:27,853 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:28,716 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:29,855 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:31,858 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:33,759 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:33,860 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:35,862 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:36,686 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:27:37,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:38,777 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:39,870 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:40,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:27:40,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:27:40,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:27:41,873 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:43,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:43,875 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:45,878 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:47,880 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:48,855 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:49,883 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:51,885 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:53,885 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:53,888 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:55,890 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:55,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:27:55,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:27:55,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:27:57,892 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:27:58,932 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:27:59,895 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:01,897 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:03,899 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:03,981 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:05,902 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:06,688 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:28:07,904 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:09,012 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:09,907 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:10,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:28:10,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:28:10,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:28:11,910 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:13,912 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:14,053 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:15,914 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:17,917 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:19,065 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:19,919 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:21,921 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:23,924 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:24,116 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:25,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:28:25,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:28:25,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:28:25,926 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:27,928 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:29,143 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:29,931 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:31,933 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:33,935 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:34,176 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:35,938 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:36,689 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:28:37,940 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:39,198 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:39,943 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:40,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:28:40,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:28:40,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:28:41,945 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:43,947 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:44,229 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:45,949 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:47,952 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:49,276 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:49,954 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:51,957 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:53,959 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:54,310 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:55,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:28:55,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:28:55,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:28:55,961 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:57,964 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:28:59,348 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:28:59,966 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:01,968 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:03,970 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:04,376 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:05,973 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:06,691 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:29:07,975 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:09,441 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:09,977 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:10,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:29:10,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:29:10,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:29:11,979 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:13,982 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:14,501 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:15,984 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:17,986 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:19,522 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:19,989 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:21,991 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:23,993 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:24,568 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:25,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:29:25,891 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:29:25,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:29:25,995 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:27,998 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:29,601 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:30,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:32,003 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:34,005 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:34,631 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:36,007 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:36,694 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:29:38,009 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:39,668 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:40,012 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:40,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:29:40,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:29:40,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:29:42,014 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:44,016 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:44,706 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:46,019 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:48,021 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:49,772 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:50,023 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:52,025 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:54,028 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:54,775 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:29:55,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:29:55,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:29:55,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:29:56,030 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:58,033 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:29:59,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:00,035 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:02,038 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:04,040 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:04,856 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:06,042 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:06,696 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:30:08,044 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:09,867 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:10,047 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:10,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:30:10,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:30:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:30:12,049 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:14,051 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:14,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:16,053 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:18,056 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:19,987 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:20,059 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:22,061 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:24,063 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:25,045 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:25,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:30:25,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:30:25,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:30:26,066 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:28,068 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:30,070 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:30,103 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:32,073 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:34,075 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:35,133 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:36,077 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:36,697 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:30:38,080 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:40,082 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:40,185 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:40,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:30:40,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:30:40,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:30:42,084 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:44,086 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:45,221 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:46,089 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:48,091 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:50,093 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:50,243 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:52,096 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:54,098 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:55,290 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:30:55,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:30:55,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:30:55,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:30:56,100 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:30:58,103 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:00,105 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:00,327 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:02,109 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:04,111 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:05,393 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:06,113 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:06,699 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:31:08,116 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:10,118 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:10,443 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:10,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:31:10,892 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:31:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:31:12,120 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:14,123 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:15,498 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:16,125 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:18,127 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:20,129 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:20,560 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:22,132 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:24,134 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:25,594 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:25,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:31:25,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:31:25,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:31:26,136 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:28,139 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:30,141 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:30,623 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:32,144 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:34,146 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:35,689 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:36,148 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:36,700 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:31:38,151 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:40,154 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:40,729 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:40,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:31:40,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:31:40,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:31:42,156 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:44,159 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:45,794 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:46,161 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:48,164 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:50,166 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:50,834 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:52,168 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:54,171 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:55,872 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:31:55,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:31:55,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:31:55,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:31:56,173 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:31:58,176 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:00,178 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:00,891 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:02,181 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:04,183 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:05,933 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:06,186 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:06,703 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:32:08,188 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:10,191 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:10,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:32:10,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:32:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:32:10,959 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:12,195 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:14,197 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:16,017 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:16,200 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:18,202 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:20,204 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:21,061 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:22,207 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:24,209 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:25,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:32:25,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:32:25,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:32:26,074 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:26,211 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:28,214 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:30,216 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:31,122 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:32,218 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:34,221 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:36,169 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:36,223 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:36,705 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:32:38,225 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:40,228 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:40,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:32:40,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:32:40,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:32:41,225 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:42,230 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:44,232 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:46,235 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:46,256 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:48,237 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:50,239 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:51,295 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:52,243 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:54,247 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:55,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:32:55,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:32:55,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:32:56,252 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:32:56,335 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:32:58,257 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:00,261 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:01,263 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:01,400 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:03,268 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:05,274 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:06,460 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:06,708 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:33:07,280 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:09,285 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:10,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:33:10,893 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:33:10,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:33:11,290 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:11,480 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:13,293 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:15,295 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:16,555 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:17,297 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:19,300 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:21,304 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:21,586 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:23,326 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:25,332 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:25,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:33:25,894 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:33:25,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:33:26,640 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:27,337 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:29,343 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:31,346 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:31,670 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:33,348 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:35,351 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:36,712 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:33:36,713 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:37,354 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:39,356 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:40,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:33:40,894 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:33:40,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:33:41,358 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:41,774 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:43,362 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:45,365 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:46,846 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:47,367 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:49,370 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:51,372 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:51,874 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:53,375 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:55,377 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:55,893 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:33:55,894 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:33:55,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:33:56,887 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:33:57,380 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:33:59,382 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:01,384 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:01,939 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:03,387 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:05,389 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:06,714 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:34:06,999 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:07,392 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:09,394 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:10,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:34:10,895 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:34:10,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:34:11,397 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:12,039 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:13,399 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:15,402 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:17,068 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:17,405 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:19,410 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:21,414 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:22,119 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:23,422 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:25,425 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:25,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:34:25,895 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:34:25,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:34:27,157 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:27,429 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:29,434 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:31,439 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:32,217 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:33,444 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:35,448 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:36,716 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:34:37,256 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:37,450 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:39,453 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:40,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:34:40,895 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:34:40,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:34:41,455 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:42,318 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:43,457 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:45,460 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:47,373 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:47,462 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:49,465 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:51,467 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:52,425 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:53,470 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:55,472 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:55,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:34:55,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:34:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:34:57,474 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:34:57,484 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:34:59,476 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:01,479 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:02,523 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:03,481 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:05,483 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:06,718 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:35:07,486 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:07,554 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:09,488 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:10,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:35:10,895 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:35:10,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:35:11,490 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:12,592 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:13,493 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:15,495 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:17,497 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:17,629 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:19,499 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:21,502 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:22,695 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:23,504 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:25,506 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:25,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:35:25,895 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:35:25,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:35:27,509 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:27,735 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:29,511 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:31,513 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:32,766 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:33,515 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:35,518 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:36,720 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:35:37,520 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:37,804 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:39,523 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:40,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:35:40,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:35:40,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:35:41,525 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:42,840 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:43,527 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:45,530 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:47,532 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:47,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:49,534 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:51,537 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:52,975 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:53,539 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:55,541 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:55,895 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:35:55,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:35:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:35:57,544 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:35:58,000 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:35:59,546 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:01,548 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:03,033 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:03,551 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:05,553 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:06,722 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:36:07,556 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:08,073 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:09,558 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:10,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:36:10,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:36:10,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:36:11,561 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:13,113 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:13,563 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:15,566 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:17,568 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:18,160 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:19,570 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:21,573 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:23,179 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:23,575 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:25,577 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:25,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:36:25,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:36:25,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:36:27,580 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:28,219 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:29,582 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:31,584 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:33,275 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:33,586 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:35,589 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:36,726 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:36:37,591 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:38,333 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:39,593 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:40,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:36:40,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:36:40,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:36:41,595 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:43,360 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:43,598 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:45,600 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:47,602 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:48,383 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:49,604 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:51,607 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:53,412 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:53,609 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:55,611 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:55,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:36:55,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:36:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:36:57,613 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:36:58,456 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:36:59,616 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:01,618 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:03,485 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:03,620 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:05,623 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:06,727 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:37:07,625 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:08,513 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:09,627 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:10,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:37:10,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:37:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:37:11,630 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:13,548 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:13,632 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:15,634 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:17,636 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:18,564 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:19,639 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:21,641 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:23,584 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:23,643 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:25,646 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:25,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:37:25,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:37:25,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:37:27,648 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:28,631 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:29,650 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:31,652 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:33,654 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:33,671 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:35,657 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:36,728 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:37:37,659 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:38,693 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:39,662 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:40,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:37:40,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:37:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:37:41,666 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:43,668 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:43,747 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:45,671 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:47,673 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:48,782 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:49,675 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:51,678 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:53,680 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:53,805 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:55,682 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:55,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:37:55,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:37:55,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:37:57,685 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:37:58,852 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:37:59,687 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:01,690 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:03,692 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:03,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:05,695 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:06,730 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:38:07,697 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:08,961 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:09,700 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:10,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:38:10,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:38:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:38:11,702 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:13,705 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:14,000 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:15,707 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:17,710 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:19,061 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:19,712 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:21,715 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:23,717 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:24,098 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:25,719 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:25,896 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:38:25,896 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:38:25,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:38:27,722 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:29,121 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:29,724 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:31,726 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:33,729 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:34,178 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:35,731 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:36,732 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:38:37,734 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:39,228 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:39,736 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:40,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:38:40,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:38:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:38:41,739 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:43,741 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:44,274 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:45,743 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:47,745 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:49,324 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:49,748 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:51,750 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:53,752 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:54,388 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:55,755 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:55,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:38:55,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:38:55,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:38:57,757 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:38:59,426 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:38:59,759 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:01,761 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:03,764 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:04,463 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:05,766 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:06,733 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:39:07,768 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:09,527 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:09,771 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:10,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:39:10,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:39:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:39:11,773 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:13,775 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:14,560 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:15,778 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:17,780 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:19,608 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:19,782 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:21,785 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:23,787 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:24,661 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:25,789 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:25,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:39:25,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:39:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:39:27,792 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:29,696 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:29,794 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:31,797 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:33,799 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:34,743 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:35,802 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:36,735 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:39:37,804 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:39,790 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:39,806 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:40,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:39:40,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:39:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:39:41,809 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:43,812 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:44,826 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:45,814 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:47,817 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:49,819 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:49,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:51,822 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:53,824 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:54,949 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:39:55,826 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:55,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:39:55,897 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:39:55,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:39:57,829 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:59,831 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:39:59,997 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:01,834 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:03,836 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:05,036 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:05,838 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:06,737 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:40:07,841 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:09,843 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:10,072 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:10,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:40:10,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:40:10,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:40:11,845 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:13,849 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:15,122 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:15,852 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:17,854 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:19,857 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:20,166 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:21,859 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:23,861 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:25,206 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:25,863 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:25,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:40:25,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:40:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:40:27,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:29,868 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:30,248 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:31,870 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:33,873 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:35,324 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:35,875 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:36,739 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:40:37,877 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:39,879 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:40,378 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:40,897 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:40:40,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:40:40,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:40:41,882 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:43,885 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:45,422 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:45,890 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:47,892 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:49,894 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:50,480 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:51,897 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:53,899 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:55,508 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:40:55,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:40:55,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:40:55,902 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:55,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:40:57,904 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:40:59,906 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:00,569 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:01,909 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:03,911 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:05,611 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:05,914 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:06,760 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:41:07,916 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:09,918 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:10,647 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:10,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:41:10,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:41:10,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:41:11,921 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:13,923 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:15,668 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:15,925 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:17,928 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:19,930 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:20,700 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:21,932 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:23,935 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:25,724 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:25,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:41:25,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:41:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:41:25,937 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:27,939 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:29,942 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:30,752 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:31,944 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:33,946 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:35,806 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:35,949 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:36,766 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:41:37,951 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:39,953 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:40,830 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:40,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:41:40,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:41:40,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:41:41,956 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:43,958 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:45,855 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:45,960 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:47,965 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:49,967 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:50,923 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:51,970 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:53,972 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:55,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:41:55,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:41:55,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:41:55,961 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:41:55,974 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:57,977 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:41:59,979 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:01,037 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:01,981 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:03,984 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:05,986 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:06,090 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:06,745 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:42:07,988 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:09,991 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:10,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:42:10,898 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:42:10,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:42:11,103 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:11,993 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:13,995 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:16,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:16,157 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:18,004 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:20,008 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:21,172 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:22,012 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:24,015 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:25,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:42:25,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:42:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:42:26,017 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:26,195 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:28,019 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:30,022 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:31,220 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:32,024 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:34,026 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:36,029 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:36,252 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:36,794 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:42:38,031 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:40,033 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:40,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:42:40,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:42:40,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:42:41,313 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:42,035 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:44,038 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:46,041 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:46,360 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:48,043 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:50,045 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:51,415 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:52,052 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:54,056 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:55,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:42:55,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:42:55,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:42:56,061 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:42:56,450 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:42:58,066 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:00,076 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:01,504 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:02,078 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:04,081 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:06,083 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:06,540 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:06,796 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:43:08,086 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:10,090 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:43:10,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:43:10,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:43:11,589 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:12,092 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:14,996 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:15,997 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:16,632 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:18,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:20,003 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:21,667 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:22,007 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:24,011 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:25,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:43:25,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:43:25,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:43:26,015 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:26,710 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:28,019 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:30,132 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:31,761 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:32,135 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:34,137 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:36,196 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:36,798 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:43:36,798 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:38,198 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:40,364 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:40,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:43:40,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:43:40,975 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:43:41,851 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:42,972 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:43,973 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:46,491 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:46,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:47,492 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:49,497 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:51,500 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:51,934 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:53,502 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:55,504 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:55,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:43:55,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:43:55,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:43:57,020 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:43:57,507 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:43:59,509 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:01,511 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:02,062 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:03,514 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:05,516 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:06,801 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:44:07,095 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:07,519 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:09,521 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:44:10,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:44:10,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:44:11,523 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:12,126 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:13,526 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:15,528 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:17,170 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:17,531 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:19,533 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:21,535 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:22,221 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:23,538 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:25,542 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:25,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:44:25,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:44:25,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:44:27,275 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:27,544 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:29,547 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:31,550 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:32,319 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:33,552 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:35,554 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:36,803 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:44:37,389 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:37,557 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:39,559 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:40,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:44:40,899 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:44:40,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:44:41,561 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:42,426 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:43,564 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:45,566 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:47,437 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:47,569 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:49,571 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:51,573 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:52,497 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:53,576 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:55,578 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:55,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:44:55,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:44:55,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:44:57,566 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:44:57,580 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:44:59,583 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:01,585 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:02,626 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:03,587 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:05,589 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:06,805 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:45:07,593 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:07,659 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:09,595 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:10,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:45:10,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:45:10,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:45:11,597 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:12,680 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:13,600 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:15,602 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:17,604 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:17,738 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:19,606 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:21,609 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:22,802 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:23,611 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:25,613 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:25,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:45:25,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:45:25,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:45:27,615 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:27,822 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:29,618 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:31,620 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:32,839 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:33,622 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:35,625 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:36,807 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:45:37,627 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:37,881 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:39,629 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:40,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:45:40,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:45:40,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:45:41,632 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:42,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:43,634 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:45,636 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:47,639 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:47,942 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:49,641 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:51,643 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:53,007 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:53,645 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:55,648 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:55,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:45:55,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:45:55,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:45:57,650 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:45:58,036 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:45:59,653 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:01,655 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:03,071 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:03,658 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:05,660 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:06,808 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:46:07,662 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:08,100 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:09,665 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:10,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:46:10,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:46:10,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:46:11,667 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:13,126 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:13,669 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:15,672 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:17,674 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:18,197 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:19,677 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:21,679 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:23,240 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:23,681 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:25,683 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:25,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:46:25,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:46:25,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:46:27,686 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:28,304 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:29,688 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:31,690 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:33,346 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:33,692 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:35,695 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:36,831 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:46:37,697 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:38,396 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:39,699 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:40,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:46:40,900 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:46:40,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:46:41,702 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:43,453 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:43,704 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:45,707 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:47,709 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:48,487 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:49,711 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:51,713 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:53,554 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:53,716 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:55,718 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:55,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:46:55,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:46:55,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:46:57,720 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:46:58,571 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:46:59,723 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:01,725 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:03,600 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:03,727 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:05,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:06,834 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:47:07,732 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:08,657 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:09,734 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:10,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:47:10,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:47:10,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:47:11,736 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:13,699 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:13,739 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:15,741 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:17,744 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:18,772 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:19,747 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:21,749 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:23,751 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:23,806 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:25,754 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:25,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:47:25,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:47:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:47:27,756 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:28,847 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:29,758 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:31,760 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:33,763 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:33,887 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:35,765 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:36,814 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:47:37,767 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:38,925 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:39,770 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:40,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:47:40,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:47:40,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:47:41,772 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:43,774 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:43,966 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:45,777 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:47,779 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:49,013 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:49,781 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:51,785 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:53,788 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:54,046 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:55,790 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:47:55,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:47:55,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:47:57,792 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:47:59,080 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:47:59,795 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:01,797 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:03,799 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:04,105 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:05,801 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:06,816 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:48:07,804 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:09,158 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:09,806 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:10,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:48:10,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:48:10,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:48:11,809 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:13,812 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:14,219 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:15,814 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:17,816 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:19,253 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:19,819 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:21,821 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:23,823 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:24,288 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:25,826 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:25,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:48:25,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:48:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:48:27,828 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:29,340 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:29,830 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:31,832 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:33,835 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:34,359 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:35,837 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:36,818 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:48:37,839 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:39,407 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:39,841 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:40,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:48:40,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:48:40,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:48:41,844 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:43,846 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:44,473 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:45,850 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:47,863 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:49,493 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:49,865 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:51,867 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:53,869 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:54,541 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:55,872 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:48:55,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:48:55,983 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:48:57,874 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:48:59,586 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:48:59,876 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:01,879 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:03,881 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:04,621 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:05,883 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:06,820 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:49:07,886 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:09,650 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:09,888 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:10,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:49:10,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:49:10,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:49:11,890 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:13,893 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:14,722 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:15,895 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:17,897 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:19,754 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:19,899 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:21,902 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:23,904 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:24,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:25,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:49:25,901 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:49:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:49:25,906 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:27,912 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:29,844 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:29,914 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:31,917 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:33,919 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:34,883 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:35,921 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:36,823 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:49:37,923 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:39,926 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:39,936 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:40,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:49:40,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:49:40,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:49:41,928 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:43,930 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:45,002 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:45,933 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:47,935 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:49,937 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:50,024 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:51,940 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:53,942 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:55,047 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:49:55,901 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:49:55,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:49:55,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:49:55,944 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:57,946 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:49:59,949 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:00,077 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:01,951 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:03,953 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:05,117 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:05,956 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:06,825 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:50:07,958 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:09,960 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:10,187 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:50:10,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:50:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:50:11,963 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:13,965 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:15,220 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:15,967 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:17,970 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:19,972 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:20,264 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:21,974 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:23,977 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:25,313 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:25,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:50:25,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:50:25,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:50:25,979 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:27,982 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:29,984 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:30,350 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:31,987 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:33,989 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:35,401 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:35,992 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:36,847 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:50:37,995 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:39,999 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:40,470 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:50:40,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:50:40,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:50:42,002 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:44,005 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:45,532 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:46,008 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:48,010 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:50,012 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:50,572 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:52,014 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:54,017 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:55,652 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:50:55,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:50:55,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:50:55,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:50:56,019 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:50:58,022 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:00,024 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:00,689 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:02,026 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:04,028 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:05,726 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:06,031 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:06,829 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:51:08,033 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:10,036 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:10,784 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:51:10,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:51:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:51:12,038 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:14,040 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:15,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:16,043 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:18,045 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:20,047 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:20,873 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:22,049 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:24,052 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:25,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:51:25,902 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:51:25,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:51:26,025 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:26,054 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:28,056 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:30,059 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:31,056 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:32,061 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:34,063 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:36,066 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:36,084 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:36,831 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:51:38,068 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:40,070 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:51:40,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:51:40,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:51:41,132 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:42,073 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:44,075 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:46,078 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:46,191 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:48,080 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:50,082 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:51,220 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:52,085 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:54,087 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:55,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:51:55,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:51:55,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:51:56,090 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:51:56,249 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:51:58,092 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:00,095 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:01,295 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:02,097 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:04,100 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:06,102 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:06,334 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:06,834 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:52:08,104 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:10,107 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:10,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:52:10,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:52:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:52:11,397 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:12,109 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:14,112 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:16,114 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:16,435 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:18,117 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:20,119 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:21,464 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:22,121 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:24,124 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:52:25,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:52:25,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:52:26,126 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:26,509 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:28,129 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:30,131 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:31,553 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:32,134 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:34,136 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:36,138 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:36,592 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:36,836 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:52:38,140 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:40,143 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:40,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:52:40,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:52:40,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:52:41,615 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:42,145 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:44,147 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:46,149 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:46,673 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:48,152 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:50,154 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:51,699 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:52,159 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:54,161 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:55,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:52:55,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:52:55,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:52:56,163 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:52:56,723 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:52:58,169 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:00,174 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:01,814 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:02,177 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:04,181 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:06,183 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:06,839 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:53:06,840 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:08,185 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:10,189 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:10,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:53:10,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:53:10,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:53:11,887 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:12,191 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:14,195 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:16,197 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:16,949 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:18,199 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:20,201 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:22,019 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:22,204 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:24,206 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:53:25,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:53:25,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:53:26,209 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:27,061 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:28,211 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:30,228 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:32,097 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:32,231 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:34,234 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:36,236 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:36,840 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:53:37,129 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:38,238 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:40,253 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:40,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:53:40,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:53:40,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:53:42,160 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:42,282 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:44,379 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:46,446 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:47,226 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:48,448 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:50,451 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:52,274 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:52,453 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:54,472 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:55,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:53:55,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:53:55,952 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:53:56,516 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:53:57,293 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:53:58,534 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:00,553 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:02,343 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:02,556 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:04,560 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:06,563 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:06,841 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:54:07,382 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:08,565 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:10,584 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:10,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:54:10,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:54:10,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:54:12,417 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:12,587 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:14,605 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:16,622 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:17,476 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:18,625 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:20,627 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:22,522 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:22,629 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:24,633 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:25,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:54:25,903 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:54:25,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:54:26,635 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:27,561 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:28,651 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:30,654 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:32,600 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:32,656 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:34,658 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:36,694 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:36,844 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:54:37,657 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:38,696 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:40,698 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:40,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:54:40,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:54:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:54:42,701 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:42,710 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:44,703 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:46,706 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:47,747 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:48,722 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:50,725 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:52,728 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:52,779 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:54,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:55,731 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:55,903 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:54:55,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:54:55,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:54:57,733 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:54:57,793 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:54:59,736 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:01,738 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:02,821 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:03,740 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:05,757 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:06,846 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:55:07,759 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:07,855 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:09,762 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:10,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:55:10,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:55:10,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:55:11,764 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:12,902 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:13,767 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:15,769 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:17,771 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:17,953 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:19,774 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:21,776 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:23,004 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:23,778 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:25,781 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:25,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:55:25,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:55:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:55:27,783 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:28,039 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:29,785 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:31,788 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:33,071 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:33,804 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:35,806 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:36,848 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:55:37,809 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:38,120 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:39,811 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:40,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:55:40,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:55:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:55:41,813 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:43,155 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:43,816 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:45,818 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:47,820 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:48,196 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:49,823 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:51,825 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:53,240 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:53,844 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:55,846 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:55,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:55:55,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:55:55,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:55:57,848 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:55:58,286 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:55:59,850 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:01,853 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:03,326 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:03,855 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:05,857 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:06,850 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:56:07,859 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:08,386 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:09,862 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:10,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:56:10,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:56:10,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:56:11,864 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:13,415 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:13,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:15,869 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:17,871 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:18,442 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:19,888 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:21,890 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:23,513 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:23,893 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:25,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:56:25,904 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:56:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:56:25,910 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:27,912 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:28,550 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:29,929 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:31,931 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:33,629 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:33,933 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:35,936 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:36,870 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:56:37,954 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:38,649 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:39,956 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:40,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:56:40,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:56:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:56:41,958 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:43,696 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:43,961 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:45,963 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:47,965 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:48,733 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:49,968 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:51,970 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:53,776 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:53,972 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:55,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:56:55,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:56:55,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:56:55,975 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:57,977 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:56:58,797 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:56:59,979 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:01,982 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:03,811 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:04,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:06,002 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:06,855 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:57:08,004 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:08,870 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:10,007 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:10,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:57:10,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:57:10,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:57:12,009 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:13,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:14,011 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:16,013 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:18,034 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:18,953 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:20,037 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:22,039 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:24,003 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:24,057 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:25,904 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:57:25,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:57:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:57:26,060 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:28,076 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:29,041 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:30,078 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:32,081 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:34,083 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:34,099 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:36,099 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:36,857 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:57:38,102 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:39,141 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:40,104 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:40,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:57:40,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:57:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:57:42,106 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:44,109 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:44,174 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:46,140 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:48,143 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:49,213 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:50,145 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:52,148 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:54,150 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:54,267 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:57:55,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:57:55,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:57:55,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:57:56,153 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:58,156 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:57:59,351 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:00,158 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:02,161 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:04,163 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:04,381 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:06,180 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:06,859 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:58:08,183 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:09,402 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:10,185 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:10,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:58:10,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:58:10,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:58:12,187 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:14,190 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:14,454 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:16,192 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:18,194 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:19,465 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:20,197 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:22,199 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:24,201 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:24,504 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:25,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:58:25,905 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:58:25,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:58:26,204 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:28,207 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:29,535 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:30,209 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:32,212 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:34,214 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:34,578 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:36,233 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:36,860 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:58:38,235 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:39,615 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:40,266 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:40,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:58:40,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:58:40,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:58:42,284 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:44,287 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:44,680 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:46,289 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:48,308 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:49,724 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:50,311 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:52,315 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:54,318 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:54,762 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:58:55,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:58:55,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:58:55,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:58:56,320 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:58,323 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:58:59,803 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:00,343 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:02,345 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:04,348 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:04,844 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:06,350 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:06,882 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:59:08,353 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:09,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:10,355 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:10,905 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:59:10,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:59:10,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:59:12,357 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:14,360 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:14,926 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:16,362 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:18,378 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:19,963 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:20,380 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:22,398 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:24,401 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:25,015 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:59:25,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:59:25,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:59:26,422 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:28,424 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:30,051 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:30,427 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:32,443 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:34,446 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:35,083 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:36,465 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:36,864 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 17:59:38,468 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:40,118 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:40,470 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:40,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:59:40,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:59:40,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:59:42,472 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:44,475 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:45,161 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:46,477 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:48,480 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:50,190 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:50,482 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:52,498 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:54,501 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:55,247 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 17:59:55,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 17:59:55,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 17:59:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 17:59:56,503 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 17:59:58,520 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:00,292 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:00,522 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:02,529 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:04,532 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:05,353 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:06,534 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:06,867 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:00:08,537 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:10,384 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:10,539 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:10,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:00:10,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:00:10,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:00:12,542 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:14,544 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:15,418 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:16,547 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:18,549 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:20,483 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:20,551 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:22,554 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:24,556 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:25,529 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:00:25,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:00:25,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:00:26,558 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:28,561 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:30,564 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:30,567 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:32,566 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:34,569 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:35,615 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:36,571 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:36,870 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:00:38,573 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:40,576 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:40,652 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:40,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:00:40,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:00:40,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:00:42,578 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:44,580 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:45,676 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:46,601 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:48,603 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:50,605 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:50,702 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:52,625 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:54,628 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:55,727 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:00:55,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:00:55,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:00:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:00:56,630 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:00:58,632 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:00,635 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:00,758 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:02,637 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:04,654 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:05,767 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:06,670 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:06,892 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:01:08,673 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:10,675 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:10,806 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:01:10,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:01:10,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:01:12,677 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:14,694 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:15,841 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:16,696 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:18,699 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:20,701 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:20,876 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:22,703 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:24,720 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:25,900 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:25,906 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:01:25,906 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:01:25,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:01:26,723 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:28,725 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:30,727 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:30,944 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:32,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:34,732 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:35,976 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:36,735 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:36,872 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:01:38,737 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:40,739 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:40,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:01:40,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:01:40,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:01:41,011 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:42,758 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:44,760 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:46,054 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:46,762 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:48,765 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:50,767 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:51,087 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:52,769 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:54,772 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:55,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:01:55,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:01:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:01:56,133 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:01:56,774 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:01:58,776 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:00,778 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:01,180 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:02,781 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:04,796 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:06,218 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:06,799 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:06,874 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:02:08,801 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:10,804 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:02:10,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:02:10,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:02:11,245 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:12,806 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:14,808 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:16,306 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:16,826 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:18,846 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:20,848 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:21,342 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:22,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:24,868 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:25,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:02:25,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:02:25,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:02:26,382 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:26,870 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:28,886 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:29,887 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:31,449 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:31,904 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:33,906 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:35,908 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:36,485 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:36,876 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:02:37,910 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:39,913 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:40,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:02:40,907 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:02:40,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:02:41,540 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:41,915 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:43,917 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:45,919 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:46,579 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:47,921 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:49,937 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:51,608 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:51,940 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:53,959 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:55,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:02:55,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:02:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:02:55,961 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:56,655 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:02:57,965 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:02:59,968 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:01,683 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:01,970 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:03,973 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:05,976 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:06,722 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:06,878 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:03:07,994 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:09,997 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:10,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:03:10,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:03:10,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:03:11,769 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:12,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:14,002 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:16,005 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:16,815 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:18,008 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:20,011 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:21,872 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:22,014 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:24,030 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:25,907 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:03:25,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:03:25,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:03:26,047 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:26,926 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:28,049 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:30,052 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:31,954 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:32,055 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:34,073 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:36,076 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:36,879 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:03:36,983 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:38,080 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:40,101 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:40,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:03:40,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:03:40,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:03:42,031 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:42,104 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:44,106 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:46,108 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:47,082 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:48,111 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:50,113 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:52,116 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:52,142 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:54,118 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:55,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:03:55,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:03:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:03:56,121 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:03:57,179 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:03:58,123 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:00,126 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:02,128 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:02,194 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:04,144 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:06,147 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:06,882 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:04:07,235 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:08,149 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:10,167 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:10,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:04:10,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:04:10,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:04:12,170 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:12,266 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:14,173 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:16,175 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:17,303 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:18,191 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:20,193 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:22,195 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:22,327 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:24,198 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:25,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:04:25,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:04:25,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:04:26,201 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:27,377 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:28,220 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:30,222 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:32,225 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:32,411 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:34,243 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:36,246 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:36,884 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:04:37,472 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:38,263 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:40,266 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:40,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:04:40,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:04:40,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:04:42,282 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:42,498 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:44,284 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:46,287 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:47,524 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:48,289 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:50,292 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:52,294 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:52,536 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:54,296 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:55,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:04:55,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:04:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:04:56,298 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:04:57,591 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:04:58,301 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:00,303 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:02,320 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:02,642 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:04,322 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:06,325 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:06,888 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:05:07,705 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:08,327 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:10,330 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:10,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:05:10,908 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:05:10,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:05:12,332 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:12,740 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:14,334 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:16,337 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:17,785 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:18,340 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:20,342 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:22,345 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:22,849 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:24,347 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:25,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:05:25,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:05:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:05:26,350 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:27,898 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:28,352 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:30,354 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:32,357 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:32,932 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:34,359 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:36,361 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:36,891 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:05:38,009 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:38,363 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:40,366 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:40,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:05:40,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:05:40,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:05:42,368 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:43,065 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:44,388 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:46,390 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:48,137 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:48,407 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:50,423 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:52,426 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:53,161 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:54,428 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:55,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:05:55,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:05:55,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:05:56,430 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:05:58,202 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:05:58,447 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:00,449 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:02,452 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:03,230 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:04,454 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:06,456 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:06,892 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:06:08,269 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:08,459 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:10,461 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:10,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:06:10,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:06:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:06:12,463 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:13,330 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:14,466 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:16,468 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:18,386 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:18,470 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:20,473 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:22,475 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:23,440 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:24,477 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:06:25,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:06:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:06:26,480 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:28,467 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:28,482 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:30,484 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:32,487 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:33,505 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:34,489 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:36,491 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:36,895 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:06:38,493 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:38,539 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:40,496 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:06:40,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:06:40,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:06:42,512 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:43,562 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:44,515 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:46,517 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:48,549 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:48,622 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:50,551 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:52,554 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:53,666 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:06:54,556 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:55,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:06:55,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:06:55,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:06:56,559 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:58,561 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:06:58,720 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:00,564 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:02,579 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:03,791 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:04,582 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:06,584 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:06,931 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:07:08,601 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:08,866 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:10,604 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:10,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:07:10,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:07:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:07:12,620 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:13,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:14,622 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:16,638 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:18,641 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:18,952 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:20,644 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:22,646 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:24,009 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:24,648 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:07:25,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:07:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:07:26,651 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:28,653 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:29,036 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:30,656 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:32,672 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:34,079 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:34,688 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:36,690 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:36,899 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:07:38,692 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:39,135 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:40,695 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:40,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:07:40,909 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:07:40,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:07:42,697 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:44,234 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:44,700 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:46,702 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:48,704 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:49,301 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:50,721 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:52,723 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:54,351 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:07:54,726 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:55,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:07:55,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:07:55,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:07:56,728 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:58,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:07:59,444 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:00,733 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:02,735 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:04,518 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:04,751 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:06,754 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:06,901 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:08:08,784 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:09,567 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:10,787 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:10,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:08:10,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:08:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:08:12,789 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:14,616 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:14,791 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:16,794 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:18,796 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:19,673 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:20,799 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:22,815 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:24,709 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:24,817 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:25,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:08:25,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:08:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:08:26,820 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:28,822 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:29,755 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:30,840 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:32,842 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:34,788 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:34,844 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:36,847 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:36,903 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:08:38,849 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:39,840 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:40,852 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:40,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:08:40,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:08:40,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:08:42,854 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:44,857 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:44,899 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:46,859 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:48,861 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:49,942 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:50,864 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:52,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:54,868 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:54,992 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:08:55,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:08:55,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:08:55,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:08:56,885 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:08:58,887 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:00,074 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:00,889 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:02,907 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:04,909 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:05,118 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:06,905 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:09:06,912 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:08,914 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:10,151 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:10,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:09:10,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:09:10,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:09:10,916 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:12,919 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:14,921 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:15,205 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:16,924 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:18,926 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:20,245 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:20,928 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:22,931 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:24,946 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:25,312 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:25,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:09:25,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:09:25,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:09:26,963 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:28,965 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:30,361 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:30,969 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:32,971 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:34,974 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:35,411 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:36,907 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:09:36,977 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:38,979 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:40,484 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:40,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:09:40,910 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:09:40,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:09:40,982 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:42,984 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:44,987 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:45,521 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:47,003 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:49,005 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:50,588 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:51,008 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:53,011 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:55,013 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:55,633 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:09:55,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:09:55,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:09:55,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:09:57,015 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:09:59,018 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:00,698 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:01,020 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:03,022 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:05,025 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:05,785 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:06,908 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:10:07,027 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:09,029 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:10,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:10,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:10:10,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:10:10,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:10:11,032 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:13,034 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:15,036 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:15,884 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:17,039 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:18,041 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:20,043 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:20,941 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:22,046 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:24,048 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:25,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:10:25,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:10:25,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:10:25,998 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:26,050 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:28,052 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:30,055 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:31,041 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:32,072 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:34,089 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:36,087 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:36,091 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:36,928 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:10:38,094 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:40,097 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:40,910 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:10:40,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:10:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:10:41,145 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:42,099 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:44,102 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:46,104 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:46,206 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:48,123 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:50,125 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:51,254 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:52,128 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:54,130 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:10:55,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:10:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:10:56,133 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:10:56,294 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:10:58,136 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:00,138 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:01,354 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:02,140 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:04,143 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:06,145 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:06,398 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:06,914 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:11:08,148 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:10,150 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:10,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:11:10,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:11:10,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:11:11,469 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:12,152 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:14,155 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:16,157 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:16,513 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:18,159 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:20,162 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:21,557 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:22,164 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:24,167 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:25,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:11:25,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:11:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:11:26,169 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:26,594 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:28,171 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:30,202 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:31,691 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:32,205 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:34,207 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:36,209 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:36,739 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:36,915 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:11:38,211 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:40,214 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:40,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:11:40,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:11:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:11:41,802 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:42,216 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:44,219 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:46,221 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:46,856 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:48,223 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:50,241 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:51,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:52,243 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:54,247 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:11:55,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:11:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:11:56,278 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:11:56,975 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:11:58,280 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:00,283 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:02,053 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:02,285 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:04,301 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:06,304 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:06,931 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:12:07,087 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:08,306 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:10,309 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:10,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:12:10,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:12:10,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:12:12,130 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:12,311 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:14,314 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:16,316 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:17,151 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:18,332 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:20,334 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:22,189 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:22,353 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:24,355 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:25,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:12:25,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:12:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:12:26,358 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:27,259 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:28,360 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:30,362 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:32,329 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:32,364 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:34,382 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:36,384 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:36,920 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:12:37,401 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:38,387 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:40,389 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:40,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:12:40,911 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:12:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:12:42,392 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:42,455 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:44,394 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:46,396 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:47,522 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:48,412 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:50,415 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:52,449 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:52,581 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:54,454 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:55,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:12:55,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:12:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:12:56,456 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:12:57,648 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:12:58,460 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:00,463 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:02,465 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:02,705 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:04,469 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:06,471 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:06,922 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:13:07,770 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:08,475 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:10,478 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:10,911 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:13:10,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:13:10,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:13:12,497 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:12,826 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:14,499 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:16,501 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:17,892 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:18,504 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:20,506 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:22,510 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:22,931 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:24,513 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:25,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:13:25,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:13:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:13:26,515 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:28,002 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:28,518 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:30,520 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:32,523 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:33,074 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:34,525 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:36,527 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:36,924 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:13:38,125 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:38,530 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:40,532 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:40,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:13:40,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:13:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:13:42,536 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:43,178 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:44,557 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:46,569 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:48,243 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:48,572 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:50,574 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:52,577 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:53,297 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:54,579 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:13:55,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:13:55,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:13:56,581 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:13:58,340 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:13:58,584 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:00,586 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:02,588 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:03,367 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:04,591 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:06,593 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:06,926 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:14:08,456 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:08,596 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:10,626 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:10,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:14:10,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:14:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:14:12,630 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:13,508 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:14,632 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:16,634 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:18,575 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:18,637 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:20,639 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:22,641 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:23,615 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:24,644 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:25,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:14:25,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:14:25,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:14:26,647 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:28,652 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:28,671 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:30,669 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:32,672 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:33,741 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:34,675 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:36,677 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:36,928 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:14:38,680 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:38,817 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:40,682 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:40,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:14:40,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:14:40,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:14:42,685 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:43,853 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:44,687 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:46,689 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:48,692 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:48,864 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:50,694 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:52,696 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:53,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:14:54,699 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:14:55,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:14:55,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:14:56,702 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:58,704 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:14:58,926 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:00,706 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:02,708 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:04,012 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:04,711 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:06,713 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:06,930 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:15:08,715 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:09,067 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:10,718 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:10,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:15:10,912 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:15:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:15:12,720 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:14,166 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:14,722 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:16,725 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:18,728 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:19,168 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:20,730 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:22,732 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:24,173 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:24,735 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:25,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:15:25,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:15:25,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:15:26,737 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:28,740 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:29,222 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:30,743 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:32,745 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:34,245 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:34,747 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:36,750 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:36,934 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:15:38,752 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:39,271 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:40,759 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:40,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:15:40,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:15:40,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:15:42,761 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:44,315 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:44,764 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:46,766 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:48,768 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:49,378 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:50,771 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:52,773 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:54,435 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:15:54,778 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:55,912 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:15:55,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:15:55,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:15:56,781 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:58,783 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:15:59,483 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:00,785 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:02,788 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:04,538 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:04,790 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:06,795 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:06,935 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:16:08,797 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:09,576 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:10,800 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:16:10,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:16:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:16:12,802 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:14,581 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:14,807 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:16,810 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:18,828 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:19,596 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:20,830 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:22,832 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:24,608 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:24,834 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:16:25,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:16:25,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:16:26,837 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:28,839 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:29,643 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:30,841 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:32,843 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:34,726 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:34,845 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:36,848 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:36,938 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:16:38,851 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:39,845 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:40,853 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:40,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:16:40,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:16:40,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:16:42,855 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:44,856 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:44,857 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:46,860 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:48,862 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:49,877 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:50,864 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:52,866 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:54,869 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:54,933 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:16:55,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:16:55,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:16:55,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:16:56,871 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:58,873 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:16:59,993 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:00,876 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:02,878 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:04,881 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:05,020 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:06,883 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:06,940 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:17:08,885 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:10,117 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:10,887 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:17:10,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:17:10,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:17:12,889 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:14,892 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:15,123 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:16,894 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:18,896 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:20,132 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:20,898 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:22,900 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:24,902 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:25,208 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:25,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:17:25,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:17:25,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:17:26,904 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:28,906 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:30,280 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:30,908 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:32,911 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:34,914 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:35,334 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:36,916 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:36,942 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:17:38,920 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:40,400 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:40,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:17:40,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:17:40,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:17:40,922 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:42,924 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:44,927 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:45,502 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:46,929 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:48,931 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:50,541 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:50,933 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:52,935 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:54,937 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:55,622 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:17:55,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:17:55,913 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:17:55,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:17:56,939 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:17:58,942 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:00,726 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:00,945 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:02,947 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:04,950 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:05,742 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:06,944 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:18:06,952 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:08,954 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:10,754 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:10,913 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:18:10,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:18:10,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:18:10,957 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:12,959 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:14,961 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:15,852 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:16,964 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:18,966 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:20,908 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:20,969 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:22,971 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:24,973 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:25,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:18:25,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:18:25,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:18:26,012 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:26,976 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:28,978 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:30,981 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:31,016 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:32,983 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:34,985 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:36,107 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:36,946 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:18:36,987 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:38,996 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:40,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:18:40,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:18:40,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:18:40,998 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:41,246 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:43,000 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:45,003 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:46,267 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:47,005 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:49,008 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:51,011 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:51,351 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:53,013 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:55,016 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:55,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:18:55,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:18:55,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:18:56,419 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:18:57,018 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:18:59,021 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:01,025 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:01,428 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:03,027 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:05,030 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:06,487 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:06,948 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:19:07,033 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:09,035 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:10,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:19:10,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:19:10,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:19:11,038 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:11,636 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:13,040 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:15,043 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:16,712 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:17,045 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:19,048 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:21,050 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:22,120 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:19:25,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:19:25,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:19:27,056 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:27,215 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:29,059 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:31,062 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:32,274 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:33,064 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:35,067 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:36,949 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:19:37,070 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:37,309 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:39,073 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:40,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:19:40,914 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:19:40,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:19:41,076 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:42,438 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:43,078 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:45,081 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:47,084 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:47,445 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:49,086 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:51,089 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:52,509 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:53,092 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:55,095 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:55,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:19:55,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:19:55,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:19:57,098 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:19:57,525 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:19:59,101 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:01,104 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:02,549 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:03,108 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:05,112 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:06,952 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:20:07,116 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:07,577 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:09,120 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:10,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:20:10,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:20:10,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:20:11,122 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:12,604 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:13,125 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:15,127 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:17,131 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:17,615 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:19,133 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:21,135 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:22,645 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:23,138 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:25,140 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:25,914 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:20:25,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:20:25,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:20:27,143 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:27,648 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:29,145 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:31,148 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:32,748 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:33,150 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:35,152 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:36,954 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:20:37,156 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:37,760 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:39,158 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:20:40,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:20:40,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:20:41,161 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:42,769 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:43,163 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:45,166 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:47,169 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:47,890 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:49,172 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:51,174 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:52,970 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:53,177 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:55,179 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:20:55,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:20:55,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:20:57,182 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:20:58,025 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:20:59,184 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:01,187 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:03,040 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:03,189 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:05,193 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:06,956 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:21:07,196 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:08,090 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:09,198 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:10,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:21:10,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:21:10,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:21:11,201 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:13,151 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:13,203 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:15,207 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:17,209 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:18,235 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:19,212 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:21,214 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:23,217 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:23,332 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:25,220 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:21:25,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:21:25,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:21:27,222 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:28,353 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:29,225 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:31,228 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:33,230 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:33,442 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:35,232 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:36,958 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:21:37,235 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:38,498 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:39,238 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:21:40,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:21:40,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:21:41,241 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:43,244 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:43,499 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:45,246 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:47,249 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:48,578 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:49,251 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:51,254 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:53,259 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:53,596 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:55,262 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:21:55,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:21:55,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:21:57,264 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:21:58,612 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:21:59,267 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:01,269 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:03,272 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:03,684 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:05,274 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:06,960 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:22:07,278 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:08,792 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:09,281 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:10,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:22:10,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:22:10,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:22:11,284 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:13,286 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:13,856 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:15,289 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:17,292 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:18,985 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:19,295 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:21,298 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:23,300 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:24,081 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:25,302 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:25,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:22:25,915 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:22:25,918 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:22:27,305 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:28,307 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:29,133 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:30,309 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:32,312 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:34,151 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:34,316 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:36,319 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:36,962 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:22:38,321 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:39,224 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:40,324 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:40,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:22:40,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:22:40,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:22:42,326 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:44,238 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:44,329 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:46,331 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:48,334 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:49,323 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:50,336 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:52,339 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:54,344 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:54,383 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:22:55,915 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:22:55,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:22:55,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:22:56,347 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:58,350 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:22:59,426 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:00,352 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:02,357 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:04,359 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:04,443 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:06,363 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:06,974 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:23:08,365 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:09,494 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:10,370 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:23:10,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:23:10,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:23:12,373 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:14,376 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:14,515 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:16,380 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:18,383 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:19,540 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:20,387 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:22,390 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:24,393 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:24,648 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:25,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:23:25,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:23:25,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:23:26,397 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:28,400 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:29,691 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:30,403 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:32,405 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:34,408 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:34,728 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:36,411 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:36,976 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:23:38,413 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:39,785 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:40,416 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:40,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:23:40,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:23:40,920 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:23:42,419 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:44,422 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:44,909 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:46,426 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:48,429 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:49,961 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:50,432 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:52,434 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:54,437 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:55,080 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:23:55,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:23:55,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:23:55,920 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:23:56,440 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:23:58,442 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:00,190 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:00,444 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:02,447 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:04,450 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:05,231 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:06,452 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:06,980 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:24:08,455 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:10,309 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:10,458 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:24:10,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:24:10,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:24:12,460 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:14,463 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:15,370 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:16,465 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:18,468 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:20,435 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:20,471 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:22,473 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:24,476 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:25,468 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:25,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:24:25,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:24:25,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:24:26,479 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:28,481 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:30,484 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:30,616 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:32,487 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:34,490 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:35,668 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:36,493 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:36,970 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:24:38,496 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:40,498 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:40,783 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:40,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:24:40,916 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:24:40,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:24:42,501 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:44,503 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:45,840 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:46,506 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:48,508 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:24:50,951 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:24:55,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:24:55,917 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:24:55,919 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:24:56,025 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:01,026 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:06,027 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:06,973 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:25:10,916 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:25:10,917 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:25:10,958 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:25:11,987 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:16,542 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:25:17,356 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:22,356 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:25,917 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: stop_status +2024-04-16 18:25:25,918 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: stop_status +2024-04-16 18:25:25,920 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:25:26,555 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:25:26,788 DEBUG SenderThread:940772 [sender.py:send():382] send: exit +2024-04-16 18:25:26,788 INFO SenderThread:940772 [sender.py:send_exit():589] handling exit code: 1 +2024-04-16 18:25:26,788 INFO SenderThread:940772 [sender.py:send_exit():591] handling runtime: 3740 +2024-04-16 18:25:26,790 INFO SenderThread:940772 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-16 18:25:26,792 INFO SenderThread:940772 [sender.py:send_exit():597] send defer +2024-04-16 18:25:26,793 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:26,793 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 0 +2024-04-16 18:25:26,793 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:26,793 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-04-16 18:25:26,793 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 1 +2024-04-16 18:25:26,793 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:26,793 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 1 +2024-04-16 18:25:26,793 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:26,793 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-04-16 18:25:26,793 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 2 +2024-04-16 18:25:26,793 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:26,793 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 2 +2024-04-16 18:25:26,793 INFO HandlerThread:940772 [system_monitor.py:finish():203] Stopping system monitor +2024-04-16 18:25:26,793 DEBUG SystemMonitor:940772 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-16 18:25:26,794 DEBUG SystemMonitor:940772 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-16 18:25:26,796 INFO HandlerThread:940772 [interfaces.py:finish():202] Joined cpu monitor +2024-04-16 18:25:26,796 INFO HandlerThread:940772 [interfaces.py:finish():202] Joined disk monitor +2024-04-16 18:25:27,559 INFO Thread-12 :940772 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-summary.json +2024-04-16 18:25:28,560 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:25:29,488 INFO HandlerThread:940772 [interfaces.py:finish():202] Joined gpu monitor +2024-04-16 18:25:29,489 INFO HandlerThread:940772 [interfaces.py:finish():202] Joined memory monitor +2024-04-16 18:25:29,489 INFO HandlerThread:940772 [interfaces.py:finish():202] Joined network monitor +2024-04-16 18:25:29,489 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-16 18:25:29,490 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:29,490 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,490 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-04-16 18:25:29,490 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 3 +2024-04-16 18:25:29,490 DEBUG SenderThread:940772 [sender.py:send():382] send: stats +2024-04-16 18:25:29,490 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:29,491 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: poll_exit +2024-04-16 18:25:29,491 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 3 +2024-04-16 18:25:29,491 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,491 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-04-16 18:25:29,491 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 4 +2024-04-16 18:25:29,491 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:29,491 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 4 +2024-04-16 18:25:29,491 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,491 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-04-16 18:25:29,491 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 5 +2024-04-16 18:25:29,491 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:29,492 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 5 +2024-04-16 18:25:29,492 DEBUG SenderThread:940772 [sender.py:send():382] send: summary +2024-04-16 18:25:29,494 INFO SenderThread:940772 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-16 18:25:29,494 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,494 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-04-16 18:25:29,494 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 6 +2024-04-16 18:25:29,494 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:29,494 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 6 +2024-04-16 18:25:29,494 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,495 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-04-16 18:25:29,495 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 7 +2024-04-16 18:25:29,495 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: status_report +2024-04-16 18:25:29,495 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:29,495 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 7 +2024-04-16 18:25:29,495 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:29,495 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-04-16 18:25:29,562 INFO Thread-12 :940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-summary.json +2024-04-16 18:25:30,369 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 8 +2024-04-16 18:25:30,369 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:30,369 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 8 +2024-04-16 18:25:30,369 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:30,369 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-04-16 18:25:30,369 INFO SenderThread:940772 [job_builder.py:build():298] Attempting to build job artifact +2024-04-16 18:25:30,371 INFO SenderThread:940772 [job_builder.py:_get_source_type():439] no source found +2024-04-16 18:25:30,371 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 9 +2024-04-16 18:25:30,371 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:30,371 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 9 +2024-04-16 18:25:30,371 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:30,371 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-04-16 18:25:30,371 INFO SenderThread:940772 [dir_watcher.py:finish():358] shutting down directory watcher +2024-04-16 18:25:30,564 INFO SenderThread:940772 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:25:30,565 INFO SenderThread:940772 [dir_watcher.py:finish():388] scan: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files +2024-04-16 18:25:30,565 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/conda-environment.yaml conda-environment.yaml +2024-04-16 18:25:30,566 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/requirements.txt requirements.txt +2024-04-16 18:25:30,566 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-summary.json wandb-summary.json +2024-04-16 18:25:30,569 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-metadata.json wandb-metadata.json +2024-04-16 18:25:30,570 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/config.yaml config.yaml +2024-04-16 18:25:30,572 INFO SenderThread:940772 [dir_watcher.py:finish():402] scan save: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log output.log +2024-04-16 18:25:30,574 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 10 +2024-04-16 18:25:30,574 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:30,576 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 10 +2024-04-16 18:25:30,576 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:30,576 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-04-16 18:25:30,576 INFO SenderThread:940772 [file_pusher.py:finish():175] shutting down file pusher +2024-04-16 18:25:30,822 INFO wandb-upload_0:940772 [upload_job.py:push():131] Uploaded file /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/conda-environment.yaml +2024-04-16 18:25:30,916 INFO wandb-upload_3:940772 [upload_job.py:push():131] Uploaded file /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/config.yaml +2024-04-16 18:25:30,932 INFO wandb-upload_2:940772 [upload_job.py:push():131] Uploaded file /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/wandb-summary.json +2024-04-16 18:25:30,933 INFO wandb-upload_4:940772 [upload_job.py:push():131] Uploaded file /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/output.log +2024-04-16 18:25:31,903 INFO wandb-upload_1:940772 [upload_job.py:push():131] Uploaded file /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/files/requirements.txt +2024-04-16 18:25:32,103 INFO Thread-11 (_thread_body):940772 [sender.py:transition_state():617] send defer: 11 +2024-04-16 18:25:32,104 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:32,104 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 11 +2024-04-16 18:25:32,104 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:32,105 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-04-16 18:25:32,105 INFO SenderThread:940772 [file_pusher.py:join():181] waiting for file pusher +2024-04-16 18:25:32,105 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 12 +2024-04-16 18:25:32,105 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:32,105 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 12 +2024-04-16 18:25:32,105 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:32,105 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-04-16 18:25:32,105 INFO SenderThread:940772 [file_stream.py:finish():595] file stream finish called +2024-04-16 18:25:32,303 INFO SenderThread:940772 [file_stream.py:finish():599] file stream finish is done +2024-04-16 18:25:32,303 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 13 +2024-04-16 18:25:32,303 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:32,303 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 13 +2024-04-16 18:25:32,303 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:32,303 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-04-16 18:25:32,303 INFO SenderThread:940772 [sender.py:transition_state():617] send defer: 14 +2024-04-16 18:25:32,303 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: defer +2024-04-16 18:25:32,303 DEBUG SenderThread:940772 [sender.py:send():382] send: final +2024-04-16 18:25:32,304 INFO HandlerThread:940772 [handler.py:handle_request_defer():172] handle defer: 14 +2024-04-16 18:25:32,304 DEBUG SenderThread:940772 [sender.py:send():382] send: footer +2024-04-16 18:25:32,304 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: defer +2024-04-16 18:25:32,304 INFO SenderThread:940772 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-04-16 18:25:32,304 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-16 18:25:32,304 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: poll_exit +2024-04-16 18:25:32,305 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: server_info +2024-04-16 18:25:32,305 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: get_summary +2024-04-16 18:25:32,305 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: server_info +2024-04-16 18:25:32,305 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: sampled_history +2024-04-16 18:25:32,307 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-16 18:25:32,307 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: job_info +2024-04-16 18:25:32,393 DEBUG SenderThread:940772 [sender.py:send_request():409] send_request: job_info +2024-04-16 18:25:32,394 INFO MainThread:940772 [wandb_run.py:_footer_history_summary_info():3837] rendering history +2024-04-16 18:25:32,394 INFO MainThread:940772 [wandb_run.py:_footer_history_summary_info():3869] rendering summary +2024-04-16 18:25:32,394 INFO MainThread:940772 [wandb_run.py:_footer_sync_info():3796] logging synced files +2024-04-16 18:25:32,394 DEBUG HandlerThread:940772 [handler.py:handle_request():146] handle_request: shutdown +2024-04-16 18:25:32,394 INFO HandlerThread:940772 [handler.py:finish():866] shutting down handler +2024-04-16 18:25:33,307 INFO WriterThread:940772 [datastore.py:close():294] close: /fsx/sanchit/mistral-debug-4bit/wandb/run-20240416_172306-uygw9yfk/run-uygw9yfk.wandb +2024-04-16 18:25:33,394 INFO SenderThread:940772 [sender.py:finish():1548] shutting down sender +2024-04-16 18:25:33,394 INFO SenderThread:940772 [file_pusher.py:finish():175] shutting down file pusher +2024-04-16 18:25:33,394 INFO SenderThread:940772 [file_pusher.py:join():181] waiting for file pusher