Model save

emissions.csv

@@ -0,0 +1,2 @@
+timestamp,experiment_id,project_name,duration,emissions,energy_consumed,country_name,country_iso_code,region,on_cloud,cloud_provider,cloud_region
+2022-02-03T07:10:25,fa24425f-fb93-453d-a3ec-d95923987a41,codecarbon,31194.02552294731,0.07735861275392851,0.3675413725101289,United States,USA,new york,N,,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b9d949d0c7831eedc1004ab137d24f1e11ab3cd37ac2db673524ace2330bca1
+oid sha256:ae4f5f543d0525d8e06addd5a81267c125c7f3dc57a189413aa2c73e36c8561a
 size 1262075377

runs/Feb02_22-30-13_brutasse/events.out.tfevents.1643841025.brutasse.21084.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9101f267030fe8e88ba749e485799d811297d513132d15d874fe00882bf4bf1e
-size 44461
+oid sha256:08218c1bd4e18144bbcb761191b0ebb1d1b7a0277e454ff7322c0bee1ceeca32
+size 45141

wandb/run-20220202_223026-1j0459xm/files/config.yaml

@@ -4802,6 +4802,26 @@ _wandb:
       5: 1
       6:
       - 1
+    - 1: train/train_runtime
+      5: 1
+      6:
+      - 1
+    - 1: train/train_samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: train/train_steps_per_second
+      5: 1
+      6:
+      - 1
+    - 1: train/total_flos
+      5: 1
+      6:
+      - 1
+    - 1: train/train_loss
+      5: 1
+      6:
+      - 1
     python_version: 3.8.12
     start_time: 1643841026
     t:

wandb/run-20220202_223026-1j0459xm/files/output.log

@@ -14468,3 +14468,156 @@ Deleting older checkpoint [checkpoint-3500] due to args.save_total_limit
  27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
 {'eval_loss': 0.27769750356674194, 'eval_wer': 0.2534487652568833, 'eval_runtime': 191.0585, 'eval_samples_per_second': 25.348, 'eval_steps_per_second': 3.172, 'epoch': 49.31}
  27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+{'loss': 1.0195, 'learning_rate': 7.43076923076923e-07, 'epoch': 49.59}
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|█████████████��█▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+{'loss': 1.0473, 'learning_rate': 2.8153846153846154e-07, 'epoch': 49.86}
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+{'train_runtime': 31196.8756, 'train_samples_per_second': 18.736, 'train_steps_per_second': 0.585, 'train_loss': 1.449225697974636, 'epoch': 50.0}
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]
+ 27%|███████████████▏                                        | 164/606 [00:41<01:41,  4.34it/s]

wandb/run-20220202_223026-1j0459xm/logs/debug-internal.log

@@ -21528,3 +21528,234 @@
 2022-02-03 07:05:05,662 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
 2022-02-03 07:05:10,913 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
 2022-02-03 07:05:10,913 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:05:25,989 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:05:25,990 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:05:27,805 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:29,805 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:31,806 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:33,806 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:35,806 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:35,939 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:05:37,807 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:39,807 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:41,084 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:05:41,084 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:05:41,807 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:43,808 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:45,808 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:47,808 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:49,809 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:51,809 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:54,810 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:56,194 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:05:56,194 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:05:56,810 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:05:58,810 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:00,810 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:02,811 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:04,811 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:06,178 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:06:08,812 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:10,812 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:11,277 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:06:11,277 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:06:12,813 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:14,813 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:16,813 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:18,813 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:20,814 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:22,814 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:24,814 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:26,354 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:06:26,354 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:06:26,815 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:28,815 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:30,815 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:32,816 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:34,816 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:36,417 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:06:36,816 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:38,817 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:40,817 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:41,463 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:06:41,463 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:06:42,817 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:44,817 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:46,818 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:48,818 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:50,818 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:52,819 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:54,819 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:56,562 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:06:56,562 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:06:56,819 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:06:58,820 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:00,820 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:02,820 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:04,821 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:06,668 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:07:06,821 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:08,821 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:10,822 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:11,652 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:07:11,652 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:07:12,822 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:14,822 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:16,823 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:18,823 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:20,823 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:22,824 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:24,824 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:26,752 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:07:26,752 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:07:26,824 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:27,595 DEBUG   SenderThread:21197 [sender.py:send():182] send: history
+2022-02-03 07:07:27,658 DEBUG   SenderThread:21197 [sender.py:send():182] send: summary
+2022-02-03 07:07:27,687 INFO    SenderThread:21197 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-02-03 07:07:27,825 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/wandb-summary.json
+2022-02-03 07:07:28,825 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:30,825 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:33,826 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:35,826 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:36,906 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:07:37,827 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:39,827 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:41,827 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:41,881 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:07:41,881 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:07:43,828 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:45,828 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:47,829 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:49,829 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:51,829 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:53,829 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:55,830 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:56,949 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:07:56,949 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:07:57,830 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:07:59,831 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:01,831 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:03,831 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:05,832 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:07,140 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:08:07,832 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:09,832 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:11,833 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:12,021 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:08:12,021 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:08:13,833 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:15,834 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:17,834 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:19,834 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:21,834 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:23,835 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:25,835 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:27,364 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:08:27,364 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:08:27,836 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:29,836 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:31,836 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:33,837 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:35,837 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:37,371 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:08:37,837 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:39,838 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:41,838 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:42,455 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:08:42,455 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:08:43,838 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:45,839 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:47,839 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:49,839 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:51,840 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:53,840 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:55,840 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:57,567 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:08:57,567 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:08:57,841 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:08:59,841 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:01,841 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:03,842 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:05,842 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:07,603 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:09:07,842 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:09,843 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:11,843 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:12,645 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:09:12,645 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:09:13,843 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:15,844 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:17,844 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:19,844 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:21,845 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:23,845 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:25,845 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:27,722 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:09:27,722 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:09:27,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:28,257 DEBUG   SenderThread:21197 [sender.py:send():182] send: history
+2022-02-03 07:09:28,300 DEBUG   SenderThread:21197 [sender.py:send():182] send: summary
+2022-02-03 07:09:28,323 INFO    SenderThread:21197 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-02-03 07:09:28,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/wandb-summary.json
+2022-02-03 07:09:28,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:29,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:31,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:33,846 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:36,847 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:37,821 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:09:38,847 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:40,848 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:42,798 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:09:42,798 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:09:42,848 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:44,848 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:46,849 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:48,849 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:50,849 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:52,849 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:54,850 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:56,850 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:09:57,874 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:09:57,874 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:09:58,850 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:00,851 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:02,851 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:04,851 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:06,852 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:08,056 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:10:08,852 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:10,852 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:12,852 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:12,943 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:10:12,944 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:10:14,853 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:16,853 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:18,853 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:20,854 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:22,782 DEBUG   SenderThread:21197 [sender.py:send():182] send: metric
+2022-02-03 07:10:22,787 DEBUG   SenderThread:21197 [sender.py:send():182] send: metric
+2022-02-03 07:10:22,797 DEBUG   SenderThread:21197 [sender.py:send():182] send: metric
+2022-02-03 07:10:22,803 DEBUG   SenderThread:21197 [sender.py:send():182] send: metric
+2022-02-03 07:10:22,806 DEBUG   SenderThread:21197 [sender.py:send():182] send: metric
+2022-02-03 07:10:22,806 DEBUG   SenderThread:21197 [sender.py:send():182] send: history
+2022-02-03 07:10:22,834 DEBUG   SenderThread:21197 [sender.py:send():182] send: summary
+2022-02-03 07:10:22,857 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/wandb-summary.json
+2022-02-03 07:10:22,857 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:22,857 INFO    SenderThread:21197 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-02-03 07:10:24,857 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:26,858 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:28,013 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:10:28,013 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:10:36,862 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/config.yaml
+2022-02-03 07:10:38,497 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:10:43,120 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:10:43,121 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:10:44,864 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:10:58,197 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:10:58,197 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:11:00,870 INFO    Thread-8  :21197 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220202_223026-1j0459xm/files/output.log
+2022-02-03 07:11:08,806 DEBUG   SenderThread:21197 [sender.py:send():182] send: stats
+2022-02-03 07:11:13,271 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:11:13,271 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status
+2022-02-03 07:11:28,350 DEBUG   HandlerThread:21197 [handler.py:handle_request():126] handle_request: stop_status
+2022-02-03 07:11:28,350 DEBUG   SenderThread:21197 [sender.py:send_request():196] send_request: stop_status

wandb/run-20220202_223026-1j0459xm/run-1j0459xm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8037bfe14ac3960a01b7ab0399f417e2382e55b3a181118f37631aa2c373e591
-size 121023901
+oid sha256:f4b1788a312bbc407ff6ca5e9124ba0d6f8863110f40fc7c6f8f37ffb41f22e4
+size 123055105