upload ViV1T model checkpoints, each trained with a different seed.

001/args.yaml

@@ -0,0 +1,87 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_eps: 1e-08
+batch_size: 6
+cache_data: false
+center_crop: 1.0
+clear_output_dir: true
+core: vivit
+core_behavior_mode: 2
+core_compile: false
+core_drop_path: 0.0505
+core_emb_dim: 112
+core_ff_activation: gelu
+core_ff_dropout: 0.0592
+core_flash_attention: 1
+core_head_dim: 48
+core_lr: 0.0048
+core_mha_dropout: 0.358
+core_mlp_dim: 136
+core_norm_qk: false
+core_num_heads: 11
+core_p_dropout: 0.1338
+core_parallel_attention: true
+core_patch_mode: 0
+core_pos_encoding: 3
+core_spatial_depth: 3
+core_spatial_patch_size: 7
+core_spatial_patch_stride: 2
+core_temporal_depth: 5
+core_temporal_patch_size: 25
+core_temporal_patch_stride: 1
+core_weight_decay: 0.1789
+criterion: poisson_correlation
+critic_mode: 0
+crop_frame: 140
+data: /home/storage/sensorium
+deterministic: false
+device: cuda
+dpi: 120
+ds_mode: 3
+ds_scale: 1
+epochs: 400
+format: svg
+git_hash: 1033c29
+grad_checkpointing: 0
+grad_norm: null
+input_shapes:
+  behavior: [2, 140]
+  pupil_center: [2, 140]
+  video: [1, 140, 36, 64]
+limit_data: null
+lr: 0.0036
+max_frame: 300
+micro_batch_size: 6
+modulator_mode: 0
+mouse_ids: [A, B, C, D, E, F, G, H, I, J]
+num_workers: 2
+output_dir: /home/storage/runs/001
+output_mode: 1
+output_shapes:
+  A: [7440, 140]
+  B: [7928, 140]
+  C: [8285, 140]
+  D: [7671, 140]
+  E: [7495, 140]
+  F: [7863, 140]
+  G: [7908, 140]
+  H: [8202, 140]
+  I: [7939, 140]
+  J: [8122, 140]
+precision: "32"
+pretrain_core: null
+readout: gaussian2d
+readout_bias_mode: 2
+readout_dropout: 0.0
+readout_grid_mode: 1
+restore: null
+save_plots: false
+seed: 2
+shifter_layers: 3
+shifter_mode: 1
+shifter_size: 5
+stat_mode: 1
+trainable_params: 12278365
+transform_mode: 2
+verbose: 1
+weight_decay: 0.3939

001/ckpt/model_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86ee492503a0f995686aba27eb0db7f8d759dd2c4b491a692740b3be3241f14a
+size 148268583

001/evaluation.yaml

@@ -0,0 +1,3 @@
+validation: {A: 0.24868205189704895, B: 0.2788656949996948, C: 0.271991103887558,
+  D: 0.2340584099292755, E: 0.23670722544193268, F: 0.23395685851573944, G: 0.25554871559143066,
+  H: 0.23508135974407196, I: 0.25329142808914185, J: 0.2589012682437897, average: 0.25070841163396834}

001/model.txt

@@ -0,0 +1,143 @@
+=======================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #
+=======================================================================================================================================
+Model                                                        --                        [1, 7440, 116]            --
+├─ViViTCore: 1-1                                             --                        [1, 112, 116, 15, 29]     --
+│    └─ViViT: 2-1                                            [1, 5, 140, 36, 64]       [1, 112, 116, 15, 29]     --
+│    │    └─Tokenizer: 3-1                                   [1, 5, 140, 36, 64]       [1, 116, 435, 112]        79,632
+│    │    │    └─Sequential: 4-1                             [1, 5, 140, 36, 64]       [1, 116, 435, 112]        --
+│    │    │    │    └─Unfold3d: 5-1                          [1, 5, 140, 36, 64]       [1, 116, 435, 6125]       --
+│    │    │    │    └─LayerNorm: 5-2                         [1, 116, 435, 6125]       [1, 116, 435, 6125]       12,250
+│    │    │    │    └─Linear: 5-3                            [1, 116, 435, 6125]       [1, 116, 435, 112]        686,112
+│    │    │    │    └─LayerNorm: 5-4                         [1, 116, 435, 112]        [1, 116, 435, 112]        224
+│    │    │    │    └─Dropout: 5-5                           [1, 116, 435, 112]        [1, 116, 435, 112]        --
+│    │    └─Transformer: 3-2                                 [116, 435, 112]           [116, 435, 112]           --
+│    │    │    └─ModuleList: 4-2                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-6          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-1                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-2                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-3                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-4                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-5                   [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-1                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-2                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-3                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-6                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-7          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-7                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-8                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-9                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-10                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-11                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-4                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-5                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-6                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-12                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-8          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-13                   [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-14                      [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-15                      [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-16                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-17                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-7                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-8                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-9                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-18                    [116, 435, 112]           [116, 435, 112]           --
+│    │    └─Transformer: 3-3                                 [435, 116, 112]           [435, 116, 112]           --
+│    │    │    └─ModuleList: 4-3                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-9          [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-19                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-20                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-21                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-22                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-23                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-10                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-11                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-12                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-24                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-10         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-25                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-26                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-27                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-28                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-29                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-13                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-14                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-15                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-30                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-11         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-31                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-32                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-33                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-34                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-35                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-16                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-17                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-18                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-36                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-12         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-37                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-38                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-39                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-40                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-41                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-19                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-20                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-21                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-42                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-13         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-43                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-44                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-45                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-46                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-47                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-22                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-23                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-24                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-48                    [435, 116, 112]           [435, 116, 112]           --
+│    │    └─Rearrange: 3-4                                   [1, 116, 435, 112]        [1, 112, 116, 15, 29]     --
+├─MLPShifters: 1-2                                           --                        [1, 2, 116]               513
+│    └─MLPShifter: 2-2                                       [116, 2]                  [116, 2]                  --
+│    │    └─Sequential: 3-5                                  [116, 2]                  [116, 2]                  --
+│    │    │    └─Linear: 4-4                                 [116, 2]                  [116, 5]                  15
+│    │    │    └─Tanh: 4-5                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-6                                 [116, 5]                  [116, 5]                  30
+│    │    │    └─Tanh: 4-7                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-8                                 [116, 5]                  [116, 2]                  12
+│    │    │    └─Tanh: 4-9                                   [116, 2]                  [116, 2]                  --
+├─Readouts: 1-3                                              [1, 112, 116, 15, 29]     [1, 7440, 116]            8,477,649
+│    └─Gaussian2DReadout2: 2-3                               [1, 112, 116, 15, 29]     [1, 7440, 116]            870,480
+│    │    └─Sequential: 3-6                                  [7440, 2]                 [7440, 2]                 --
+│    │    │    └─Linear: 4-10                                [7440, 2]                 [7440, 30]                90
+│    │    │    └─ELU: 4-11                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-12                                [7440, 30]                [7440, 2]                 62
+│    │    │    └─Tanh: 4-13                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-7                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-14                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-15                                   [7440, 30]                [7440, 30]                --
+��    │    │    └─Linear: 4-16                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-17                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-8                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-18                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-19                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-20                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-21                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-9                                  [116, 4]                  [116, 112]                --
+│    │    │    └─Linear: 4-22                                [116, 4]                  [116, 112]                560
+│    │    │    └─LayerNorm: 4-23                             [116, 112]                [116, 112]                224
+│    │    │    └─GELU: 4-24                                  [116, 112]                [116, 112]                --
+│    │    │    └─Dropout: 4-25                               [116, 112]                [116, 112]                --
+│    │    │    └─Linear: 4-26                                [116, 112]                [116, 112]                12,656
+│    │    │    └─Tanh: 4-27                                  [116, 112]                [116, 112]                --
+├─Exponential: 1-4                                           [1, 7440, 116]            [1, 7440, 116]            --
+=======================================================================================================================================
+Total params: 12,278,365
+Trainable params: 12,278,365
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 679.88
+=======================================================================================================================================
+Input size (MB): 1.29
+Forward/backward pass size (MB): 9208.73
+Params size (MB): 11.40
+Estimated Total Size (MB): 9221.42
+=======================================================================================================================================

001/output.log

@@ -0,0 +1,816 @@
+Use bfloat16 for core module.
+Use parallel attention and MLP in ViViT.
+
+Epoch 001/400
+Train		loss: 113924008.00	correlation: 0.0107
+Validation	loss: 200107760.00	correlation: 0.0263
+Elapse: 605.46s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 002/400
+Train		loss: 97896768.00	correlation: 0.0340
+Validation	loss: 199235856.00	correlation: 0.0378
+Elapse: 556.86s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 003/400
+Train		loss: 96883144.00	correlation: 0.0443
+Validation	loss: 198767200.00	correlation: 0.0405
+Elapse: 560.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 004/400
+Train		loss: 96317000.00	correlation: 0.0499
+Validation	loss: 198194576.00	correlation: 0.0449
+Elapse: 560.78s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 005/400
+Train		loss: 95592096.00	correlation: 0.0577
+Validation	loss: 197382576.00	correlation: 0.0493
+Elapse: 563.47s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 006/400
+Train		loss: 94455696.00	correlation: 0.0692
+Validation	loss: 195917728.00	correlation: 0.0602
+Elapse: 565.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 007/400
+Train		loss: 93257144.00	correlation: 0.0821
+Validation	loss: 193956304.00	correlation: 0.0721
+Elapse: 566.60s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 008/400
+Train		loss: 92014728.00	correlation: 0.0951
+Validation	loss: 192039424.00	correlation: 0.0839
+Elapse: 566.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 009/400
+Train		loss: 91067848.00	correlation: 0.1053
+Validation	loss: 190701360.00	correlation: 0.0935
+Elapse: 563.36s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 010/400
+Train		loss: 90106464.00	correlation: 0.1149
+Validation	loss: 189259712.00	correlation: 0.1026
+Elapse: 562.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 011/400
+Train		loss: 89183952.00	correlation: 0.1243
+Validation	loss: 187589920.00	correlation: 0.1130
+Elapse: 560.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 012/400
+Train		loss: 88171952.00	correlation: 0.1348
+Validation	loss: 186131232.00	correlation: 0.1219
+Elapse: 559.08s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 013/400
+Train		loss: 87220088.00	correlation: 0.1442
+Validation	loss: 184294144.00	correlation: 0.1328
+Elapse: 564.57s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 014/400
+Train		loss: 86305888.00	correlation: 0.1535
+Validation	loss: 182766432.00	correlation: 0.1425
+Elapse: 564.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 015/400
+Train		loss: 85306032.00	correlation: 0.1632
+Validation	loss: 181567952.00	correlation: 0.1505
+Elapse: 562.48s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 016/400
+Train		loss: 84622736.00	correlation: 0.1701
+Validation	loss: 180351280.00	correlation: 0.1587
+Elapse: 561.75s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 017/400
+Train		loss: 83839824.00	correlation: 0.1780
+Validation	loss: 179434864.00	correlation: 0.1644
+Elapse: 561.90s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 018/400
+Train		loss: 83213000.00	correlation: 0.1840
+Validation	loss: 178816208.00	correlation: 0.1695
+Elapse: 562.07s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 019/400
+Train		loss: 82673560.00	correlation: 0.1895
+Validation	loss: 178169568.00	correlation: 0.1740
+Elapse: 562.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 020/400
+Train		loss: 82081376.00	correlation: 0.1949
+Validation	loss: 177435600.00	correlation: 0.1787
+Elapse: 561.87s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 021/400
+Train		loss: 81644440.00	correlation: 0.1992
+Validation	loss: 176873408.00	correlation: 0.1824
+Elapse: 562.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 022/400
+Train		loss: 81300944.00	correlation: 0.2027
+Validation	loss: 176298464.00	correlation: 0.1855
+Elapse: 562.57s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 023/400
+Train		loss: 80864992.00	correlation: 0.2072
+Validation	loss: 175630304.00	correlation: 0.1905
+Elapse: 562.55s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 024/400
+Train		loss: 80516536.00	correlation: 0.2102
+Validation	loss: 175099184.00	correlation: 0.1937
+Elapse: 561.74s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 025/400
+Train		loss: 80141312.00	correlation: 0.2140
+Validation	loss: 174953952.00	correlation: 0.1954
+Elapse: 563.63s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 026/400
+Train		loss: 79914320.00	correlation: 0.2162
+Validation	loss: 174397456.00	correlation: 0.1975
+Elapse: 563.20s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 027/400
+Train		loss: 79520984.00	correlation: 0.2200
+Validation	loss: 174134560.00	correlation: 0.2014
+Elapse: 562.71s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 028/400
+Train		loss: 79212208.00	correlation: 0.2229
+Validation	loss: 173471920.00	correlation: 0.2046
+Elapse: 562.79s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 029/400
+Train		loss: 78973040.00	correlation: 0.2252
+Validation	loss: 173183520.00	correlation: 0.2064
+Elapse: 563.08s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 030/400
+Train		loss: 78870784.00	correlation: 0.2264
+Validation	loss: 172775088.00	correlation: 0.2096
+Elapse: 562.82s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 031/400
+Train		loss: 78490200.00	correlation: 0.2296
+Validation	loss: 172664992.00	correlation: 0.2107
+Elapse: 563.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 032/400
+Train		loss: 78317200.00	correlation: 0.2318
+Validation	loss: 172019824.00	correlation: 0.2132
+Elapse: 562.89s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 033/400
+Train		loss: 78137128.00	correlation: 0.2333
+Validation	loss: 171897024.00	correlation: 0.2150
+Elapse: 562.71s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 034/400
+Train		loss: 77950032.00	correlation: 0.2354
+Validation	loss: 171802656.00	correlation: 0.2157
+Elapse: 562.68s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 035/400
+Train		loss: 77766832.00	correlation: 0.2367
+Validation	loss: 171322160.00	correlation: 0.2180
+Elapse: 561.92s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 036/400
+Train		loss: 77637216.00	correlation: 0.2384
+Validation	loss: 171034880.00	correlation: 0.2209
+Elapse: 562.15s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 037/400
+Train		loss: 77482304.00	correlation: 0.2397
+Validation	loss: 171174608.00	correlation: 0.2203
+Elapse: 564.93s
+
+Epoch 038/400
+Train		loss: 77429408.00	correlation: 0.2402
+Validation	loss: 170815888.00	correlation: 0.2218
+Elapse: 564.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 039/400
+Train		loss: 77292792.00	correlation: 0.2412
+Validation	loss: 170692576.00	correlation: 0.2224
+Elapse: 563.94s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 040/400
+Train		loss: 77122352.00	correlation: 0.2426
+Validation	loss: 170492064.00	correlation: 0.2256
+Elapse: 565.03s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 041/400
+Train		loss: 77124456.00	correlation: 0.2430
+Validation	loss: 170532256.00	correlation: 0.2245
+Elapse: 564.39s
+
+Epoch 042/400
+Train		loss: 76964592.00	correlation: 0.2445
+Validation	loss: 170406688.00	correlation: 0.2257
+Elapse: 565.12s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 043/400
+Train		loss: 76869008.00	correlation: 0.2454
+Validation	loss: 170592080.00	correlation: 0.2248
+Elapse: 563.98s
+
+Epoch 044/400
+Train		loss: 76798544.00	correlation: 0.2461
+Validation	loss: 170285664.00	correlation: 0.2253
+Elapse: 564.88s
+
+Epoch 045/400
+Train		loss: 76706736.00	correlation: 0.2469
+Validation	loss: 170137504.00	correlation: 0.2276
+Elapse: 564.78s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 046/400
+Train		loss: 76661312.00	correlation: 0.2474
+Validation	loss: 170203360.00	correlation: 0.2258
+Elapse: 563.90s
+
+Epoch 047/400
+Train		loss: 76653904.00	correlation: 0.2479
+Validation	loss: 169920448.00	correlation: 0.2283
+Elapse: 565.79s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 048/400
+Train		loss: 76510704.00	correlation: 0.2491
+Validation	loss: 169954384.00	correlation: 0.2279
+Elapse: 566.41s
+
+Epoch 049/400
+Train		loss: 76493416.00	correlation: 0.2492
+Validation	loss: 169840224.00	correlation: 0.2290
+Elapse: 566.13s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 050/400
+Train		loss: 76429016.00	correlation: 0.2495
+Validation	loss: 170036560.00	correlation: 0.2272
+Elapse: 567.15s
+
+Epoch 051/400
+Train		loss: 76434184.00	correlation: 0.2500
+Validation	loss: 169668848.00	correlation: 0.2298
+Elapse: 563.78s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 052/400
+Train		loss: 76368824.00	correlation: 0.2505
+Validation	loss: 169862640.00	correlation: 0.2292
+Elapse: 563.52s
+
+Epoch 053/400
+Train		loss: 76235120.00	correlation: 0.2516
+Validation	loss: 169524560.00	correlation: 0.2315
+Elapse: 565.24s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 054/400
+Train		loss: 76236176.00	correlation: 0.2518
+Validation	loss: 169235696.00	correlation: 0.2324
+Elapse: 564.43s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 055/400
+Train		loss: 76272208.00	correlation: 0.2514
+Validation	loss: 169566032.00	correlation: 0.2303
+Elapse: 564.62s
+
+Epoch 056/400
+Train		loss: 76095872.00	correlation: 0.2528
+Validation	loss: 169257008.00	correlation: 0.2329
+Elapse: 564.52s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 057/400
+Train		loss: 76150160.00	correlation: 0.2523
+Validation	loss: 169306656.00	correlation: 0.2323
+Elapse: 564.42s
+
+Epoch 058/400
+Train		loss: 75971168.00	correlation: 0.2540
+Validation	loss: 169281824.00	correlation: 0.2326
+Elapse: 565.79s
+
+Epoch 059/400
+Train		loss: 76016632.00	correlation: 0.2537
+Validation	loss: 169175984.00	correlation: 0.2330
+Elapse: 567.53s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 060/400
+Train		loss: 75961288.00	correlation: 0.2543
+Validation	loss: 169244832.00	correlation: 0.2312
+Elapse: 564.74s
+
+Epoch 061/400
+Train		loss: 75918368.00	correlation: 0.2545
+Validation	loss: 169277088.00	correlation: 0.2310
+Elapse: 564.73s
+
+Epoch 062/400
+Train		loss: 75869072.00	correlation: 0.2550
+Validation	loss: 169328944.00	correlation: 0.2320
+Elapse: 564.39s
+
+Epoch 063/400
+Train		loss: 75762192.00	correlation: 0.2561
+Validation	loss: 169258112.00	correlation: 0.2316
+Elapse: 564.19s
+
+Epoch 064/400
+Train		loss: 75825328.00	correlation: 0.2556
+Validation	loss: 169071840.00	correlation: 0.2332
+Elapse: 564.93s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 065/400
+Train		loss: 75713256.00	correlation: 0.2566
+Validation	loss: 169214000.00	correlation: 0.2329
+Elapse: 564.84s
+
+Epoch 066/400
+Train		loss: 75745160.00	correlation: 0.2561
+Validation	loss: 169041248.00	correlation: 0.2327
+Elapse: 564.78s
+
+Epoch 067/400
+Train		loss: 75648152.00	correlation: 0.2572
+Validation	loss: 169051616.00	correlation: 0.2327
+Elapse: 565.20s
+
+Epoch 068/400
+Train		loss: 75614368.00	correlation: 0.2577
+Validation	loss: 169221120.00	correlation: 0.2327
+Elapse: 564.71s
+
+Epoch 069/400
+Train		loss: 75625776.00	correlation: 0.2574
+Validation	loss: 169122256.00	correlation: 0.2330
+Elapse: 564.46s
+
+Loaded checkpoint from epoch 64 (correlation: 0.2332).
+
+Reduce learning rate of core to 1.4400e-03 (num. reduce: 1).
+Reduce learning rate of readouts to 1.0800e-03 (num. reduce: 1).
+Reduce learning rate of shifters to 1.0800e-03 (num. reduce: 1).
+
+Epoch 070/400
+Train		loss: 73759920.00	correlation: 0.2731
+Validation	loss: 167411552.00	correlation: 0.2444
+Elapse: 565.27s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 071/400
+Train		loss: 73162624.00	correlation: 0.2785
+Validation	loss: 167276400.00	correlation: 0.2454
+Elapse: 565.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 072/400
+Train		loss: 73062080.00	correlation: 0.2797
+Validation	loss: 167303840.00	correlation: 0.2453
+Elapse: 565.70s
+
+Epoch 073/400
+Train		loss: 72859248.00	correlation: 0.2818
+Validation	loss: 167279840.00	correlation: 0.2448
+Elapse: 566.21s
+
+Epoch 074/400
+Train		loss: 72867944.00	correlation: 0.2819
+Validation	loss: 167328304.00	correlation: 0.2445
+Elapse: 566.10s
+
+Epoch 075/400
+Train		loss: 72806064.00	correlation: 0.2823
+Validation	loss: 167367392.00	correlation: 0.2443
+Elapse: 566.39s
+
+Epoch 076/400
+Train		loss: 72707504.00	correlation: 0.2836
+Validation	loss: 167176992.00	correlation: 0.2450
+Elapse: 566.85s
+
+Loaded checkpoint from epoch 71 (correlation: 0.2454).
+
+Reduce learning rate of core to 4.3200e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 3.2400e-04 (num. reduce: 1).
+Reduce learning rate of shifters to 3.2400e-04 (num. reduce: 1).
+
+Epoch 077/400
+Train		loss: 72382144.00	correlation: 0.2856
+Validation	loss: 166842576.00	correlation: 0.2479
+Elapse: 566.50s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 078/400
+Train		loss: 72078376.00	correlation: 0.2881
+Validation	loss: 166822496.00	correlation: 0.2483
+Elapse: 567.42s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 079/400
+Train		loss: 71958248.00	correlation: 0.2896
+Validation	loss: 166738624.00	correlation: 0.2488
+Elapse: 567.48s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 080/400
+Train		loss: 71895880.00	correlation: 0.2900
+Validation	loss: 166642640.00	correlation: 0.2490
+Elapse: 567.47s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 081/400
+Train		loss: 71789800.00	correlation: 0.2911
+Validation	loss: 166705248.00	correlation: 0.2491
+Elapse: 566.97s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 082/400
+Train		loss: 71770424.00	correlation: 0.2915
+Validation	loss: 166628368.00	correlation: 0.2487
+Elapse: 567.08s
+
+Epoch 083/400
+Train		loss: 71742960.00	correlation: 0.2915
+Validation	loss: 166623744.00	correlation: 0.2491
+Elapse: 567.32s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 084/400
+Train		loss: 71726080.00	correlation: 0.2915
+Validation	loss: 166593008.00	correlation: 0.2489
+Elapse: 566.52s
+
+Epoch 085/400
+Train		loss: 71608472.00	correlation: 0.2930
+Validation	loss: 166582448.00	correlation: 0.2494
+Elapse: 566.39s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 086/400
+Train		loss: 71622656.00	correlation: 0.2929
+Validation	loss: 166651488.00	correlation: 0.2488
+Elapse: 567.27s
+
+Epoch 087/400
+Train		loss: 71550640.00	correlation: 0.2936
+Validation	loss: 166630768.00	correlation: 0.2490
+Elapse: 566.70s
+
+Epoch 088/400
+Train		loss: 71491864.00	correlation: 0.2941
+Validation	loss: 166610080.00	correlation: 0.2493
+Elapse: 567.65s
+
+Epoch 089/400
+Train		loss: 71458584.00	correlation: 0.2944
+Validation	loss: 166546464.00	correlation: 0.2492
+Elapse: 566.68s
+
+Epoch 090/400
+Train		loss: 71462792.00	correlation: 0.2945
+Validation	loss: 166668320.00	correlation: 0.2492
+Elapse: 566.84s
+
+Loaded checkpoint from epoch 85 (correlation: 0.2494).
+
+Reduce learning rate of core to 1.2960e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 9.7200e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 9.7200e-05 (num. reduce: 1).
+
+Epoch 091/400
+Train		loss: 71288968.00	correlation: 0.2953
+Validation	loss: 166494880.00	correlation: 0.2500
+Elapse: 567.12s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 092/400
+Train		loss: 71275904.00	correlation: 0.2958
+Validation	loss: 166504256.00	correlation: 0.2498
+Elapse: 566.80s
+
+Epoch 093/400
+Train		loss: 71229392.00	correlation: 0.2962
+Validation	loss: 166490080.00	correlation: 0.2502
+Elapse: 567.91s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 094/400
+Train		loss: 71289360.00	correlation: 0.2955
+Validation	loss: 166481824.00	correlation: 0.2501
+Elapse: 567.35s
+
+Epoch 095/400
+Train		loss: 71209928.00	correlation: 0.2962
+Validation	loss: 166463984.00	correlation: 0.2501
+Elapse: 567.29s
+
+Epoch 096/400
+Train		loss: 71155936.00	correlation: 0.2969
+Validation	loss: 166447504.00	correlation: 0.2502
+Elapse: 567.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 097/400
+Train		loss: 71163080.00	correlation: 0.2967
+Validation	loss: 166476448.00	correlation: 0.2501
+Elapse: 567.23s
+
+Epoch 098/400
+Train		loss: 71090216.00	correlation: 0.2976
+Validation	loss: 166447952.00	correlation: 0.2502
+Elapse: 567.15s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 099/400
+Train		loss: 71146160.00	correlation: 0.2971
+Validation	loss: 166436720.00	correlation: 0.2504
+Elapse: 566.92s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 100/400
+Train		loss: 71064016.00	correlation: 0.2977
+Validation	loss: 166422944.00	correlation: 0.2502
+Elapse: 567.03s
+
+Epoch 101/400
+Train		loss: 71056432.00	correlation: 0.2979
+Validation	loss: 166403840.00	correlation: 0.2505
+Elapse: 567.52s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 102/400
+Train		loss: 71051576.00	correlation: 0.2978
+Validation	loss: 166443872.00	correlation: 0.2504
+Elapse: 567.13s
+
+Epoch 103/400
+Train		loss: 71046608.00	correlation: 0.2979
+Validation	loss: 166479680.00	correlation: 0.2502
+Elapse: 566.67s
+
+Epoch 104/400
+Train		loss: 71038120.00	correlation: 0.2981
+Validation	loss: 166454112.00	correlation: 0.2501
+Elapse: 564.93s
+
+Epoch 105/400
+Train		loss: 70978192.00	correlation: 0.2988
+Validation	loss: 166414432.00	correlation: 0.2503
+Elapse: 564.91s
+
+Epoch 106/400
+Train		loss: 70976960.00	correlation: 0.2990
+Validation	loss: 166474720.00	correlation: 0.2500
+Elapse: 566.07s
+
+Loaded checkpoint from epoch 101 (correlation: 0.2505).
+
+Reduce learning rate of core to 3.8880e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 2.9160e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 2.9160e-05 (num. reduce: 1).
+
+Epoch 107/400
+Train		loss: 70972624.00	correlation: 0.2986
+Validation	loss: 166398144.00	correlation: 0.2506
+Elapse: 566.75s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 108/400
+Train		loss: 70964176.00	correlation: 0.2987
+Validation	loss: 166382512.00	correlation: 0.2506
+Elapse: 566.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 109/400
+Train		loss: 70976968.00	correlation: 0.2987
+Validation	loss: 166404432.00	correlation: 0.2505
+Elapse: 567.32s
+
+Epoch 110/400
+Train		loss: 70951736.00	correlation: 0.2985
+Validation	loss: 166396896.00	correlation: 0.2506
+Elapse: 567.77s
+
+Epoch 111/400
+Train		loss: 70895944.00	correlation: 0.2997
+Validation	loss: 166385856.00	correlation: 0.2506
+Elapse: 566.89s
+
+Epoch 112/400
+Train		loss: 70880200.00	correlation: 0.2995
+Validation	loss: 166380672.00	correlation: 0.2507
+Elapse: 566.94s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/002/ckpt/model_state.pt.
+
+Epoch 113/400
+Train		loss: 70861304.00	correlation: 0.2998
+Validation	loss: 166391184.00	correlation: 0.2506
+Elapse: 566.54s
+
+Epoch 114/400
+Train		loss: 70921912.00	correlation: 0.2989
+Validation	loss: 166417776.00	correlation: 0.2504
+Elapse: 567.26s
+
+Epoch 115/400
+Train		loss: 70914312.00	correlation: 0.2989
+Validation	loss: 166403376.00	correlation: 0.2506
+Elapse: 566.92s
+
+Epoch 116/400
+Train		loss: 70903784.00	correlation: 0.2992
+Validation	loss: 166403360.00	correlation: 0.2506
+Elapse: 567.11s
+
+Epoch 117/400
+Train		loss: 70866896.00	correlation: 0.2994
+Validation	loss: 166382688.00	correlation: 0.2505
+Elapse: 566.94s
+
+Loaded checkpoint from epoch 112 (correlation: 0.2507).
+
+Reduce learning rate of core to 1.1664e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 8.7480e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 8.7480e-06 (num. reduce: 1).
+
+Epoch 118/400
+Train		loss: 70926000.00	correlation: 0.2989
+Validation	loss: 166380768.00	correlation: 0.2506
+Elapse: 567.50s
+
+Epoch 119/400
+Train		loss: 70900808.00	correlation: 0.2993
+Validation	loss: 166386016.00	correlation: 0.2506
+Elapse: 567.35s
+
+Epoch 120/400
+Train		loss: 70881328.00	correlation: 0.2993
+Validation	loss: 166387568.00	correlation: 0.2506
+Elapse: 567.65s
+
+Epoch 121/400
+Train		loss: 70937256.00	correlation: 0.2984
+Validation	loss: 166393568.00	correlation: 0.2506
+Elapse: 566.97s
+
+Epoch 122/400
+Train		loss: 70855888.00	correlation: 0.2996
+Validation	loss: 166378592.00	correlation: 0.2507
+Elapse: 567.50s
+
+Loaded checkpoint from epoch 112 (correlation: 0.2507).
+
+Reduce learning rate of core to 3.4992e-06 (num. reduce: 2).
+Reduce learning rate of readouts to 2.6244e-06 (num. reduce: 2).
+Reduce learning rate of shifters to 2.6244e-06 (num. reduce: 2).
+
+Epoch 123/400
+Train		loss: 70938496.00	correlation: 0.2988
+Validation	loss: 166375792.00	correlation: 0.2507
+Elapse: 567.50s
+
+Epoch 124/400
+Train		loss: 70966320.00	correlation: 0.2986
+Validation	loss: 166383664.00	correlation: 0.2506
+Elapse: 567.77s
+
+Epoch 125/400
+Train		loss: 70872080.00	correlation: 0.2990
+Validation	loss: 166383328.00	correlation: 0.2506
+Elapse: 567.87s
+
+Epoch 126/400
+Train		loss: 70866264.00	correlation: 0.2997
+Validation	loss: 166383136.00	correlation: 0.2506
+Elapse: 567.21s
+
+Epoch 127/400
+Train		loss: 70836352.00	correlation: 0.2996
+Validation	loss: 166386816.00	correlation: 0.2506
+Elapse: 566.90s
+
+Model has not improved after 2 LR reductions.
+
+Loaded checkpoint from epoch 112 (correlation: 0.2507).
+
+
+ValidationA: 0.2487	B: 0.2789	C: 0.2720	D: 0.2341	E: 0.2367	F: 0.2340	G: 0.2555	H: 0.2351	I: 0.2533	J: 0.2589	average: 0.2507
+
+Results saved to /home/storage/runs/vivit_ensemble/002.

002/args.yaml

@@ -0,0 +1,87 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_eps: 1e-08
+batch_size: 6
+cache_data: false
+center_crop: 1.0
+clear_output_dir: true
+core: vivit
+core_behavior_mode: 2
+core_compile: false
+core_drop_path: 0.0505
+core_emb_dim: 112
+core_ff_activation: gelu
+core_ff_dropout: 0.0592
+core_flash_attention: 1
+core_head_dim: 48
+core_lr: 0.0048
+core_mha_dropout: 0.358
+core_mlp_dim: 136
+core_norm_qk: false
+core_num_heads: 11
+core_p_dropout: 0.1338
+core_parallel_attention: true
+core_patch_mode: 0
+core_pos_encoding: 3
+core_spatial_depth: 3
+core_spatial_patch_size: 7
+core_spatial_patch_stride: 2
+core_temporal_depth: 5
+core_temporal_patch_size: 25
+core_temporal_patch_stride: 1
+core_weight_decay: 0.1789
+criterion: poisson_correlation
+critic_mode: 0
+crop_frame: 140
+data: /home/sensorium2023
+deterministic: false
+device: cuda
+dpi: 120
+ds_mode: 3
+ds_scale: 1
+epochs: 400
+format: svg
+git_hash: 1033c29
+grad_checkpointing: 0
+grad_norm: null
+input_shapes:
+  behavior: [2, 140]
+  pupil_center: [2, 140]
+  video: [1, 140, 36, 64]
+limit_data: null
+lr: 0.0036
+max_frame: 300
+micro_batch_size: 6
+modulator_mode: 0
+mouse_ids: [A, B, C, D, E, F, G, H, I, J]
+num_workers: 2
+output_dir: /home/storage/runs/002
+output_mode: 1
+output_shapes:
+  A: [7440, 140]
+  B: [7928, 140]
+  C: [8285, 140]
+  D: [7671, 140]
+  E: [7495, 140]
+  F: [7863, 140]
+  G: [7908, 140]
+  H: [8202, 140]
+  I: [7939, 140]
+  J: [8122, 140]
+precision: "32"
+pretrain_core: null
+readout: gaussian2d
+readout_bias_mode: 2
+readout_dropout: 0.0
+readout_grid_mode: 1
+restore: null
+save_plots: false
+seed: 9
+shifter_layers: 3
+shifter_mode: 1
+shifter_size: 5
+stat_mode: 1
+trainable_params: 12278365
+transform_mode: 2
+verbose: 1
+weight_decay: 0.3939

002/ckpt/model_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beef5725438551d35727d0a1a42f996cce8ce0c760b74e7f7228171b34283e11
+size 148268583

002/evaluation.yaml

@@ -0,0 +1,3 @@
+validation: {A: 0.25081300735473633, B: 0.2804259657859802, C: 0.27504464983940125,
+  D: 0.23932193219661713, E: 0.23903165757656097, F: 0.23461945354938507, G: 0.25730910897254944,
+  H: 0.2378837764263153, I: 0.2555205523967743, J: 0.25993871688842773, average: 0.25299088209867476}

002/model.txt

@@ -0,0 +1,143 @@
+=======================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #
+=======================================================================================================================================
+Model                                                        --                        [1, 7440, 116]            --
+├─ViViTCore: 1-1                                             --                        [1, 112, 116, 15, 29]     --
+│    └─ViViT: 2-1                                            [1, 5, 140, 36, 64]       [1, 112, 116, 15, 29]     --
+│    │    └─Tokenizer: 3-1                                   [1, 5, 140, 36, 64]       [1, 116, 435, 112]        79,632
+│    │    │    └─Sequential: 4-1                             [1, 5, 140, 36, 64]       [1, 116, 435, 112]        --
+│    │    │    │    └─Unfold3d: 5-1                          [1, 5, 140, 36, 64]       [1, 116, 435, 6125]       --
+│    │    │    │    └─LayerNorm: 5-2                         [1, 116, 435, 6125]       [1, 116, 435, 6125]       12,250
+│    │    │    │    └─Linear: 5-3                            [1, 116, 435, 6125]       [1, 116, 435, 112]        686,112
+│    │    │    │    └─LayerNorm: 5-4                         [1, 116, 435, 112]        [1, 116, 435, 112]        224
+│    │    │    │    └─Dropout: 5-5                           [1, 116, 435, 112]        [1, 116, 435, 112]        --
+│    │    └─Transformer: 3-2                                 [116, 435, 112]           [116, 435, 112]           --
+│    │    │    └─ModuleList: 4-2                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-6          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-1                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-2                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-3                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-4                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-5                   [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-1                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-2                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-3                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-6                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-7          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-7                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-8                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-9                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-10                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-11                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-4                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-5                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-6                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-12                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-8          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-13                   [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-14                      [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-15                      [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-16                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-17                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-7                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-8                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-9                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-18                    [116, 435, 112]           [116, 435, 112]           --
+│    │    └─Transformer: 3-3                                 [435, 116, 112]           [435, 116, 112]           --
+│    │    │    └─ModuleList: 4-3                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-9          [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-19                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-20                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-21                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-22                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-23                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-10                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-11                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-12                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-24                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-10         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-25                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-26                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-27                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-28                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-29                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-13                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-14                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-15                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-30                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-11         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-31                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-32                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-33                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-34                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-35                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-16                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-17                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-18                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-36                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-12         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-37                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-38                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-39                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-40                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-41                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-19                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-20                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-21                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-42                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-13         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-43                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-44                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-45                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-46                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-47                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-22                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-23                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-24                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-48                    [435, 116, 112]           [435, 116, 112]           --
+│    │    └─Rearrange: 3-4                                   [1, 116, 435, 112]        [1, 112, 116, 15, 29]     --
+├─MLPShifters: 1-2                                           --                        [1, 2, 116]               513
+│    └─MLPShifter: 2-2                                       [116, 2]                  [116, 2]                  --
+│    │    └─Sequential: 3-5                                  [116, 2]                  [116, 2]                  --
+│    │    │    └─Linear: 4-4                                 [116, 2]                  [116, 5]                  15
+│    │    │    └─Tanh: 4-5                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-6                                 [116, 5]                  [116, 5]                  30
+│    │    │    └─Tanh: 4-7                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-8                                 [116, 5]                  [116, 2]                  12
+│    │    │    └─Tanh: 4-9                                   [116, 2]                  [116, 2]                  --
+├─Readouts: 1-3                                              [1, 112, 116, 15, 29]     [1, 7440, 116]            8,477,649
+│    └─Gaussian2DReadout2: 2-3                               [1, 112, 116, 15, 29]     [1, 7440, 116]            870,480
+│    │    └─Sequential: 3-6                                  [7440, 2]                 [7440, 2]                 --
+│    │    │    └─Linear: 4-10                                [7440, 2]                 [7440, 30]                90
+│    │    │    └─ELU: 4-11                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-12                                [7440, 30]                [7440, 2]                 62
+│    │    │    └─Tanh: 4-13                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-7                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-14                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-15                                   [7440, 30]                [7440, 30]                --
+��    │    │    └─Linear: 4-16                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-17                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-8                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-18                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-19                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-20                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-21                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-9                                  [116, 4]                  [116, 112]                --
+│    │    │    └─Linear: 4-22                                [116, 4]                  [116, 112]                560
+│    │    │    └─LayerNorm: 4-23                             [116, 112]                [116, 112]                224
+│    │    │    └─GELU: 4-24                                  [116, 112]                [116, 112]                --
+│    │    │    └─Dropout: 4-25                               [116, 112]                [116, 112]                --
+│    │    │    └─Linear: 4-26                                [116, 112]                [116, 112]                12,656
+│    │    │    └─Tanh: 4-27                                  [116, 112]                [116, 112]                --
+├─Exponential: 1-4                                           [1, 7440, 116]            [1, 7440, 116]            --
+=======================================================================================================================================
+Total params: 12,278,365
+Trainable params: 12,278,365
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 679.88
+=======================================================================================================================================
+Input size (MB): 1.29
+Forward/backward pass size (MB): 9208.73
+Params size (MB): 11.40
+Estimated Total Size (MB): 9221.42
+=======================================================================================================================================

002/output.log

@@ -0,0 +1,942 @@
+Use bfloat16 for core module.
+Use parallel attention and MLP in ViViT.
+
+Epoch 001/400
+Train		loss: 121388096.00	correlation: 0.0109
+Validation	loss: 200734000.00	correlation: 0.0221
+Elapse: 522.91s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 002/400
+Train		loss: 98318960.00	correlation: 0.0308
+Validation	loss: 199955104.00	correlation: 0.0339
+Elapse: 525.64s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 003/400
+Train		loss: 97097640.00	correlation: 0.0413
+Validation	loss: 199270752.00	correlation: 0.0396
+Elapse: 531.95s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 004/400
+Train		loss: 96457752.00	correlation: 0.0479
+Validation	loss: 198723456.00	correlation: 0.0421
+Elapse: 536.86s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 005/400
+Train		loss: 95920056.00	correlation: 0.0541
+Validation	loss: 197835248.00	correlation: 0.0461
+Elapse: 538.89s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 006/400
+Train		loss: 95131240.00	correlation: 0.0621
+Validation	loss: 196873360.00	correlation: 0.0519
+Elapse: 542.34s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 007/400
+Train		loss: 94226616.00	correlation: 0.0710
+Validation	loss: 195726576.00	correlation: 0.0578
+Elapse: 541.43s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 008/400
+Train		loss: 93506680.00	correlation: 0.0783
+Validation	loss: 194520016.00	correlation: 0.0654
+Elapse: 541.56s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 009/400
+Train		loss: 92727504.00	correlation: 0.0871
+Validation	loss: 193465776.00	correlation: 0.0716
+Elapse: 543.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 010/400
+Train		loss: 91843728.00	correlation: 0.0966
+Validation	loss: 192176672.00	correlation: 0.0791
+Elapse: 541.68s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 011/400
+Train		loss: 90936304.00	correlation: 0.1063
+Validation	loss: 190695968.00	correlation: 0.0928
+Elapse: 540.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 012/400
+Train		loss: 90051912.00	correlation: 0.1153
+Validation	loss: 189604784.00	correlation: 0.1011
+Elapse: 542.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 013/400
+Train		loss: 89180568.00	correlation: 0.1245
+Validation	loss: 188031648.00	correlation: 0.1112
+Elapse: 542.62s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 014/400
+Train		loss: 88307792.00	correlation: 0.1337
+Validation	loss: 186159440.00	correlation: 0.1226
+Elapse: 540.81s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 015/400
+Train		loss: 87278960.00	correlation: 0.1436
+Validation	loss: 184683440.00	correlation: 0.1313
+Elapse: 543.63s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 016/400
+Train		loss: 86609264.00	correlation: 0.1505
+Validation	loss: 183679264.00	correlation: 0.1374
+Elapse: 544.07s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 017/400
+Train		loss: 85933704.00	correlation: 0.1573
+Validation	loss: 182858960.00	correlation: 0.1427
+Elapse: 543.60s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 018/400
+Train		loss: 85316224.00	correlation: 0.1634
+Validation	loss: 181758608.00	correlation: 0.1502
+Elapse: 543.09s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 019/400
+Train		loss: 84800144.00	correlation: 0.1686
+Validation	loss: 181267248.00	correlation: 0.1538
+Elapse: 545.10s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 020/400
+Train		loss: 84276656.00	correlation: 0.1737
+Validation	loss: 180435344.00	correlation: 0.1578
+Elapse: 545.34s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 021/400
+Train		loss: 83717728.00	correlation: 0.1793
+Validation	loss: 179796752.00	correlation: 0.1639
+Elapse: 545.93s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 022/400
+Train		loss: 83306056.00	correlation: 0.1836
+Validation	loss: 178887952.00	correlation: 0.1689
+Elapse: 545.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 023/400
+Train		loss: 82787680.00	correlation: 0.1887
+Validation	loss: 177999440.00	correlation: 0.1749
+Elapse: 545.47s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 024/400
+Train		loss: 82335960.00	correlation: 0.1933
+Validation	loss: 177650256.00	correlation: 0.1774
+Elapse: 545.75s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 025/400
+Train		loss: 82013392.00	correlation: 0.1962
+Validation	loss: 177572752.00	correlation: 0.1786
+Elapse: 545.69s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 026/400
+Train		loss: 81675472.00	correlation: 0.1996
+Validation	loss: 176845120.00	correlation: 0.1841
+Elapse: 545.58s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 027/400
+Train		loss: 81393008.00	correlation: 0.2025
+Validation	loss: 176056032.00	correlation: 0.1886
+Elapse: 545.76s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 028/400
+Train		loss: 80972248.00	correlation: 0.2065
+Validation	loss: 175799008.00	correlation: 0.1898
+Elapse: 545.76s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 029/400
+Train		loss: 80749744.00	correlation: 0.2086
+Validation	loss: 175376608.00	correlation: 0.1918
+Elapse: 546.06s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 030/400
+Train		loss: 80453840.00	correlation: 0.2117
+Validation	loss: 174830400.00	correlation: 0.1967
+Elapse: 546.10s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 031/400
+Train		loss: 80186704.00	correlation: 0.2145
+Validation	loss: 174322544.00	correlation: 0.1984
+Elapse: 545.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 032/400
+Train		loss: 79856432.00	correlation: 0.2176
+Validation	loss: 174104672.00	correlation: 0.2003
+Elapse: 545.77s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 033/400
+Train		loss: 79589576.00	correlation: 0.2200
+Validation	loss: 173607248.00	correlation: 0.2031
+Elapse: 545.80s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 034/400
+Train		loss: 79380120.00	correlation: 0.2221
+Validation	loss: 173319472.00	correlation: 0.2055
+Elapse: 545.70s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 035/400
+Train		loss: 79158640.00	correlation: 0.2243
+Validation	loss: 172901472.00	correlation: 0.2089
+Elapse: 545.61s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 036/400
+Train		loss: 78927200.00	correlation: 0.2266
+Validation	loss: 172636560.00	correlation: 0.2097
+Elapse: 546.53s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 037/400
+Train		loss: 78682176.00	correlation: 0.2288
+Validation	loss: 172407184.00	correlation: 0.2118
+Elapse: 545.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 038/400
+Train		loss: 78582504.00	correlation: 0.2299
+Validation	loss: 172238496.00	correlation: 0.2137
+Elapse: 542.71s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 039/400
+Train		loss: 78397632.00	correlation: 0.2314
+Validation	loss: 171969648.00	correlation: 0.2147
+Elapse: 545.53s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 040/400
+Train		loss: 78234600.00	correlation: 0.2330
+Validation	loss: 171758000.00	correlation: 0.2170
+Elapse: 545.77s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 041/400
+Train		loss: 78047656.00	correlation: 0.2349
+Validation	loss: 171651168.00	correlation: 0.2175
+Elapse: 545.10s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 042/400
+Train		loss: 77868808.00	correlation: 0.2362
+Validation	loss: 171537312.00	correlation: 0.2173
+Elapse: 545.58s
+
+Epoch 043/400
+Train		loss: 77839128.00	correlation: 0.2370
+Validation	loss: 171300016.00	correlation: 0.2185
+Elapse: 546.30s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 044/400
+Train		loss: 77629288.00	correlation: 0.2390
+Validation	loss: 171374944.00	correlation: 0.2191
+Elapse: 546.42s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 045/400
+Train		loss: 77550776.00	correlation: 0.2396
+Validation	loss: 171207344.00	correlation: 0.2194
+Elapse: 545.96s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 046/400
+Train		loss: 77437640.00	correlation: 0.2408
+Validation	loss: 171176240.00	correlation: 0.2198
+Elapse: 546.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 047/400
+Train		loss: 77397712.00	correlation: 0.2412
+Validation	loss: 170808800.00	correlation: 0.2224
+Elapse: 545.84s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 048/400
+Train		loss: 77307432.00	correlation: 0.2422
+Validation	loss: 170735520.00	correlation: 0.2233
+Elapse: 546.33s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 049/400
+Train		loss: 77119432.00	correlation: 0.2439
+Validation	loss: 170442368.00	correlation: 0.2245
+Elapse: 545.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 050/400
+Train		loss: 76988832.00	correlation: 0.2449
+Validation	loss: 170672960.00	correlation: 0.2232
+Elapse: 544.34s
+
+Epoch 051/400
+Train		loss: 76992464.00	correlation: 0.2450
+Validation	loss: 170562112.00	correlation: 0.2244
+Elapse: 545.56s
+
+Epoch 052/400
+Train		loss: 76925112.00	correlation: 0.2452
+Validation	loss: 170404016.00	correlation: 0.2248
+Elapse: 545.72s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 053/400
+Train		loss: 76844464.00	correlation: 0.2460
+Validation	loss: 170137952.00	correlation: 0.2261
+Elapse: 545.92s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 054/400
+Train		loss: 76812920.00	correlation: 0.2469
+Validation	loss: 170023120.00	correlation: 0.2259
+Elapse: 545.56s
+
+Epoch 055/400
+Train		loss: 76702536.00	correlation: 0.2475
+Validation	loss: 170054384.00	correlation: 0.2263
+Elapse: 544.95s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 056/400
+Train		loss: 76612752.00	correlation: 0.2485
+Validation	loss: 170032864.00	correlation: 0.2272
+Elapse: 545.49s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 057/400
+Train		loss: 76617696.00	correlation: 0.2485
+Validation	loss: 169630816.00	correlation: 0.2293
+Elapse: 545.90s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 058/400
+Train		loss: 76465880.00	correlation: 0.2499
+Validation	loss: 169825600.00	correlation: 0.2283
+Elapse: 545.37s
+
+Epoch 059/400
+Train		loss: 76456912.00	correlation: 0.2499
+Validation	loss: 169766688.00	correlation: 0.2284
+Elapse: 545.61s
+
+Epoch 060/400
+Train		loss: 76426872.00	correlation: 0.2501
+Validation	loss: 169651072.00	correlation: 0.2298
+Elapse: 545.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 061/400
+Train		loss: 76272176.00	correlation: 0.2515
+Validation	loss: 169694896.00	correlation: 0.2290
+Elapse: 544.26s
+
+Epoch 062/400
+Train		loss: 76381040.00	correlation: 0.2506
+Validation	loss: 169396208.00	correlation: 0.2316
+Elapse: 544.91s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 063/400
+Train		loss: 76206744.00	correlation: 0.2523
+Validation	loss: 169683888.00	correlation: 0.2297
+Elapse: 544.69s
+
+Epoch 064/400
+Train		loss: 76167792.00	correlation: 0.2527
+Validation	loss: 169194928.00	correlation: 0.2329
+Elapse: 545.55s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 065/400
+Train		loss: 76127384.00	correlation: 0.2528
+Validation	loss: 169763808.00	correlation: 0.2287
+Elapse: 545.54s
+
+Epoch 066/400
+Train		loss: 76104336.00	correlation: 0.2531
+Validation	loss: 169165280.00	correlation: 0.2327
+Elapse: 545.65s
+
+Epoch 067/400
+Train		loss: 76021512.00	correlation: 0.2539
+Validation	loss: 169219872.00	correlation: 0.2317
+Elapse: 545.30s
+
+Epoch 068/400
+Train		loss: 76148160.00	correlation: 0.2528
+Validation	loss: 169344656.00	correlation: 0.2314
+Elapse: 545.59s
+
+Epoch 069/400
+Train		loss: 75939984.00	correlation: 0.2545
+Validation	loss: 169043376.00	correlation: 0.2339
+Elapse: 545.18s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 070/400
+Train		loss: 75822872.00	correlation: 0.2557
+Validation	loss: 169167760.00	correlation: 0.2328
+Elapse: 544.88s
+
+Epoch 071/400
+Train		loss: 75873072.00	correlation: 0.2551
+Validation	loss: 169123712.00	correlation: 0.2336
+Elapse: 545.60s
+
+Epoch 072/400
+Train		loss: 75867984.00	correlation: 0.2552
+Validation	loss: 168947232.00	correlation: 0.2344
+Elapse: 545.61s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 073/400
+Train		loss: 75840912.00	correlation: 0.2555
+Validation	loss: 169026784.00	correlation: 0.2348
+Elapse: 545.60s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 074/400
+Train		loss: 75855384.00	correlation: 0.2555
+Validation	loss: 168842592.00	correlation: 0.2341
+Elapse: 545.99s
+
+Epoch 075/400
+Train		loss: 75737744.00	correlation: 0.2565
+Validation	loss: 168971312.00	correlation: 0.2336
+Elapse: 545.40s
+
+Epoch 076/400
+Train		loss: 75735656.00	correlation: 0.2565
+Validation	loss: 168829920.00	correlation: 0.2338
+Elapse: 546.00s
+
+Epoch 077/400
+Train		loss: 75607888.00	correlation: 0.2576
+Validation	loss: 168935968.00	correlation: 0.2349
+Elapse: 545.47s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 078/400
+Train		loss: 75679128.00	correlation: 0.2571
+Validation	loss: 168987280.00	correlation: 0.2328
+Elapse: 545.11s
+
+Epoch 079/400
+Train		loss: 75621152.00	correlation: 0.2574
+Validation	loss: 168822512.00	correlation: 0.2347
+Elapse: 545.59s
+
+Epoch 080/400
+Train		loss: 75483728.00	correlation: 0.2590
+Validation	loss: 168774416.00	correlation: 0.2347
+Elapse: 545.47s
+
+Epoch 081/400
+Train		loss: 75531304.00	correlation: 0.2586
+Validation	loss: 169126304.00	correlation: 0.2331
+Elapse: 545.29s
+
+Epoch 082/400
+Train		loss: 75558736.00	correlation: 0.2583
+Validation	loss: 168670160.00	correlation: 0.2354
+Elapse: 545.72s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 083/400
+Train		loss: 75590176.00	correlation: 0.2579
+Validation	loss: 169091472.00	correlation: 0.2330
+Elapse: 545.76s
+
+Epoch 084/400
+Train		loss: 75406416.00	correlation: 0.2595
+Validation	loss: 169084192.00	correlation: 0.2329
+Elapse: 545.78s
+
+Epoch 085/400
+Train		loss: 75376912.00	correlation: 0.2599
+Validation	loss: 168861952.00	correlation: 0.2341
+Elapse: 545.89s
+
+Epoch 086/400
+Train		loss: 75349048.00	correlation: 0.2604
+Validation	loss: 168882304.00	correlation: 0.2349
+Elapse: 545.73s
+
+Epoch 087/400
+Train		loss: 75302984.00	correlation: 0.2607
+Validation	loss: 169006784.00	correlation: 0.2333
+Elapse: 545.26s
+
+Loaded checkpoint from epoch 82 (correlation: 0.2354).
+
+Reduce learning rate of core to 1.4400e-03 (num. reduce: 1).
+Reduce learning rate of readouts to 1.0800e-03 (num. reduce: 1).
+Reduce learning rate of shifters to 1.0800e-03 (num. reduce: 1).
+
+Epoch 088/400
+Train		loss: 73472912.00	correlation: 0.2760
+Validation	loss: 166967968.00	correlation: 0.2466
+Elapse: 546.46s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 089/400
+Train		loss: 72834528.00	correlation: 0.2820
+Validation	loss: 166834160.00	correlation: 0.2485
+Elapse: 546.50s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 090/400
+Train		loss: 72607408.00	correlation: 0.2844
+Validation	loss: 166764992.00	correlation: 0.2490
+Elapse: 546.18s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 091/400
+Train		loss: 72571872.00	correlation: 0.2846
+Validation	loss: 166824368.00	correlation: 0.2476
+Elapse: 546.25s
+
+Epoch 092/400
+Train		loss: 72491600.00	correlation: 0.2854
+Validation	loss: 166791872.00	correlation: 0.2474
+Elapse: 545.90s
+
+Epoch 093/400
+Train		loss: 72479560.00	correlation: 0.2856
+Validation	loss: 166900064.00	correlation: 0.2473
+Elapse: 546.88s
+
+Epoch 094/400
+Train		loss: 72391640.00	correlation: 0.2867
+Validation	loss: 166895872.00	correlation: 0.2471
+Elapse: 546.67s
+
+Epoch 095/400
+Train		loss: 72406800.00	correlation: 0.2865
+Validation	loss: 166785824.00	correlation: 0.2477
+Elapse: 546.76s
+
+Loaded checkpoint from epoch 90 (correlation: 0.2490).
+
+Reduce learning rate of core to 4.3200e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 3.2400e-04 (num. reduce: 1).
+Reduce learning rate of shifters to 3.2400e-04 (num. reduce: 1).
+
+Epoch 096/400
+Train		loss: 71820392.00	correlation: 0.2914
+Validation	loss: 166357664.00	correlation: 0.2510
+Elapse: 546.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 097/400
+Train		loss: 71646128.00	correlation: 0.2926
+Validation	loss: 166358480.00	correlation: 0.2512
+Elapse: 546.63s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 098/400
+Train		loss: 71469944.00	correlation: 0.2943
+Validation	loss: 166261168.00	correlation: 0.2520
+Elapse: 547.33s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 099/400
+Train		loss: 71457032.00	correlation: 0.2945
+Validation	loss: 166242256.00	correlation: 0.2517
+Elapse: 547.22s
+
+Epoch 100/400
+Train		loss: 71381520.00	correlation: 0.2953
+Validation	loss: 166269072.00	correlation: 0.2516
+Elapse: 546.93s
+
+Epoch 101/400
+Train		loss: 71365904.00	correlation: 0.2954
+Validation	loss: 166306016.00	correlation: 0.2512
+Elapse: 547.14s
+
+Epoch 102/400
+Train		loss: 71313408.00	correlation: 0.2956
+Validation	loss: 166202720.00	correlation: 0.2520
+Elapse: 547.18s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 103/400
+Train		loss: 71246520.00	correlation: 0.2963
+Validation	loss: 166201664.00	correlation: 0.2519
+Elapse: 546.59s
+
+Epoch 104/400
+Train		loss: 71237544.00	correlation: 0.2967
+Validation	loss: 166225280.00	correlation: 0.2516
+Elapse: 546.47s
+
+Epoch 105/400
+Train		loss: 71175248.00	correlation: 0.2973
+Validation	loss: 166234304.00	correlation: 0.2515
+Elapse: 546.73s
+
+Epoch 106/400
+Train		loss: 71139096.00	correlation: 0.2976
+Validation	loss: 166179840.00	correlation: 0.2518
+Elapse: 547.27s
+
+Epoch 107/400
+Train		loss: 71152664.00	correlation: 0.2972
+Validation	loss: 166277328.00	correlation: 0.2509
+Elapse: 547.28s
+
+Loaded checkpoint from epoch 102 (correlation: 0.2520).
+
+Reduce learning rate of core to 1.2960e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 9.7200e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 9.7200e-05 (num. reduce: 1).
+
+Epoch 108/400
+Train		loss: 70975272.00	correlation: 0.2990
+Validation	loss: 166102416.00	correlation: 0.2526
+Elapse: 547.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 109/400
+Train		loss: 70927400.00	correlation: 0.2990
+Validation	loss: 166090848.00	correlation: 0.2526
+Elapse: 547.10s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 110/400
+Train		loss: 70928176.00	correlation: 0.2993
+Validation	loss: 166087520.00	correlation: 0.2526
+Elapse: 547.24s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 111/400
+Train		loss: 70948912.00	correlation: 0.2989
+Validation	loss: 166067424.00	correlation: 0.2526
+Elapse: 547.30s
+
+Epoch 112/400
+Train		loss: 70843856.00	correlation: 0.2999
+Validation	loss: 166089904.00	correlation: 0.2527
+Elapse: 547.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 113/400
+Train		loss: 70930864.00	correlation: 0.2991
+Validation	loss: 166048192.00	correlation: 0.2527
+Elapse: 547.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 114/400
+Train		loss: 70888224.00	correlation: 0.2998
+Validation	loss: 166076336.00	correlation: 0.2525
+Elapse: 547.07s
+
+Epoch 115/400
+Train		loss: 70807400.00	correlation: 0.3003
+Validation	loss: 166078992.00	correlation: 0.2524
+Elapse: 547.34s
+
+Epoch 116/400
+Train		loss: 70784824.00	correlation: 0.3006
+Validation	loss: 166036512.00	correlation: 0.2528
+Elapse: 547.33s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 117/400
+Train		loss: 70760888.00	correlation: 0.3007
+Validation	loss: 166109792.00	correlation: 0.2523
+Elapse: 547.36s
+
+Epoch 118/400
+Train		loss: 70793224.00	correlation: 0.3003
+Validation	loss: 166033088.00	correlation: 0.2527
+Elapse: 547.66s
+
+Epoch 119/400
+Train		loss: 70801416.00	correlation: 0.3002
+Validation	loss: 166062960.00	correlation: 0.2529
+Elapse: 547.68s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 120/400
+Train		loss: 70748944.00	correlation: 0.3009
+Validation	loss: 166063152.00	correlation: 0.2524
+Elapse: 547.18s
+
+Epoch 121/400
+Train		loss: 70736464.00	correlation: 0.3010
+Validation	loss: 166058976.00	correlation: 0.2528
+Elapse: 547.53s
+
+Epoch 122/400
+Train		loss: 70653008.00	correlation: 0.3019
+Validation	loss: 166041920.00	correlation: 0.2527
+Elapse: 547.68s
+
+Epoch 123/400
+Train		loss: 70723656.00	correlation: 0.3011
+Validation	loss: 166053344.00	correlation: 0.2525
+Elapse: 547.37s
+
+Epoch 124/400
+Train		loss: 70689936.00	correlation: 0.3016
+Validation	loss: 166053472.00	correlation: 0.2526
+Elapse: 547.93s
+
+Loaded checkpoint from epoch 119 (correlation: 0.2529).
+
+Reduce learning rate of core to 3.8880e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 2.9160e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 2.9160e-05 (num. reduce: 1).
+
+Epoch 125/400
+Train		loss: 70604440.00	correlation: 0.3023
+Validation	loss: 166025312.00	correlation: 0.2529
+Elapse: 547.63s
+
+Epoch 126/400
+Train		loss: 70608160.00	correlation: 0.3021
+Validation	loss: 166010512.00	correlation: 0.2530
+Elapse: 547.36s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 127/400
+Train		loss: 70626448.00	correlation: 0.3020
+Validation	loss: 166023648.00	correlation: 0.2528
+Elapse: 547.13s
+
+Epoch 128/400
+Train		loss: 70612096.00	correlation: 0.3018
+Validation	loss: 166023216.00	correlation: 0.2529
+Elapse: 546.99s
+
+Epoch 129/400
+Train		loss: 70635640.00	correlation: 0.3020
+Validation	loss: 166036208.00	correlation: 0.2528
+Elapse: 547.23s
+
+Epoch 130/400
+Train		loss: 70666920.00	correlation: 0.3015
+Validation	loss: 166035808.00	correlation: 0.2527
+Elapse: 546.99s
+
+Epoch 131/400
+Train		loss: 70564592.00	correlation: 0.3028
+Validation	loss: 166005856.00	correlation: 0.2528
+Elapse: 547.30s
+
+Loaded checkpoint from epoch 126 (correlation: 0.2530).
+
+Reduce learning rate of core to 1.1664e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 8.7480e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 8.7480e-06 (num. reduce: 1).
+
+Epoch 132/400
+Train		loss: 70603848.00	correlation: 0.3022
+Validation	loss: 166004944.00	correlation: 0.2530
+Elapse: 547.32s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/009/ckpt/model_state.pt.
+
+Epoch 133/400
+Train		loss: 70533872.00	correlation: 0.3031
+Validation	loss: 166019360.00	correlation: 0.2529
+Elapse: 547.50s
+
+Epoch 134/400
+Train		loss: 70635248.00	correlation: 0.3019
+Validation	loss: 166021888.00	correlation: 0.2529
+Elapse: 547.34s
+
+Epoch 135/400
+Train		loss: 70620176.00	correlation: 0.3020
+Validation	loss: 166009888.00	correlation: 0.2529
+Elapse: 546.38s
+
+Epoch 136/400
+Train		loss: 70572368.00	correlation: 0.3023
+Validation	loss: 166012144.00	correlation: 0.2529
+Elapse: 547.10s
+
+Epoch 137/400
+Train		loss: 70658640.00	correlation: 0.3016
+Validation	loss: 166018080.00	correlation: 0.2529
+Elapse: 546.90s
+
+Loaded checkpoint from epoch 132 (correlation: 0.2530).
+
+Reduce learning rate of core to 3.4992e-06 (num. reduce: 1).
+Reduce learning rate of readouts to 2.6244e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 2.6244e-06 (num. reduce: 1).
+
+Epoch 138/400
+Train		loss: 70545520.00	correlation: 0.3026
+Validation	loss: 166011936.00	correlation: 0.2530
+Elapse: 546.72s
+
+Epoch 139/400
+Train		loss: 70622800.00	correlation: 0.3018
+Validation	loss: 166013088.00	correlation: 0.2529
+Elapse: 547.01s
+
+Epoch 140/400
+Train		loss: 70572936.00	correlation: 0.3025
+Validation	loss: 166013696.00	correlation: 0.2529
+Elapse: 547.38s
+
+Epoch 141/400
+Train		loss: 70624016.00	correlation: 0.3020
+Validation	loss: 166012496.00	correlation: 0.2530
+Elapse: 546.98s
+
+Epoch 142/400
+Train		loss: 70651392.00	correlation: 0.3018
+Validation	loss: 166017120.00	correlation: 0.2529
+Elapse: 547.57s
+
+Loaded checkpoint from epoch 132 (correlation: 0.2530).
+
+Reduce learning rate of core to 1.0498e-06 (num. reduce: 2).
+Reduce learning rate of readouts to 7.8732e-07 (num. reduce: 2).
+Reduce learning rate of shifters to 7.8732e-07 (num. reduce: 2).
+
+Epoch 143/400
+Train		loss: 70616608.00	correlation: 0.3018
+Validation	loss: 166009488.00	correlation: 0.2530
+Elapse: 547.46s
+
+Epoch 144/400
+Train		loss: 70557224.00	correlation: 0.3027
+Validation	loss: 166010240.00	correlation: 0.2530
+Elapse: 547.42s
+
+Epoch 145/400
+Train		loss: 70617064.00	correlation: 0.3020
+Validation	loss: 166011824.00	correlation: 0.2530
+Elapse: 547.48s
+
+Epoch 146/400
+Train		loss: 70601720.00	correlation: 0.3022
+Validation	loss: 166010880.00	correlation: 0.2530
+Elapse: 547.26s
+
+Epoch 147/400
+Train		loss: 70616768.00	correlation: 0.3021
+Validation	loss: 166012944.00	correlation: 0.2530
+Elapse: 547.43s
+
+Model has not improved after 2 LR reductions.
+
+Loaded checkpoint from epoch 132 (correlation: 0.2530).
+
+
+ValidationA: 0.2508	B: 0.2804	C: 0.2750	D: 0.2393	E: 0.2390	F: 0.2346	G: 0.2573	H: 0.2379	I: 0.2555	J: 0.2599	average: 0.2530
+
+Results saved to /home/storage/runs/vivit_ensemble/009.

003/args.yaml

@@ -0,0 +1,87 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_eps: 1e-08
+batch_size: 6
+cache_data: false
+center_crop: 1.0
+clear_output_dir: true
+core: vivit
+core_behavior_mode: 2
+core_compile: false
+core_drop_path: 0.0505
+core_emb_dim: 112
+core_ff_activation: gelu
+core_ff_dropout: 0.0592
+core_flash_attention: 1
+core_head_dim: 48
+core_lr: 0.0048
+core_mha_dropout: 0.358
+core_mlp_dim: 136
+core_norm_qk: false
+core_num_heads: 11
+core_p_dropout: 0.1338
+core_parallel_attention: true
+core_patch_mode: 0
+core_pos_encoding: 3
+core_spatial_depth: 3
+core_spatial_patch_size: 7
+core_spatial_patch_stride: 2
+core_temporal_depth: 5
+core_temporal_patch_size: 25
+core_temporal_patch_stride: 1
+core_weight_decay: 0.1789
+criterion: poisson_correlation
+critic_mode: 0
+crop_frame: 140
+data: /home/sensorium2023
+deterministic: false
+device: cuda
+dpi: 120
+ds_mode: 3
+ds_scale: 1
+epochs: 400
+format: svg
+git_hash: 1033c29
+grad_checkpointing: 0
+grad_norm: null
+input_shapes:
+  behavior: [2, 140]
+  pupil_center: [2, 140]
+  video: [1, 140, 36, 64]
+limit_data: null
+lr: 0.0036
+max_frame: 300
+micro_batch_size: 6
+modulator_mode: 0
+mouse_ids: [A, B, C, D, E, F, G, H, I, J]
+num_workers: 2
+output_dir: /home/storage/runs/003
+output_mode: 1
+output_shapes:
+  A: [7440, 140]
+  B: [7928, 140]
+  C: [8285, 140]
+  D: [7671, 140]
+  E: [7495, 140]
+  F: [7863, 140]
+  G: [7908, 140]
+  H: [8202, 140]
+  I: [7939, 140]
+  J: [8122, 140]
+precision: "32"
+pretrain_core: null
+readout: gaussian2d
+readout_bias_mode: 2
+readout_dropout: 0.0
+readout_grid_mode: 1
+restore: null
+save_plots: false
+seed: 12
+shifter_layers: 3
+shifter_mode: 1
+shifter_size: 5
+stat_mode: 1
+trainable_params: 12278365
+transform_mode: 2
+verbose: 1
+weight_decay: 0.3939

003/ckpt/model_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09874ebf6e03ea617ccd08af30de997d9ca2d372fd68f9fe26696e58b675f025
+size 148268583

003/evaluation.yaml

@@ -0,0 +1,3 @@
+validation: {A: 0.24746963381767273, B: 0.27505889534950256, C: 0.27303239703178406,
+  D: 0.23642510175704956, E: 0.23546315729618073, F: 0.23425357043743134, G: 0.2543371021747589,
+  H: 0.23437707126140594, I: 0.2602153420448303, J: 0.2588430643081665, average: 0.25094753354787824}

003/model.txt

@@ -0,0 +1,143 @@
+=======================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #
+=======================================================================================================================================
+Model                                                        --                        [1, 7440, 116]            --
+├─ViViTCore: 1-1                                             --                        [1, 112, 116, 15, 29]     --
+│    └─ViViT: 2-1                                            [1, 5, 140, 36, 64]       [1, 112, 116, 15, 29]     --
+│    │    └─Tokenizer: 3-1                                   [1, 5, 140, 36, 64]       [1, 116, 435, 112]        79,632
+│    │    │    └─Sequential: 4-1                             [1, 5, 140, 36, 64]       [1, 116, 435, 112]        --
+│    │    │    │    └─Unfold3d: 5-1                          [1, 5, 140, 36, 64]       [1, 116, 435, 6125]       --
+│    │    │    │    └─LayerNorm: 5-2                         [1, 116, 435, 6125]       [1, 116, 435, 6125]       12,250
+│    │    │    │    └─Linear: 5-3                            [1, 116, 435, 6125]       [1, 116, 435, 112]        686,112
+│    │    │    │    └─LayerNorm: 5-4                         [1, 116, 435, 112]        [1, 116, 435, 112]        224
+│    │    │    │    └─Dropout: 5-5                           [1, 116, 435, 112]        [1, 116, 435, 112]        --
+│    │    └─Transformer: 3-2                                 [116, 435, 112]           [116, 435, 112]           --
+│    │    │    └─ModuleList: 4-2                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-6          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-1                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-2                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-3                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-4                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-5                   [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-1                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-2                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-3                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-6                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-7          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-7                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-8                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-9                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-10                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-11                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-4                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-5                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-6                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-12                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-8          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-13                   [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-14                      [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-15                      [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-16                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-17                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-7                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-8                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-9                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-18                    [116, 435, 112]           [116, 435, 112]           --
+│    │    └─Transformer: 3-3                                 [435, 116, 112]           [435, 116, 112]           --
+│    │    │    └─ModuleList: 4-3                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-9          [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-19                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-20                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-21                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-22                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-23                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-10                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-11                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-12                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-24                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-10         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-25                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-26                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-27                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-28                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-29                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-13                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-14                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-15                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-30                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-11         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-31                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-32                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-33                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-34                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-35                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-16                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-17                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-18                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-36                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-12         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-37                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-38                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-39                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-40                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-41                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-19                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-20                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-21                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-42                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-13         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-43                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-44                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-45                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-46                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-47                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-22                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-23                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-24                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-48                    [435, 116, 112]           [435, 116, 112]           --
+│    │    └─Rearrange: 3-4                                   [1, 116, 435, 112]        [1, 112, 116, 15, 29]     --
+├─MLPShifters: 1-2                                           --                        [1, 2, 116]               513
+│    └─MLPShifter: 2-2                                       [116, 2]                  [116, 2]                  --
+│    │    └─Sequential: 3-5                                  [116, 2]                  [116, 2]                  --
+│    │    │    └─Linear: 4-4                                 [116, 2]                  [116, 5]                  15
+│    │    │    └─Tanh: 4-5                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-6                                 [116, 5]                  [116, 5]                  30
+│    │    │    └─Tanh: 4-7                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-8                                 [116, 5]                  [116, 2]                  12
+│    │    │    └─Tanh: 4-9                                   [116, 2]                  [116, 2]                  --
+├─Readouts: 1-3                                              [1, 112, 116, 15, 29]     [1, 7440, 116]            8,477,649
+│    └─Gaussian2DReadout2: 2-3                               [1, 112, 116, 15, 29]     [1, 7440, 116]            870,480
+│    │    └─Sequential: 3-6                                  [7440, 2]                 [7440, 2]                 --
+│    │    │    └─Linear: 4-10                                [7440, 2]                 [7440, 30]                90
+│    │    │    └─ELU: 4-11                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-12                                [7440, 30]                [7440, 2]                 62
+│    │    │    └─Tanh: 4-13                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-7                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-14                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-15                                   [7440, 30]                [7440, 30]                --
+��    │    │    └─Linear: 4-16                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-17                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-8                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-18                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-19                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-20                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-21                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-9                                  [116, 4]                  [116, 112]                --
+│    │    │    └─Linear: 4-22                                [116, 4]                  [116, 112]                560
+│    │    │    └─LayerNorm: 4-23                             [116, 112]                [116, 112]                224
+│    │    │    └─GELU: 4-24                                  [116, 112]                [116, 112]                --
+│    │    │    └─Dropout: 4-25                               [116, 112]                [116, 112]                --
+│    │    │    └─Linear: 4-26                                [116, 112]                [116, 112]                12,656
+│    │    │    └─Tanh: 4-27                                  [116, 112]                [116, 112]                --
+├─Exponential: 1-4                                           [1, 7440, 116]            [1, 7440, 116]            --
+=======================================================================================================================================
+Total params: 12,278,365
+Trainable params: 12,278,365
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 679.88
+=======================================================================================================================================
+Input size (MB): 1.29
+Forward/backward pass size (MB): 9208.73
+Params size (MB): 11.40
+Estimated Total Size (MB): 9221.42
+=======================================================================================================================================

003/output.log

@@ -0,0 +1,789 @@
+Use bfloat16 for core module.
+Use parallel attention and MLP in ViViT.
+
+Epoch 001/400
+Train		loss: 112704208.00	correlation: 0.0126
+Validation	loss: 199545376.00	correlation: 0.0293
+Elapse: 540.35s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 002/400
+Train		loss: 97414480.00	correlation: 0.0393
+Validation	loss: 198820640.00	correlation: 0.0410
+Elapse: 549.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 003/400
+Train		loss: 95851936.00	correlation: 0.0545
+Validation	loss: 197542912.00	correlation: 0.0493
+Elapse: 552.73s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 004/400
+Train		loss: 94762680.00	correlation: 0.0658
+Validation	loss: 195978896.00	correlation: 0.0592
+Elapse: 553.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 005/400
+Train		loss: 93366904.00	correlation: 0.0807
+Validation	loss: 193138784.00	correlation: 0.0786
+Elapse: 551.37s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 006/400
+Train		loss: 91530000.00	correlation: 0.0996
+Validation	loss: 190414640.00	correlation: 0.0957
+Elapse: 548.87s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 007/400
+Train		loss: 90052112.00	correlation: 0.1151
+Validation	loss: 187958800.00	correlation: 0.1117
+Elapse: 545.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 008/400
+Train		loss: 88550296.00	correlation: 0.1303
+Validation	loss: 185819104.00	correlation: 0.1231
+Elapse: 542.13s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 009/400
+Train		loss: 87211088.00	correlation: 0.1444
+Validation	loss: 183781008.00	correlation: 0.1379
+Elapse: 540.39s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 010/400
+Train		loss: 85912744.00	correlation: 0.1575
+Validation	loss: 182160864.00	correlation: 0.1483
+Elapse: 539.93s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 011/400
+Train		loss: 84825776.00	correlation: 0.1681
+Validation	loss: 180796960.00	correlation: 0.1574
+Elapse: 539.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 012/400
+Train		loss: 83932096.00	correlation: 0.1768
+Validation	loss: 179665872.00	correlation: 0.1651
+Elapse: 539.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 013/400
+Train		loss: 83185960.00	correlation: 0.1841
+Validation	loss: 178867920.00	correlation: 0.1706
+Elapse: 540.49s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 014/400
+Train		loss: 82619600.00	correlation: 0.1896
+Validation	loss: 178183616.00	correlation: 0.1749
+Elapse: 540.25s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 015/400
+Train		loss: 82049104.00	correlation: 0.1953
+Validation	loss: 177195872.00	correlation: 0.1811
+Elapse: 539.98s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 016/400
+Train		loss: 81438744.00	correlation: 0.2013
+Validation	loss: 176732480.00	correlation: 0.1848
+Elapse: 539.92s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 017/400
+Train		loss: 80981184.00	correlation: 0.2058
+Validation	loss: 175869328.00	correlation: 0.1895
+Elapse: 540.34s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 018/400
+Train		loss: 80463984.00	correlation: 0.2108
+Validation	loss: 175137504.00	correlation: 0.1941
+Elapse: 540.52s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 019/400
+Train		loss: 80076456.00	correlation: 0.2145
+Validation	loss: 174743216.00	correlation: 0.1976
+Elapse: 540.79s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 020/400
+Train		loss: 79770240.00	correlation: 0.2180
+Validation	loss: 174354656.00	correlation: 0.1999
+Elapse: 541.02s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 021/400
+Train		loss: 79389680.00	correlation: 0.2210
+Validation	loss: 173737072.00	correlation: 0.2049
+Elapse: 540.85s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 022/400
+Train		loss: 79099264.00	correlation: 0.2238
+Validation	loss: 173426080.00	correlation: 0.2057
+Elapse: 542.08s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 023/400
+Train		loss: 78893584.00	correlation: 0.2259
+Validation	loss: 172965088.00	correlation: 0.2094
+Elapse: 542.73s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 024/400
+Train		loss: 78610448.00	correlation: 0.2286
+Validation	loss: 172757408.00	correlation: 0.2102
+Elapse: 543.17s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 025/400
+Train		loss: 78350984.00	correlation: 0.2311
+Validation	loss: 172301040.00	correlation: 0.2136
+Elapse: 543.01s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 026/400
+Train		loss: 78082256.00	correlation: 0.2338
+Validation	loss: 172086432.00	correlation: 0.2145
+Elapse: 543.35s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 027/400
+Train		loss: 77827456.00	correlation: 0.2363
+Validation	loss: 171786464.00	correlation: 0.2169
+Elapse: 543.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 028/400
+Train		loss: 77638136.00	correlation: 0.2379
+Validation	loss: 171793264.00	correlation: 0.2171
+Elapse: 543.62s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 029/400
+Train		loss: 77553520.00	correlation: 0.2390
+Validation	loss: 171183824.00	correlation: 0.2206
+Elapse: 543.14s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 030/400
+Train		loss: 77293184.00	correlation: 0.2414
+Validation	loss: 171063472.00	correlation: 0.2218
+Elapse: 542.93s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 031/400
+Train		loss: 77137136.00	correlation: 0.2425
+Validation	loss: 170927232.00	correlation: 0.2215
+Elapse: 542.90s
+
+Epoch 032/400
+Train		loss: 77036008.00	correlation: 0.2439
+Validation	loss: 170678112.00	correlation: 0.2241
+Elapse: 544.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 033/400
+Train		loss: 76850512.00	correlation: 0.2458
+Validation	loss: 170585168.00	correlation: 0.2249
+Elapse: 543.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 034/400
+Train		loss: 76745528.00	correlation: 0.2467
+Validation	loss: 170361008.00	correlation: 0.2261
+Elapse: 543.46s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 035/400
+Train		loss: 76631640.00	correlation: 0.2479
+Validation	loss: 170159952.00	correlation: 0.2272
+Elapse: 544.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 036/400
+Train		loss: 76572216.00	correlation: 0.2483
+Validation	loss: 170068624.00	correlation: 0.2280
+Elapse: 544.03s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 037/400
+Train		loss: 76306000.00	correlation: 0.2511
+Validation	loss: 170078272.00	correlation: 0.2281
+Elapse: 543.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 038/400
+Train		loss: 76521968.00	correlation: 0.2488
+Validation	loss: 170034304.00	correlation: 0.2284
+Elapse: 543.78s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 039/400
+Train		loss: 76277552.00	correlation: 0.2512
+Validation	loss: 169955232.00	correlation: 0.2283
+Elapse: 544.11s
+
+Epoch 040/400
+Train		loss: 76083856.00	correlation: 0.2532
+Validation	loss: 169547568.00	correlation: 0.2316
+Elapse: 544.27s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 041/400
+Train		loss: 76034800.00	correlation: 0.2536
+Validation	loss: 169778400.00	correlation: 0.2301
+Elapse: 544.31s
+
+Epoch 042/400
+Train		loss: 75948104.00	correlation: 0.2542
+Validation	loss: 169703472.00	correlation: 0.2304
+Elapse: 543.89s
+
+Epoch 043/400
+Train		loss: 75900336.00	correlation: 0.2549
+Validation	loss: 169460816.00	correlation: 0.2314
+Elapse: 543.95s
+
+Epoch 044/400
+Train		loss: 75856872.00	correlation: 0.2554
+Validation	loss: 169446544.00	correlation: 0.2314
+Elapse: 543.85s
+
+Epoch 045/400
+Train		loss: 75771024.00	correlation: 0.2563
+Validation	loss: 169373568.00	correlation: 0.2320
+Elapse: 544.58s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 046/400
+Train		loss: 75747888.00	correlation: 0.2564
+Validation	loss: 169484960.00	correlation: 0.2316
+Elapse: 544.29s
+
+Epoch 047/400
+Train		loss: 75614488.00	correlation: 0.2578
+Validation	loss: 169374048.00	correlation: 0.2332
+Elapse: 544.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 048/400
+Train		loss: 75621768.00	correlation: 0.2577
+Validation	loss: 169829152.00	correlation: 0.2292
+Elapse: 544.60s
+
+Epoch 049/400
+Train		loss: 75540576.00	correlation: 0.2584
+Validation	loss: 169151584.00	correlation: 0.2322
+Elapse: 545.07s
+
+Epoch 050/400
+Train		loss: 75552112.00	correlation: 0.2587
+Validation	loss: 169218528.00	correlation: 0.2327
+Elapse: 544.47s
+
+Epoch 051/400
+Train		loss: 75440728.00	correlation: 0.2597
+Validation	loss: 169177248.00	correlation: 0.2329
+Elapse: 544.75s
+
+Epoch 052/400
+Train		loss: 75499728.00	correlation: 0.2591
+Validation	loss: 169380720.00	correlation: 0.2327
+Elapse: 544.77s
+
+Loaded checkpoint from epoch 47 (correlation: 0.2332).
+
+Reduce learning rate of core to 1.4400e-03 (num. reduce: 1).
+Reduce learning rate of readouts to 1.0800e-03 (num. reduce: 1).
+Reduce learning rate of shifters to 1.0800e-03 (num. reduce: 1).
+
+Epoch 053/400
+Train		loss: 73633064.00	correlation: 0.2750
+Validation	loss: 167526240.00	correlation: 0.2446
+Elapse: 545.49s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 054/400
+Train		loss: 72992880.00	correlation: 0.2809
+Validation	loss: 167505792.00	correlation: 0.2447
+Elapse: 545.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 055/400
+Train		loss: 72791024.00	correlation: 0.2827
+Validation	loss: 167337248.00	correlation: 0.2451
+Elapse: 545.82s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 056/400
+Train		loss: 72699672.00	correlation: 0.2837
+Validation	loss: 167396864.00	correlation: 0.2453
+Elapse: 545.65s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 057/400
+Train		loss: 72666128.00	correlation: 0.2841
+Validation	loss: 167320224.00	correlation: 0.2457
+Elapse: 546.33s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 058/400
+Train		loss: 72662680.00	correlation: 0.2840
+Validation	loss: 167358624.00	correlation: 0.2457
+Elapse: 545.38s
+
+Epoch 059/400
+Train		loss: 72528456.00	correlation: 0.2854
+Validation	loss: 167347344.00	correlation: 0.2456
+Elapse: 546.44s
+
+Epoch 060/400
+Train		loss: 72553424.00	correlation: 0.2854
+Validation	loss: 167294624.00	correlation: 0.2462
+Elapse: 546.17s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 061/400
+Train		loss: 72496144.00	correlation: 0.2860
+Validation	loss: 167428896.00	correlation: 0.2444
+Elapse: 545.96s
+
+Epoch 062/400
+Train		loss: 72479344.00	correlation: 0.2859
+Validation	loss: 167235104.00	correlation: 0.2454
+Elapse: 546.11s
+
+Epoch 063/400
+Train		loss: 72437912.00	correlation: 0.2863
+Validation	loss: 167384528.00	correlation: 0.2447
+Elapse: 546.05s
+
+Epoch 064/400
+Train		loss: 72443952.00	correlation: 0.2864
+Validation	loss: 167276464.00	correlation: 0.2452
+Elapse: 546.23s
+
+Epoch 065/400
+Train		loss: 72421648.00	correlation: 0.2869
+Validation	loss: 167359664.00	correlation: 0.2448
+Elapse: 546.55s
+
+Loaded checkpoint from epoch 60 (correlation: 0.2462).
+
+Reduce learning rate of core to 4.3200e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 3.2400e-04 (num. reduce: 1).
+Reduce learning rate of shifters to 3.2400e-04 (num. reduce: 1).
+
+Epoch 066/400
+Train		loss: 71631360.00	correlation: 0.2929
+Validation	loss: 166762592.00	correlation: 0.2488
+Elapse: 546.04s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 067/400
+Train		loss: 71301752.00	correlation: 0.2963
+Validation	loss: 166766816.00	correlation: 0.2491
+Elapse: 546.30s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 068/400
+Train		loss: 71239384.00	correlation: 0.2970
+Validation	loss: 166698608.00	correlation: 0.2493
+Elapse: 546.54s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 069/400
+Train		loss: 71140904.00	correlation: 0.2977
+Validation	loss: 166688480.00	correlation: 0.2493
+Elapse: 546.57s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 070/400
+Train		loss: 71157128.00	correlation: 0.2979
+Validation	loss: 166701680.00	correlation: 0.2493
+Elapse: 546.93s
+
+Epoch 071/400
+Train		loss: 71109936.00	correlation: 0.2983
+Validation	loss: 166666112.00	correlation: 0.2493
+Elapse: 546.33s
+
+Epoch 072/400
+Train		loss: 71014384.00	correlation: 0.2990
+Validation	loss: 166637680.00	correlation: 0.2493
+Elapse: 546.25s
+
+Epoch 073/400
+Train		loss: 71012144.00	correlation: 0.2989
+Validation	loss: 166633760.00	correlation: 0.2495
+Elapse: 546.14s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 074/400
+Train		loss: 70962312.00	correlation: 0.2996
+Validation	loss: 166724848.00	correlation: 0.2491
+Elapse: 546.66s
+
+Epoch 075/400
+Train		loss: 70909096.00	correlation: 0.3002
+Validation	loss: 166680192.00	correlation: 0.2492
+Elapse: 546.39s
+
+Epoch 076/400
+Train		loss: 70859696.00	correlation: 0.3007
+Validation	loss: 166666160.00	correlation: 0.2493
+Elapse: 546.76s
+
+Epoch 077/400
+Train		loss: 70862616.00	correlation: 0.3006
+Validation	loss: 166644512.00	correlation: 0.2493
+Elapse: 546.35s
+
+Epoch 078/400
+Train		loss: 70866920.00	correlation: 0.3008
+Validation	loss: 166689088.00	correlation: 0.2489
+Elapse: 546.43s
+
+Loaded checkpoint from epoch 73 (correlation: 0.2495).
+
+Reduce learning rate of core to 1.2960e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 9.7200e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 9.7200e-05 (num. reduce: 1).
+
+Epoch 079/400
+Train		loss: 70659088.00	correlation: 0.3023
+Validation	loss: 166495968.00	correlation: 0.2504
+Elapse: 546.32s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 080/400
+Train		loss: 70583640.00	correlation: 0.3029
+Validation	loss: 166499952.00	correlation: 0.2503
+Elapse: 546.55s
+
+Epoch 081/400
+Train		loss: 70541232.00	correlation: 0.3031
+Validation	loss: 166486608.00	correlation: 0.2504
+Elapse: 546.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 082/400
+Train		loss: 70561208.00	correlation: 0.3030
+Validation	loss: 166498080.00	correlation: 0.2505
+Elapse: 546.48s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 083/400
+Train		loss: 70504848.00	correlation: 0.3037
+Validation	loss: 166479744.00	correlation: 0.2504
+Elapse: 545.91s
+
+Epoch 084/400
+Train		loss: 70483280.00	correlation: 0.3037
+Validation	loss: 166527136.00	correlation: 0.2504
+Elapse: 546.39s
+
+Epoch 085/400
+Train		loss: 70486104.00	correlation: 0.3039
+Validation	loss: 166507024.00	correlation: 0.2504
+Elapse: 546.66s
+
+Epoch 086/400
+Train		loss: 70448464.00	correlation: 0.3042
+Validation	loss: 166480160.00	correlation: 0.2505
+Elapse: 546.13s
+
+Epoch 087/400
+Train		loss: 70433528.00	correlation: 0.3043
+Validation	loss: 166521664.00	correlation: 0.2504
+Elapse: 546.51s
+
+Loaded checkpoint from epoch 82 (correlation: 0.2505).
+
+Reduce learning rate of core to 3.8880e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 2.9160e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 2.9160e-05 (num. reduce: 1).
+
+Epoch 088/400
+Train		loss: 70419248.00	correlation: 0.3041
+Validation	loss: 166470848.00	correlation: 0.2507
+Elapse: 546.72s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 089/400
+Train		loss: 70456936.00	correlation: 0.3039
+Validation	loss: 166478544.00	correlation: 0.2507
+Elapse: 546.31s
+
+Epoch 090/400
+Train		loss: 70377392.00	correlation: 0.3047
+Validation	loss: 166470880.00	correlation: 0.2508
+Elapse: 546.64s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 091/400
+Train		loss: 70370720.00	correlation: 0.3048
+Validation	loss: 166471712.00	correlation: 0.2507
+Elapse: 546.55s
+
+Epoch 092/400
+Train		loss: 70332624.00	correlation: 0.3052
+Validation	loss: 166468176.00	correlation: 0.2506
+Elapse: 546.51s
+
+Epoch 093/400
+Train		loss: 70375248.00	correlation: 0.3046
+Validation	loss: 166466304.00	correlation: 0.2507
+Elapse: 547.09s
+
+Epoch 094/400
+Train		loss: 70312208.00	correlation: 0.3053
+Validation	loss: 166426080.00	correlation: 0.2509
+Elapse: 546.36s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 095/400
+Train		loss: 70306024.00	correlation: 0.3054
+Validation	loss: 166448992.00	correlation: 0.2509
+Elapse: 546.42s
+
+Epoch 096/400
+Train		loss: 70301760.00	correlation: 0.3053
+Validation	loss: 166460400.00	correlation: 0.2508
+Elapse: 545.99s
+
+Epoch 097/400
+Train		loss: 70339392.00	correlation: 0.3050
+Validation	loss: 166448912.00	correlation: 0.2507
+Elapse: 546.74s
+
+Epoch 098/400
+Train		loss: 70365064.00	correlation: 0.3048
+Validation	loss: 166476400.00	correlation: 0.2506
+Elapse: 546.17s
+
+Epoch 099/400
+Train		loss: 70304792.00	correlation: 0.3053
+Validation	loss: 166451168.00	correlation: 0.2507
+Elapse: 545.82s
+
+Loaded checkpoint from epoch 94 (correlation: 0.2509).
+
+Reduce learning rate of core to 1.1664e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 8.7480e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 8.7480e-06 (num. reduce: 1).
+
+Epoch 100/400
+Train		loss: 70282520.00	correlation: 0.3055
+Validation	loss: 166444240.00	correlation: 0.2509
+Elapse: 546.60s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 101/400
+Train		loss: 70321056.00	correlation: 0.3050
+Validation	loss: 166451424.00	correlation: 0.2509
+Elapse: 546.61s
+
+Epoch 102/400
+Train		loss: 70312536.00	correlation: 0.3052
+Validation	loss: 166450992.00	correlation: 0.2509
+Elapse: 546.64s
+
+Epoch 103/400
+Train		loss: 70294928.00	correlation: 0.3057
+Validation	loss: 166437664.00	correlation: 0.2509
+Elapse: 546.47s
+
+Epoch 104/400
+Train		loss: 70265648.00	correlation: 0.3058
+Validation	loss: 166443488.00	correlation: 0.2509
+Elapse: 546.86s
+
+Epoch 105/400
+Train		loss: 70228080.00	correlation: 0.3058
+Validation	loss: 166447312.00	correlation: 0.2508
+Elapse: 546.77s
+
+Loaded checkpoint from epoch 100 (correlation: 0.2509).
+
+Reduce learning rate of core to 3.4992e-06 (num. reduce: 1).
+Reduce learning rate of readouts to 2.6244e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 2.6244e-06 (num. reduce: 1).
+
+Epoch 106/400
+Train		loss: 70303288.00	correlation: 0.3053
+Validation	loss: 166441600.00	correlation: 0.2509
+Elapse: 546.61s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 107/400
+Train		loss: 70329360.00	correlation: 0.3052
+Validation	loss: 166438976.00	correlation: 0.2509
+Elapse: 546.25s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/012/ckpt/model_state.pt.
+
+Epoch 108/400
+Train		loss: 70249752.00	correlation: 0.3063
+Validation	loss: 166440032.00	correlation: 0.2509
+Elapse: 546.40s
+
+Epoch 109/400
+Train		loss: 70245592.00	correlation: 0.3059
+Validation	loss: 166436832.00	correlation: 0.2509
+Elapse: 546.34s
+
+Epoch 110/400
+Train		loss: 70323520.00	correlation: 0.3050
+Validation	loss: 166439552.00	correlation: 0.2509
+Elapse: 546.33s
+
+Epoch 111/400
+Train		loss: 70291648.00	correlation: 0.3053
+Validation	loss: 166438448.00	correlation: 0.2509
+Elapse: 546.45s
+
+Epoch 112/400
+Train		loss: 70337264.00	correlation: 0.3050
+Validation	loss: 166435040.00	correlation: 0.2509
+Elapse: 546.37s
+
+Loaded checkpoint from epoch 107 (correlation: 0.2509).
+
+Reduce learning rate of core to 1.0498e-06 (num. reduce: 1).
+Reduce learning rate of readouts to 7.8732e-07 (num. reduce: 1).
+Reduce learning rate of shifters to 7.8732e-07 (num. reduce: 1).
+
+Epoch 113/400
+Train		loss: 70326192.00	correlation: 0.3050
+Validation	loss: 166438976.00	correlation: 0.2509
+Elapse: 546.86s
+
+Epoch 114/400
+Train		loss: 70297808.00	correlation: 0.3056
+Validation	loss: 166439680.00	correlation: 0.2509
+Elapse: 546.99s
+
+Epoch 115/400
+Train		loss: 70375312.00	correlation: 0.3045
+Validation	loss: 166439312.00	correlation: 0.2509
+Elapse: 547.26s
+
+Epoch 116/400
+Train		loss: 70331832.00	correlation: 0.3050
+Validation	loss: 166439392.00	correlation: 0.2509
+Elapse: 546.75s
+
+Epoch 117/400
+Train		loss: 70241608.00	correlation: 0.3060
+Validation	loss: 166441104.00	correlation: 0.2509
+Elapse: 546.43s
+
+Loaded checkpoint from epoch 107 (correlation: 0.2509).
+
+Reduce learning rate of core to 3.1493e-07 (num. reduce: 2).
+Reduce learning rate of readouts to 2.3620e-07 (num. reduce: 2).
+Reduce learning rate of shifters to 2.3620e-07 (num. reduce: 2).
+
+Epoch 118/400
+Train		loss: 70344976.00	correlation: 0.3049
+Validation	loss: 166439680.00	correlation: 0.2509
+Elapse: 546.75s
+
+Epoch 119/400
+Train		loss: 70278184.00	correlation: 0.3054
+Validation	loss: 166439840.00	correlation: 0.2509
+Elapse: 547.33s
+
+Epoch 120/400
+Train		loss: 70224856.00	correlation: 0.3062
+Validation	loss: 166439440.00	correlation: 0.2509
+Elapse: 546.94s
+
+Epoch 121/400
+Train		loss: 70317744.00	correlation: 0.3049
+Validation	loss: 166439280.00	correlation: 0.2509
+Elapse: 547.22s
+
+Epoch 122/400
+Train		loss: 70322624.00	correlation: 0.3052
+Validation	loss: 166438784.00	correlation: 0.2509
+Elapse: 546.83s
+
+Model has not improved after 2 LR reductions.
+
+Loaded checkpoint from epoch 107 (correlation: 0.2509).
+
+
+ValidationA: 0.2475	B: 0.2751	C: 0.2730	D: 0.2364	E: 0.2355	F: 0.2343	G: 0.2543	H: 0.2344	I: 0.2602	J: 0.2588	average: 0.2509
+
+Results saved to /home/storage/runs/vivit_ensemble/012.

004/args.yaml

@@ -0,0 +1,87 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_eps: 1e-08
+batch_size: 6
+cache_data: false
+center_crop: 1.0
+clear_output_dir: true
+core: vivit
+core_behavior_mode: 2
+core_compile: false
+core_drop_path: 0.0505
+core_emb_dim: 112
+core_ff_activation: gelu
+core_ff_dropout: 0.0592
+core_flash_attention: 1
+core_head_dim: 48
+core_lr: 0.0048
+core_mha_dropout: 0.358
+core_mlp_dim: 136
+core_norm_qk: false
+core_num_heads: 11
+core_p_dropout: 0.1338
+core_parallel_attention: true
+core_patch_mode: 0
+core_pos_encoding: 3
+core_spatial_depth: 3
+core_spatial_patch_size: 7
+core_spatial_patch_stride: 2
+core_temporal_depth: 5
+core_temporal_patch_size: 25
+core_temporal_patch_stride: 1
+core_weight_decay: 0.1789
+criterion: poisson_correlation
+critic_mode: 0
+crop_frame: 140
+data: /home/sensorium2023
+deterministic: false
+device: cuda
+dpi: 120
+ds_mode: 3
+ds_scale: 1
+epochs: 400
+format: svg
+git_hash: 1033c29
+grad_checkpointing: 0
+grad_norm: null
+input_shapes:
+  behavior: [2, 140]
+  pupil_center: [2, 140]
+  video: [1, 140, 36, 64]
+limit_data: null
+lr: 0.0036
+max_frame: 300
+micro_batch_size: 6
+modulator_mode: 0
+mouse_ids: [A, B, C, D, E, F, G, H, I, J]
+num_workers: 2
+output_dir: /home/storage/runs/004
+output_mode: 1
+output_shapes:
+  A: [7440, 140]
+  B: [7928, 140]
+  C: [8285, 140]
+  D: [7671, 140]
+  E: [7495, 140]
+  F: [7863, 140]
+  G: [7908, 140]
+  H: [8202, 140]
+  I: [7939, 140]
+  J: [8122, 140]
+precision: "32"
+pretrain_core: null
+readout: gaussian2d
+readout_bias_mode: 2
+readout_dropout: 0.0
+readout_grid_mode: 1
+restore: null
+save_plots: false
+seed: 15
+shifter_layers: 3
+shifter_mode: 1
+shifter_size: 5
+stat_mode: 1
+trainable_params: 12278365
+transform_mode: 2
+verbose: 1
+weight_decay: 0.3939

004/ckpt/model_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8538df1cffe25f014e11f5db7e89d7b4e8167dd9545be98a14aa7f78e7bc6b42
+size 148268583

004/evaluation.yaml

@@ -0,0 +1,3 @@
+validation: {A: 0.24283820390701294, B: 0.2745824456214905, C: 0.2716088593006134,
+  D: 0.2359682023525238, E: 0.2350907325744629, F: 0.2335391342639923, G: 0.25313541293144226,
+  H: 0.23314021527767181, I: 0.25676068663597107, J: 0.2575773298740387, average: 0.24942412227392197}

004/model.txt

@@ -0,0 +1,143 @@
+=======================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #
+=======================================================================================================================================
+Model                                                        --                        [1, 7440, 116]            --
+├─ViViTCore: 1-1                                             --                        [1, 112, 116, 15, 29]     --
+│    └─ViViT: 2-1                                            [1, 5, 140, 36, 64]       [1, 112, 116, 15, 29]     --
+│    │    └─Tokenizer: 3-1                                   [1, 5, 140, 36, 64]       [1, 116, 435, 112]        79,632
+│    │    │    └─Sequential: 4-1                             [1, 5, 140, 36, 64]       [1, 116, 435, 112]        --
+│    │    │    │    └─Unfold3d: 5-1                          [1, 5, 140, 36, 64]       [1, 116, 435, 6125]       --
+│    │    │    │    └─LayerNorm: 5-2                         [1, 116, 435, 6125]       [1, 116, 435, 6125]       12,250
+│    │    │    │    └─Linear: 5-3                            [1, 116, 435, 6125]       [1, 116, 435, 112]        686,112
+│    │    │    │    └─LayerNorm: 5-4                         [1, 116, 435, 112]        [1, 116, 435, 112]        224
+│    │    │    │    └─Dropout: 5-5                           [1, 116, 435, 112]        [1, 116, 435, 112]        --
+│    │    └─Transformer: 3-2                                 [116, 435, 112]           [116, 435, 112]           --
+│    │    │    └─ModuleList: 4-2                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-6          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-1                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-2                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-3                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-4                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-5                   [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-1                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-2                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-3                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-6                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-7          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-7                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-8                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-9                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-10                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-11                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-4                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-5                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-6                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-12                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-8          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-13                   [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-14                      [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-15                      [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-16                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-17                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-7                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-8                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-9                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-18                    [116, 435, 112]           [116, 435, 112]           --
+│    │    └─Transformer: 3-3                                 [435, 116, 112]           [435, 116, 112]           --
+│    │    │    └─ModuleList: 4-3                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-9          [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-19                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-20                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-21                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-22                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-23                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-10                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-11                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-12                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-24                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-10         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-25                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-26                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-27                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-28                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-29                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-13                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-14                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-15                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-30                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-11         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-31                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-32                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-33                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-34                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-35                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-16                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-17                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-18                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-36                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-12         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-37                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-38                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-39                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-40                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-41                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-19                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-20                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-21                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-42                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-13         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-43                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-44                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-45                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-46                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-47                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-22                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-23                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-24                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-48                    [435, 116, 112]           [435, 116, 112]           --
+│    │    └─Rearrange: 3-4                                   [1, 116, 435, 112]        [1, 112, 116, 15, 29]     --
+├─MLPShifters: 1-2                                           --                        [1, 2, 116]               513
+│    └─MLPShifter: 2-2                                       [116, 2]                  [116, 2]                  --
+│    │    └─Sequential: 3-5                                  [116, 2]                  [116, 2]                  --
+│    │    │    └─Linear: 4-4                                 [116, 2]                  [116, 5]                  15
+│    │    │    └─Tanh: 4-5                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-6                                 [116, 5]                  [116, 5]                  30
+│    │    │    └─Tanh: 4-7                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-8                                 [116, 5]                  [116, 2]                  12
+│    │    │    └─Tanh: 4-9                                   [116, 2]                  [116, 2]                  --
+├─Readouts: 1-3                                              [1, 112, 116, 15, 29]     [1, 7440, 116]            8,477,649
+│    └─Gaussian2DReadout2: 2-3                               [1, 112, 116, 15, 29]     [1, 7440, 116]            870,480
+│    │    └─Sequential: 3-6                                  [7440, 2]                 [7440, 2]                 --
+│    │    │    └─Linear: 4-10                                [7440, 2]                 [7440, 30]                90
+│    │    │    └─ELU: 4-11                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-12                                [7440, 30]                [7440, 2]                 62
+│    │    │    └─Tanh: 4-13                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-7                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-14                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-15                                   [7440, 30]                [7440, 30]                --
+��    │    │    └─Linear: 4-16                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-17                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-8                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-18                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-19                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-20                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-21                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-9                                  [116, 4]                  [116, 112]                --
+│    │    │    └─Linear: 4-22                                [116, 4]                  [116, 112]                560
+│    │    │    └─LayerNorm: 4-23                             [116, 112]                [116, 112]                224
+│    │    │    └─GELU: 4-24                                  [116, 112]                [116, 112]                --
+│    │    │    └─Dropout: 4-25                               [116, 112]                [116, 112]                --
+│    │    │    └─Linear: 4-26                                [116, 112]                [116, 112]                12,656
+│    │    │    └─Tanh: 4-27                                  [116, 112]                [116, 112]                --
+├─Exponential: 1-4                                           [1, 7440, 116]            [1, 7440, 116]            --
+=======================================================================================================================================
+Total params: 12,278,365
+Trainable params: 12,278,365
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 679.88
+=======================================================================================================================================
+Input size (MB): 1.29
+Forward/backward pass size (MB): 9208.73
+Params size (MB): 11.40
+Estimated Total Size (MB): 9221.42
+=======================================================================================================================================

004/output.log

@@ -0,0 +1,770 @@
+Use bfloat16 for core module.
+Use parallel attention and MLP in ViViT.
+
+Epoch 001/400
+Train		loss: 108875712.00	correlation: 0.0142
+Validation	loss: 200009632.00	correlation: 0.0280
+Elapse: 536.31s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 002/400
+Train		loss: 97286888.00	correlation: 0.0397
+Validation	loss: 199179056.00	correlation: 0.0389
+Elapse: 540.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 003/400
+Train		loss: 96065840.00	correlation: 0.0525
+Validation	loss: 198705632.00	correlation: 0.0414
+Elapse: 548.06s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 004/400
+Train		loss: 95196720.00	correlation: 0.0616
+Validation	loss: 197267888.00	correlation: 0.0520
+Elapse: 550.50s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 005/400
+Train		loss: 94175976.00	correlation: 0.0728
+Validation	loss: 195531712.00	correlation: 0.0637
+Elapse: 549.52s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 006/400
+Train		loss: 92696320.00	correlation: 0.0883
+Validation	loss: 193088208.00	correlation: 0.0800
+Elapse: 547.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 007/400
+Train		loss: 91328912.00	correlation: 0.1027
+Validation	loss: 190954368.00	correlation: 0.0931
+Elapse: 546.41s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 008/400
+Train		loss: 90154608.00	correlation: 0.1149
+Validation	loss: 189186224.00	correlation: 0.1027
+Elapse: 544.21s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 009/400
+Train		loss: 88984904.00	correlation: 0.1267
+Validation	loss: 187096768.00	correlation: 0.1152
+Elapse: 542.67s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 010/400
+Train		loss: 87719928.00	correlation: 0.1389
+Validation	loss: 185224656.00	correlation: 0.1287
+Elapse: 541.24s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 011/400
+Train		loss: 86455896.00	correlation: 0.1522
+Validation	loss: 183495936.00	correlation: 0.1386
+Elapse: 540.40s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 012/400
+Train		loss: 85286208.00	correlation: 0.1638
+Validation	loss: 181750576.00	correlation: 0.1502
+Elapse: 539.78s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 013/400
+Train		loss: 84359632.00	correlation: 0.1728
+Validation	loss: 180806192.00	correlation: 0.1570
+Elapse: 540.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 014/400
+Train		loss: 83654496.00	correlation: 0.1800
+Validation	loss: 179662112.00	correlation: 0.1648
+Elapse: 539.99s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 015/400
+Train		loss: 82936160.00	correlation: 0.1870
+Validation	loss: 178832416.00	correlation: 0.1697
+Elapse: 540.96s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 016/400
+Train		loss: 82256432.00	correlation: 0.1933
+Validation	loss: 177940560.00	correlation: 0.1755
+Elapse: 541.08s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 017/400
+Train		loss: 81816704.00	correlation: 0.1974
+Validation	loss: 177331792.00	correlation: 0.1789
+Elapse: 541.24s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 018/400
+Train		loss: 81317624.00	correlation: 0.2024
+Validation	loss: 176620096.00	correlation: 0.1839
+Elapse: 541.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 019/400
+Train		loss: 80761904.00	correlation: 0.2073
+Validation	loss: 175956080.00	correlation: 0.1882
+Elapse: 540.85s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 020/400
+Train		loss: 80425104.00	correlation: 0.2111
+Validation	loss: 175399376.00	correlation: 0.1923
+Elapse: 541.23s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 021/400
+Train		loss: 80006704.00	correlation: 0.2149
+Validation	loss: 174810944.00	correlation: 0.1963
+Elapse: 540.64s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 022/400
+Train		loss: 79605576.00	correlation: 0.2188
+Validation	loss: 174441312.00	correlation: 0.1990
+Elapse: 541.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 023/400
+Train		loss: 79238384.00	correlation: 0.2225
+Validation	loss: 173628144.00	correlation: 0.2041
+Elapse: 541.23s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 024/400
+Train		loss: 78942312.00	correlation: 0.2253
+Validation	loss: 173239408.00	correlation: 0.2069
+Elapse: 541.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 025/400
+Train		loss: 78635512.00	correlation: 0.2284
+Validation	loss: 172979872.00	correlation: 0.2084
+Elapse: 541.43s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 026/400
+Train		loss: 78411792.00	correlation: 0.2303
+Validation	loss: 172513792.00	correlation: 0.2110
+Elapse: 541.42s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 027/400
+Train		loss: 78185872.00	correlation: 0.2329
+Validation	loss: 172230208.00	correlation: 0.2128
+Elapse: 540.87s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 028/400
+Train		loss: 77904656.00	correlation: 0.2353
+Validation	loss: 172162992.00	correlation: 0.2139
+Elapse: 541.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 029/400
+Train		loss: 77741728.00	correlation: 0.2368
+Validation	loss: 171699152.00	correlation: 0.2166
+Elapse: 542.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 030/400
+Train		loss: 77624216.00	correlation: 0.2381
+Validation	loss: 171523792.00	correlation: 0.2179
+Elapse: 541.70s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 031/400
+Train		loss: 77405640.00	correlation: 0.2398
+Validation	loss: 171512912.00	correlation: 0.2182
+Elapse: 541.80s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 032/400
+Train		loss: 77315544.00	correlation: 0.2411
+Validation	loss: 171059296.00	correlation: 0.2209
+Elapse: 541.92s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 033/400
+Train		loss: 77214704.00	correlation: 0.2420
+Validation	loss: 171228784.00	correlation: 0.2201
+Elapse: 541.77s
+
+Epoch 034/400
+Train		loss: 77108704.00	correlation: 0.2433
+Validation	loss: 170971952.00	correlation: 0.2219
+Elapse: 541.81s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 035/400
+Train		loss: 76970384.00	correlation: 0.2445
+Validation	loss: 170653312.00	correlation: 0.2236
+Elapse: 542.50s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 036/400
+Train		loss: 76829184.00	correlation: 0.2459
+Validation	loss: 170723072.00	correlation: 0.2228
+Elapse: 542.11s
+
+Epoch 037/400
+Train		loss: 76814728.00	correlation: 0.2461
+Validation	loss: 170317536.00	correlation: 0.2260
+Elapse: 542.26s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 038/400
+Train		loss: 76648232.00	correlation: 0.2476
+Validation	loss: 170352096.00	correlation: 0.2263
+Elapse: 542.30s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 039/400
+Train		loss: 76570480.00	correlation: 0.2485
+Validation	loss: 170048544.00	correlation: 0.2276
+Elapse: 542.30s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 040/400
+Train		loss: 76481936.00	correlation: 0.2492
+Validation	loss: 170178064.00	correlation: 0.2271
+Elapse: 542.72s
+
+Epoch 041/400
+Train		loss: 76392648.00	correlation: 0.2501
+Validation	loss: 169916672.00	correlation: 0.2292
+Elapse: 542.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 042/400
+Train		loss: 76277856.00	correlation: 0.2512
+Validation	loss: 169939984.00	correlation: 0.2288
+Elapse: 542.30s
+
+Epoch 043/400
+Train		loss: 76200912.00	correlation: 0.2518
+Validation	loss: 169966848.00	correlation: 0.2293
+Elapse: 543.01s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 044/400
+Train		loss: 76180752.00	correlation: 0.2520
+Validation	loss: 169804672.00	correlation: 0.2301
+Elapse: 541.51s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 045/400
+Train		loss: 76124944.00	correlation: 0.2529
+Validation	loss: 169699056.00	correlation: 0.2297
+Elapse: 541.18s
+
+Epoch 046/400
+Train		loss: 75982544.00	correlation: 0.2542
+Validation	loss: 169554976.00	correlation: 0.2308
+Elapse: 541.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 047/400
+Train		loss: 75966320.00	correlation: 0.2542
+Validation	loss: 169422160.00	correlation: 0.2317
+Elapse: 540.36s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 048/400
+Train		loss: 75923248.00	correlation: 0.2547
+Validation	loss: 169780352.00	correlation: 0.2287
+Elapse: 540.75s
+
+Epoch 049/400
+Train		loss: 75947968.00	correlation: 0.2547
+Validation	loss: 169837856.00	correlation: 0.2298
+Elapse: 541.57s
+
+Epoch 050/400
+Train		loss: 75809040.00	correlation: 0.2556
+Validation	loss: 169605200.00	correlation: 0.2306
+Elapse: 541.98s
+
+Epoch 051/400
+Train		loss: 75789808.00	correlation: 0.2558
+Validation	loss: 169401440.00	correlation: 0.2307
+Elapse: 542.15s
+
+Epoch 052/400
+Train		loss: 75764560.00	correlation: 0.2561
+Validation	loss: 169899840.00	correlation: 0.2291
+Elapse: 541.98s
+
+Loaded checkpoint from epoch 47 (correlation: 0.2317).
+
+Reduce learning rate of core to 1.4400e-03 (num. reduce: 1).
+Reduce learning rate of readouts to 1.0800e-03 (num. reduce: 1).
+Reduce learning rate of shifters to 1.0800e-03 (num. reduce: 1).
+
+Epoch 053/400
+Train		loss: 73947496.00	correlation: 0.2716
+Validation	loss: 167854176.00	correlation: 0.2423
+Elapse: 542.40s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 054/400
+Train		loss: 73334280.00	correlation: 0.2771
+Validation	loss: 167810464.00	correlation: 0.2430
+Elapse: 543.06s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 055/400
+Train		loss: 73178416.00	correlation: 0.2788
+Validation	loss: 167610272.00	correlation: 0.2431
+Elapse: 543.11s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 056/400
+Train		loss: 73106944.00	correlation: 0.2793
+Validation	loss: 167676960.00	correlation: 0.2432
+Elapse: 543.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 057/400
+Train		loss: 73098200.00	correlation: 0.2798
+Validation	loss: 167731616.00	correlation: 0.2427
+Elapse: 543.05s
+
+Epoch 058/400
+Train		loss: 73030688.00	correlation: 0.2803
+Validation	loss: 167715872.00	correlation: 0.2427
+Elapse: 543.72s
+
+Epoch 059/400
+Train		loss: 72944768.00	correlation: 0.2814
+Validation	loss: 167514192.00	correlation: 0.2440
+Elapse: 543.63s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 060/400
+Train		loss: 72929872.00	correlation: 0.2815
+Validation	loss: 167771024.00	correlation: 0.2423
+Elapse: 543.29s
+
+Epoch 061/400
+Train		loss: 72848184.00	correlation: 0.2822
+Validation	loss: 167556976.00	correlation: 0.2437
+Elapse: 543.81s
+
+Epoch 062/400
+Train		loss: 72894440.00	correlation: 0.2819
+Validation	loss: 167687056.00	correlation: 0.2429
+Elapse: 543.05s
+
+Epoch 063/400
+Train		loss: 72843624.00	correlation: 0.2824
+Validation	loss: 167582688.00	correlation: 0.2431
+Elapse: 544.00s
+
+Epoch 064/400
+Train		loss: 72864712.00	correlation: 0.2818
+Validation	loss: 167485088.00	correlation: 0.2435
+Elapse: 543.20s
+
+Loaded checkpoint from epoch 59 (correlation: 0.2440).
+
+Reduce learning rate of core to 4.3200e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 3.2400e-04 (num. reduce: 1).
+Reduce learning rate of shifters to 3.2400e-04 (num. reduce: 1).
+
+Epoch 065/400
+Train		loss: 72100880.00	correlation: 0.2884
+Validation	loss: 167043168.00	correlation: 0.2470
+Elapse: 543.83s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 066/400
+Train		loss: 71817264.00	correlation: 0.2907
+Validation	loss: 166928480.00	correlation: 0.2480
+Elapse: 543.10s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 067/400
+Train		loss: 71765488.00	correlation: 0.2914
+Validation	loss: 166935472.00	correlation: 0.2473
+Elapse: 543.05s
+
+Epoch 068/400
+Train		loss: 71709384.00	correlation: 0.2919
+Validation	loss: 166836192.00	correlation: 0.2479
+Elapse: 544.05s
+
+Epoch 069/400
+Train		loss: 71609624.00	correlation: 0.2931
+Validation	loss: 166932704.00	correlation: 0.2478
+Elapse: 544.51s
+
+Epoch 070/400
+Train		loss: 71532048.00	correlation: 0.2937
+Validation	loss: 166911168.00	correlation: 0.2475
+Elapse: 543.81s
+
+Epoch 071/400
+Train		loss: 71531912.00	correlation: 0.2938
+Validation	loss: 166931488.00	correlation: 0.2474
+Elapse: 543.44s
+
+Loaded checkpoint from epoch 66 (correlation: 0.2480).
+
+Reduce learning rate of core to 1.2960e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 9.7200e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 9.7200e-05 (num. reduce: 1).
+
+Epoch 072/400
+Train		loss: 71521936.00	correlation: 0.2938
+Validation	loss: 166835152.00	correlation: 0.2481
+Elapse: 544.28s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 073/400
+Train		loss: 71422952.00	correlation: 0.2945
+Validation	loss: 166809568.00	correlation: 0.2486
+Elapse: 542.98s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 074/400
+Train		loss: 71402352.00	correlation: 0.2945
+Validation	loss: 166765920.00	correlation: 0.2487
+Elapse: 544.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 075/400
+Train		loss: 71357648.00	correlation: 0.2949
+Validation	loss: 166766240.00	correlation: 0.2487
+Elapse: 544.05s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 076/400
+Train		loss: 71315664.00	correlation: 0.2955
+Validation	loss: 166761216.00	correlation: 0.2489
+Elapse: 544.69s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 077/400
+Train		loss: 71363472.00	correlation: 0.2948
+Validation	loss: 166736352.00	correlation: 0.2489
+Elapse: 544.27s
+
+Epoch 078/400
+Train		loss: 71227736.00	correlation: 0.2964
+Validation	loss: 166733360.00	correlation: 0.2489
+Elapse: 544.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 079/400
+Train		loss: 71277464.00	correlation: 0.2955
+Validation	loss: 166735664.00	correlation: 0.2488
+Elapse: 543.94s
+
+Epoch 080/400
+Train		loss: 71206488.00	correlation: 0.2965
+Validation	loss: 166742432.00	correlation: 0.2489
+Elapse: 544.10s
+
+Epoch 081/400
+Train		loss: 71163264.00	correlation: 0.2968
+Validation	loss: 166738592.00	correlation: 0.2489
+Elapse: 544.08s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 082/400
+Train		loss: 71186960.00	correlation: 0.2968
+Validation	loss: 166722656.00	correlation: 0.2489
+Elapse: 544.32s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 083/400
+Train		loss: 71185200.00	correlation: 0.2968
+Validation	loss: 166706704.00	correlation: 0.2489
+Elapse: 543.20s
+
+Epoch 084/400
+Train		loss: 71172336.00	correlation: 0.2970
+Validation	loss: 166667952.00	correlation: 0.2490
+Elapse: 544.33s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 085/400
+Train		loss: 71094784.00	correlation: 0.2977
+Validation	loss: 166736224.00	correlation: 0.2490
+Elapse: 544.02s
+
+Epoch 086/400
+Train		loss: 71121680.00	correlation: 0.2972
+Validation	loss: 166702080.00	correlation: 0.2491
+Elapse: 544.15s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 087/400
+Train		loss: 71068680.00	correlation: 0.2980
+Validation	loss: 166759776.00	correlation: 0.2488
+Elapse: 544.35s
+
+Epoch 088/400
+Train		loss: 71037376.00	correlation: 0.2982
+Validation	loss: 166717840.00	correlation: 0.2488
+Elapse: 544.48s
+
+Epoch 089/400
+Train		loss: 71044336.00	correlation: 0.2979
+Validation	loss: 166688608.00	correlation: 0.2491
+Elapse: 543.63s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 090/400
+Train		loss: 71028720.00	correlation: 0.2984
+Validation	loss: 166697808.00	correlation: 0.2488
+Elapse: 544.01s
+
+Epoch 091/400
+Train		loss: 70956424.00	correlation: 0.2991
+Validation	loss: 166683104.00	correlation: 0.2491
+Elapse: 544.36s
+
+Epoch 092/400
+Train		loss: 71034488.00	correlation: 0.2982
+Validation	loss: 166691024.00	correlation: 0.2491
+Elapse: 544.53s
+
+Epoch 093/400
+Train		loss: 71008336.00	correlation: 0.2981
+Validation	loss: 166667472.00	correlation: 0.2493
+Elapse: 544.24s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 094/400
+Train		loss: 70955584.00	correlation: 0.2990
+Validation	loss: 166707392.00	correlation: 0.2487
+Elapse: 543.78s
+
+Epoch 095/400
+Train		loss: 70927976.00	correlation: 0.2993
+Validation	loss: 166647504.00	correlation: 0.2491
+Elapse: 543.94s
+
+Epoch 096/400
+Train		loss: 70877440.00	correlation: 0.2996
+Validation	loss: 166701728.00	correlation: 0.2486
+Elapse: 543.45s
+
+Epoch 097/400
+Train		loss: 70917616.00	correlation: 0.2991
+Validation	loss: 166641920.00	correlation: 0.2492
+Elapse: 544.10s
+
+Epoch 098/400
+Train		loss: 70867216.00	correlation: 0.2999
+Validation	loss: 166664160.00	correlation: 0.2492
+Elapse: 544.30s
+
+Loaded checkpoint from epoch 93 (correlation: 0.2493).
+
+Reduce learning rate of core to 3.8880e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 2.9160e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 2.9160e-05 (num. reduce: 1).
+
+Epoch 099/400
+Train		loss: 70901904.00	correlation: 0.2993
+Validation	loss: 166624560.00	correlation: 0.2493
+Elapse: 544.44s
+
+Epoch 100/400
+Train		loss: 70815976.00	correlation: 0.3000
+Validation	loss: 166631168.00	correlation: 0.2494
+Elapse: 544.02s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 101/400
+Train		loss: 70876152.00	correlation: 0.2993
+Validation	loss: 166642400.00	correlation: 0.2492
+Elapse: 544.50s
+
+Epoch 102/400
+Train		loss: 70783744.00	correlation: 0.3006
+Validation	loss: 166643056.00	correlation: 0.2492
+Elapse: 544.34s
+
+Epoch 103/400
+Train		loss: 70822832.00	correlation: 0.3002
+Validation	loss: 166621408.00	correlation: 0.2494
+Elapse: 544.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 104/400
+Train		loss: 70795760.00	correlation: 0.3005
+Validation	loss: 166626384.00	correlation: 0.2494
+Elapse: 544.65s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/015/ckpt/model_state.pt.
+
+Epoch 105/400
+Train		loss: 70758416.00	correlation: 0.3004
+Validation	loss: 166603600.00	correlation: 0.2494
+Elapse: 544.30s
+
+Epoch 106/400
+Train		loss: 70836856.00	correlation: 0.2999
+Validation	loss: 166631360.00	correlation: 0.2493
+Elapse: 544.28s
+
+Epoch 107/400
+Train		loss: 70783560.00	correlation: 0.3006
+Validation	loss: 166638304.00	correlation: 0.2493
+Elapse: 544.44s
+
+Epoch 108/400
+Train		loss: 70793304.00	correlation: 0.3004
+Validation	loss: 166632480.00	correlation: 0.2492
+Elapse: 544.19s
+
+Epoch 109/400
+Train		loss: 70730064.00	correlation: 0.3012
+Validation	loss: 166617632.00	correlation: 0.2493
+Elapse: 543.89s
+
+Loaded checkpoint from epoch 104 (correlation: 0.2494).
+
+Reduce learning rate of core to 1.1664e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 8.7480e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 8.7480e-06 (num. reduce: 1).
+
+Epoch 110/400
+Train		loss: 70741584.00	correlation: 0.3007
+Validation	loss: 166614400.00	correlation: 0.2493
+Elapse: 544.26s
+
+Epoch 111/400
+Train		loss: 70763320.00	correlation: 0.3005
+Validation	loss: 166633072.00	correlation: 0.2493
+Elapse: 544.57s
+
+Epoch 112/400
+Train		loss: 70721984.00	correlation: 0.3011
+Validation	loss: 166633552.00	correlation: 0.2493
+Elapse: 544.64s
+
+Epoch 113/400
+Train		loss: 70737856.00	correlation: 0.3009
+Validation	loss: 166608208.00	correlation: 0.2494
+Elapse: 544.39s
+
+Epoch 114/400
+Train		loss: 70798224.00	correlation: 0.3000
+Validation	loss: 166619792.00	correlation: 0.2494
+Elapse: 543.95s
+
+Loaded checkpoint from epoch 104 (correlation: 0.2494).
+
+Reduce learning rate of core to 3.4992e-06 (num. reduce: 2).
+Reduce learning rate of readouts to 2.6244e-06 (num. reduce: 2).
+Reduce learning rate of shifters to 2.6244e-06 (num. reduce: 2).
+
+Epoch 115/400
+Train		loss: 70736776.00	correlation: 0.3008
+Validation	loss: 166615600.00	correlation: 0.2494
+Elapse: 544.31s
+
+Epoch 116/400
+Train		loss: 70760608.00	correlation: 0.3006
+Validation	loss: 166622592.00	correlation: 0.2493
+Elapse: 544.61s
+
+Epoch 117/400
+Train		loss: 70707912.00	correlation: 0.3011
+Validation	loss: 166622416.00	correlation: 0.2494
+Elapse: 544.73s
+
+Epoch 118/400
+Train		loss: 70797264.00	correlation: 0.3002
+Validation	loss: 166620240.00	correlation: 0.2493
+Elapse: 545.29s
+
+Epoch 119/400
+Train		loss: 70782240.00	correlation: 0.3003
+Validation	loss: 166620000.00	correlation: 0.2493
+Elapse: 544.33s
+
+Model has not improved after 2 LR reductions.
+
+Loaded checkpoint from epoch 104 (correlation: 0.2494).
+
+
+ValidationA: 0.2428	B: 0.2746	C: 0.2716	D: 0.2360	E: 0.2351	F: 0.2335	G: 0.2531	H: 0.2331	I: 0.2568	J: 0.2576	average: 0.2494
+
+Results saved to /home/storage/runs/vivit_ensemble/015.

005/args.yaml

@@ -0,0 +1,87 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_eps: 1e-08
+batch_size: 6
+cache_data: false
+center_crop: 1.0
+clear_output_dir: true
+core: vivit
+core_behavior_mode: 2
+core_compile: false
+core_drop_path: 0.0505
+core_emb_dim: 112
+core_ff_activation: gelu
+core_ff_dropout: 0.0592
+core_flash_attention: 1
+core_head_dim: 48
+core_lr: 0.0048
+core_mha_dropout: 0.358
+core_mlp_dim: 136
+core_norm_qk: false
+core_num_heads: 11
+core_p_dropout: 0.1338
+core_parallel_attention: true
+core_patch_mode: 0
+core_pos_encoding: 3
+core_spatial_depth: 3
+core_spatial_patch_size: 7
+core_spatial_patch_stride: 2
+core_temporal_depth: 5
+core_temporal_patch_size: 25
+core_temporal_patch_stride: 1
+core_weight_decay: 0.1789
+criterion: poisson_correlation
+critic_mode: 0
+crop_frame: 140
+data: /home/sensorium2023
+deterministic: false
+device: cuda
+dpi: 120
+ds_mode: 3
+ds_scale: 1
+epochs: 400
+format: svg
+git_hash: 3cd580b
+grad_checkpointing: 0
+grad_norm: null
+input_shapes:
+  behavior: [2, 140]
+  pupil_center: [2, 140]
+  video: [1, 140, 36, 64]
+limit_data: null
+lr: 0.0036
+max_frame: 300
+micro_batch_size: 6
+modulator_mode: 0
+mouse_ids: [A, B, C, D, E, F, G, H, I, J]
+num_workers: 2
+output_dir: /home/storage/runs/005
+output_mode: 1
+output_shapes:
+  A: [7440, 140]
+  B: [7928, 140]
+  C: [8285, 140]
+  D: [7671, 140]
+  E: [7495, 140]
+  F: [7863, 140]
+  G: [7908, 140]
+  H: [8202, 140]
+  I: [7939, 140]
+  J: [8122, 140]
+precision: "32"
+pretrain_core: null
+readout: gaussian2d
+readout_bias_mode: 2
+readout_dropout: 0.0
+readout_grid_mode: 1
+restore: null
+save_plots: false
+seed: 28
+shifter_layers: 3
+shifter_mode: 1
+shifter_size: 5
+stat_mode: 1
+trainable_params: 12278365
+transform_mode: 2
+verbose: 1
+weight_decay: 0.3939

005/ckpt/model_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f8586d53aa03f24b6b533d7a172c6c33aefaf02e346d810dbb171fcf6202e2b
+size 148268583

005/evaluation.yaml

@@ -0,0 +1,3 @@
+validation: {A: 0.2465260773897171, B: 0.27831149101257324, C: 0.2688469886779785,
+  D: 0.22918999195098877, E: 0.23662497103214264, F: 0.2340845912694931, G: 0.254902184009552,
+  H: 0.23301741480827332, I: 0.25673478841781616, J: 0.2570823132991791, average: 0.2495320811867714}

005/model.txt

@@ -0,0 +1,143 @@
+=======================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #
+=======================================================================================================================================
+Model                                                        --                        [1, 7440, 116]            --
+├─ViViTCore: 1-1                                             --                        [1, 112, 116, 15, 29]     --
+│    └─ViViT: 2-1                                            [1, 5, 140, 36, 64]       [1, 112, 116, 15, 29]     --
+│    │    └─Tokenizer: 3-1                                   [1, 5, 140, 36, 64]       [1, 116, 435, 112]        79,632
+│    │    │    └─Sequential: 4-1                             [1, 5, 140, 36, 64]       [1, 116, 435, 112]        --
+│    │    │    │    └─Unfold3d: 5-1                          [1, 5, 140, 36, 64]       [1, 116, 435, 6125]       --
+│    │    │    │    └─LayerNorm: 5-2                         [1, 116, 435, 6125]       [1, 116, 435, 6125]       12,250
+│    │    │    │    └─Linear: 5-3                            [1, 116, 435, 6125]       [1, 116, 435, 112]        686,112
+│    │    │    │    └─LayerNorm: 5-4                         [1, 116, 435, 112]        [1, 116, 435, 112]        224
+│    │    │    │    └─Dropout: 5-5                           [1, 116, 435, 112]        [1, 116, 435, 112]        --
+│    │    └─Transformer: 3-2                                 [116, 435, 112]           [116, 435, 112]           --
+│    │    │    └─ModuleList: 4-2                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-6          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-1                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-2                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-3                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-4                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-5                   [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-1                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-2                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-3                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-6                     [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-7          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-7                    [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-8                       [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-9                       [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-10                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-11                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-4                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-5                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-6                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-12                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-8          [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-13                   [116, 435, 112]           [116, 435, 112]           224
+│    │    │    │    │    └─Linear: 6-14                      [116, 435, 112]           [116, 435, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-15                      [116, 435, 528]           [116, 435, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-16                    [116, 435, 112]           [116, 435, 112]           --
+│    │    │    │    │    └─Sequential: 6-17                  [116, 435, 136]           [116, 435, 112]           --
+│    │    │    │    │    │    └─GELU: 7-7                    [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-8                 [116, 435, 136]           [116, 435, 136]           --
+│    │    │    │    │    │    └─Linear: 7-9                  [116, 435, 136]           [116, 435, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-18                    [116, 435, 112]           [116, 435, 112]           --
+│    │    └─Transformer: 3-3                                 [435, 116, 112]           [435, 116, 112]           --
+│    │    │    └─ModuleList: 4-3                             --                        --                        --
+│    │    │    │    └─ParallelTransformerBlock: 5-9          [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-19                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-20                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-21                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-22                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-23                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-10                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-11                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-12                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-24                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-10         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-25                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-26                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-27                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-28                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-29                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-13                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-14                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-15                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-30                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-11         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-31                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-32                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-33                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-34                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-35                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-16                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-17                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-18                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-36                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-12         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-37                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-38                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-39                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-40                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-41                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-19                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-20                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-21                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-42                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    └─ParallelTransformerBlock: 5-13         [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─LayerNorm: 6-43                   [435, 116, 112]           [435, 116, 112]           224
+│    │    │    │    │    └─Linear: 6-44                      [435, 116, 112]           [435, 116, 1720]          192,640
+│    │    │    │    │    └─Linear: 6-45                      [435, 116, 528]           [435, 116, 112]           59,136
+│    │    │    │    │    └─DropPath: 6-46                    [435, 116, 112]           [435, 116, 112]           --
+│    │    │    │    │    └─Sequential: 6-47                  [435, 116, 136]           [435, 116, 112]           --
+│    │    │    │    │    │    └─GELU: 7-22                   [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Dropout: 7-23                [435, 116, 136]           [435, 116, 136]           --
+│    │    │    │    │    │    └─Linear: 7-24                 [435, 116, 136]           [435, 116, 112]           15,232
+│    │    │    │    │    └─DropPath: 6-48                    [435, 116, 112]           [435, 116, 112]           --
+│    │    └─Rearrange: 3-4                                   [1, 116, 435, 112]        [1, 112, 116, 15, 29]     --
+├─MLPShifters: 1-2                                           --                        [1, 2, 116]               513
+│    └─MLPShifter: 2-2                                       [116, 2]                  [116, 2]                  --
+│    │    └─Sequential: 3-5                                  [116, 2]                  [116, 2]                  --
+│    │    │    └─Linear: 4-4                                 [116, 2]                  [116, 5]                  15
+│    │    │    └─Tanh: 4-5                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-6                                 [116, 5]                  [116, 5]                  30
+│    │    │    └─Tanh: 4-7                                   [116, 5]                  [116, 5]                  --
+│    │    │    └─Linear: 4-8                                 [116, 5]                  [116, 2]                  12
+│    │    │    └─Tanh: 4-9                                   [116, 2]                  [116, 2]                  --
+├─Readouts: 1-3                                              [1, 112, 116, 15, 29]     [1, 7440, 116]            8,477,649
+│    └─Gaussian2DReadout2: 2-3                               [1, 112, 116, 15, 29]     [1, 7440, 116]            870,480
+│    │    └─Sequential: 3-6                                  [7440, 2]                 [7440, 2]                 --
+│    │    │    └─Linear: 4-10                                [7440, 2]                 [7440, 30]                90
+│    │    │    └─ELU: 4-11                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-12                                [7440, 30]                [7440, 2]                 62
+│    │    │    └─Tanh: 4-13                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-7                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-14                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-15                                   [7440, 30]                [7440, 30]                --
+��    │    │    └─Linear: 4-16                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-17                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-8                                  [7440, 2]                 [7440, 2]                 (recursive)
+│    │    │    └─Linear: 4-18                                [7440, 2]                 [7440, 30]                (recursive)
+│    │    │    └─ELU: 4-19                                   [7440, 30]                [7440, 30]                --
+│    │    │    └─Linear: 4-20                                [7440, 30]                [7440, 2]                 (recursive)
+│    │    │    └─Tanh: 4-21                                  [7440, 2]                 [7440, 2]                 --
+│    │    └─Sequential: 3-9                                  [116, 4]                  [116, 112]                --
+│    │    │    └─Linear: 4-22                                [116, 4]                  [116, 112]                560
+│    │    │    └─LayerNorm: 4-23                             [116, 112]                [116, 112]                224
+│    │    │    └─GELU: 4-24                                  [116, 112]                [116, 112]                --
+│    │    │    └─Dropout: 4-25                               [116, 112]                [116, 112]                --
+│    │    │    └─Linear: 4-26                                [116, 112]                [116, 112]                12,656
+│    │    │    └─Tanh: 4-27                                  [116, 112]                [116, 112]                --
+├─Exponential: 1-4                                           [1, 7440, 116]            [1, 7440, 116]            --
+=======================================================================================================================================
+Total params: 12,278,365
+Trainable params: 12,278,365
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 679.88
+=======================================================================================================================================
+Input size (MB): 1.29
+Forward/backward pass size (MB): 9208.73
+Params size (MB): 11.40
+Estimated Total Size (MB): 9221.42
+=======================================================================================================================================

005/output.log

@@ -0,0 +1,822 @@
+Use bfloat16 for core module.
+Use parallel attention and MLP in ViViT.
+
+Epoch 001/400
+Train		loss: 111647336.00	correlation: 0.0122
+Validation	loss: 200300080.00	correlation: 0.0267
+Elapse: 537.13s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 002/400
+Train		loss: 97648256.00	correlation: 0.0367
+Validation	loss: 199289632.00	correlation: 0.0355
+Elapse: 539.72s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 003/400
+Train		loss: 96321792.00	correlation: 0.0500
+Validation	loss: 198299696.00	correlation: 0.0446
+Elapse: 548.37s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 004/400
+Train		loss: 95213544.00	correlation: 0.0617
+Validation	loss: 196983504.00	correlation: 0.0527
+Elapse: 552.04s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 005/400
+Train		loss: 93915056.00	correlation: 0.0756
+Validation	loss: 195022704.00	correlation: 0.0660
+Elapse: 551.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 006/400
+Train		loss: 92500376.00	correlation: 0.0909
+Validation	loss: 193045536.00	correlation: 0.0815
+Elapse: 549.45s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 007/400
+Train		loss: 91511136.00	correlation: 0.1012
+Validation	loss: 191545056.00	correlation: 0.0905
+Elapse: 547.11s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 008/400
+Train		loss: 90428944.00	correlation: 0.1113
+Validation	loss: 189976016.00	correlation: 0.0998
+Elapse: 544.76s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 009/400
+Train		loss: 89301664.00	correlation: 0.1228
+Validation	loss: 188514336.00	correlation: 0.1074
+Elapse: 541.79s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 010/400
+Train		loss: 88238616.00	correlation: 0.1335
+Validation	loss: 186730592.00	correlation: 0.1194
+Elapse: 540.44s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 011/400
+Train		loss: 87239336.00	correlation: 0.1436
+Validation	loss: 185287104.00	correlation: 0.1296
+Elapse: 539.71s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 012/400
+Train		loss: 86205256.00	correlation: 0.1540
+Validation	loss: 183237792.00	correlation: 0.1419
+Elapse: 540.02s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 013/400
+Train		loss: 85068472.00	correlation: 0.1655
+Validation	loss: 181721296.00	correlation: 0.1515
+Elapse: 539.81s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 014/400
+Train		loss: 84160560.00	correlation: 0.1745
+Validation	loss: 180469664.00	correlation: 0.1595
+Elapse: 540.34s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 015/400
+Train		loss: 83386128.00	correlation: 0.1821
+Validation	loss: 179517792.00	correlation: 0.1658
+Elapse: 540.81s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 016/400
+Train		loss: 82805200.00	correlation: 0.1877
+Validation	loss: 178801584.00	correlation: 0.1698
+Elapse: 540.70s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 017/400
+Train		loss: 82275736.00	correlation: 0.1926
+Validation	loss: 177842080.00	correlation: 0.1761
+Elapse: 540.76s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 018/400
+Train		loss: 81662840.00	correlation: 0.1987
+Validation	loss: 176971248.00	correlation: 0.1820
+Elapse: 541.07s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 019/400
+Train		loss: 81147088.00	correlation: 0.2038
+Validation	loss: 176423600.00	correlation: 0.1858
+Elapse: 541.20s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 020/400
+Train		loss: 80748784.00	correlation: 0.2079
+Validation	loss: 175690976.00	correlation: 0.1908
+Elapse: 541.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 021/400
+Train		loss: 80313880.00	correlation: 0.2122
+Validation	loss: 175389216.00	correlation: 0.1929
+Elapse: 540.65s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 022/400
+Train		loss: 79957744.00	correlation: 0.2153
+Validation	loss: 174862864.00	correlation: 0.1968
+Elapse: 540.96s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 023/400
+Train		loss: 79621360.00	correlation: 0.2186
+Validation	loss: 174437632.00	correlation: 0.1996
+Elapse: 541.11s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 024/400
+Train		loss: 79374368.00	correlation: 0.2211
+Validation	loss: 173783968.00	correlation: 0.2031
+Elapse: 542.03s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 025/400
+Train		loss: 79085072.00	correlation: 0.2240
+Validation	loss: 173357520.00	correlation: 0.2059
+Elapse: 541.66s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 026/400
+Train		loss: 78723728.00	correlation: 0.2273
+Validation	loss: 173125120.00	correlation: 0.2087
+Elapse: 542.13s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 027/400
+Train		loss: 78456624.00	correlation: 0.2299
+Validation	loss: 172740432.00	correlation: 0.2102
+Elapse: 541.72s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 028/400
+Train		loss: 78264240.00	correlation: 0.2316
+Validation	loss: 172426416.00	correlation: 0.2125
+Elapse: 541.90s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 029/400
+Train		loss: 78095968.00	correlation: 0.2335
+Validation	loss: 172338688.00	correlation: 0.2139
+Elapse: 542.04s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 030/400
+Train		loss: 77880056.00	correlation: 0.2357
+Validation	loss: 171972944.00	correlation: 0.2151
+Elapse: 541.96s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 031/400
+Train		loss: 77782920.00	correlation: 0.2367
+Validation	loss: 171662176.00	correlation: 0.2171
+Elapse: 542.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 032/400
+Train		loss: 77596272.00	correlation: 0.2382
+Validation	loss: 171484624.00	correlation: 0.2187
+Elapse: 543.51s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 033/400
+Train		loss: 77378984.00	correlation: 0.2403
+Validation	loss: 171180896.00	correlation: 0.2208
+Elapse: 542.09s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 034/400
+Train		loss: 77329272.00	correlation: 0.2411
+Validation	loss: 171382000.00	correlation: 0.2191
+Elapse: 541.49s
+
+Epoch 035/400
+Train		loss: 77173400.00	correlation: 0.2426
+Validation	loss: 170990288.00	correlation: 0.2215
+Elapse: 542.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 036/400
+Train		loss: 77103352.00	correlation: 0.2430
+Validation	loss: 171064224.00	correlation: 0.2212
+Elapse: 542.85s
+
+Epoch 037/400
+Train		loss: 76961264.00	correlation: 0.2447
+Validation	loss: 170693952.00	correlation: 0.2239
+Elapse: 542.80s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 038/400
+Train		loss: 76838424.00	correlation: 0.2456
+Validation	loss: 170481408.00	correlation: 0.2246
+Elapse: 542.40s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 039/400
+Train		loss: 76726736.00	correlation: 0.2469
+Validation	loss: 170405888.00	correlation: 0.2257
+Elapse: 541.86s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 040/400
+Train		loss: 76625736.00	correlation: 0.2481
+Validation	loss: 170183216.00	correlation: 0.2271
+Elapse: 542.67s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 041/400
+Train		loss: 76633712.00	correlation: 0.2477
+Validation	loss: 170194400.00	correlation: 0.2270
+Elapse: 542.30s
+
+Epoch 042/400
+Train		loss: 76442136.00	correlation: 0.2493
+Validation	loss: 170452384.00	correlation: 0.2263
+Elapse: 542.26s
+
+Epoch 043/400
+Train		loss: 76492200.00	correlation: 0.2491
+Validation	loss: 170249952.00	correlation: 0.2260
+Elapse: 542.10s
+
+Epoch 044/400
+Train		loss: 76279480.00	correlation: 0.2510
+Validation	loss: 169784496.00	correlation: 0.2295
+Elapse: 541.77s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 045/400
+Train		loss: 76193080.00	correlation: 0.2520
+Validation	loss: 169944928.00	correlation: 0.2278
+Elapse: 542.23s
+
+Epoch 046/400
+Train		loss: 76107936.00	correlation: 0.2526
+Validation	loss: 169860016.00	correlation: 0.2283
+Elapse: 542.90s
+
+Epoch 047/400
+Train		loss: 76111024.00	correlation: 0.2527
+Validation	loss: 169548304.00	correlation: 0.2308
+Elapse: 541.89s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 048/400
+Train		loss: 76049672.00	correlation: 0.2534
+Validation	loss: 169801264.00	correlation: 0.2296
+Elapse: 543.95s
+
+Epoch 049/400
+Train		loss: 75997232.00	correlation: 0.2537
+Validation	loss: 169721248.00	correlation: 0.2301
+Elapse: 543.00s
+
+Epoch 050/400
+Train		loss: 75961488.00	correlation: 0.2545
+Validation	loss: 169722912.00	correlation: 0.2293
+Elapse: 542.91s
+
+Epoch 051/400
+Train		loss: 75908376.00	correlation: 0.2548
+Validation	loss: 169474704.00	correlation: 0.2310
+Elapse: 542.56s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 052/400
+Train		loss: 75771856.00	correlation: 0.2561
+Validation	loss: 170132720.00	correlation: 0.2272
+Elapse: 542.05s
+
+Epoch 053/400
+Train		loss: 75913696.00	correlation: 0.2547
+Validation	loss: 169491888.00	correlation: 0.2309
+Elapse: 542.57s
+
+Epoch 054/400
+Train		loss: 75764272.00	correlation: 0.2560
+Validation	loss: 169266704.00	correlation: 0.2330
+Elapse: 542.91s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 055/400
+Train		loss: 75670816.00	correlation: 0.2569
+Validation	loss: 169200320.00	correlation: 0.2327
+Elapse: 542.21s
+
+Epoch 056/400
+Train		loss: 75680240.00	correlation: 0.2570
+Validation	loss: 169253344.00	correlation: 0.2322
+Elapse: 542.73s
+
+Epoch 057/400
+Train		loss: 75623792.00	correlation: 0.2573
+Validation	loss: 169158704.00	correlation: 0.2336
+Elapse: 544.29s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 058/400
+Train		loss: 75518968.00	correlation: 0.2584
+Validation	loss: 169124832.00	correlation: 0.2339
+Elapse: 542.64s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 059/400
+Train		loss: 75452928.00	correlation: 0.2591
+Validation	loss: 169483008.00	correlation: 0.2320
+Elapse: 542.34s
+
+Epoch 060/400
+Train		loss: 75478992.00	correlation: 0.2590
+Validation	loss: 169223728.00	correlation: 0.2337
+Elapse: 543.01s
+
+Epoch 061/400
+Train		loss: 75414704.00	correlation: 0.2597
+Validation	loss: 169588672.00	correlation: 0.2307
+Elapse: 542.79s
+
+Epoch 062/400
+Train		loss: 75631880.00	correlation: 0.2578
+Validation	loss: 169325440.00	correlation: 0.2326
+Elapse: 543.75s
+
+Epoch 063/400
+Train		loss: 75359640.00	correlation: 0.2602
+Validation	loss: 169091040.00	correlation: 0.2339
+Elapse: 543.16s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 064/400
+Train		loss: 75319080.00	correlation: 0.2605
+Validation	loss: 169416560.00	correlation: 0.2321
+Elapse: 543.52s
+
+Epoch 065/400
+Train		loss: 75306104.00	correlation: 0.2609
+Validation	loss: 169176224.00	correlation: 0.2331
+Elapse: 542.80s
+
+Epoch 066/400
+Train		loss: 75301592.00	correlation: 0.2609
+Validation	loss: 169371616.00	correlation: 0.2327
+Elapse: 543.10s
+
+Epoch 067/400
+Train		loss: 75389672.00	correlation: 0.2601
+Validation	loss: 169169312.00	correlation: 0.2332
+Elapse: 544.11s
+
+Epoch 068/400
+Train		loss: 75361008.00	correlation: 0.2604
+Validation	loss: 169366960.00	correlation: 0.2315
+Elapse: 543.42s
+
+Loaded checkpoint from epoch 63 (correlation: 0.2339).
+
+Reduce learning rate of core to 1.4400e-03 (num. reduce: 1).
+Reduce learning rate of readouts to 1.0800e-03 (num. reduce: 1).
+Reduce learning rate of shifters to 1.0800e-03 (num. reduce: 1).
+
+Epoch 069/400
+Train		loss: 73278464.00	correlation: 0.2784
+Validation	loss: 167482848.00	correlation: 0.2447
+Elapse: 544.59s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 070/400
+Train		loss: 72562424.00	correlation: 0.2849
+Validation	loss: 167277872.00	correlation: 0.2456
+Elapse: 544.90s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 071/400
+Train		loss: 72429480.00	correlation: 0.2862
+Validation	loss: 167337568.00	correlation: 0.2452
+Elapse: 544.84s
+
+Epoch 072/400
+Train		loss: 72339856.00	correlation: 0.2872
+Validation	loss: 167472656.00	correlation: 0.2442
+Elapse: 544.89s
+
+Epoch 073/400
+Train		loss: 72270912.00	correlation: 0.2877
+Validation	loss: 167356592.00	correlation: 0.2443
+Elapse: 545.34s
+
+Epoch 074/400
+Train		loss: 72207960.00	correlation: 0.2884
+Validation	loss: 167467040.00	correlation: 0.2434
+Elapse: 545.38s
+
+Epoch 075/400
+Train		loss: 72186752.00	correlation: 0.2885
+Validation	loss: 167473600.00	correlation: 0.2434
+Elapse: 545.13s
+
+Loaded checkpoint from epoch 70 (correlation: 0.2456).
+
+Reduce learning rate of core to 4.3200e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 3.2400e-04 (num. reduce: 1).
+Reduce learning rate of shifters to 3.2400e-04 (num. reduce: 1).
+
+Epoch 076/400
+Train		loss: 71743648.00	correlation: 0.2919
+Validation	loss: 166945040.00	correlation: 0.2479
+Elapse: 544.86s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 077/400
+Train		loss: 71453328.00	correlation: 0.2949
+Validation	loss: 166933984.00	correlation: 0.2480
+Elapse: 545.20s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 078/400
+Train		loss: 71407184.00	correlation: 0.2951
+Validation	loss: 166839616.00	correlation: 0.2480
+Elapse: 545.38s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 079/400
+Train		loss: 71308560.00	correlation: 0.2960
+Validation	loss: 166820688.00	correlation: 0.2484
+Elapse: 545.26s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 080/400
+Train		loss: 71223520.00	correlation: 0.2968
+Validation	loss: 166801616.00	correlation: 0.2487
+Elapse: 545.81s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 081/400
+Train		loss: 71151600.00	correlation: 0.2975
+Validation	loss: 166777584.00	correlation: 0.2485
+Elapse: 545.15s
+
+Epoch 082/400
+Train		loss: 71166128.00	correlation: 0.2973
+Validation	loss: 166806736.00	correlation: 0.2484
+Elapse: 545.51s
+
+Epoch 083/400
+Train		loss: 71003768.00	correlation: 0.2991
+Validation	loss: 166794000.00	correlation: 0.2487
+Elapse: 545.39s
+
+Epoch 084/400
+Train		loss: 71028680.00	correlation: 0.2987
+Validation	loss: 166740976.00	correlation: 0.2486
+Elapse: 545.50s
+
+Epoch 085/400
+Train		loss: 70987920.00	correlation: 0.2989
+Validation	loss: 166771344.00	correlation: 0.2483
+Elapse: 545.38s
+
+Loaded checkpoint from epoch 80 (correlation: 0.2487).
+
+Reduce learning rate of core to 1.2960e-04 (num. reduce: 1).
+Reduce learning rate of readouts to 9.7200e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 9.7200e-05 (num. reduce: 1).
+
+Epoch 086/400
+Train		loss: 70886192.00	correlation: 0.2998
+Validation	loss: 166696544.00	correlation: 0.2492
+Elapse: 545.68s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 087/400
+Train		loss: 70891072.00	correlation: 0.2998
+Validation	loss: 166700640.00	correlation: 0.2493
+Elapse: 545.36s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 088/400
+Train		loss: 70786208.00	correlation: 0.3009
+Validation	loss: 166720928.00	correlation: 0.2490
+Elapse: 545.75s
+
+Epoch 089/400
+Train		loss: 70840864.00	correlation: 0.3002
+Validation	loss: 166705888.00	correlation: 0.2491
+Elapse: 546.01s
+
+Epoch 090/400
+Train		loss: 70781008.00	correlation: 0.3008
+Validation	loss: 166757248.00	correlation: 0.2489
+Elapse: 545.02s
+
+Epoch 091/400
+Train		loss: 70768960.00	correlation: 0.3009
+Validation	loss: 166698640.00	correlation: 0.2489
+Elapse: 545.07s
+
+Epoch 092/400
+Train		loss: 70700664.00	correlation: 0.3016
+Validation	loss: 166705392.00	correlation: 0.2491
+Elapse: 545.85s
+
+Loaded checkpoint from epoch 87 (correlation: 0.2493).
+
+Reduce learning rate of core to 3.8880e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 2.9160e-05 (num. reduce: 1).
+Reduce learning rate of shifters to 2.9160e-05 (num. reduce: 1).
+
+Epoch 093/400
+Train		loss: 70743624.00	correlation: 0.3012
+Validation	loss: 166685456.00	correlation: 0.2494
+Elapse: 545.69s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 094/400
+Train		loss: 70752552.00	correlation: 0.3012
+Validation	loss: 166676048.00	correlation: 0.2493
+Elapse: 545.53s
+
+Epoch 095/400
+Train		loss: 70729328.00	correlation: 0.3013
+Validation	loss: 166677072.00	correlation: 0.2494
+Elapse: 545.12s
+
+Epoch 096/400
+Train		loss: 70661304.00	correlation: 0.3018
+Validation	loss: 166687824.00	correlation: 0.2493
+Elapse: 545.30s
+
+Epoch 097/400
+Train		loss: 70768896.00	correlation: 0.3006
+Validation	loss: 166658272.00	correlation: 0.2492
+Elapse: 545.47s
+
+Epoch 098/400
+Train		loss: 70673976.00	correlation: 0.3017
+Validation	loss: 166648288.00	correlation: 0.2495
+Elapse: 545.22s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 099/400
+Train		loss: 70635672.00	correlation: 0.3022
+Validation	loss: 166648032.00	correlation: 0.2495
+Elapse: 545.17s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 100/400
+Train		loss: 70748008.00	correlation: 0.3010
+Validation	loss: 166647312.00	correlation: 0.2495
+Elapse: 545.61s
+
+Epoch 101/400
+Train		loss: 70682408.00	correlation: 0.3016
+Validation	loss: 166662208.00	correlation: 0.2494
+Elapse: 545.56s
+
+Epoch 102/400
+Train		loss: 70642688.00	correlation: 0.3020
+Validation	loss: 166662624.00	correlation: 0.2495
+Elapse: 545.17s
+
+Epoch 103/400
+Train		loss: 70638368.00	correlation: 0.3019
+Validation	loss: 166645120.00	correlation: 0.2495
+Elapse: 545.25s
+
+Epoch 104/400
+Train		loss: 70563872.00	correlation: 0.3027
+Validation	loss: 166649696.00	correlation: 0.2495
+Elapse: 545.70s
+
+Loaded checkpoint from epoch 99 (correlation: 0.2495).
+
+Reduce learning rate of core to 1.1664e-05 (num. reduce: 1).
+Reduce learning rate of readouts to 8.7480e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 8.7480e-06 (num. reduce: 1).
+
+Epoch 105/400
+Train		loss: 70679696.00	correlation: 0.3019
+Validation	loss: 166646944.00	correlation: 0.2495
+Elapse: 545.44s
+
+Epoch 106/400
+Train		loss: 70623656.00	correlation: 0.3021
+Validation	loss: 166651056.00	correlation: 0.2495
+Elapse: 545.46s
+
+Epoch 107/400
+Train		loss: 70689912.00	correlation: 0.3013
+Validation	loss: 166642080.00	correlation: 0.2495
+Elapse: 545.17s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 108/400
+Train		loss: 70636352.00	correlation: 0.3019
+Validation	loss: 166643376.00	correlation: 0.2495
+Elapse: 545.12s
+
+Epoch 109/400
+Train		loss: 70622056.00	correlation: 0.3021
+Validation	loss: 166645664.00	correlation: 0.2495
+Elapse: 545.91s
+
+Epoch 110/400
+Train		loss: 70689848.00	correlation: 0.3013
+Validation	loss: 166646544.00	correlation: 0.2495
+Elapse: 544.95s
+
+Epoch 111/400
+Train		loss: 70594792.00	correlation: 0.3025
+Validation	loss: 166655344.00	correlation: 0.2494
+Elapse: 545.66s
+
+Epoch 112/400
+Train		loss: 70636904.00	correlation: 0.3020
+Validation	loss: 166638224.00	correlation: 0.2495
+Elapse: 544.74s
+
+Loaded checkpoint from epoch 107 (correlation: 0.2495).
+
+Reduce learning rate of core to 3.4992e-06 (num. reduce: 1).
+Reduce learning rate of readouts to 2.6244e-06 (num. reduce: 1).
+Reduce learning rate of shifters to 2.6244e-06 (num. reduce: 1).
+
+Epoch 113/400
+Train		loss: 70679760.00	correlation: 0.3016
+Validation	loss: 166645488.00	correlation: 0.2495
+Elapse: 543.62s
+
+Epoch 114/400
+Train		loss: 70609840.00	correlation: 0.3026
+Validation	loss: 166641568.00	correlation: 0.2495
+Elapse: 543.57s
+
+Checkpoint saved to /home/storage/runs/vivit_ensemble/028/ckpt/model_state.pt.
+
+Epoch 115/400
+Train		loss: 70676264.00	correlation: 0.3017
+Validation	loss: 166645824.00	correlation: 0.2495
+Elapse: 545.22s
+
+Epoch 116/400
+Train		loss: 70640760.00	correlation: 0.3023
+Validation	loss: 166649888.00	correlation: 0.2495
+Elapse: 545.14s
+
+Epoch 117/400
+Train		loss: 70651464.00	correlation: 0.3017
+Validation	loss: 166644896.00	correlation: 0.2495
+Elapse: 545.75s
+
+Epoch 118/400
+Train		loss: 70647664.00	correlation: 0.3020
+Validation	loss: 166642016.00	correlation: 0.2495
+Elapse: 551.10s
+
+Epoch 119/400
+Train		loss: 70647632.00	correlation: 0.3017
+Validation	loss: 166642992.00	correlation: 0.2495
+Elapse: 545.02s
+
+Loaded checkpoint from epoch 114 (correlation: 0.2495).
+
+Reduce learning rate of core to 1.0498e-06 (num. reduce: 1).
+Reduce learning rate of readouts to 7.8732e-07 (num. reduce: 1).
+Reduce learning rate of shifters to 7.8732e-07 (num. reduce: 1).
+
+Epoch 120/400
+Train		loss: 70673976.00	correlation: 0.3014
+Validation	loss: 166641376.00	correlation: 0.2495
+Elapse: 544.88s
+
+Epoch 121/400
+Train		loss: 70599064.00	correlation: 0.3023
+Validation	loss: 166642464.00	correlation: 0.2495
+Elapse: 544.91s
+
+Epoch 122/400
+Train		loss: 70592552.00	correlation: 0.3025
+Validation	loss: 166642720.00	correlation: 0.2495
+Elapse: 544.80s
+
+Epoch 123/400
+Train		loss: 70644184.00	correlation: 0.3019
+Validation	loss: 166643072.00	correlation: 0.2495
+Elapse: 544.57s
+
+Epoch 124/400
+Train		loss: 70629504.00	correlation: 0.3022
+Validation	loss: 166642208.00	correlation: 0.2495
+Elapse: 545.38s
+
+Loaded checkpoint from epoch 114 (correlation: 0.2495).
+
+Reduce learning rate of core to 3.1493e-07 (num. reduce: 2).
+Reduce learning rate of readouts to 2.3620e-07 (num. reduce: 2).
+Reduce learning rate of shifters to 2.3620e-07 (num. reduce: 2).
+
+Epoch 125/400
+Train		loss: 70617168.00	correlation: 0.3023
+Validation	loss: 166641952.00	correlation: 0.2495
+Elapse: 544.97s
+
+Epoch 126/400
+Train		loss: 70613328.00	correlation: 0.3026
+Validation	loss: 166642704.00	correlation: 0.2495
+Elapse: 544.92s
+
+Epoch 127/400
+Train		loss: 70681712.00	correlation: 0.3016
+Validation	loss: 166642848.00	correlation: 0.2495
+Elapse: 544.98s
+
+Epoch 128/400
+Train		loss: 70630672.00	correlation: 0.3021
+Validation	loss: 166642912.00	correlation: 0.2495
+Elapse: 544.96s
+
+Epoch 129/400
+Train		loss: 70600632.00	correlation: 0.3025
+Validation	loss: 166642848.00	correlation: 0.2495
+Elapse: 545.27s
+
+Model has not improved after 2 LR reductions.
+
+Loaded checkpoint from epoch 114 (correlation: 0.2495).
+
+
+ValidationA: 0.2465	B: 0.2783	C: 0.2688	D: 0.2292	E: 0.2366	F: 0.2341	G: 0.2549	H: 0.2330	I: 0.2567	J: 0.2571	average: 0.2495
+
+Results saved to /home/storage/runs/vivit_ensemble/028.