HaileyStorm
commited on
Update chess-gpt-eval-contrastive/mamba_module.py
Browse files
chess-gpt-eval-contrastive/mamba_module.py
CHANGED
@@ -321,9 +321,9 @@ class MambaPlayer:
|
|
321 |
def train_linear_probes(self):
|
322 |
criterion = nn.MSELoss()
|
323 |
self.wandb_step += 1
|
324 |
-
decay_iters = 2000 *
|
325 |
-
learning_rate = 0.
|
326 |
-
min_lr = 0.
|
327 |
|
328 |
coeff = 0.5 * (1.0 + math.cos(math.pi * min(self.wandb_step / decay_iters, 1.0)))
|
329 |
lr = min_lr + coeff * (learning_rate - min_lr)
|
|
|
321 |
def train_linear_probes(self):
|
322 |
criterion = nn.MSELoss()
|
323 |
self.wandb_step += 1
|
324 |
+
decay_iters = 2000 * 43
|
325 |
+
learning_rate = 0.00025
|
326 |
+
min_lr = 0.000025
|
327 |
|
328 |
coeff = 0.5 * (1.0 + math.cos(math.pi * min(self.wandb_step / decay_iters, 1.0)))
|
329 |
lr = min_lr + coeff * (learning_rate - min_lr)
|