HaileyStorm commited on
Commit
fbb8b5e
·
verified ·
1 Parent(s): 03f501f

Update chess-gpt-eval-contrastive/mamba_module.py

Browse files
chess-gpt-eval-contrastive/mamba_module.py CHANGED
@@ -321,9 +321,9 @@ class MambaPlayer:
321
  def train_linear_probes(self):
322
  criterion = nn.MSELoss()
323
  self.wandb_step += 1
324
- decay_iters = 2000 * 40
325
- learning_rate = 0.01
326
- min_lr = 0.0001
327
 
328
  coeff = 0.5 * (1.0 + math.cos(math.pi * min(self.wandb_step / decay_iters, 1.0)))
329
  lr = min_lr + coeff * (learning_rate - min_lr)
 
321
  def train_linear_probes(self):
322
  criterion = nn.MSELoss()
323
  self.wandb_step += 1
324
+ decay_iters = 2000 * 43
325
+ learning_rate = 0.00025
326
+ min_lr = 0.000025
327
 
328
  coeff = 0.5 * (1.0 + math.cos(math.pi * min(self.wandb_step / decay_iters, 1.0)))
329
  lr = min_lr + coeff * (learning_rate - min_lr)