sophia_opt.SophiaG

scripts/pretrain-model.yaml

@@ -65,10 +65,12 @@ train:
   log_interval: 1
 
   # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
-  global_batch_size: 512
+  # global_batch_size: 512
+  global_batch_size: 256
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 2
+  # micro_batch_size: 2
+  micro_batch_size: 6
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 2000
@@ -100,7 +102,8 @@ train:
 # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
 eval:
   # Number of optimizer steps between evaluation calls (type: int, default: 1000)
-  interval: 100
+  # interval: 100
+  interval: 20
 
   # Number of tokens to generate (type: Optional[int], default: null)
   max_new_tokens: