mode: pt
device: gpu
precision: bf16
eval_only: false
predict_only: false
seed: 2137
model:
  klass: hf_t5
  name: /home/jovyan/bert-train/nanot5/templates/base_slavic_120k_sub
  overwrite:
    dropout_rate: 0.1
  add_config:
    is_bf16: true
  checkpoint_path: ''
  random_init: true
  compile: false
data:
  input_length: 512
  mlm_probability: 0.15
  mean_noise_span_length: 3.0
  num_workers: 8
optim:
  name: adafactor
  base_lr: 0.02
  batch_size: 128
  total_steps: 120000
  epochs: -1
  warmup_steps: 10000
  lr_scheduler: legacy
  weight_decay: 0.0
  grad_clip: 1.0
  grad_acc: 8
  final_cosine: 1.0e-05
eval:
  every_steps: 5000
  steps: 500
checkpoint:
  every_steps: 10000
logging:
  neptune: false
  neptune_creds:
    project: null
    api_token: null
    tags: ''
  every_steps: 100
  grad_l2: true
  weights_l2: true