| _config_type: haystacks.embeddings.train_batch.TrainConfig | |
| accelerator: gpu | |
| accumulate_grad_batches: 1 | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.95 | |
| base_save_dir: /home/sabri/code/haystacks/checkpoints | |
| check_val_every_n_epoch: null | |
| ckpt_path: null | |
| devices: 8 | |
| dtype: bfloat16 | |
| embedding_module: | |
| _config_type: haystacks.embeddings.modeling.TokenEmbeddingModuleConfig | |
| batch_attention_layers: [] | |
| embedding_dim: 8192 | |
| kwargs: {} | |
| target: | |
| _is_type: true | |
| name: haystacks.embeddings.modeling.TokenEmbeddingModule | |
| enable_checkpointing: false | |
| foreach: null | |
| gradient_clip_val: null | |
| launch_id: null | |
| learnable_bias: true | |
| learnable_temp: true | |
| limit_train_batches: 1.0 | |
| limit_val_batches: 1.0 | |
| load_hub: true | |
| log_every_n_steps: 4 | |
| log_grad_norms: false | |
| loss_comparison: matched | |
| loss_token_idxs: | |
| - 64 | |
| - 128 | |
| - 256 | |
| - 512 | |
| - 1024 | |
| lr: 0.0001 | |
| lr_scheduler: null | |
| manual_save_epochs: null | |
| manual_save_steps: 8192 | |
| max_epochs: 512 | |
| max_hidden_layers: null | |
| max_problems: null | |
| max_seq_len: 1024 | |
| max_steps: -1 | |
| model_name: meta-llama/Llama-3.2-1B-Instruct | |
| name: no_batch-attention-lr0.0001-bs32-d8192-new | |
| num_sanity_val_steps: null | |
| num_workers: 0 | |
| objective: cross_entropy | |
| output_dir: null | |
| overfit_batches: 0.0 | |
| precision: bf16 | |
| reload_dataloaders_every_n_epochs: 0 | |
| run_dir: null | |
| run_id: null | |
| samples_per_batch: 32 | |
| save_intermediates: true | |
| script_id: null | |
| seed: 42 | |
| train_batch_size: 1 | |
| train_data_path: ScalingIntelligence/math-train-l3.2-3Bi-meta-n128 | |
| use_wandb: false | |
| val_batch_size: 1 | |
| val_check_interval: 64 | |
| val_data_path: ScalingIntelligence/math-test-l3.2-3Bi-meta-n128 | |
| val_rollout_data_path: null | |
| val_samples_per_batch: 32 | |
| validate_before_train: true | |
| wandb: | |
| _config_type: haystacks.embeddings.train_batch.WandbLoggerConfig | |
| group: '' | |
| id: null | |
| job_type: train | |
| kwargs: {} | |
| log_model: false | |
| mode: online | |
| name: null | |
| prefix: '' | |
| project: haystacks | |
| save_dir: . | |
| tags: [] | |
| target: | |
| _is_type: true | |
| name: pytorch_lightning.loggers.wandb.WandbLogger | |
| weight_decay: 0.1 | |
| weights_only: true | |