Upload training_config.yml with huggingface_hub

training_config.yml

@@ -11,10 +11,10 @@ model:
   use_clip: false
 tokenizer:
   _component_: models.a2a_tokenizer
-  path: checkpoints/Meta-Llama-3-8B-Instruct/tokenizer.model
+  path: models/tokenizer.model
 checkpointer:
   _component_: torchtune.utils.FullModelMetaCheckpointer
-  checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/
+  checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
   checkpoint_files:
   - consolidated.00.pth
   adapter_checkpoint: null
@@ -25,10 +25,11 @@ resume_from_checkpoint: false
 interim_checkpoint_steps: 5000
 interim_gen_steps: null
 max_new_tokens: 100
-temperature: 0.8
+temperature: 0.6
 top_k: 300
 dataset:
   _component_: ds.EvenBatcher
+  buffer_size: 36
   dataset:
     _component_: ds.RoundRobinDataset
     datasets:
@@ -45,21 +46,21 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 2
+batch_size: 4
 optimizer:
   _component_: torch.optim.AdamW
   weight_decay: 0.01
-  lr: 3.0e-06
+  lr: 0.0003
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
   num_warmup_steps: 100
 loss:
   _component_: torch.nn.CrossEntropyLoss
-epochs: 1
+epochs: 5
 max_steps_per_epoch: null
-gradient_accumulation_steps: 32
+gradient_accumulation_steps: 16
 compile: false
-output_dir: /home/user/omegalabs-anytoany-bittensor/modelss
+output_dir: /tmp/lora_finetune_output
 metric_logger:
   _component_: torchtune.utils.metric_logging.DiskLogger
   log_dir: ${output_dir}
@@ -77,6 +78,7 @@ inference:
 
     Caption the previous video.'
   max_new_tokens: 300
-  temperature: 0.8
+  temperature: 0.6
   top_k: 300
   quantizer: null
+gradient-accumulation-steps: 32