Upload 2 files

scripts/full.yaml

@@ -1,7 +1,7 @@
 # Tokenizer
 tokenizer:
   _component_: torchtune.models.llama3.llama3_tokenizer
-  path: ../../slice_with_mergekit/merged/original/tokenizer.model
+  path: ../original/tokenizer.model
 
 # Dataset and Sampler
 dataset:
@@ -11,7 +11,7 @@ dataset:
 #  _component_: torchtune.datasets.stack_exchanged_paired_dataset
 seed: 42
 shuffle: True
-batch_size: 1
+batch_size: 2
 
 # Model Arguments
 model:
@@ -29,25 +29,25 @@ model:
 
 checkpointer:
   _component_: torchtune.utils.FullModelHFCheckpointer
-  checkpoint_dir: ../../slice_with_mergekit/merged/
+  checkpoint_dir: ../merged/
   checkpoint_files: [
     model-00001-of-00003.safetensors,
     model-00002-of-00003.safetensors,
     model-00003-of-00003.safetensors
   ]
   recipe_checkpoint: null
-  output_dir: /media/hailey/More/AI/PruneMe/train/torchtune/llama3-5b/
+  output_dir: ./llama3-5b/
   model_type: LLAMA3
 resume_from_checkpoint: False
 
 # Fine-tuning arguments
-epochs: 1
+epochs: 5
 optimizer:
   _component_: torch.optim.AdamW #bitsandbytes.optim.PagedAdamW8bit
-  lr: 5e-6
+  lr: 3e-6
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 1000
+  num_warmup_steps: 1500
 #loss:
 #  _component_: torchtune.modules.loss.DPOLoss
 #  beta: 0.1
@@ -57,8 +57,8 @@ loss:
   _component_: torch.nn.CrossEntropyLoss
 
 max_steps_per_epoch: null
-gradient_accumulation_steps: 1
-optimizer_in_bwd: True  # False if grad accum > 1
+gradient_accumulation_steps: 2
+optimizer_in_bwd: False  # False if grad accum > 1
 compile: False
 
 # Training environment
@@ -68,7 +68,7 @@ device: cuda
 enable_activation_checkpointing: True
 
 # Reduced precision
-dtype: fp32
+dtype: bf16 #fp32
 
 # Logging
 # enable logging to the built-in WandBLogger
@@ -76,6 +76,6 @@ metric_logger:
   _component_: torchtune.utils.metric_logging.WandBLogger
   # the W&B project to log to
   project: llama3-5b
-output_dir: /media/hailey/More/AI/PruneMe/train/torchtune/llama3-5b-dpo/
+output_dir: ./logs/
 log_every_n_steps: 1
 log_peak_memory_stats: False

scripts/test_generation.py

@@ -2,7 +2,7 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
 # Define your model path
-model_path = "./merged"  # or the path/model_name you have
+model_path = "./llama3-5b/hf"  # or the path/model_name you have
 
 # Your custom quantization configuration
 quantization_config = None