Upload folder using huggingface_hub

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/flyteav9zvdg1/local_flytekit/3dc3fdcbe70c98829ce1a05b3cd622c0",
+  "_name_or_path": "togethercomputer/RedPajama-INCITE-Base-3B-v1",
   "architectures": [
     "GPTNeoXForCausalLM"
   ],
@@ -7,20 +7,20 @@
   "classifier_dropout": 0.1,
   "eos_token_id": 0,
   "hidden_act": "gelu",
-  "hidden_size": 2048,
+  "hidden_size": 2560,
   "initializer_range": 0.02,
-  "intermediate_size": 8192,
+  "intermediate_size": 10240,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 2048,
   "model_type": "gpt_neox",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 16,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
   "rotary_emb_base": 10000,
-  "rotary_pct": 0.25,
+  "rotary_pct": 1.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
+  "torch_dtype": "float16",
   "transformers_version": "4.29.2",
   "use_cache": true,
-  "use_parallel_residual": true,
-  "vocab_size": 50304
+  "use_parallel_residual": false,
+  "vocab_size": 50432
 }

flyte_training_config.json

@@ -1 +1 @@
-{"base_model": "EleutherAI/pythia-1B-deduped", "data_path": "wikipedia", "data_name": "20220301.simple", "num_epochs": 1, "max_steps": 10, "learning_rate": 2e-05, "weight_decay": 0.02, "warmup_ratio": 0.03, "lr_scheduler_type": "cosine", "batch_size": 8, "micro_batch_size": 1, "val_set_size": 0, "group_by_length": false, "instruction_key": "instruction", "input_key": "input", "output_key": "output", "device_map": "auto", "cache_dir": null, "optim": "adamw_torch", "model_max_length": 512, "debug_mode": false, "debug_train_data_size": 1024, "wandb_project": ""}
+{"base_model": "togethercomputer/RedPajama-INCITE-Base-3B-v1", "data_path": "wikipedia", "data_name": "20220301.simple", "num_epochs": 1, "max_steps": 10, "learning_rate": 2e-05, "weight_decay": 0.02, "warmup_ratio": 0.03, "lr_scheduler_type": "cosine", "batch_size": 16, "micro_batch_size": 1, "val_set_size": 0, "group_by_length": false, "instruction_key": "instruction", "input_key": "input", "output_key": "output", "device_map": "auto", "cache_dir": null, "optim": "adamw_torch", "model_max_length": 512, "debug_mode": false, "debug_train_data_size": 1024, "wandb_project": ""}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ff095431efbef5d7a3b7e02f6a0888af48d5e1c2fe8de04c43c1c810c7e1b49
-size 4114316253
+oid sha256:747543e20635e50ec287e6ab96d89482a765b1459a8ca932e77b9d5033d1acd9
+size 5686106713

trainer_state.json

@@ -1,31 +1,31 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.3333333333333335,
+  "epoch": 6.666666666666667,
   "global_step": 10,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 3.33,
+      "epoch": 6.67,
       "learning_rate": 2e-05,
-      "loss": 2.1837,
+      "loss": 2.0296,
       "step": 10
     },
     {
-      "epoch": 3.33,
+      "epoch": 6.67,
       "step": 10,
-      "total_flos": 835002040320.0,
-      "train_loss": 2.1836517333984373,
-      "train_runtime": 74.2827,
-      "train_samples_per_second": 8.616,
-      "train_steps_per_second": 0.135
+      "total_flos": 4181427486720.0,
+      "train_loss": 2.029572677612305,
+      "train_runtime": 431.4795,
+      "train_samples_per_second": 2.967,
+      "train_steps_per_second": 0.023
     }
   ],
   "max_steps": 10,
-  "num_train_epochs": 4,
-  "total_flos": 835002040320.0,
+  "num_train_epochs": 10,
+  "total_flos": 4181427486720.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dccd10791eeca2765d0dab04a01995f27d811561179ad9f79d8ca91bbc1bae49
+oid sha256:ea233ffe22ae0f54efe9e7c78d909a4e8222d25cd04ce62ed9ddfb7cf3ee00a8
 size 5499