Upload run_gpt2_350M_edu_hermes.sh with huggingface_hub

run_gpt2_350M_edu_hermes.sh

@@ -0,0 +1,44 @@
+# GPT-3 (124M) repro on FineWeb
+# 124M parameter model on 300B tokens
+# note context length: 1024 -> 2048 for GPT-3
+# => 6 * 124e6 * 300e9 = 7.44e18 ~= 2.2e20 capability model
+# 565,950 steps of 524,288 tokens/step
+# on 8X A100 80GB SXM ($14/hr) steps in ~300ms/iter
+# => training time 565,950 * 300ms ~= 47 hours ~= $658
+
+make train_gpt2cu USE_CUDNN=1
+out_dir="log_gpt2_350M_edu_hermes"
+done_file="$out_dir/DONE_00019622"
+
+while true; do
+
+    # exit condition is that optimization has finished
+    if [ -f "$done_file" ]; then
+        echo "File $done_file exists. Exiting the loop."
+        break
+    fi
+
+    # run python dev/data/fineweb_edu_hermes.py --version 10B to prepro data
+    # run python dev/data/hellaswag.py to prepro hellaswag eval
+    mpirun -np 2 ./train_gpt2cu \
+                -i "dev/data/edu_fineweb10B_hermes/edu_fineweb_hermes_train_*.bin" \
+                -j "dev/data/edu_fineweb10B_hermes/edu_fineweb_hermes_val_*.bin" \
+                -o $out_dir \
+                -v 250 -s 5000 -g 144 \
+                -h 1 \
+                -b 16 -t 1024 \
+                -d 524288 \
+                -r 0 \
+                -z 1 \
+                -c 0.1 \
+                -l 0.0003 \
+                -q 0.0 \
+                -u 700 \
+                -n 5000 \
+                -sl 7.0 -sg 7.0 \
+                -y 1 \
+                -x 19622 \
+                -e "d24"
+
+    sleep 1
+done