Upload 2 files

llm_metaeval_eval_harness_pub.ipynb

@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "gpuType": "L4",
+      "gpuType": "T4",
       "machine_shape": "hm"
     },
     "kernelspec": {
@@ -121,7 +121,7 @@
         "fewshot_split: test\n",
         "fewshot_config:\n",
         "  sampler: first_n\n",
-        "num_fewshot: 10\n",
+        "num_fewshot: 5\n",
         "output_type: multiple_choice\n",
         "doc_to_text: \"{{pretext.strip()}}\\n Options:\\n__options__\\nAnswer:\"\n",
         "doc_to_choice: \"{{options}}\"\n",
@@ -130,9 +130,6 @@
         "  - metric: acc\n",
         "    aggregation: mean\n",
         "    higher_is_better: true\n",
-        "  - metric: acc_norm\n",
-        "    aggregation: mean\n",
-        "    higher_is_better: true\n",
         "\"\"\"\n",
         "tasks = []\n",
         "for t in YAML_template_pub_tasks:\n",
@@ -166,11 +163,11 @@
     {
       "cell_type": "code",
       "source": [
-        "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
-        "--model hf --model_args pretrained=meta-llama/Llama-3.2-1B-Instruct,parallelize=True \\\n",
+        "!for i in $(echo $TASKS|tr ',' ' '); do lm_eval \\\n",
+        "--model hf --model_args pretrained=meta-llama/Llama-3.2-1B-Instruct,revision=d0a2081ed47e20ce524e8bc5d132f3fad2f69ff0,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",
         "--tasks $i \\\n",
-        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --use_cache cache --log_samples \\\n",
-        "--batch_size 8; done"
+        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
+        "--batch_size auto; done &> run.log"
       ],
       "metadata": {
         "id": "NOwy6ZlY3Mw7"
@@ -192,11 +189,11 @@
     {
       "cell_type": "code",
       "source": [
-        "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
-        "--model hf --model_args pretrained=meta-llama/Llama-3.2-3B-Instruct,parallelize=True \\\n",
+        "!for i in $(echo $TASKS|tr ',' ' '); do lm_eval \\\n",
+        "--model hf --model_args pretrained=meta-llama/Llama-3.2-3B-Instruct,revision=392a143b624368100f77a3eafaa4a2468ba50a72,trust_remote_code=False,dtype=bfloat16,parallelize=False \\\n",
         "--tasks $i \\\n",
-        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --use_cache cache --log_samples \\\n",
-        "--batch_size 8; done"
+        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
+        "--batch_size auto; done &> run.log"
       ],
       "metadata": {
         "id": "oIACOAhDW5ow"
@@ -218,11 +215,11 @@
     {
       "cell_type": "code",
       "source": [
-        "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
-        "--model hf --model_args pretrained=meta-llama/Meta-Llama-3-8B,parallelize=True \\\n",
+        "!for i in $(echo $TASKS|tr ',' ' '); do lm_eval \\\n",
+        "--model hf --model_args pretrained=meta-llama/Meta-Llama-3-8B,revision=62bd457b6fe961a42a631306577e622c83876cb6,trust_remote_code=False,dtype=bfloat16,parallelize=False \\\n",
         "--tasks $i \\\n",
-        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --use_cache cache --log_samples \\\n",
-        "--batch_size 8; done"
+        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
+        "--batch_size auto; done &> run.log"
       ],
       "metadata": {
         "id": "1Nxw4WNxZUyb"
@@ -253,11 +250,11 @@
     {
       "cell_type": "code",
       "source": [
-        "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
-        "--model hf --model_args pretrained=mistralai/Mixtral-8x7B-Instruct-v0.1,parallelize=True \\\n",
+        "!for i in $(echo $TASKS|tr ',' ' '); do lm_eval \\\n",
+        "--model hf --model_args pretrained=mistralai/Mixtral-8x7B-Instruct-v0.1,revision=41bd4c9e7e4fb318ca40e721131d4933966c2cc1,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",
         "--tasks $i \\\n",
-        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --use_cache cache --log_samples \\\n",
-        "--batch_size 8; done"
+        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
+        "--batch_size auto; done &> run.log"
       ],
       "metadata": {
         "id": "E3dBWV1V9C-O"
@@ -279,11 +276,11 @@
     {
       "cell_type": "code",
       "source": [
-        "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
-        "--model hf --model_args pretrained=mistralai/Mixtral-8x22B-v0.1,parallelize=True \\\n",
+        "!for i in $(echo $TASKS|tr ',' ' '); do lm_eval \\\n",
+        "--model hf --model_args pretrained=mistralai/Mixtral-8x22B-v0.1,revision=b03e260818710044a2f088d88fab12bb220884fb,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",
         "--tasks $i \\\n",
-        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --use_cache cache --log_samples \\\n",
-        "--batch_size 8; done"
+        "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
+        "--batch_size auto; done &> run.log"
       ],
       "metadata": {
         "id": "LPqTo2z29RKx"