Add files

.gitattributes

@@ -30,3 +30,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step1750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step2250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step2750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseqglobal_step750/tokenizer.json filter=lfs diff=lfs merge=lfs -text

xp3capmixnewcodelonglossseqglobal_step1000/config.json

@@ -0,0 +1,30 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "attention_dropout": 0.0,
+  "architectures": [
+    "BloomModel"
+  ],
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "unk_token_id": 0,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 1024,
+  "n_inner": null,
+  "n_layer": 24,
+  "num_attention_heads": 16,
+  "offset_alibi": 100,
+  "pretraining_tp": 1,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "transformers_version": "4.20.0",
+  "use_cache": true,
+  "vocab_size": 250880
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/en/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4516
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/en/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.401
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/en/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3026
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/es/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3546
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/es/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3324
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/es/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2362
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/fr/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3872
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/fr/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3582
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/fr/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.265
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/zh/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3732
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/zh/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3506
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/amazon_reviews_multi/zh/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.276
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/aqua_rat/raw/Answer_questions_from_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "Answer questions from options",
+  "evaluation": {
+    "accuracy": 0.25196850393700787
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/aqua_rat/raw/answer_quiz/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "answer_quiz",
+  "evaluation": {
+    "accuracy": 0.2637795275590551
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/aqua_rat/raw/select_the_best_option/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "select_the_best_option",
+  "evaluation": {
+    "accuracy": 0.18110236220472442
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/art/choose_hypothesis/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis",
+  "evaluation": {
+    "accuracy": 0.5032637075718016
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/art/choose_hypothesis_believable/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_believable",
+  "evaluation": {
+    "accuracy": 0.4993472584856397
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/art/choose_hypothesis_desc/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_desc",
+  "evaluation": {
+    "accuracy": 0.49477806788511747
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/art/choose_hypothesis_likely/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_likely",
+  "evaluation": {
+    "accuracy": 0.5071801566579635
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/art/choose_hypothesis_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_options",
+  "evaluation": {
+    "accuracy": 0.5006527415143603
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/banking77/direct_to_which_department/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "direct_to_which_department",
+  "evaluation": {
+    "accuracy": 0.14642857142857144
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/banking77/help_page_topic/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "help_page_topic",
+  "evaluation": {
+    "accuracy": 0.15681818181818183
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/banking77/rephrase_as_banking_term/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "rephrase_as_banking_term",
+  "evaluation": {
+    "accuracy": 0.16396103896103897
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blbooksgenre/title_genre_classifiction/classify/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "classify",
+  "evaluation": {
+    "accuracy": 0.2540322580645161
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blbooksgenre/title_genre_classifiction/multi-choice/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "multi-choice",
+  "evaluation": {
+    "accuracy": 0.25057603686635943
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blbooksgenre/title_genre_classifiction/premise_context_first/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "premise_context_first",
+  "evaluation": {
+    "accuracy": 0.744815668202765
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blimp/adjunct_island/grammatical_between_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_1_2",
+  "evaluation": {
+    "accuracy": 0.486
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blimp/adjunct_island/grammatical_between_A_B/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_A_B",
+  "evaluation": {
+    "accuracy": 0.674
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blimp/adjunct_island/grammatical_which_one_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_which_one_1_2",
+  "evaluation": {
+    "accuracy": 0.494
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blimp/adjunct_island/single_sentence_bad_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_bad_yes_no",
+  "evaluation": {
+    "accuracy": 0.495
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/blimp/adjunct_island/single_sentence_good_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_good_yes_no",
+  "evaluation": {
+    "accuracy": 0.524
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/climate_fever/claim_and_all_supporting_evidences/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "claim_and_all_supporting_evidences",
+  "evaluation": {
+    "accuracy": 0.1719869706840391
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/climate_fever/fifth_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "fifth_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.11205211726384365
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/climate_fever/first_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "first_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.10814332247557003
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/climate_fever/second_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "second_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.11465798045602606
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/climate_fever/third_evidence_claim_pair/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "third_evidence_claim_pair",
+  "evaluation": {
+    "accuracy": 0.41107491856677525
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/codah/codah/affirmative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.25756484149855907
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/codah/codah/affirmative_instruction_before_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_before_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.2478386167146974
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/codah/codah/interrogative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "interrogative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.2510806916426513
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/commonsense_qa/answer_given_question_without_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "answer_given_question_without_options",
+  "evaluation": {
+    "accuracy": 0.3759213759213759
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/commonsense_qa/most_suitable_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "most_suitable_answer",
+  "evaluation": {
+    "accuracy": 0.3955773955773956
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/commonsense_qa/question_answering/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "question_answering",
+  "evaluation": {
+    "accuracy": 0.371007371007371
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/conv_ai_3/ambiguous/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "ambiguous",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/conv_ai_3/clarification_needed/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "clarification_needed",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/conv_ai_3/directly_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "directly_answer",
+  "evaluation": {
+    "accuracy": 0.6095979247730221
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/conv_ai_3/score_give_number/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_give_number",
+  "evaluation": {
+    "accuracy": 0.21444012105490703
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/conv_ai_3/score_how_much/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_how_much",
+  "evaluation": {
+    "accuracy": 0.21444012105490703
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseqglobal_step1000/evaluation/craigslist_bargains/best_deal/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "craigslist_bargains",
+  "dataset_config_name": null,
+  "template_name": "best deal",
+  "evaluation": {
+    "accuracy": 0.5192629815745393
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/350mt0/xp3capmixnewcodelonglossseqglobal_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)"
+}