Spaces:
Running
Running
MLLM_leaderboard
/
eval-results
/WhoTookMyAmogusNickname
/NewHope_HF_not_official
/results_2023-09-17T06-38-00.301208.json
{ | |
"config_general": { | |
"model_name": "WhoTookMyAmogusNickname/NewHope_HF_not_official", | |
"model_sha": "f587f4a31de6818f4200d9cdc7f116ca8ba1cdc2", | |
"model_size": "24.28 GB", | |
"model_dtype": "torch.float16", | |
"lighteval_sha": "0f318ecf002208468154899217b3ba7c6ae09374", | |
"num_few_shot_default": 0, | |
"num_fewshot_seeds": 1, | |
"override_batch_size": 1, | |
"max_samples": null, | |
"job_id": "" | |
}, | |
"results": { | |
"harness|drop|3": { | |
"em": 0.19693791946308725, | |
"em_stderr": 0.004072666833657848, | |
"f1": 0.2666285654362424, | |
"f1_stderr": 0.004068431318455121 | |
}, | |
"harness|gsm8k|5": { | |
"acc": 0.15845337376800606, | |
"acc_stderr": 0.010058474790238971 | |
}, | |
"harness|winogrande|5": { | |
"acc": 0.749802683504341, | |
"acc_stderr": 0.012173009642449151 | |
}, | |
"all": { | |
"em": 0.19693791946308725, | |
"em_stderr": 0.004072666833657848, | |
"f1": 0.2666285654362424, | |
"f1_stderr": 0.004068431318455121, | |
"acc": 0.4541280286361735, | |
"acc_stderr": 0.011115742216344062 | |
} | |
}, | |
"versions": { | |
"harness|drop|3": 1, | |
"harness|gsm8k|5": 0, | |
"harness|winogrande|5": 0, | |
"all": 0 | |
}, | |
"config_tasks": { | |
"harness|drop": "LM Harness task", | |
"harness|gsm8k": "LM Harness task", | |
"harness|winogrande": "LM Harness task" | |
}, | |
"summary_tasks": { | |
"harness|drop|3": { | |
"hashes": { | |
"hash_examples": "1d27416e8324e9a3", | |
"hash_full_prompts": "a5513ff9a741b385", | |
"hash_input_tokens": "61b608e0b5ceed76", | |
"hash_cont_tokens": "5bcc6a0d3257f718" | |
}, | |
"truncated": 1263, | |
"non-truncated": 8273, | |
"padded": 0, | |
"non-padded": 9536, | |
"effective_few_shots": 3.0, | |
"num_truncated_few_shots": 0 | |
}, | |
"harness|gsm8k|5": { | |
"hashes": { | |
"hash_examples": "4c0843a5d99bcfdc", | |
"hash_full_prompts": "41d55e83abc0e02d", | |
"hash_input_tokens": "bda342e47b5099b2", | |
"hash_cont_tokens": "5bbe8ff839da2365" | |
}, | |
"truncated": 0, | |
"non-truncated": 1319, | |
"padded": 0, | |
"non-padded": 1319, | |
"effective_few_shots": 5.0, | |
"num_truncated_few_shots": 0 | |
}, | |
"harness|winogrande|5": { | |
"hashes": { | |
"hash_examples": "aada0a176fd81218", | |
"hash_full_prompts": "c8655cbd12de8409", | |
"hash_input_tokens": "c0bedf98cb040854", | |
"hash_cont_tokens": "f08975ad6f2d5864" | |
}, | |
"truncated": 0, | |
"non-truncated": 2534, | |
"padded": 2432, | |
"non-padded": 102, | |
"effective_few_shots": 5.0, | |
"num_truncated_few_shots": 0 | |
} | |
}, | |
"summary_general": { | |
"hashes": { | |
"hash_examples": "9b4d8993161e637d", | |
"hash_full_prompts": "08215e527b7e60a5", | |
"hash_input_tokens": "80afe720f936f8d2", | |
"hash_cont_tokens": "978d0727f4e66702" | |
}, | |
"total_evaluation_time_secondes": "53089.931958675385", | |
"truncated": 1263, | |
"non-truncated": 12126, | |
"padded": 2432, | |
"non-padded": 10957, | |
"num_truncated_few_shots": 0 | |
} | |
} |