Updated results files

Signed-off-by: Jonathan Bnayahu <[email protected]>

results/bluebench/2025-06-22T14-01-49_evaluation_results.json

@@ -0,0 +1,1283 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-06-22T18:01:46.346556Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/meta-llama/llama-3-2-11b-vision-instruct,max_tokens=256",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/meta-llama/llama-3-2-11b-vision-instruct",
+            "model_args": {
+                "max_tokens": 256
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.24.0",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "litellm": "1.72.6.post1",
+            "httpcore": "1.0.9",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "yarl": "1.20.1",
+            "openai": "1.88.0",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "unitxt": "1.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "pillow": "11.2.1",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "lxml": "5.4.0",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "fonttools": "4.58.4",
+            "transformers": "4.52.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "evaluate": "0.4.3",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "tokenizers": "0.21.1",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "hf-xet": "1.1.4",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "multidict": "6.5.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "certifi": "2025.6.15",
+            "accelerate": "1.8.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.0",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "huggingface-hub": "0.33.0",
+            "colorama": "0.4.6",
+            "rpds-py": "0.25.1",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6555555555555556,
+                "accuracy_ci_high": 0.8333333333333334,
+                "score_name": "accuracy",
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.8333333333333334,
+                "score_ci_low": 0.6555555555555556,
+                "num_of_instances": 90
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.8666666666666667,
+                "score_name": "accuracy",
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8666666666666667,
+                "score_ci_low": 0.7,
+                "num_of_instances": 90
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.9111111111111111,
+                "accuracy_ci_low": 0.8444444444444444,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9111111111111111,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8444444444444444,
+                "num_of_instances": 90
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.8666666666666667,
+                "score_name": "accuracy",
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8666666666666667,
+                "score_ci_low": 0.7,
+                "num_of_instances": 90
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 0.8111111111111111,
+                "accuracy_ci_low": 0.7222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.8111111111111111,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7222222222222222,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.8222222222222222,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8222222222222222,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 0.9666666666666667,
+                "accuracy_ci_low": 0.9111111111111111,
+                "accuracy_ci_high": 0.9888888888888889,
+                "score_name": "accuracy",
+                "score": 0.9666666666666667,
+                "score_ci_high": 0.9888888888888889,
+                "score_ci_low": 0.9111111111111111,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.7888888888888889,
+                "accuracy_ci_high": 0.9333333333333333,
+                "score_name": "accuracy",
+                "score": 0.8666666666666667,
+                "score_ci_high": 0.9333333333333333,
+                "score_ci_low": 0.7888888888888889,
+                "num_of_instances": 90
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.9111111111111111,
+                "accuracy_ci_low": 0.8412016500028439,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9111111111111111,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8412016500028439,
+                "num_of_instances": 90
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.8444444444444444,
+                "accuracy_ci_low": 0.7666666666666667,
+                "accuracy_ci_high": 0.9111111111111111,
+                "score_name": "accuracy",
+                "score": 0.8444444444444444,
+                "score_ci_high": 0.9111111111111111,
+                "score_ci_low": 0.7666666666666667,
+                "num_of_instances": 90
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.9111111111111111,
+                "accuracy_ci_low": 0.8333333333333334,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9111111111111111,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8333333333333334,
+                "num_of_instances": 90
+            },
+            "score": 0.8585858585858586,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 500,
+                "llama_3_70b_instruct_template_arena_hard": 0.07721639656816015,
+                "score": 0.07721639656816015,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.07721639656816015,
+            "score_name": "subsets_mean",
+            "num_of_instances": 500
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 1000,
+                "f1_Person": 0.48831168831168825,
+                "f1_Organization": 0.35220125786163525,
+                "f1_Location": 0.3775100401606426,
+                "f1_macro": 0.406007662111322,
+                "recall_macro": 0.3667818453974414,
+                "precision_macro": 0.4584981753989352,
+                "in_classes_support": 0.7834862385321101,
+                "f1_micro": 0.3682242990654206,
+                "recall_micro": 0.37523809523809526,
+                "precision_micro": 0.3614678899082569,
+                "score": 0.3682242990654206,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.32532095178317566,
+                "score_ci_high": 0.4180775144242145,
+                "f1_micro_ci_low": 0.32532095178317566,
+                "f1_micro_ci_high": 0.4180775144242145
+            },
+            "score": 0.3682242990654206,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.676056338028169,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4507042253521127,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
+                "score_name": "accuracy",
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.352112676056338,
+                "score_name": "accuracy",
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.352112676056338,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.36619718309859156,
+                "accuracy_ci_low": 0.2535211267605634,
+                "accuracy_ci_high": 0.4788732394366197,
+                "score_name": "accuracy",
+                "score": 0.36619718309859156,
+                "score_ci_high": 0.4788732394366197,
+                "score_ci_low": 0.2535211267605634,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.5492957746478874,
+                "accuracy_ci_low": 0.43661971830985913,
+                "accuracy_ci_high": 0.6619718309859155,
+                "score_name": "accuracy",
+                "score": 0.5492957746478874,
+                "score_ci_high": 0.6619718309859155,
+                "score_ci_low": 0.43661971830985913,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.352112676056338,
+                "score_name": "accuracy",
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.352112676056338,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.4788732394366197,
+                "accuracy_ci_low": 0.36619718309859156,
+                "accuracy_ci_high": 0.6056338028169014,
+                "score_name": "accuracy",
+                "score": 0.4788732394366197,
+                "score_ci_high": 0.6056338028169014,
+                "score_ci_low": 0.36619718309859156,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.5070422535211268,
+                "accuracy_ci_low": 0.39436619718309857,
+                "accuracy_ci_high": 0.6197183098591549,
+                "score_name": "accuracy",
+                "score": 0.5070422535211268,
+                "score_ci_high": 0.6197183098591549,
+                "score_ci_low": 0.39436619718309857,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.30985915492957744,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.42820969566908634,
+                "score_name": "accuracy",
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.42820969566908634,
+                "score_ci_low": 0.2112676056338028,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.14084507042253522,
+                "accuracy_ci_low": 0.07042253521126761,
+                "accuracy_ci_high": 0.22535211267605634,
+                "score_name": "accuracy",
+                "score": 0.14084507042253522,
+                "score_ci_high": 0.22535211267605634,
+                "score_ci_low": 0.07042253521126761,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.28169014084507044,
+                "accuracy_ci_low": 0.18309859154929578,
+                "accuracy_ci_high": 0.39436619718309857,
+                "score_name": "accuracy",
+                "score": 0.28169014084507044,
+                "score_ci_high": 0.39436619718309857,
+                "score_ci_low": 0.18309859154929578,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.4507042253521127,
+                "accuracy_ci_low": 0.323943661971831,
+                "accuracy_ci_high": 0.5633802816901409,
+                "score_name": "accuracy",
+                "score": 0.4507042253521127,
+                "score_ci_high": 0.5633802816901409,
+                "score_ci_low": 0.323943661971831,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.16901408450704225,
+                "accuracy_ci_high": 0.36619718309859156,
+                "score_name": "accuracy",
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.16901408450704225,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.5774647887323944,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.6894343225712088,
+                "score_name": "accuracy",
+                "score": 0.5774647887323944,
+                "score_ci_high": 0.6894343225712088,
+                "score_ci_low": 0.4507042253521127,
+                "num_of_instances": 71
+            },
+            "score": 0.3722334004024145,
+            "score_name": "subsets_mean",
+            "num_of_instances": 994
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.5662558356676004,
+                "f1_suggestive": 0.4666666666666667,
+                "f1_arbitrary": 0.4444444444444444,
+                "f1_generic": 0.8571428571428571,
+                "f1_fanciful": 0.35714285714285715,
+                "f1_descriptive": 0.7058823529411765,
+                "f1_macro_ci_low": 0.47410052522342583,
+                "f1_macro_ci_high": 0.6713730404881563,
+                "score_name": "f1_micro",
+                "score": 0.5575757575757576,
+                "score_ci_high": 0.6506589298059469,
+                "score_ci_low": 0.4457831325301205,
+                "num_of_instances": 85,
+                "accuracy": 0.5411764705882353,
+                "accuracy_ci_low": 0.43529411764705883,
+                "accuracy_ci_high": 0.6352941176470588,
+                "f1_micro": 0.5575757575757576,
+                "f1_micro_ci_low": 0.4457831325301205,
+                "f1_micro_ci_high": 0.6506589298059469
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.575166034793364,
+                "f1_no": 0.6877470355731226,
+                "f1_yes": 0.46258503401360546,
+                "f1_macro_ci_low": 0.5066495066495067,
+                "f1_macro_ci_high": 0.6496773446094443,
+                "score_name": "f1_micro",
+                "score": 0.605,
+                "score_ci_high": 0.67,
+                "score_ci_low": 0.535,
+                "num_of_instances": 200,
+                "accuracy": 0.605,
+                "accuracy_ci_low": 0.535,
+                "accuracy_ci_high": 0.67,
+                "f1_micro": 0.605,
+                "f1_micro_ci_low": 0.535,
+                "f1_micro_ci_high": 0.67
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.3240051765352555,
+                "f1_conclusion": 0.0975609756097561,
+                "f1_analysis": 0.509090909090909,
+                "f1_decree": 0.34285714285714286,
+                "f1_issue": 0.22641509433962265,
+                "f1_procedural history": 0.29850746268656714,
+                "f1_facts": 0.4186046511627907,
+                "f1_rule": 0.375,
+                "f1_macro_ci_low": 0.2737291340244584,
+                "f1_macro_ci_high": 0.39709087675818633,
+                "score_name": "f1_micro",
+                "score": 0.3526448362720403,
+                "score_ci_high": 0.42317380352644834,
+                "score_ci_low": 0.29292929292929293,
+                "num_of_instances": 200,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.29,
+                "accuracy_ci_high": 0.42,
+                "f1_micro": 0.3526448362720403,
+                "f1_micro_ci_low": 0.29292929292929293,
+                "f1_micro_ci_high": 0.42317380352644834
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.5769837972579975,
+                "f1_yes": 0.45517241379310347,
+                "f1_no": 0.6987951807228916,
+                "f1_macro_ci_low": 0.5127178863190986,
+                "f1_macro_ci_high": 0.6553872211311121,
+                "score_name": "f1_micro",
+                "score": 0.6091370558375635,
+                "score_ci_high": 0.6785772255666204,
+                "score_ci_low": 0.5449871465295629,
+                "num_of_instances": 200,
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.535,
+                "accuracy_ci_high": 0.67,
+                "f1_micro": 0.6091370558375635,
+                "f1_micro_ci_low": 0.5449871465295629,
+                "f1_micro_ci_high": 0.6785772255666204
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.9404761904761905,
+                "f1_yes": 0.9523809523809523,
+                "f1_no": 0.9285714285714286,
+                "f1_macro_ci_low": 0.8717038360531253,
+                "f1_macro_ci_high": 0.9763503609021853,
+                "score_name": "f1_micro",
+                "score": 0.9404761904761905,
+                "score_ci_high": 0.9764705882352941,
+                "score_ci_low": 0.8724795930656631,
+                "num_of_instances": 85,
+                "accuracy": 0.9294117647058824,
+                "accuracy_ci_low": 0.8470588235294118,
+                "accuracy_ci_high": 0.9764705882352941,
+                "f1_micro": 0.9404761904761905,
+                "f1_micro_ci_low": 0.8724795930656631,
+                "f1_micro_ci_high": 0.9764705882352941
+            },
+            "score": 0.6129667680323103,
+            "score_name": "subsets_mean",
+            "num_of_instances": 770
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.5153316959213506,
+                "f1_cars": 0.7317073170731707,
+                "f1_windows x": 0.08450704225352113,
+                "f1_computer graphics": 0.4948453608247423,
+                "f1_atheism": 0.2978723404255319,
+                "f1_religion": 0.05263157894736842,
+                "f1_medicine": 0.7733333333333333,
+                "f1_christianity": 0.5806451612903226,
+                "f1_microsoft windows": 0.4507042253521127,
+                "f1_middle east": 0.32727272727272727,
+                "f1_politics": 0.4132231404958678,
+                "f1_motorcycles": 0.7058823529411765,
+                "f1_pc hardware": 0.48520710059171596,
+                "f1_mac hardware": 0.5057471264367817,
+                "f1_electronics": 0.48739495798319327,
+                "f1_for sale": 0.5,
+                "f1_guns": 0.28125,
+                "f1_space": 0.7659574468085106,
+                "f1_cryptography": 0.6,
+                "f1_baseball": 0.8813559322033898,
+                "f1_hockey": 0.8870967741935484,
+                "f1_macro_ci_low": 0.4890786960094656,
+                "f1_macro_ci_high": 0.5464781246183315,
+                "score_name": "f1_micro",
+                "score": 0.5437325905292479,
+                "score_ci_high": 0.5741315636296753,
+                "score_ci_low": 0.5090753018614114,
+                "num_of_instances": 1000,
+                "accuracy": 0.488,
+                "accuracy_ci_low": 0.454,
+                "accuracy_ci_high": 0.519,
+                "f1_micro": 0.5437325905292479,
+                "f1_micro_ci_low": 0.5090753018614114,
+                "f1_micro_ci_high": 0.5741315636296753
+            },
+            "score": 0.5437325905292479,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.6685227589041403,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9225251076040172,
+                "f1_checking or savings account": 0.5806451612903226,
+                "f1_debt collection": 0.5274725274725275,
+                "f1_credit card or prepaid card": 0.6371681415929203,
+                "f1_mortgage": 0.8059701492537313,
+                "f1_student loan": 0.8571428571428571,
+                "f1_money transfer or virtual currency or money service": 0.6181818181818182,
+                "f1_vehicle loan or lease": 0.6060606060606061,
+                "f1_payday loan or title loan or personal loan": 0.46153846153846156,
+                "f1_macro_ci_low": 0.6111841538128283,
+                "f1_macro_ci_high": 0.7335266591830523,
+                "score_name": "f1_micro",
+                "score": 0.8321536905965622,
+                "score_ci_high": 0.85326682230999,
+                "score_ci_low": 0.80760586975502,
+                "num_of_instances": 1000,
+                "accuracy": 0.823,
+                "accuracy_ci_low": 0.7962032615906698,
+                "accuracy_ci_high": 0.8449169646606582,
+                "f1_micro": 0.8321536905965622,
+                "f1_micro_ci_low": 0.80760586975502,
+                "f1_micro_ci_high": 0.85326682230999
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.6712830729866164,
+                "f1_mortgages and loans": 0.7439024390243902,
+                "f1_credit card": 0.7777777777777778,
+                "f1_debt collection": 0.6571428571428571,
+                "f1_credit reporting": 0.7817589576547231,
+                "f1_retail banking": 0.3958333333333333,
+                "f1_macro_ci_low": 0.6287200378375363,
+                "f1_macro_ci_high": 0.7180257299728254,
+                "score_name": "f1_micro",
+                "score": 0.7097435897435898,
+                "score_ci_high": 0.7484617342104366,
+                "score_ci_low": 0.6680812073559,
+                "num_of_instances": 500,
+                "accuracy": 0.692,
+                "accuracy_ci_low": 0.648,
+                "accuracy_ci_high": 0.732,
+                "f1_micro": 0.7097435897435898,
+                "f1_micro_ci_low": 0.6680812073559,
+                "f1_micro_ci_high": 0.7484617342104366
+            },
+            "score": 0.770948640170076,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 1000,
+                "program_accuracy": 0.063,
+                "score": 0.063,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.053,
+                "program_accuracy_ci_low": 0.049,
+                "program_accuracy_ci_high": 0.07883525503658394,
+                "score_ci_low": 0.049,
+                "score_ci_high": 0.07883525503658394,
+                "execution_accuracy_ci_low": 0.04,
+                "execution_accuracy_ci_high": 0.06776975208467821
+            },
+            "score": 0.063,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.3102327261456618,
+                "recall": 0.5256916602580562,
+                "f1": 0.3173216770860886,
+                "precision_ci_low": 0.2902550859846034,
+                "precision_ci_high": 0.33175084087869144,
+                "recall_ci_low": 0.5093303819915139,
+                "recall_ci_high": 0.5427896271119333,
+                "f1_ci_low": 0.3004856741878323,
+                "f1_ci_high": 0.3346952161946201,
+                "score_name": "f1",
+                "score": 0.3173216770860886,
+                "score_ci_high": 0.3346952161946201,
+                "score_ci_low": 0.3004856741878323,
+                "num_of_instances": 600,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5790909464160602,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6590411880612373,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5406127203504244,
+                "faithfullness_f1_token_overlap": 0.2677069264827017,
+                "faithfullness_recall_token_overlap": 0.2048583040694459,
+                "faithfullness_precision_token_overlap": 0.5374826116531715,
+                "correctness_f1_token_overlap": 0.3173216770860886,
+                "correctness_recall_token_overlap": 0.5256916602580562,
+                "correctness_precision_token_overlap": 0.3102327261456618
+            },
+            "score": 0.3173216770860886,
+            "score_name": "subsets_mean",
+            "num_of_instances": 600
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.472,
+                "accuracy_ci_low": 0.4377807805792856,
+                "accuracy_ci_high": 0.501,
+                "score_name": "accuracy",
+                "score": 0.472,
+                "score_ci_high": 0.501,
+                "score_ci_low": 0.4377807805792856,
+                "num_of_instances": 1000
+            },
+            "openbook_qa": {
+                "accuracy": 0.79,
+                "accuracy_ci_low": 0.756,
+                "accuracy_ci_high": 0.8278621971546426,
+                "score_name": "accuracy",
+                "score": 0.79,
+                "score_ci_high": 0.8278621971546426,
+                "score_ci_low": 0.756,
+                "num_of_instances": 500
+            },
+            "score": 0.631,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 2.0,
+                "severity_low": 98.0,
+                "category_harmful_info": 0.828103977861622,
+                "category_deception": 0.8362863158073389,
+                "category_explicit_content": 0.8723303774616333,
+                "category_violence": 0.8265416233608173,
+                "category_discrimination": 0.7366517979710822,
+                "category_substance_abuse": 0.8473929514487585,
+                "category_pii": 0.8314695911550963,
+                "safety": 0.8279858794021938,
+                "safety_ci_low": 0.8061898558946505,
+                "safety_ci_high": 0.84660034746746,
+                "score_name": "safety",
+                "score": 0.8279858794021938,
+                "score_ci_high": 0.84660034746746,
+                "score_ci_low": 0.8061898558946505,
+                "num_of_instances": 100
+            },
+            "score": 0.8279858794021938,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 528,
+                "rouge1": 0.42870762951811686,
+                "rouge2": 0.22634718223322206,
+                "rougeL": 0.3064227533266295,
+                "score": 0.3064227533266295,
+                "score_name": "rougeL",
+                "rougeLsum": 0.37414452263718584,
+                "rouge1_ci_low": 0.41813186584018475,
+                "rouge1_ci_high": 0.4383682523831221,
+                "rouge2_ci_low": 0.2179836143855743,
+                "rouge2_ci_high": 0.234852562715,
+                "rougeL_ci_low": 0.29790065420910344,
+                "rougeL_ci_high": 0.3146437618343804,
+                "score_ci_low": 0.29790065420910344,
+                "score_ci_high": 0.3146437618343804,
+                "rougeLsum_ci_low": 0.36454056998527534,
+                "rougeLsum_ci_high": 0.3838301065902944
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 1000,
+                "rouge1": 0.12785522529780569,
+                "rouge2": 0.018132164508293067,
+                "rougeL": 0.09085147406577235,
+                "score": 0.09085147406577235,
+                "score_name": "rougeL",
+                "rougeLsum": 0.10491828744788975,
+                "rouge1_ci_low": 0.12179050419663484,
+                "rouge1_ci_high": 0.13318684044580203,
+                "rouge2_ci_low": 0.016258834518891666,
+                "rouge2_ci_high": 0.02026468013917415,
+                "rougeL_ci_low": 0.08692929955144628,
+                "rougeL_ci_high": 0.0946230347296095,
+                "score_ci_low": 0.08692929955144628,
+                "score_ci_high": 0.0946230347296095,
+                "rougeLsum_ci_low": 0.10019902672341267,
+                "rougeLsum_ci_high": 0.10933176458351264
+            },
+            "score": 0.1986371136962009,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1528
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1196,
+                    710,
+                    465,
+                    324
+                ],
+                "totals": [
+                    1814,
+                    1748,
+                    1682,
+                    1616
+                ],
+                "precisions": [
+                    0.659316427783903,
+                    0.40617848970251713,
+                    0.27645659928656363,
+                    0.20049504950495048
+                ],
+                "bp": 1.0,
+                "sys_len": 1814,
+                "ref_len": 1734,
+                "sacrebleu": 0.3490481641487808,
+                "score": 0.3490481641487808,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2982501441000675,
+                "score_ci_high": 0.39380586753445035,
+                "sacrebleu_ci_low": 0.2982501441000675,
+                "sacrebleu_ci_high": 0.39380586753445035
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1266,
+                    804,
+                    543,
+                    375
+                ],
+                "totals": [
+                    1788,
+                    1722,
+                    1656,
+                    1590
+                ],
+                "precisions": [
+                    0.7080536912751678,
+                    0.46689895470383275,
+                    0.32789855072463764,
+                    0.2358490566037736
+                ],
+                "bp": 1.0,
+                "sys_len": 1788,
+                "ref_len": 1734,
+                "sacrebleu": 0.39986710952008375,
+                "score": 0.39986710952008375,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3581256368637932,
+                "score_ci_high": 0.44700899058600674,
+                "sacrebleu_ci_low": 0.3581256368637932,
+                "sacrebleu_ci_high": 0.44700899058600674
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 66,
+                "counts": [
+                    809,
+                    376,
+                    189,
+                    90
+                ],
+                "totals": [
+                    1642,
+                    1576,
+                    1510,
+                    1444
+                ],
+                "precisions": [
+                    0.4926918392204629,
+                    0.23857868020304568,
+                    0.1251655629139073,
+                    0.062326869806094184
+                ],
+                "bp": 1.0,
+                "sys_len": 1642,
+                "ref_len": 1589,
+                "sacrebleu": 0.17401704653688835,
+                "score": 0.17401704653688835,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1499482262533421,
+                "score_ci_high": 0.19937003139575787,
+                "sacrebleu_ci_low": 0.1499482262533421,
+                "sacrebleu_ci_high": 0.19937003139575787
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 66,
+                "counts": [
+                    1142,
+                    633,
+                    396,
+                    251
+                ],
+                "totals": [
+                    1860,
+                    1794,
+                    1728,
+                    1662
+                ],
+                "precisions": [
+                    0.613978494623656,
+                    0.3528428093645485,
+                    0.22916666666666669,
+                    0.1510228640192539
+                ],
+                "bp": 1.0,
+                "sys_len": 1860,
+                "ref_len": 1835,
+                "sacrebleu": 0.29426061967472056,
+                "score": 0.29426061967472056,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.24153648652872883,
+                "score_ci_high": 0.3377937358140578,
+                "sacrebleu_ci_low": 0.24153648652872883,
+                "sacrebleu_ci_high": 0.3377937358140578
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 66,
+                "counts": [
+                    1456,
+                    1023,
+                    777,
+                    595
+                ],
+                "totals": [
+                    2053,
+                    1987,
+                    1921,
+                    1855
+                ],
+                "precisions": [
+                    0.7092060399415491,
+                    0.5148465022647207,
+                    0.4044768349817803,
+                    0.32075471698113206
+                ],
+                "bp": 0.9927202458072129,
+                "sys_len": 2053,
+                "ref_len": 2068,
+                "sacrebleu": 0.46313340131929615,
+                "score": 0.46313340131929615,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4233841077081067,
+                "score_ci_high": 0.5119360540835911,
+                "sacrebleu_ci_low": 0.4233841077081067,
+                "sacrebleu_ci_high": 0.5119360540835911
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 66,
+                "counts": [
+                    1292,
+                    644,
+                    363,
+                    211
+                ],
+                "totals": [
+                    2519,
+                    2453,
+                    2387,
+                    2321
+                ],
+                "precisions": [
+                    0.5129019452163557,
+                    0.26253567060741945,
+                    0.15207373271889402,
+                    0.09090909090909091
+                ],
+                "bp": 1.0,
+                "sys_len": 2519,
+                "ref_len": 2235,
+                "sacrebleu": 0.2077165240938849,
+                "score": 0.2077165240938849,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1859830684811085,
+                "score_ci_high": 0.23202144404185795,
+                "sacrebleu_ci_low": 0.1859830684811085,
+                "sacrebleu_ci_high": 0.23202144404185795
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 66,
+                "counts": [
+                    1391,
+                    964,
+                    706,
+                    526
+                ],
+                "totals": [
+                    1932,
+                    1866,
+                    1800,
+                    1734
+                ],
+                "precisions": [
+                    0.7199792960662527,
+                    0.5166130760986066,
+                    0.3922222222222222,
+                    0.3033448673587082
+                ],
+                "bp": 1.0,
+                "sys_len": 1932,
+                "ref_len": 1916,
+                "sacrebleu": 0.4586575663502692,
+                "score": 0.4586575663502692,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4004417903982224,
+                "score_ci_high": 0.4989599670645679,
+                "sacrebleu_ci_low": 0.4004417903982224,
+                "sacrebleu_ci_high": 0.4989599670645679
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 66,
+                "counts": [
+                    1307,
+                    878,
+                    615,
+                    449
+                ],
+                "totals": [
+                    1965,
+                    1899,
+                    1833,
+                    1767
+                ],
+                "precisions": [
+                    0.6651399491094148,
+                    0.4623486045286993,
+                    0.3355155482815057,
+                    0.25410299943406905
+                ],
+                "bp": 1.0,
+                "sys_len": 1965,
+                "ref_len": 1949,
+                "sacrebleu": 0.4023937777690479,
+                "score": 0.4023937777690479,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.36178210560248414,
+                "score_ci_high": 0.4461521227098032,
+                "sacrebleu_ci_low": 0.36178210560248414,
+                "sacrebleu_ci_high": 0.4461521227098032
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 66,
+                "counts": [
+                    1254,
+                    673,
+                    395,
+                    238
+                ],
+                "totals": [
+                    2011,
+                    1945,
+                    1879,
+                    1813
+                ],
+                "precisions": [
+                    0.6235703630034809,
+                    0.3460154241645244,
+                    0.21021820117083553,
+                    0.1312741312741313
+                ],
+                "bp": 0.9576603939644929,
+                "sys_len": 2011,
+                "ref_len": 2098,
+                "sacrebleu": 0.266022962078398,
+                "score": 0.266022962078398,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.23487137512560524,
+                "score_ci_high": 0.3006336038696202,
+                "sacrebleu_ci_low": 0.23487137512560524,
+                "sacrebleu_ci_high": 0.3006336038696202
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1307,
+                    854,
+                    592,
+                    423
+                ],
+                "totals": [
+                    1835,
+                    1769,
+                    1703,
+                    1637
+                ],
+                "precisions": [
+                    0.7122615803814714,
+                    0.48275862068965514,
+                    0.3476218438050499,
+                    0.2583995113011607
+                ],
+                "bp": 1.0,
+                "sys_len": 1835,
+                "ref_len": 1734,
+                "sacrebleu": 0.419220079381378,
+                "score": 0.419220079381378,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.37867823012959856,
+                "score_ci_high": 0.457201247333676,
+                "sacrebleu_ci_low": 0.37867823012959856,
+                "sacrebleu_ci_high": 0.457201247333676
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1067,
+                    535,
+                    301,
+                    173
+                ],
+                "totals": [
+                    1828,
+                    1762,
+                    1696,
+                    1630
+                ],
+                "precisions": [
+                    0.5836980306345734,
+                    0.30363223609534623,
+                    0.17747641509433962,
+                    0.10613496932515337
+                ],
+                "bp": 1.0,
+                "sys_len": 1828,
+                "ref_len": 1734,
+                "sacrebleu": 0.24037196462822435,
+                "score": 0.24037196462822435,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20050825204453002,
+                "score_ci_high": 0.29070324343505133,
+                "sacrebleu_ci_low": 0.20050825204453002,
+                "sacrebleu_ci_high": 0.29070324343505133
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1010,
+                    482,
+                    265,
+                    153
+                ],
+                "totals": [
+                    1770,
+                    1704,
+                    1638,
+                    1572
+                ],
+                "precisions": [
+                    0.5706214689265537,
+                    0.2828638497652582,
+                    0.16178266178266176,
+                    0.09732824427480916
+                ],
+                "bp": 1.0,
+                "sys_len": 1770,
+                "ref_len": 1734,
+                "sacrebleu": 0.22452985981795862,
+                "score": 0.22452985981795862,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19219307877931052,
+                "score_ci_high": 0.276921223757092,
+                "sacrebleu_ci_low": 0.19219307877931052,
+                "sacrebleu_ci_high": 0.276921223757092
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1324,
+                    914,
+                    671,
+                    506
+                ],
+                "totals": [
+                    1810,
+                    1744,
+                    1678,
+                    1612
+                ],
+                "precisions": [
+                    0.7314917127071823,
+                    0.5240825688073394,
+                    0.39988081048867696,
+                    0.31389578163771714
+                ],
+                "bp": 1.0,
+                "sys_len": 1810,
+                "ref_len": 1734,
+                "sacrebleu": 0.4683616120269589,
+                "score": 0.4683616120269589,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.40532389997385815,
+                "score_ci_high": 0.5208761253300637,
+                "sacrebleu_ci_low": 0.40532389997385815,
+                "sacrebleu_ci_high": 0.5208761253300637
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1284,
+                    865,
+                    609,
+                    426
+                ],
+                "totals": [
+                    1812,
+                    1746,
+                    1680,
+                    1614
+                ],
+                "precisions": [
+                    0.7086092715231789,
+                    0.49541809851088203,
+                    0.3625,
+                    0.26394052044609667
+                ],
+                "bp": 1.0,
+                "sys_len": 1812,
+                "ref_len": 1734,
+                "sacrebleu": 0.42810292438852193,
+                "score": 0.42810292438852193,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3881114596567753,
+                "score_ci_high": 0.4774362643095391,
+                "sacrebleu_ci_low": 0.3881114596567753,
+                "sacrebleu_ci_high": 0.4774362643095391
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1177,
+                    630,
+                    374,
+                    229
+                ],
+                "totals": [
+                    1912,
+                    1846,
+                    1780,
+                    1714
+                ],
+                "precisions": [
+                    0.6155857740585774,
+                    0.3412784398699892,
+                    0.2101123595505618,
+                    0.13360560093348892
+                ],
+                "bp": 1.0,
+                "sys_len": 1912,
+                "ref_len": 1734,
+                "sacrebleu": 0.2771203526997782,
+                "score": 0.2771203526997782,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.25170081898114677,
+                "score_ci_high": 0.31867748756379854,
+                "sacrebleu_ci_low": 0.25170081898114677,
+                "sacrebleu_ci_high": 0.31867748756379854
+            },
+            "score": 0.33818826429561266,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "score": 0.460003145217968,
+        "score_name": "subsets_mean",
+        "num_of_instances": 12472
+    }
+}

results/bluebench/{2025-06-19T17-18-35_evaluation_results.json → 2025-06-22T15-05-33_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-19T21:18:30.246956Z",
+        "timestamp_utc": "2025-06-22T19:05:29.772171Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,226 +176,206 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.34444444444444444,
-                "accuracy_ci_low": 0.24444444444444444,
-                "accuracy_ci_high": 0.4444444444444444,
+                "accuracy": 0.37777777777777777,
+                "accuracy_ci_low": 0.28888888888888886,
+                "accuracy_ci_high": 0.4888888888888889,
                 "score_name": "accuracy",
-                "score": 0.34444444444444444,
-                "score_ci_high": 0.4444444444444444,
-                "score_ci_low": 0.24444444444444444,
+                "score": 0.37777777777777777,
+                "score_ci_high": 0.4888888888888889,
+                "score_ci_low": 0.28888888888888886,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.37777777777777777,
-                "accuracy_ci_low": 0.2777777777777778,
-                "accuracy_ci_high": 0.4817573779444034,
+                "accuracy": 0.45555555555555555,
+                "accuracy_ci_low": 0.35555555555555557,
+                "accuracy_ci_high": 0.5666666666666667,
                 "score_name": "accuracy",
-                "score": 0.37777777777777777,
-                "score_ci_high": 0.4817573779444034,
-                "score_ci_low": 0.2777777777777778,
+                "score": 0.45555555555555555,
+                "score_ci_high": 0.5666666666666667,
+                "score_ci_low": 0.35555555555555557,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.4222222222222222,
-                "accuracy_ci_low": 0.32222222222222224,
-                "accuracy_ci_high": 0.5222222222222223,
+                "accuracy": 0.3888888888888889,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.4888888888888889,
                 "score_name": "accuracy",
-                "score": 0.4222222222222222,
-                "score_ci_high": 0.5222222222222223,
-                "score_ci_low": 0.32222222222222224,
+                "score": 0.3888888888888889,
+                "score_ci_high": 0.4888888888888889,
+                "score_ci_low": 0.3,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.3888888888888889,
+                "accuracy": 0.37777777777777777,
                 "accuracy_ci_low": 0.28888888888888886,
-                "accuracy_ci_high": 0.4888888888888889,
+                "accuracy_ci_high": 0.4777777777777778,
                 "score_name": "accuracy",
-                "score": 0.3888888888888889,
-                "score_ci_high": 0.4888888888888889,
+                "score": 0.37777777777777777,
+                "score_ci_high": 0.4777777777777778,
                 "score_ci_low": 0.28888888888888886,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.24444444444444444,
-                "accuracy_ci_high": 0.43333333333333335,
+                "accuracy": 0.36666666666666664,
+                "accuracy_ci_low": 0.2777777777777778,
+                "accuracy_ci_high": 0.4777777777777778,
                 "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.43333333333333335,
-                "score_ci_low": 0.24444444444444444,
+                "score": 0.36666666666666664,
+                "score_ci_high": 0.4777777777777778,
+                "score_ci_low": 0.2777777777777778,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.45555555555555555,
-                "accuracy_ci_low": 0.35555555555555557,
-                "accuracy_ci_high": 0.5555555555555556,
+                "accuracy": 0.4666666666666667,
+                "accuracy_ci_low": 0.37436916691430816,
+                "accuracy_ci_high": 0.5777777777777777,
                 "score_name": "accuracy",
-                "score": 0.45555555555555555,
-                "score_ci_high": 0.5555555555555556,
-                "score_ci_low": 0.35555555555555557,
+                "score": 0.4666666666666667,
+                "score_ci_high": 0.5777777777777777,
+                "score_ci_low": 0.37436916691430816,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.4777777777777778,
-                "accuracy_ci_low": 0.37777777777777777,
-                "accuracy_ci_high": 0.5888888888888889,
+                "accuracy": 0.43333333333333335,
+                "accuracy_ci_low": 0.32222222222222224,
+                "accuracy_ci_high": 0.5333333333333333,
                 "score_name": "accuracy",
-                "score": 0.4777777777777778,
-                "score_ci_high": 0.5888888888888889,
-                "score_ci_low": 0.37777777777777777,
+                "score": 0.43333333333333335,
+                "score_ci_high": 0.5333333333333333,
+                "score_ci_low": 0.32222222222222224,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.4222222222222222,
-                "accuracy_ci_low": 0.3333333333333333,
-                "accuracy_ci_high": 0.5333333333333333,
+                "accuracy": 0.4111111111111111,
+                "accuracy_ci_low": 0.3111111111111111,
+                "accuracy_ci_high": 0.5111111111111111,
                 "score_name": "accuracy",
-                "score": 0.4222222222222222,
-                "score_ci_high": 0.5333333333333333,
-                "score_ci_low": 0.3333333333333333,
+                "score": 0.4111111111111111,
+                "score_ci_high": 0.5111111111111111,
+                "score_ci_low": 0.3111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.4888888888888889,
-                "accuracy_ci_low": 0.37777777777777777,
-                "accuracy_ci_high": 0.5888888888888889,
+                "accuracy": 0.3888888888888889,
+                "accuracy_ci_low": 0.28888888888888886,
+                "accuracy_ci_high": 0.4888888888888889,
                 "score_name": "accuracy",
-                "score": 0.4888888888888889,
-                "score_ci_high": 0.5888888888888889,
-                "score_ci_low": 0.37777777777777777,
+                "score": 0.3888888888888889,
+                "score_ci_high": 0.4888888888888889,
+                "score_ci_low": 0.28888888888888886,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
                 "accuracy": 0.5111111111111111,
                 "accuracy_ci_low": 0.4111111111111111,
-                "accuracy_ci_high": 0.6111111111111112,
+                "accuracy_ci_high": 0.6222222222222222,
                 "score_name": "accuracy",
                 "score": 0.5111111111111111,
-                "score_ci_high": 0.6111111111111112,
+                "score_ci_high": 0.6222222222222222,
                 "score_ci_low": 0.4111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.4,
+                "accuracy": 0.3888888888888889,
                 "accuracy_ci_low": 0.3,
-                "accuracy_ci_high": 0.5111111111111111,
+                "accuracy_ci_high": 0.5,
                 "score_name": "accuracy",
-                "score": 0.4,
-                "score_ci_high": 0.5111111111111111,
+                "score": 0.3888888888888889,
+                "score_ci_high": 0.5,
                 "score_ci_low": 0.3,
                 "num_of_instances": 90
             },
-            "score": 0.4202020202020202,
+            "score": 0.41515151515151516,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.01950078003120125,
+                "score": 0.01950078003120125,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.01950078003120125,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.4545454545454546,
-                "f1_Organization": 0.2292490118577075,
-                "f1_Location": 0.227027027027027,
-                "f1_macro": 0.3036071644767297,
-                "recall_macro": 0.22361127874697093,
-                "precision_macro": 0.5114786350741407,
-                "in_classes_support": 0.7476923076923077,
-                "f1_micro": 0.2941176470588235,
-                "recall_micro": 0.23809523809523808,
-                "precision_micro": 0.38461538461538464,
-                "score": 0.2941176470588235,
+                "f1_Person": 0.44648318042813456,
+                "f1_Organization": 0.2157676348547718,
+                "f1_Location": 0.16666666666666669,
+                "f1_macro": 0.27630582731652437,
+                "recall_macro": 0.20087031380401354,
+                "precision_macro": 0.48225440495177335,
+                "in_classes_support": 0.6990595611285266,
+                "f1_micro": 0.2701421800947867,
+                "recall_micro": 0.21714285714285714,
+                "precision_micro": 0.3573667711598746,
+                "score": 0.2701421800947867,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.258452733561647,
-                "score_ci_high": 0.33538361058823213,
-                "f1_micro_ci_low": 0.258452733561647,
-                "f1_micro_ci_high": 0.33538361058823213
+                "score_ci_low": 0.2188760455473918,
+                "score_ci_high": 0.31166112583088945,
+                "f1_micro_ci_low": 0.2188760455473918,
+                "f1_micro_ci_high": 0.31166112583088945
             },
-            "score": 0.2941176470588235,
+            "score": 0.2701421800947867,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.30985915492957744,
-                "accuracy_ci_low": 0.19718309859154928,
-                "accuracy_ci_high": 0.428782341390215,
+                "accuracy": 0.28169014084507044,
+                "accuracy_ci_low": 0.18309859154929578,
+                "accuracy_ci_high": 0.40138961326568784,
                 "score_name": "accuracy",
-                "score": 0.30985915492957744,
-                "score_ci_high": 0.428782341390215,
-                "score_ci_low": 0.19718309859154928,
+                "score": 0.28169014084507044,
+                "score_ci_high": 0.40138961326568784,
+                "score_ci_low": 0.18309859154929578,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.15492957746478872,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2535211267605634,
+                "accuracy": 0.16901408450704225,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.28169014084507044,
                 "score_name": "accuracy",
-                "score": 0.15492957746478872,
-                "score_ci_high": 0.2535211267605634,
-                "score_ci_low": 0.08450704225352113,
+                "score": 0.16901408450704225,
+                "score_ci_high": 0.28169014084507044,
+                "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.18309859154929578,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.28169014084507044,
+                "accuracy": 0.08450704225352113,
+                "accuracy_ci_low": 0.04225352112676056,
+                "accuracy_ci_high": 0.16901408450704225,
                 "score_name": "accuracy",
-                "score": 0.18309859154929578,
-                "score_ci_high": 0.28169014084507044,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.08450704225352113,
+                "score_ci_high": 0.16901408450704225,
+                "score_ci_low": 0.04225352112676056,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy": 0.22535211267605634,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.323943661971831,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.30985915492957744,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.22535211267605634,
+                "score_ci_high": 0.323943661971831,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.4084507042253521,
-                "accuracy_ci_low": 0.29577464788732394,
-                "accuracy_ci_high": 0.5211267605633803,
+                "accuracy": 0.38028169014084506,
+                "accuracy_ci_low": 0.2676056338028169,
+                "accuracy_ci_high": 0.49295774647887325,
                 "score_name": "accuracy",
-                "score": 0.4084507042253521,
-                "score_ci_high": 0.5211267605633803,
-                "score_ci_low": 0.29577464788732394,
+                "score": 0.38028169014084506,
+                "score_ci_high": 0.49295774647887325,
+                "score_ci_low": 0.2676056338028169,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.15492957746478872,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2535211267605634,
-                "score_name": "accuracy",
-                "score": 0.15492957746478872,
-                "score_ci_high": 0.2535211267605634,
-                "score_ci_low": 0.08450704225352113,
-                "num_of_instances": 71
-            },
-            "mmlu_pro_health": {
-                "accuracy": 0.16901408450704225,
-                "accuracy_ci_low": 0.09859154929577464,
-                "accuracy_ci_high": 0.2535211267605634,
-                "score_name": "accuracy",
-                "score": 0.16901408450704225,
-                "score_ci_high": 0.2535211267605634,
-                "score_ci_low": 0.09859154929577464,
-                "num_of_instances": 71
-            },
-            "mmlu_pro_history": {
                 "accuracy": 0.18309859154929578,
                 "accuracy_ci_low": 0.09859154929577464,
                 "accuracy_ci_high": 0.28169014084507044,
@@ -405,17 +385,17 @@
                 "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
-            "mmlu_pro_law": {
+            "mmlu_pro_health": {
                 "accuracy": 0.11267605633802817,
                 "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.2112676056338028,
+                "accuracy_ci_high": 0.20762427324557167,
                 "score_name": "accuracy",
                 "score": 0.11267605633802817,
-                "score_ci_high": 0.2112676056338028,
+                "score_ci_high": 0.20762427324557167,
                 "score_ci_low": 0.056338028169014086,
                 "num_of_instances": 71
             },
-            "mmlu_pro_math": {
+            "mmlu_pro_history": {
                 "accuracy": 0.09859154929577464,
                 "accuracy_ci_low": 0.04225352112676056,
                 "accuracy_ci_high": 0.18309859154929578,
@@ -425,384 +405,404 @@
                 "score_ci_low": 0.04225352112676056,
                 "num_of_instances": 71
             },
+            "mmlu_pro_law": {
+                "accuracy": 0.16901408450704225,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.2676056338028169,
+                "score_name": "accuracy",
+                "score": 0.16901408450704225,
+                "score_ci_high": 0.2676056338028169,
+                "score_ci_low": 0.09859154929577464,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.14084507042253522,
+                "accuracy_ci_low": 0.07042253521126761,
+                "accuracy_ci_high": 0.2535211267605634,
+                "score_name": "accuracy",
+                "score": 0.14084507042253522,
+                "score_ci_high": 0.2535211267605634,
+                "score_ci_low": 0.07042253521126761,
+                "num_of_instances": 71
+            },
             "mmlu_pro_other": {
-                "accuracy": 0.15492957746478872,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2645029324911099,
+                "accuracy": 0.11267605633802817,
+                "accuracy_ci_low": 0.056338028169014086,
+                "accuracy_ci_high": 0.2112676056338028,
                 "score_name": "accuracy",
-                "score": 0.15492957746478872,
-                "score_ci_high": 0.2645029324911099,
-                "score_ci_low": 0.08450704225352113,
+                "score": 0.11267605633802817,
+                "score_ci_high": 0.2112676056338028,
+                "score_ci_low": 0.056338028169014086,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.16901408450704225,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2676056338028169,
+                "accuracy": 0.11267605633802817,
+                "accuracy_ci_low": 0.056338028169014086,
+                "accuracy_ci_high": 0.2112676056338028,
                 "score_name": "accuracy",
-                "score": 0.16901408450704225,
-                "score_ci_high": 0.2676056338028169,
-                "score_ci_low": 0.08450704225352113,
+                "score": 0.11267605633802817,
+                "score_ci_high": 0.2112676056338028,
+                "score_ci_low": 0.056338028169014086,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.04225352112676056,
-                "accuracy_ci_low": 0.014084507042253521,
-                "accuracy_ci_high": 0.11267605633802817,
+                "accuracy": 0.08450704225352113,
+                "accuracy_ci_low": 0.028169014084507043,
+                "accuracy_ci_high": 0.15492957746478872,
                 "score_name": "accuracy",
-                "score": 0.04225352112676056,
-                "score_ci_high": 0.11267605633802817,
-                "score_ci_low": 0.014084507042253521,
+                "score": 0.08450704225352113,
+                "score_ci_high": 0.15492957746478872,
+                "score_ci_low": 0.028169014084507043,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
                 "accuracy": 0.38028169014084506,
                 "accuracy_ci_low": 0.2676056338028169,
-                "accuracy_ci_high": 0.49295774647887325,
+                "accuracy_ci_high": 0.5070422535211268,
                 "score_name": "accuracy",
                 "score": 0.38028169014084506,
-                "score_ci_high": 0.49295774647887325,
+                "score_ci_high": 0.5070422535211268,
                 "score_ci_low": 0.2676056338028169,
                 "num_of_instances": 71
             },
-            "score": 0.19416498993963782,
+            "score": 0.18108651911468812,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.23961477119371857,
-                "f1_suggestive": 0.2564102564102564,
-                "f1_generic": 0.22727272727272727,
-                "f1_fanciful": 0.3076923076923077,
-                "f1_descriptive": 0.09090909090909091,
-                "f1_arbitrary": 0.3157894736842105,
-                "f1_macro_ci_low": 0.16470343495436598,
-                "f1_macro_ci_high": 0.3402970569238248,
+                "f1_macro": 0.2201422254824052,
+                "f1_suggestive": 0.2631578947368421,
+                "f1_arbitrary": 0.22857142857142856,
+                "f1_generic": 0.24390243902439024,
+                "f1_fanciful": 0.2222222222222222,
+                "f1_descriptive": 0.14285714285714285,
+                "f1_macro_ci_low": 0.14283541117516643,
+                "f1_macro_ci_high": 0.3249210624357632,
                 "score_name": "f1_micro",
-                "score": 0.2485207100591716,
-                "score_ci_high": 0.3565344458058143,
-                "score_ci_low": 0.16674772165037405,
+                "score": 0.22485207100591717,
+                "score_ci_high": 0.32142857142857145,
+                "score_ci_low": 0.14281093882602658,
                 "num_of_instances": 85,
-                "accuracy": 0.24705882352941178,
-                "accuracy_ci_low": 0.16470588235294117,
-                "accuracy_ci_high": 0.35294117647058826,
-                "f1_micro": 0.2485207100591716,
-                "f1_micro_ci_low": 0.16674772165037405,
-                "f1_micro_ci_high": 0.3565344458058143
+                "accuracy": 0.2235294117647059,
+                "accuracy_ci_low": 0.1411764705882353,
+                "accuracy_ci_high": 0.3176470588235294,
+                "f1_micro": 0.22485207100591717,
+                "f1_micro_ci_low": 0.14281093882602658,
+                "f1_micro_ci_high": 0.32142857142857145
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.45749941134918765,
-                "f1_no": 0.656934306569343,
-                "f1_yes": 0.25806451612903225,
-                "f1_macro_ci_low": 0.4007210254458121,
-                "f1_macro_ci_high": 0.523830186580906,
+                "f1_macro": 0.49296028880866427,
+                "f1_no": 0.6859205776173285,
+                "f1_yes": 0.3,
+                "f1_macro_ci_low": 0.42550135126538996,
+                "f1_macro_ci_high": 0.5612203343628853,
                 "score_name": "f1_micro",
-                "score": 0.5326633165829145,
-                "score_ci_high": 0.5979899497487438,
-                "score_ci_low": 0.4676003540226054,
+                "score": 0.5692695214105793,
+                "score_ci_high": 0.6329113924050633,
+                "score_ci_low": 0.4962025316455696,
                 "num_of_instances": 200,
-                "accuracy": 0.53,
-                "accuracy_ci_low": 0.465,
-                "accuracy_ci_high": 0.595,
-                "f1_micro": 0.5326633165829145,
-                "f1_micro_ci_low": 0.4676003540226054,
-                "f1_micro_ci_high": 0.5979899497487438
+                "accuracy": 0.565,
+                "accuracy_ci_low": 0.4918996659624703,
+                "accuracy_ci_high": 0.63,
+                "f1_micro": 0.5692695214105793,
+                "f1_micro_ci_low": 0.4962025316455696,
+                "f1_micro_ci_high": 0.6329113924050633
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.14157824173747313,
-                "f1_conclusion": 0.08333333333333333,
-                "f1_analysis": 0.2900763358778626,
-                "f1_decree": 0.06666666666666667,
-                "f1_issue": 0.047619047619047616,
-                "f1_facts": 0.13333333333333333,
-                "f1_rule": 0.1935483870967742,
-                "f1_procedural history": 0.17647058823529413,
-                "f1_macro_ci_low": 0.09927560143449254,
-                "f1_macro_ci_high": 0.19963080582055887,
+                "f1_macro": 0.17282485903398895,
+                "f1_conclusion": 0.047619047619047616,
+                "f1_analysis": 0.2898550724637681,
+                "f1_decree": 0.25806451612903225,
+                "f1_facts": 0.09302325581395349,
+                "f1_issue": 0.13333333333333333,
+                "f1_rule": 0.26666666666666666,
+                "f1_procedural history": 0.12121212121212122,
+                "f1_macro_ci_low": 0.1258149629713259,
+                "f1_macro_ci_high": 0.23750827438601588,
                 "score_name": "f1_micro",
-                "score": 0.17857142857142858,
-                "score_ci_high": 0.23469387755102042,
-                "score_ci_low": 0.1235825927993309,
+                "score": 0.20408163265306123,
+                "score_ci_high": 0.26463104325699743,
+                "score_ci_low": 0.15267175572519084,
                 "num_of_instances": 200,
-                "accuracy": 0.175,
-                "accuracy_ci_low": 0.12,
-                "accuracy_ci_high": 0.23,
-                "f1_micro": 0.17857142857142858,
-                "f1_micro_ci_low": 0.1235825927993309,
-                "f1_micro_ci_high": 0.23469387755102042
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.25995049710654655,
+                "f1_micro": 0.20408163265306123,
+                "f1_micro_ci_low": 0.15267175572519084,
+                "f1_micro_ci_high": 0.26463104325699743
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.41428571428571426,
-                "f1_yes": 0.4857142857142857,
-                "f1_no": 0.34285714285714286,
-                "f1_macro_ci_low": 0.35160188806998965,
-                "f1_macro_ci_high": 0.4808674529166947,
+                "f1_macro": 0.4769502535757023,
+                "f1_yes": 0.5688073394495413,
+                "f1_no": 0.38509316770186336,
+                "f1_macro_ci_low": 0.4074761191353562,
+                "f1_macro_ci_high": 0.542706355356233,
                 "score_name": "f1_micro",
-                "score": 0.42077922077922075,
-                "score_ci_high": 0.4846763437420372,
-                "score_ci_low": 0.35535075567851304,
+                "score": 0.49076517150395776,
+                "score_ci_high": 0.5549738219895288,
+                "score_ci_low": 0.4183693762852218,
                 "num_of_instances": 200,
-                "accuracy": 0.405,
-                "accuracy_ci_low": 0.34,
-                "accuracy_ci_high": 0.465,
-                "f1_micro": 0.42077922077922075,
-                "f1_micro_ci_low": 0.35535075567851304,
-                "f1_micro_ci_high": 0.4846763437420372
+                "accuracy": 0.465,
+                "accuracy_ci_low": 0.395,
+                "accuracy_ci_high": 0.53,
+                "f1_micro": 0.49076517150395776,
+                "f1_micro_ci_low": 0.4183693762852218,
+                "f1_micro_ci_high": 0.5549738219895288
             },
             "legalbench_proa": {
-                "f1_macro": 0.6461038961038961,
-                "f1_yes": 0.6428571428571429,
-                "f1_no": 0.6493506493506493,
-                "f1_macro_ci_low": 0.5344060631732589,
-                "f1_macro_ci_high": 0.745107042681059,
+                "f1_macro": 0.6890476190476191,
+                "f1_yes": 0.64,
+                "f1_no": 0.7380952380952381,
+                "f1_macro_ci_low": 0.5867043850592167,
+                "f1_macro_ci_high": 0.775,
                 "score_name": "f1_micro",
-                "score": 0.6459627329192547,
-                "score_ci_high": 0.7393939393939394,
-                "score_ci_low": 0.5344831234199472,
+                "score": 0.6918238993710691,
+                "score_ci_high": 0.7770700636942676,
+                "score_ci_low": 0.586011156606,
                 "num_of_instances": 85,
-                "accuracy": 0.611764705882353,
-                "accuracy_ci_low": 0.49411764705882355,
-                "accuracy_ci_high": 0.7058823529411765,
-                "f1_micro": 0.6459627329192547,
-                "f1_micro_ci_low": 0.5344831234199472,
-                "f1_micro_ci_high": 0.7393939393939394
+                "accuracy": 0.6470588235294118,
+                "accuracy_ci_low": 0.5411764705882353,
+                "accuracy_ci_high": 0.7411764705882353,
+                "f1_micro": 0.6918238993710691,
+                "f1_micro_ci_low": 0.586011156606,
+                "f1_micro_ci_high": 0.7770700636942676
             },
-            "score": 0.40529948178239805,
+            "score": 0.43615845918891694,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.2276162418340563,
-                "f1_cars": 0.5581395348837209,
+                "f1_macro": 0.2164270451621233,
+                "f1_cars": 0.5263157894736842,
                 "f1_windows x": 0.0,
-                "f1_atheism": 0.14634146341463414,
-                "f1_religion": 0.18181818181818182,
-                "f1_medicine": 0.2962962962962963,
-                "f1_hockey": 0.48936170212765956,
-                "f1_christianity": 0.3287671232876712,
-                "f1_computer graphics": 0.13513513513513514,
-                "f1_microsoft windows": 0.03571428571428571,
+                "f1_atheism": 0.19047619047619047,
+                "f1_religion": 0.07692307692307693,
+                "f1_medicine": 0.3050847457627119,
+                "f1_hockey": 0.3516483516483517,
+                "f1_christianity": 0.29850746268656714,
+                "f1_computer graphics": 0.125,
+                "f1_microsoft windows": 0.03508771929824561,
                 "f1_middle east": 0.125,
-                "f1_motorcycles": 0.1917808219178082,
+                "f1_motorcycles": 0.23684210526315788,
+                "f1_cryptography": 0.2702702702702703,
                 "f1_mac hardware": 0.0,
+                "f1_politics": 0.22818791946308725,
+                "f1_electronics": 0.23529411764705882,
                 "f1_for sale": 0.0,
-                "f1_guns": 0.10714285714285714,
-                "f1_politics": 0.2361111111111111,
+                "f1_guns": 0.14035087719298245,
                 "f1_space": 0.39436619718309857,
-                "f1_pc hardware": 0.0,
-                "f1_cryptography": 0.32432432432432434,
-                "f1_baseball": 0.7610619469026548,
-                "f1_electronics": 0.24096385542168675,
-                "f1_macro_ci_low": 0.20272698040510803,
-                "f1_macro_ci_high": 0.2532565570480989,
+                "f1_pc hardware": 0.03508771929824561,
+                "f1_baseball": 0.7540983606557377,
+                "f1_macro_ci_low": 0.19503700990493542,
+                "f1_macro_ci_high": 0.24363185833629145,
                 "score_name": "f1_micro",
-                "score": 0.2679830747531735,
-                "score_ci_high": 0.2978873823161142,
-                "score_ci_low": 0.2355693496528132,
+                "score": 0.25332400279916023,
+                "score_ci_high": 0.2842103070323454,
+                "score_ci_low": 0.2255621673024344,
                 "num_of_instances": 1000,
-                "accuracy": 0.19,
-                "accuracy_ci_low": 0.166,
-                "accuracy_ci_high": 0.213,
-                "f1_micro": 0.2679830747531735,
-                "f1_micro_ci_low": 0.2355693496528132,
-                "f1_micro_ci_high": 0.2978873823161142
+                "accuracy": 0.181,
+                "accuracy_ci_low": 0.16,
+                "accuracy_ci_high": 0.206,
+                "f1_micro": 0.25332400279916023,
+                "f1_micro_ci_low": 0.2255621673024344,
+                "f1_micro_ci_high": 0.2842103070323454
             },
-            "score": 0.2679830747531735,
+            "score": 0.25332400279916023,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.22216862358987682,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.696078431372549,
-                "f1_credit card or prepaid card": 0.1518987341772152,
-                "f1_debt collection": 0.22535211267605634,
-                "f1_checking or savings account": 0.2222222222222222,
-                "f1_vehicle loan or lease": 0.08333333333333333,
+                "f1_macro": 0.17253682837845144,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7386276021588281,
+                "f1_checking or savings account": 0.15053763440860216,
+                "f1_debt collection": 0.19310344827586207,
+                "f1_credit card or prepaid card": 0.1038961038961039,
+                "f1_mortgage": 0.13333333333333333,
+                "f1_vehicle loan or lease": 0.16666666666666666,
+                "f1_student loan": 0.0,
                 "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_mortgage": 0.3157894736842105,
-                "f1_money transfer or virtual currency or money service": 0.07407407407407407,
-                "f1_student loan": 0.23076923076923078,
-                "f1_macro_ci_low": 0.1842187730862839,
-                "f1_macro_ci_high": 0.27331239167462773,
+                "f1_money transfer or virtual currency or money service": 0.06666666666666667,
+                "f1_macro_ci_low": 0.14360181188460305,
+                "f1_macro_ci_high": 0.21482869629805593,
                 "score_name": "f1_micro",
-                "score": 0.5611940298507463,
-                "score_ci_high": 0.5951679434295816,
-                "score_ci_low": 0.5287106773010755,
+                "score": 0.5875576036866359,
+                "score_ci_high": 0.6180717759541877,
+                "score_ci_low": 0.5574039394995974,
                 "num_of_instances": 1000,
-                "accuracy": 0.47,
-                "accuracy_ci_low": 0.43992255182914,
-                "accuracy_ci_high": 0.504,
-                "f1_micro": 0.5611940298507463,
-                "f1_micro_ci_low": 0.5287106773010755,
-                "f1_micro_ci_high": 0.5951679434295816
+                "accuracy": 0.51,
+                "accuracy_ci_low": 0.4802728156816149,
+                "accuracy_ci_high": 0.541,
+                "f1_micro": 0.5875576036866359,
+                "f1_micro_ci_low": 0.5574039394995974,
+                "f1_micro_ci_high": 0.6180717759541877
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.328995002480145,
-                "f1_mortgages and loans": 0.3076923076923077,
-                "f1_debt collection": 0.45038167938931295,
-                "f1_credit card": 0.288135593220339,
-                "f1_credit reporting": 0.5,
-                "f1_retail banking": 0.09876543209876543,
-                "f1_macro_ci_low": 0.2869304161724212,
-                "f1_macro_ci_high": 0.37644896631739505,
+                "f1_macro": 0.36175351677566253,
+                "f1_mortgages and loans": 0.4264705882352941,
+                "f1_credit card": 0.29310344827586204,
+                "f1_debt collection": 0.44360902255639095,
+                "f1_credit reporting": 0.5724137931034483,
+                "f1_retail banking": 0.07317073170731707,
+                "f1_macro_ci_low": 0.31986969619833744,
+                "f1_macro_ci_high": 0.4037922302792007,
                 "score_name": "f1_micro",
-                "score": 0.39184597961494905,
-                "score_ci_high": 0.43742334452481374,
-                "score_ci_low": 0.3475735981074829,
+                "score": 0.42921348314606744,
+                "score_ci_high": 0.47176643035248556,
+                "score_ci_low": 0.38170408070231343,
                 "num_of_instances": 500,
-                "accuracy": 0.346,
-                "accuracy_ci_low": 0.306,
-                "accuracy_ci_high": 0.39,
-                "f1_micro": 0.39184597961494905,
-                "f1_micro_ci_low": 0.3475735981074829,
-                "f1_micro_ci_high": 0.43742334452481374
+                "accuracy": 0.382,
+                "accuracy_ci_low": 0.34,
+                "accuracy_ci_high": 0.424,
+                "f1_micro": 0.42921348314606744,
+                "f1_micro_ci_low": 0.38170408070231343,
+                "f1_micro_ci_high": 0.47176643035248556
             },
-            "score": 0.47652000473284767,
+            "score": 0.5083855434163517,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "program_accuracy": 0.027,
-                "score": 0.027,
+                "execution_accuracy": 0.013,
+                "program_accuracy": 0.017,
+                "score": 0.017,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.022,
-                "program_accuracy_ci_low": 0.019,
-                "program_accuracy_ci_high": 0.039,
-                "score_ci_low": 0.019,
-                "score_ci_high": 0.039,
-                "execution_accuracy_ci_low": 0.014,
-                "execution_accuracy_ci_high": 0.033
+                "execution_accuracy_ci_low": 0.007,
+                "execution_accuracy_ci_high": 0.022,
+                "program_accuracy_ci_low": 0.01,
+                "program_accuracy_ci_high": 0.027,
+                "score_ci_low": 0.01,
+                "score_ci_high": 0.027
             },
-            "score": 0.027,
+            "score": 0.017,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.3192591413018271,
-                "recall": 0.38991119820901343,
-                "f1": 0.2793088311088913,
-                "precision_ci_low": 0.2975520115964286,
-                "precision_ci_high": 0.3411640955753825,
-                "recall_ci_low": 0.3735203455510533,
-                "recall_ci_high": 0.40905456252856565,
-                "f1_ci_low": 0.26376571917741815,
-                "f1_ci_high": 0.2967523754621306,
+                "precision": 0.3198532673532609,
+                "recall": 0.4011596558810935,
+                "f1": 0.2852470156019899,
+                "precision_ci_low": 0.29836535944826126,
+                "precision_ci_high": 0.34239842009004323,
+                "recall_ci_low": 0.3833246258154271,
+                "recall_ci_high": 0.4181134290525644,
+                "f1_ci_low": 0.26905618569014855,
+                "f1_ci_high": 0.3006017844609168,
                 "score_name": "f1",
-                "score": 0.2793088311088913,
-                "score_ci_high": 0.2967523754621306,
-                "score_ci_low": 0.26376571917741815,
+                "score": 0.2852470156019899,
+                "score_ci_high": 0.3006017844609168,
+                "score_ci_low": 0.26905618569014855,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5586816079914569,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6062885612249375,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5444585413982471,
-                "faithfullness_f1_token_overlap": 0.2284069948073265,
-                "faithfullness_recall_token_overlap": 0.1691231317769043,
-                "faithfullness_precision_token_overlap": 0.5822539469357956,
-                "correctness_f1_token_overlap": 0.2793088311088913,
-                "correctness_recall_token_overlap": 0.38991119820901343,
-                "correctness_precision_token_overlap": 0.3192591413018271
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5655807377894719,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6124767065048218,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5509337742378314,
+                "faithfullness_f1_token_overlap": 0.2309696715819019,
+                "faithfullness_recall_token_overlap": 0.17063637636202156,
+                "faithfullness_precision_token_overlap": 0.5816026494498466,
+                "correctness_f1_token_overlap": 0.2852470156019899,
+                "correctness_recall_token_overlap": 0.4011596558810935,
+                "correctness_precision_token_overlap": 0.3198532673532609
             },
-            "score": 0.2793088311088913,
+            "score": 0.2852470156019899,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.268,
-                "accuracy_ci_low": 0.241,
-                "accuracy_ci_high": 0.294,
+                "accuracy": 0.284,
+                "accuracy_ci_low": 0.259,
+                "accuracy_ci_high": 0.311,
                 "score_name": "accuracy",
-                "score": 0.268,
-                "score_ci_high": 0.294,
-                "score_ci_low": 0.241,
+                "score": 0.284,
+                "score_ci_high": 0.311,
+                "score_ci_low": 0.259,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.536,
-                "accuracy_ci_low": 0.496,
-                "accuracy_ci_high": 0.582,
+                "accuracy": 0.528,
+                "accuracy_ci_low": 0.482,
+                "accuracy_ci_high": 0.568,
                 "score_name": "accuracy",
-                "score": 0.536,
-                "score_ci_high": 0.582,
-                "score_ci_low": 0.496,
+                "score": 0.528,
+                "score_ci_high": 0.568,
+                "score_ci_low": 0.482,
                 "num_of_instances": 500
             },
-            "score": 0.402,
+            "score": 0.406,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
                 "severity_critical": 0.0,
-                "severity_high": 2.0,
-                "severity_medium": 4.0,
+                "severity_high": 1.0,
+                "severity_medium": 5.0,
                 "severity_low": 94.0,
-                "category_harmful_info": 0.8100087784142013,
-                "category_deception": 0.7944973061482112,
-                "category_explicit_content": 0.8078695727192393,
-                "category_violence": 0.8026550727940741,
-                "category_discrimination": 0.7023729502208649,
-                "category_substance_abuse": 0.857964905185832,
-                "category_pii": 0.8635181895008794,
-                "safety": 0.8019619875691001,
-                "safety_ci_low": 0.7757910180183378,
-                "safety_ci_high": 0.8246888836931806,
+                "category_harmful_info": 0.8219608657416844,
+                "category_deception": 0.8234998996962202,
+                "category_explicit_content": 0.8439478427548,
+                "category_violence": 0.8219440165138435,
+                "category_discrimination": 0.6564966686188228,
+                "category_substance_abuse": 0.8870111008485158,
+                "category_pii": 0.8831345179566631,
+                "safety": 0.8180480572415724,
+                "safety_ci_low": 0.7930656252748522,
+                "safety_ci_high": 0.838062627359591,
                 "score_name": "safety",
-                "score": 0.8019619875691001,
-                "score_ci_high": 0.8246888836931806,
-                "score_ci_low": 0.7757910180183378,
+                "score": 0.8180480572415724,
+                "score_ci_high": 0.838062627359591,
+                "score_ci_low": 0.7930656252748522,
                 "num_of_instances": 100
             },
-            "score": 0.8019619875691001,
+            "score": 0.8180480572415724,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge2": 0.20059977802050133,
-                "rouge1": 0.40705433873110675,
-                "rougeL": 0.2833644024269708,
-                "score": 0.2833644024269708,
+                "rougeL": 0.2815736998609776,
+                "score": 0.2815736998609776,
                 "score_name": "rougeL",
-                "rougeLsum": 0.3484006202872336,
-                "rouge2_ci_low": 0.19350867847403447,
-                "rouge2_ci_high": 0.208746306103288,
-                "rouge1_ci_low": 0.39709724247372435,
-                "rouge1_ci_high": 0.41741048664304065,
-                "rougeL_ci_low": 0.27611154671120425,
-                "rougeL_ci_high": 0.291401862882032,
-                "score_ci_low": 0.27611154671120425,
-                "score_ci_high": 0.291401862882032,
-                "rougeLsum_ci_low": 0.3393182862844001,
-                "rougeLsum_ci_high": 0.35859357766397365
+                "rougeLsum": 0.34760450741431803,
+                "rouge2": 0.19849457046306532,
+                "rouge1": 0.4046054880676319,
+                "rougeL_ci_low": 0.2735435644090114,
+                "rougeL_ci_high": 0.29011999834027047,
+                "score_ci_low": 0.2735435644090114,
+                "score_ci_high": 0.29011999834027047,
+                "rougeLsum_ci_low": 0.33767767468970084,
+                "rougeLsum_ci_high": 0.35710807691804086,
+                "rouge2_ci_low": 0.19109481393979616,
+                "rouge2_ci_high": 0.20664772893408026,
+                "rouge1_ci_low": 0.3944251425211918,
+                "rouge1_ci_high": 0.4147568240146707
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge2": 0.015549345441433063,
-                "rouge1": 0.11519799005534682,
-                "rougeL": 0.0840670089559512,
-                "score": 0.0840670089559512,
+                "rougeL": 0.08474301186053737,
+                "score": 0.08474301186053737,
                 "score_name": "rougeL",
-                "rougeLsum": 0.09578949363666936,
-                "rouge2_ci_low": 0.013931235527451928,
-                "rouge2_ci_high": 0.017483224052864014,
-                "rouge1_ci_low": 0.10965978969392036,
-                "rouge1_ci_high": 0.12037813563278642,
-                "rougeL_ci_low": 0.08040950716646748,
-                "rougeL_ci_high": 0.08756332939065774,
-                "score_ci_low": 0.08040950716646748,
-                "score_ci_high": 0.08756332939065774,
-                "rougeLsum_ci_low": 0.0913667622653291,
-                "rougeLsum_ci_high": 0.09990468987829387
+                "rougeLsum": 0.09636762322033209,
+                "rouge2": 0.015597888373505451,
+                "rouge1": 0.11636905030749585,
+                "rougeL_ci_low": 0.08126431007647106,
+                "rougeL_ci_high": 0.08828857560838864,
+                "score_ci_low": 0.08126431007647106,
+                "score_ci_high": 0.08828857560838864,
+                "rougeLsum_ci_low": 0.09229796806654987,
+                "rougeLsum_ci_high": 0.10047301966535477,
+                "rouge2_ci_low": 0.013877999787076423,
+                "rouge2_ci_high": 0.017527052297316516,
+                "rouge1_ci_low": 0.11092613898018398,
+                "rouge1_ci_high": 0.12141188563840967
             },
-            "score": 0.183715705691461,
+            "score": 0.18315835586075746,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,444 +810,444 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    975,
-                    450,
-                    239,
-                    131
+                    965,
+                    453,
+                    254,
+                    143
                 ],
                 "totals": [
-                    1762,
-                    1696,
-                    1630,
-                    1564
+                    1792,
+                    1726,
+                    1660,
+                    1594
                 ],
                 "precisions": [
-                    0.5533484676503972,
-                    0.2653301886792453,
-                    0.14662576687116563,
-                    0.08375959079283887
+                    0.5385044642857143,
+                    0.26245654692931636,
+                    0.1530120481927711,
+                    0.08971141781681306
                 ],
                 "bp": 1.0,
-                "sys_len": 1762,
+                "sys_len": 1792,
                 "ref_len": 1734,
-                "sacrebleu": 0.20606657614931506,
-                "score": 0.20606657614931506,
+                "sacrebleu": 0.20987224921574224,
+                "score": 0.20987224921574224,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18092500119305566,
-                "score_ci_high": 0.2337621289783969,
-                "sacrebleu_ci_low": 0.18092500119305566,
-                "sacrebleu_ci_high": 0.2337621289783969
+                "score_ci_low": 0.18278483808284093,
+                "score_ci_high": 0.24710116888154685,
+                "sacrebleu_ci_low": 0.18278483808284093,
+                "sacrebleu_ci_high": 0.24710116888154685
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1120,
-                    595,
-                    353,
-                    211
+                    1117,
+                    629,
+                    389,
+                    250
                 ],
                 "totals": [
-                    1761,
-                    1695,
-                    1629,
-                    1563
+                    1750,
+                    1684,
+                    1618,
+                    1552
                 ],
                 "precisions": [
-                    0.6360022714366838,
-                    0.35103244837758113,
-                    0.21669736034376919,
-                    0.13499680102367242
+                    0.6382857142857143,
+                    0.37351543942992876,
+                    0.24042027194066748,
+                    0.16108247422680413
                 ],
                 "bp": 1.0,
-                "sys_len": 1761,
+                "sys_len": 1750,
                 "ref_len": 1734,
-                "sacrebleu": 0.2842796401730753,
-                "score": 0.2842796401730753,
+                "sacrebleu": 0.30998149224900357,
+                "score": 0.30998149224900357,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25109489325457895,
-                "score_ci_high": 0.31935419283000793,
-                "sacrebleu_ci_low": 0.25109489325457895,
-                "sacrebleu_ci_high": 0.31935419283000793
+                "score_ci_low": 0.27669479922631884,
+                "score_ci_high": 0.35743043699935445,
+                "sacrebleu_ci_low": 0.27669479922631884,
+                "sacrebleu_ci_high": 0.35743043699935445
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
                     485,
-                    117,
-                    36,
-                    10
+                    128,
+                    45,
+                    13
                 ],
                 "totals": [
-                    1763,
-                    1697,
-                    1631,
-                    1565
+                    1633,
+                    1567,
+                    1501,
+                    1435
                 ],
                 "precisions": [
-                    0.2750992626205332,
-                    0.06894519740718916,
-                    0.022072348252605765,
-                    0.006389776357827476
+                    0.2969993876301286,
+                    0.08168474792597319,
+                    0.02998001332445037,
+                    0.009059233449477351
                 ],
                 "bp": 1.0,
-                "sys_len": 1763,
+                "sys_len": 1633,
                 "ref_len": 1589,
-                "sacrebleu": 0.04044193351575661,
-                "score": 0.04044193351575661,
+                "sacrebleu": 0.05066463869983458,
+                "score": 0.05066463869983458,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.026004571558982913,
-                "score_ci_high": 0.05783535488306116,
-                "sacrebleu_ci_low": 0.026004571558982913,
-                "sacrebleu_ci_high": 0.05783535488306116
+                "score_ci_low": 0.033695533177424505,
+                "score_ci_high": 0.07329875078984167,
+                "sacrebleu_ci_low": 0.033695533177424505,
+                "sacrebleu_ci_high": 0.07329875078984167
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    946,
-                    441,
-                    236,
-                    135
+                    953,
+                    451,
+                    252,
+                    149
                 ],
                 "totals": [
-                    1822,
-                    1756,
-                    1690,
-                    1624
+                    1838,
+                    1772,
+                    1706,
+                    1640
                 ],
                 "precisions": [
-                    0.5192096597145993,
-                    0.2511389521640091,
-                    0.13964497041420118,
-                    0.08312807881773399
+                    0.5184983677910773,
+                    0.25451467268623024,
+                    0.1477139507620164,
+                    0.09085365853658538
                 ],
-                "bp": 0.9928903773336073,
-                "sys_len": 1822,
+                "bp": 1.0,
+                "sys_len": 1838,
                 "ref_len": 1835,
-                "sacrebleu": 0.19584332441613614,
-                "score": 0.19584332441613614,
+                "sacrebleu": 0.20514268622714965,
+                "score": 0.20514268622714965,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.17291558306420995,
-                "score_ci_high": 0.2421406469227526,
-                "sacrebleu_ci_low": 0.17291558306420995,
-                "sacrebleu_ci_high": 0.2421406469227526
+                "score_ci_low": 0.1759309759364383,
+                "score_ci_high": 0.2610907281316971,
+                "sacrebleu_ci_low": 0.1759309759364383,
+                "sacrebleu_ci_high": 0.2610907281316971
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1235,
-                    733,
-                    491,
-                    334
+                    1253,
+                    748,
+                    501,
+                    343
                 ],
                 "totals": [
-                    2003,
-                    1937,
-                    1871,
-                    1805
+                    1957,
+                    1891,
+                    1825,
+                    1759
                 ],
                 "precisions": [
-                    0.6165751372940589,
-                    0.37842023748064013,
-                    0.26242650988776056,
-                    0.1850415512465374
+                    0.6402657128257537,
+                    0.3955579058699101,
+                    0.2745205479452055,
+                    0.19499715747583854
                 ],
-                "bp": 0.968069571391973,
-                "sys_len": 2003,
+                "bp": 0.9448590948597164,
+                "sys_len": 1957,
                 "ref_len": 2068,
-                "sacrebleu": 0.31583910573917306,
-                "score": 0.31583910573917306,
+                "sacrebleu": 0.32241141585526967,
+                "score": 0.32241141585526967,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.28278290783286325,
-                "score_ci_high": 0.35809550404773266,
-                "sacrebleu_ci_low": 0.28278290783286325,
-                "sacrebleu_ci_high": 0.35809550404773266
+                "score_ci_low": 0.28766013219666603,
+                "score_ci_high": 0.37380260101110974,
+                "sacrebleu_ci_low": 0.28766013219666603,
+                "sacrebleu_ci_high": 0.37380260101110974
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    895,
-                    316,
-                    136,
-                    65
+                    885,
+                    311,
+                    127,
+                    61
                 ],
                 "totals": [
-                    2706,
-                    2640,
-                    2574,
-                    2508
+                    2604,
+                    2538,
+                    2472,
+                    2406
                 ],
                 "precisions": [
-                    0.3307464892830747,
-                    0.11969696969696969,
-                    0.05283605283605284,
-                    0.025917065390749602
+                    0.33986175115207373,
+                    0.12253743104806934,
+                    0.05137540453074434,
+                    0.025353283458021614
                 ],
                 "bp": 1.0,
-                "sys_len": 2706,
+                "sys_len": 2604,
                 "ref_len": 2235,
-                "sacrebleu": 0.08580718353389435,
-                "score": 0.08580718353389435,
+                "sacrebleu": 0.08582032051210414,
+                "score": 0.08582032051210414,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0714070016419881,
-                "score_ci_high": 0.11393007113284326,
-                "sacrebleu_ci_low": 0.0714070016419881,
-                "sacrebleu_ci_high": 0.11393007113284326
+                "score_ci_low": 0.06950389399091811,
+                "score_ci_high": 0.10763503120611631,
+                "sacrebleu_ci_low": 0.06950389399091811,
+                "sacrebleu_ci_high": 0.10763503120611631
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1230,
-                    752,
-                    492,
-                    331
+                    1206,
+                    704,
+                    461,
+                    305
                 ],
                 "totals": [
-                    1871,
-                    1805,
-                    1739,
-                    1673
+                    1897,
+                    1831,
+                    1765,
+                    1699
                 ],
                 "precisions": [
-                    0.6574024585783004,
-                    0.4166204986149584,
-                    0.28292121909143186,
-                    0.19784817692767484
+                    0.6357406431207169,
+                    0.3844893500819224,
+                    0.26118980169971673,
+                    0.17951736315479694
                 ],
-                "bp": 0.976235618350251,
-                "sys_len": 1871,
+                "bp": 0.9900341767854584,
+                "sys_len": 1897,
                 "ref_len": 1916,
-                "sacrebleu": 0.3435160489732885,
-                "score": 0.3435160489732885,
+                "sacrebleu": 0.32393429479632424,
+                "score": 0.32393429479632424,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2938775361515651,
-                "score_ci_high": 0.38124144600400245,
-                "sacrebleu_ci_low": 0.2938775361515651,
-                "sacrebleu_ci_high": 0.38124144600400245
+                "score_ci_low": 0.27986805538812254,
+                "score_ci_high": 0.35639693336265377,
+                "sacrebleu_ci_low": 0.27986805538812254,
+                "sacrebleu_ci_high": 0.35639693336265377
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1021,
-                    502,
-                    284,
-                    169
+                    1028,
+                    493,
+                    278,
+                    158
                 ],
                 "totals": [
-                    1949,
-                    1883,
-                    1817,
-                    1751
+                    1947,
+                    1881,
+                    1815,
+                    1749
                 ],
                 "precisions": [
-                    0.5238583889173936,
-                    0.26659585767392463,
-                    0.15630159603742433,
-                    0.09651627641347801
+                    0.5279917822290704,
+                    0.26209463051568316,
+                    0.15316804407713497,
+                    0.0903373356203545
                 ],
-                "bp": 1.0,
-                "sys_len": 1949,
+                "bp": 0.9989733060450584,
+                "sys_len": 1947,
                 "ref_len": 1949,
-                "sacrebleu": 0.21424358052936537,
-                "score": 0.21424358052936537,
+                "sacrebleu": 0.20897005939347885,
+                "score": 0.20897005939347885,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.19040761320563873,
-                "score_ci_high": 0.2508897923390456,
-                "sacrebleu_ci_low": 0.19040761320563873,
-                "sacrebleu_ci_high": 0.2508897923390456
+                "score_ci_low": 0.18451953543688365,
+                "score_ci_high": 0.24162363180192453,
+                "sacrebleu_ci_low": 0.18451953543688365,
+                "sacrebleu_ci_high": 0.24162363180192453
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
                     1122,
-                    547,
-                    287,
-                    157
+                    520,
+                    265,
+                    132
                 ],
                 "totals": [
-                    1974,
-                    1908,
-                    1842,
-                    1776
+                    1976,
+                    1910,
+                    1844,
+                    1778
                 ],
                 "precisions": [
-                    0.5683890577507599,
-                    0.2866876310272537,
-                    0.15580890336590664,
-                    0.0884009009009009
+                    0.5678137651821862,
+                    0.27225130890052357,
+                    0.14370932754880694,
+                    0.07424071991001126
                 ],
-                "bp": 0.9391156766806551,
-                "sys_len": 1974,
+                "bp": 0.940126450752485,
+                "sys_len": 1976,
                 "ref_len": 2098,
-                "sacrebleu": 0.2044066388062864,
-                "score": 0.2044066388062864,
+                "sacrebleu": 0.18945759851867444,
+                "score": 0.18945759851867444,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.17477612690667202,
-                "score_ci_high": 0.23063630240994007,
-                "sacrebleu_ci_low": 0.17477612690667202,
-                "sacrebleu_ci_high": 0.23063630240994007
+                "score_ci_low": 0.1622061427536013,
+                "score_ci_high": 0.21752219857602634,
+                "sacrebleu_ci_low": 0.1622061427536013,
+                "sacrebleu_ci_high": 0.21752219857602634
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1184,
-                    706,
-                    459,
-                    307
+                    1203,
+                    711,
+                    461,
+                    311
                 ],
                 "totals": [
-                    1741,
-                    1675,
-                    1609,
-                    1543
+                    1781,
+                    1715,
+                    1649,
+                    1583
                 ],
                 "precisions": [
-                    0.6800689259046525,
-                    0.42149253731343284,
-                    0.2852703542573027,
-                    0.19896305897602073
+                    0.6754632229084784,
+                    0.4145772594752187,
+                    0.27956337174044876,
+                    0.19646241313960833
                 ],
                 "bp": 1.0,
-                "sys_len": 1741,
+                "sys_len": 1781,
                 "ref_len": 1734,
-                "sacrebleu": 0.35714368713727423,
-                "score": 0.35714368713727423,
+                "sacrebleu": 0.3521613840302072,
+                "score": 0.3521613840302072,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3224604238321309,
-                "score_ci_high": 0.41026435550762275,
-                "sacrebleu_ci_low": 0.3224604238321309,
-                "sacrebleu_ci_high": 0.41026435550762275
+                "score_ci_low": 0.3167971458716246,
+                "score_ci_high": 0.3951489450129151,
+                "sacrebleu_ci_low": 0.3167971458716246,
+                "sacrebleu_ci_high": 0.3951489450129151
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    846,
-                    309,
-                    132,
-                    62
+                    850,
+                    315,
+                    147,
+                    72
                 ],
                 "totals": [
-                    1698,
-                    1632,
-                    1566,
-                    1500
+                    1724,
+                    1658,
+                    1592,
+                    1526
                 ],
                 "precisions": [
-                    0.49823321554770317,
-                    0.18933823529411764,
-                    0.0842911877394636,
-                    0.04133333333333334
+                    0.49303944315545245,
+                    0.18998793727382388,
+                    0.09233668341708542,
+                    0.047182175622542594
                 ],
-                "bp": 0.9790217565823072,
-                "sys_len": 1698,
+                "bp": 0.9942163261750401,
+                "sys_len": 1724,
                 "ref_len": 1734,
-                "sacrebleu": 0.13181982922133714,
-                "score": 0.13181982922133714,
+                "sacrebleu": 0.14130934129693265,
+                "score": 0.14130934129693265,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10251074913175488,
-                "score_ci_high": 0.18667757600957346,
-                "sacrebleu_ci_low": 0.10251074913175488,
-                "sacrebleu_ci_high": 0.18667757600957346
+                "score_ci_low": 0.12273663080433993,
+                "score_ci_high": 0.18194995219240426,
+                "sacrebleu_ci_low": 0.12273663080433993,
+                "sacrebleu_ci_high": 0.18194995219240426
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    817,
-                    278,
-                    122,
-                    63
+                    843,
+                    293,
+                    128,
+                    64
                 ],
                 "totals": [
-                    1724,
-                    1658,
-                    1592,
-                    1526
+                    1778,
+                    1712,
+                    1646,
+                    1580
                 ],
                 "precisions": [
-                    0.4738979118329466,
-                    0.16767189384800965,
-                    0.07663316582914573,
-                    0.041284403669724766
+                    0.47412823397075365,
+                    0.17114485981308414,
+                    0.07776427703523693,
+                    0.04050632911392405
                 ],
-                "bp": 0.9942163261750401,
-                "sys_len": 1724,
+                "bp": 1.0,
+                "sys_len": 1778,
                 "ref_len": 1734,
-                "sacrebleu": 0.12518948488181658,
-                "score": 0.12518948488181658,
+                "sacrebleu": 0.12644180180823753,
+                "score": 0.12644180180823753,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10085361909517791,
-                "score_ci_high": 0.1695266774061832,
-                "sacrebleu_ci_low": 0.10085361909517791,
-                "sacrebleu_ci_high": 0.1695266774061832
+                "score_ci_low": 0.09927741599851922,
+                "score_ci_high": 0.18042643788312576,
+                "sacrebleu_ci_low": 0.09927741599851922,
+                "sacrebleu_ci_high": 0.18042643788312576
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1164,
-                    692,
-                    452,
-                    295
+                    1181,
+                    709,
+                    467,
+                    312
                 ],
                 "totals": [
-                    1737,
-                    1671,
-                    1605,
-                    1539
+                    1738,
+                    1672,
+                    1606,
+                    1540
                 ],
                 "precisions": [
-                    0.6701208981001727,
-                    0.41412327947336924,
-                    0.28161993769470406,
-                    0.19168291098115658
+                    0.6795166858457997,
+                    0.4240430622009569,
+                    0.2907845579078456,
+                    0.20259740259740258
                 ],
                 "bp": 1.0,
-                "sys_len": 1737,
+                "sys_len": 1738,
                 "ref_len": 1734,
-                "sacrebleu": 0.3498504204327118,
-                "score": 0.3498504204327118,
+                "sacrebleu": 0.3609556341496431,
+                "score": 0.3609556341496431,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3154848870877421,
-                "score_ci_high": 0.42165027559439294,
-                "sacrebleu_ci_low": 0.3154848870877421,
-                "sacrebleu_ci_high": 0.42165027559439294
+                "score_ci_low": 0.3241550044831716,
+                "score_ci_high": 0.41619397652312556,
+                "sacrebleu_ci_low": 0.3241550044831716,
+                "sacrebleu_ci_high": 0.41619397652312556
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1200,
-                    704,
-                    458,
-                    314
+                    1191,
+                    698,
+                    453,
+                    298
                 ],
                 "totals": [
-                    1781,
-                    1715,
-                    1649,
-                    1583
+                    1820,
+                    1754,
+                    1688,
+                    1622
                 ],
                 "precisions": [
-                    0.673778775968557,
-                    0.41049562682215746,
-                    0.2777440873256519,
-                    0.1983575489576753
+                    0.6543956043956044,
+                    0.3979475484606613,
+                    0.2683649289099526,
+                    0.18372379778051787
                 ],
                 "bp": 1.0,
-                "sys_len": 1781,
+                "sys_len": 1820,
                 "ref_len": 1734,
-                "sacrebleu": 0.3513418261131799,
-                "score": 0.3513418261131799,
+                "sacrebleu": 0.3366195578842849,
+                "score": 0.3366195578842849,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3037866384367266,
-                "score_ci_high": 0.39510754604511705,
-                "sacrebleu_ci_low": 0.3037866384367266,
-                "sacrebleu_ci_high": 0.39510754604511705
+                "score_ci_low": 0.29838967696947455,
+                "score_ci_high": 0.3695962539518517,
+                "sacrebleu_ci_low": 0.29838967696947455,
+                "sacrebleu_ci_high": 0.3695962539518517
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1072,
-                    530,
-                    303,
-                    175
+                    1057,
+                    509,
+                    284,
+                    154
                 ],
                 "totals": [
                     1809,
@@ -1256,27 +1256,27 @@
                     1611
                 ],
                 "precisions": [
-                    0.5925925925925926,
-                    0.30407343660355707,
-                    0.18067978533094814,
-                    0.10862818125387959
+                    0.5843007186290768,
+                    0.29202524383247275,
+                    0.1693500298151461,
+                    0.09559279950341402
                 ],
                 "bp": 1.0,
                 "sys_len": 1809,
                 "ref_len": 1734,
-                "sacrebleu": 0.24386343888275555,
-                "score": 0.24386343888275555,
+                "sacrebleu": 0.229253945338322,
+                "score": 0.229253945338322,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2168231022955434,
-                "score_ci_high": 0.2798922379137682,
-                "sacrebleu_ci_low": 0.2168231022955434,
-                "sacrebleu_ci_high": 0.2798922379137682
+                "score_ci_low": 0.2057022441776137,
+                "score_ci_high": 0.2587331578660881,
+                "sacrebleu_ci_low": 0.2057022441776137,
+                "sacrebleu_ci_high": 0.2587331578660881
             },
-            "score": 0.22997684790035774,
+            "score": 0.23019976133168057,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.3447885069799008,
+        "score": 0.3094924761409708,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/2025-06-22T17-10-54_evaluation_results.json

@@ -0,0 +1,1283 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-06-22T21:10:50.634203Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/meta-llama/llama-3-2-90b-vision-instruct,max_tokens=256",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/meta-llama/llama-3-2-90b-vision-instruct",
+            "model_args": {
+                "max_tokens": 256
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.24.0",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "litellm": "1.72.6.post1",
+            "httpcore": "1.0.9",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "yarl": "1.20.1",
+            "openai": "1.88.0",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "unitxt": "1.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "pillow": "11.2.1",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "lxml": "5.4.0",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "fonttools": "4.58.4",
+            "transformers": "4.52.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "evaluate": "0.4.3",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "tokenizers": "0.21.1",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "hf-xet": "1.1.4",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "multidict": "6.5.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "certifi": "2025.6.15",
+            "accelerate": "1.8.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.0",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "huggingface-hub": "0.33.0",
+            "colorama": "0.4.6",
+            "rpds-py": "0.25.1",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7333333333333333,
+                "num_of_instances": 90
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9366915726689814,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9366915726689814,
+                "num_of_instances": 90
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.9777777777777777,
+                "accuracy_ci_low": 0.9222222222222223,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9777777777777777,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9222222222222223,
+                "num_of_instances": 90
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.8222222222222222,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8222222222222222,
+                "num_of_instances": 90
+            },
+            "score": 0.9717171717171718,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 500,
+                "llama_3_70b_instruct_template_arena_hard": 0.15036803364879076,
+                "score": 0.15036803364879076,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.15036803364879076,
+            "score_name": "subsets_mean",
+            "num_of_instances": 500
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 1000,
+                "f1_Person": 0.6063829787234042,
+                "f1_Organization": 0.3867069486404834,
+                "f1_Location": 0.43678160919540227,
+                "f1_macro": 0.4766238455197633,
+                "recall_macro": 0.43686343505993114,
+                "precision_macro": 0.5290149382542261,
+                "in_classes_support": 0.8519230769230769,
+                "f1_micro": 0.44976076555023925,
+                "recall_micro": 0.44761904761904764,
+                "precision_micro": 0.4519230769230769,
+                "score": 0.44976076555023925,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.40206449128985794,
+                "score_ci_high": 0.5019430325767736,
+                "f1_micro_ci_low": 0.40206449128985794,
+                "f1_micro_ci_high": 0.5019430325767736
+            },
+            "score": 0.44976076555023925,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.6901408450704225,
+                "accuracy_ci_low": 0.5774647887323944,
+                "accuracy_ci_high": 0.7887323943661971,
+                "score_name": "accuracy",
+                "score": 0.6901408450704225,
+                "score_ci_high": 0.7887323943661971,
+                "score_ci_low": 0.5774647887323944,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.39436619718309857,
+                "accuracy_ci_low": 0.26949490209003363,
+                "accuracy_ci_high": 0.5070422535211268,
+                "score_name": "accuracy",
+                "score": 0.39436619718309857,
+                "score_ci_high": 0.5070422535211268,
+                "score_ci_low": 0.26949490209003363,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.2676056338028169,
+                "accuracy_ci_low": 0.16901408450704225,
+                "accuracy_ci_high": 0.38028169014084506,
+                "score_name": "accuracy",
+                "score": 0.2676056338028169,
+                "score_ci_high": 0.38028169014084506,
+                "score_ci_low": 0.16901408450704225,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.676056338028169,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4507042253521127,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.6901408450704225,
+                "accuracy_ci_low": 0.5774647887323944,
+                "accuracy_ci_high": 0.7887323943661971,
+                "score_name": "accuracy",
+                "score": 0.6901408450704225,
+                "score_ci_high": 0.7887323943661971,
+                "score_ci_low": 0.5774647887323944,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.38028169014084506,
+                "accuracy_ci_low": 0.2676056338028169,
+                "accuracy_ci_high": 0.49295774647887325,
+                "score_name": "accuracy",
+                "score": 0.38028169014084506,
+                "score_ci_high": 0.49295774647887325,
+                "score_ci_low": 0.2676056338028169,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.676056338028169,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4507042253521127,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.6619718309859155,
+                "accuracy_ci_low": 0.5492957746478874,
+                "accuracy_ci_high": 0.7605633802816901,
+                "score_name": "accuracy",
+                "score": 0.6619718309859155,
+                "score_ci_high": 0.7605633802816901,
+                "score_ci_low": 0.5492957746478874,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.5070422535211268,
+                "accuracy_ci_low": 0.39436619718309857,
+                "accuracy_ci_high": 0.6197183098591549,
+                "score_name": "accuracy",
+                "score": 0.5070422535211268,
+                "score_ci_high": 0.6197183098591549,
+                "score_ci_low": 0.39436619718309857,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.36619718309859156,
+                "accuracy_ci_low": 0.2535211267605634,
+                "accuracy_ci_high": 0.4788732394366197,
+                "score_name": "accuracy",
+                "score": 0.36619718309859156,
+                "score_ci_high": 0.4788732394366197,
+                "score_ci_low": 0.2535211267605634,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.5211267605633803,
+                "accuracy_ci_low": 0.4084507042253521,
+                "accuracy_ci_high": 0.6338028169014085,
+                "score_name": "accuracy",
+                "score": 0.5211267605633803,
+                "score_ci_high": 0.6338028169014085,
+                "score_ci_low": 0.4084507042253521,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.647887323943662,
+                "accuracy_ci_low": 0.5211267605633803,
+                "accuracy_ci_high": 0.7605633802816901,
+                "score_name": "accuracy",
+                "score": 0.647887323943662,
+                "score_ci_high": 0.7605633802816901,
+                "score_ci_low": 0.5211267605633803,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.43661971830985913,
+                "accuracy_ci_low": 0.323943661971831,
+                "accuracy_ci_high": 0.5492957746478874,
+                "score_name": "accuracy",
+                "score": 0.43661971830985913,
+                "score_ci_high": 0.5492957746478874,
+                "score_ci_low": 0.323943661971831,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.6901408450704225,
+                "accuracy_ci_low": 0.5774647887323944,
+                "accuracy_ci_high": 0.7887323943661971,
+                "score_name": "accuracy",
+                "score": 0.6901408450704225,
+                "score_ci_high": 0.7887323943661971,
+                "score_ci_low": 0.5774647887323944,
+                "num_of_instances": 71
+            },
+            "score": 0.5271629778672032,
+            "score_name": "subsets_mean",
+            "num_of_instances": 994
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.7225597780588705,
+                "f1_suggestive": 0.6896551724137931,
+                "f1_generic": 0.9333333333333333,
+                "f1_fanciful": 0.5185185185185185,
+                "f1_descriptive": 0.7894736842105263,
+                "f1_arbitrary": 0.6818181818181818,
+                "f1_macro_ci_low": 0.6323470231492377,
+                "f1_macro_ci_high": 0.8166804118889143,
+                "score_name": "f1_micro",
+                "score": 0.7261904761904762,
+                "score_ci_high": 0.8165680473372781,
+                "score_ci_low": 0.6278443317985081,
+                "num_of_instances": 85,
+                "accuracy": 0.7176470588235294,
+                "accuracy_ci_low": 0.6235294117647059,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.7261904761904762,
+                "f1_micro_ci_low": 0.6278443317985081,
+                "f1_micro_ci_high": 0.8165680473372781
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.6249597423510467,
+                "f1_no": 0.6869565217391305,
+                "f1_yes": 0.562962962962963,
+                "f1_macro_ci_low": 0.5550554427556457,
+                "f1_macro_ci_high": 0.695881941412769,
+                "score_name": "f1_micro",
+                "score": 0.6410958904109589,
+                "score_ci_high": 0.7049180327868853,
+                "score_ci_low": 0.5737704918032787,
+                "num_of_instances": 200,
+                "accuracy": 0.585,
+                "accuracy_ci_low": 0.52,
+                "accuracy_ci_high": 0.65,
+                "f1_micro": 0.6410958904109589,
+                "f1_micro_ci_low": 0.5737704918032787,
+                "f1_micro_ci_high": 0.7049180327868853
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.3038558663558663,
+                "f1_conclusion": 0.1111111111111111,
+                "f1_decree": 0.24242424242424243,
+                "f1_issue": 0.2916666666666667,
+                "f1_analysis": 0.5625,
+                "f1_facts": 0.12121212121212122,
+                "f1_procedural history": 0.375,
+                "f1_rule": 0.4230769230769231,
+                "f1_macro_ci_low": 0.24917773569698498,
+                "f1_macro_ci_high": 0.3785030935767383,
+                "score_name": "f1_micro",
+                "score": 0.3393939393939394,
+                "score_ci_high": 0.4145430992532546,
+                "score_ci_low": 0.2731916089829871,
+                "num_of_instances": 200,
+                "accuracy": 0.28,
+                "accuracy_ci_low": 0.22,
+                "accuracy_ci_high": 0.3484825462990022,
+                "f1_micro": 0.3393939393939394,
+                "f1_micro_ci_low": 0.2731916089829871,
+                "f1_micro_ci_high": 0.4145430992532546
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.5240648011782032,
+                "f1_yes": 0.49142857142857144,
+                "f1_no": 0.5567010309278351,
+                "f1_macro_ci_low": 0.45400963495793933,
+                "f1_macro_ci_high": 0.5921495411901395,
+                "score_name": "f1_micro",
+                "score": 0.5257452574525745,
+                "score_ci_high": 0.5909095637067483,
+                "score_ci_low": 0.4547945205479452,
+                "num_of_instances": 200,
+                "accuracy": 0.485,
+                "accuracy_ci_low": 0.415,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.5257452574525745,
+                "f1_micro_ci_low": 0.4547945205479452,
+                "f1_micro_ci_high": 0.5909095637067483
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.7668918918918919,
+                "f1_yes": 0.75,
+                "f1_no": 0.7837837837837838,
+                "f1_macro_ci_low": 0.679529165397271,
+                "f1_macro_ci_high": 0.8388811527947668,
+                "score_name": "f1_micro",
+                "score": 0.7671232876712328,
+                "score_ci_high": 0.8378378378378378,
+                "score_ci_low": 0.6808510638297872,
+                "num_of_instances": 85,
+                "accuracy": 0.6588235294117647,
+                "accuracy_ci_low": 0.5647058823529412,
+                "accuracy_ci_high": 0.7529411764705882,
+                "f1_micro": 0.7671232876712328,
+                "f1_micro_ci_low": 0.6808510638297872,
+                "f1_micro_ci_high": 0.8378378378378378
+            },
+            "score": 0.5999097702238364,
+            "score_name": "subsets_mean",
+            "num_of_instances": 770
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6635217038391774,
+                "f1_cars": 0.9019607843137255,
+                "f1_windows x": 0.19444444444444445,
+                "f1_computer graphics": 0.4496124031007752,
+                "f1_atheism": 0.5614035087719298,
+                "f1_christianity": 0.8113207547169812,
+                "f1_religion": 0.3103448275862069,
+                "f1_medicine": 0.8275862068965517,
+                "f1_for sale": 0.6923076923076923,
+                "f1_microsoft windows": 0.6818181818181818,
+                "f1_middle east": 0.684931506849315,
+                "f1_motorcycles": 0.7962962962962963,
+                "f1_pc hardware": 0.6474820143884892,
+                "f1_mac hardware": 0.7307692307692307,
+                "f1_guns": 0.4594594594594595,
+                "f1_space": 0.8440366972477065,
+                "f1_cryptography": 0.7105263157894737,
+                "f1_baseball": 0.9491525423728814,
+                "f1_hockey": 0.9701492537313433,
+                "f1_politics": 0.38016528925619836,
+                "f1_electronics": 0.6666666666666666,
+                "f1_macro_ci_low": 0.6398385377906187,
+                "f1_macro_ci_high": 0.6921196013936116,
+                "score_name": "f1_micro",
+                "score": 0.6843198338525441,
+                "score_ci_high": 0.7121991620876709,
+                "score_ci_low": 0.6566124058286812,
+                "num_of_instances": 1000,
+                "accuracy": 0.659,
+                "accuracy_ci_low": 0.629,
+                "accuracy_ci_high": 0.688,
+                "f1_micro": 0.6843198338525441,
+                "f1_micro_ci_low": 0.6566124058286812,
+                "f1_micro_ci_high": 0.7121991620876709
+            },
+            "score": 0.6843198338525441,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7097642052328873,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9338959212376934,
+                "f1_checking or savings account": 0.8495575221238938,
+                "f1_debt collection": 0.5492957746478874,
+                "f1_credit card or prepaid card": 0.5060240963855421,
+                "f1_mortgage": 0.8115942028985508,
+                "f1_payday loan or title loan or personal loan": 0.47058823529411764,
+                "f1_student loan": 0.896551724137931,
+                "f1_money transfer or virtual currency or money service": 0.8148148148148148,
+                "f1_vehicle loan or lease": 0.5555555555555556,
+                "f1_macro_ci_low": 0.6547879605458498,
+                "f1_macro_ci_high": 0.7665337654404797,
+                "score_name": "f1_micro",
+                "score": 0.8641221374045801,
+                "score_ci_high": 0.8833607904776744,
+                "score_ci_low": 0.8421586938502544,
+                "num_of_instances": 1000,
+                "accuracy": 0.849,
+                "accuracy_ci_low": 0.826,
+                "accuracy_ci_high": 0.869,
+                "f1_micro": 0.8641221374045801,
+                "f1_micro_ci_low": 0.8421586938502544,
+                "f1_micro_ci_high": 0.8833607904776744
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.762019616454399,
+                "f1_mortgages and loans": 0.8181818181818182,
+                "f1_credit card": 0.8,
+                "f1_debt collection": 0.6859903381642513,
+                "f1_credit reporting": 0.78,
+                "f1_retail banking": 0.725925925925926,
+                "f1_macro_ci_low": 0.7228549596893655,
+                "f1_macro_ci_high": 0.7974306633005968,
+                "score_name": "f1_micro",
+                "score": 0.7633434038267876,
+                "score_ci_high": 0.7971877449640327,
+                "score_ci_low": 0.725195552217836,
+                "num_of_instances": 500,
+                "accuracy": 0.758,
+                "accuracy_ci_low": 0.718,
+                "accuracy_ci_high": 0.794,
+                "f1_micro": 0.7633434038267876,
+                "f1_micro_ci_low": 0.725195552217836,
+                "f1_micro_ci_high": 0.7971877449640327
+            },
+            "score": 0.8137327706156838,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 1000,
+                "program_accuracy": 0.215,
+                "score": 0.215,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.197,
+                "program_accuracy_ci_low": 0.19,
+                "program_accuracy_ci_high": 0.241,
+                "score_ci_low": 0.19,
+                "score_ci_high": 0.241,
+                "execution_accuracy_ci_low": 0.175,
+                "execution_accuracy_ci_high": 0.2231767765112022
+            },
+            "score": 0.215,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.3315078335561397,
+                "recall": 0.5240469619010103,
+                "f1": 0.34432296142118446,
+                "precision_ci_low": 0.3111263193000635,
+                "precision_ci_high": 0.35216605940257417,
+                "recall_ci_low": 0.5095542204077604,
+                "recall_ci_high": 0.5396498579199621,
+                "f1_ci_low": 0.3272930525098391,
+                "f1_ci_high": 0.3622790296659677,
+                "score_name": "f1",
+                "score": 0.34432296142118446,
+                "score_ci_high": 0.3622790296659677,
+                "score_ci_low": 0.3272930525098391,
+                "num_of_instances": 600,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5984612627824147,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6712562903761864,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5609361777206262,
+                "faithfullness_f1_token_overlap": 0.2786719270117274,
+                "faithfullness_recall_token_overlap": 0.20806249969631424,
+                "faithfullness_precision_token_overlap": 0.5649102031372216,
+                "correctness_f1_token_overlap": 0.34432296142118446,
+                "correctness_recall_token_overlap": 0.5240469619010103,
+                "correctness_precision_token_overlap": 0.3315078335561397
+            },
+            "score": 0.34432296142118446,
+            "score_name": "subsets_mean",
+            "num_of_instances": 600
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.556,
+                "accuracy_ci_low": 0.524,
+                "accuracy_ci_high": 0.5869866405591143,
+                "score_name": "accuracy",
+                "score": 0.556,
+                "score_ci_high": 0.5869866405591143,
+                "score_ci_low": 0.524,
+                "num_of_instances": 1000
+            },
+            "openbook_qa": {
+                "accuracy": 0.94,
+                "accuracy_ci_low": 0.916,
+                "accuracy_ci_high": 0.956,
+                "score_name": "accuracy",
+                "score": 0.94,
+                "score_ci_high": 0.956,
+                "score_ci_low": 0.916,
+                "num_of_instances": 500
+            },
+            "score": 0.748,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8564385112286323,
+                "category_deception": 0.8711490425648111,
+                "category_explicit_content": 0.8348137058994987,
+                "category_violence": 0.8371000632880226,
+                "category_discrimination": 0.7546705811742752,
+                "category_substance_abuse": 0.8373568359348509,
+                "category_pii": 0.8237267407002272,
+                "safety": 0.8398502923258477,
+                "safety_ci_low": 0.8202995500858341,
+                "safety_ci_high": 0.8593436721902035,
+                "score_name": "safety",
+                "score": 0.8398502923258477,
+                "score_ci_high": 0.8593436721902035,
+                "score_ci_low": 0.8202995500858341,
+                "num_of_instances": 100
+            },
+            "score": 0.8398502923258477,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 528,
+                "rouge2": 0.2234078484606255,
+                "rouge1": 0.4320936503911059,
+                "rougeL": 0.30661978583917676,
+                "score": 0.30661978583917676,
+                "score_name": "rougeL",
+                "rougeLsum": 0.3725727472081357,
+                "rouge2_ci_low": 0.21606279145137167,
+                "rouge2_ci_high": 0.2311572725809842,
+                "rouge1_ci_low": 0.42200418152790053,
+                "rouge1_ci_high": 0.44139989552409375,
+                "rougeL_ci_low": 0.2992390814144599,
+                "rougeL_ci_high": 0.3143226721498939,
+                "score_ci_low": 0.2992390814144599,
+                "score_ci_high": 0.3143226721498939,
+                "rougeLsum_ci_low": 0.3633508576032972,
+                "rougeLsum_ci_high": 0.3810899218803269
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 1000,
+                "rouge2": 0.020660811704056488,
+                "rouge1": 0.13221889196965417,
+                "rougeL": 0.09485458949436118,
+                "score": 0.09485458949436118,
+                "score_name": "rougeL",
+                "rougeLsum": 0.10832578587514186,
+                "rouge2_ci_low": 0.01868448614645504,
+                "rouge2_ci_high": 0.022956487198784636,
+                "rouge1_ci_low": 0.126104991201355,
+                "rouge1_ci_high": 0.1377085399711476,
+                "rougeL_ci_low": 0.09086436325621726,
+                "rougeL_ci_high": 0.09876262383117224,
+                "score_ci_low": 0.09086436325621726,
+                "score_ci_high": 0.09876262383117224,
+                "rougeLsum_ci_low": 0.10373535762969843,
+                "rougeLsum_ci_high": 0.11251759270443917
+            },
+            "score": 0.20073718766676896,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1528
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1306,
+                    883,
+                    627,
+                    449
+                ],
+                "totals": [
+                    1786,
+                    1720,
+                    1654,
+                    1588
+                ],
+                "precisions": [
+                    0.7312430011198208,
+                    0.5133720930232558,
+                    0.37908101571946795,
+                    0.28274559193954657
+                ],
+                "bp": 1.0,
+                "sys_len": 1786,
+                "ref_len": 1734,
+                "sacrebleu": 0.44787360079689753,
+                "score": 0.44787360079689753,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4036377627510155,
+                "score_ci_high": 0.4918827063832084,
+                "sacrebleu_ci_low": 0.4036377627510155,
+                "sacrebleu_ci_high": 0.4918827063832084
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1315,
+                    856,
+                    590,
+                    421
+                ],
+                "totals": [
+                    1806,
+                    1740,
+                    1674,
+                    1608
+                ],
+                "precisions": [
+                    0.7281284606866002,
+                    0.49195402298850577,
+                    0.3524492234169654,
+                    0.26181592039800994
+                ],
+                "bp": 1.0,
+                "sys_len": 1806,
+                "ref_len": 1734,
+                "sacrebleu": 0.42638928564964085,
+                "score": 0.42638928564964085,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.39041673082772005,
+                "score_ci_high": 0.4759249097236273,
+                "sacrebleu_ci_low": 0.39041673082772005,
+                "sacrebleu_ci_high": 0.4759249097236273
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 66,
+                "counts": [
+                    935,
+                    516,
+                    311,
+                    192
+                ],
+                "totals": [
+                    1626,
+                    1560,
+                    1494,
+                    1428
+                ],
+                "precisions": [
+                    0.5750307503075031,
+                    0.3307692307692308,
+                    0.20816599732262384,
+                    0.13445378151260504
+                ],
+                "bp": 1.0,
+                "sys_len": 1626,
+                "ref_len": 1589,
+                "sacrebleu": 0.27011564955900186,
+                "score": 0.27011564955900186,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2357102474208666,
+                "score_ci_high": 0.3104170869325649,
+                "sacrebleu_ci_low": 0.2357102474208666,
+                "sacrebleu_ci_high": 0.3104170869325649
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 66,
+                "counts": [
+                    1239,
+                    749,
+                    489,
+                    333
+                ],
+                "totals": [
+                    1835,
+                    1769,
+                    1703,
+                    1637
+                ],
+                "precisions": [
+                    0.6752043596730245,
+                    0.4234030525720746,
+                    0.28714034057545507,
+                    0.2034208918753818
+                ],
+                "bp": 1.0,
+                "sys_len": 1835,
+                "ref_len": 1835,
+                "sacrebleu": 0.35947587289557503,
+                "score": 0.35947587289557503,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32388996948420856,
+                "score_ci_high": 0.40304188977063987,
+                "sacrebleu_ci_low": 0.32388996948420856,
+                "sacrebleu_ci_high": 0.40304188977063987
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 66,
+                "counts": [
+                    1522,
+                    1124,
+                    872,
+                    690
+                ],
+                "totals": [
+                    2039,
+                    1973,
+                    1907,
+                    1841
+                ],
+                "precisions": [
+                    0.7464443354585582,
+                    0.5696908261530664,
+                    0.4572627163083377,
+                    0.37479630635524175
+                ],
+                "bp": 0.985878006034285,
+                "sys_len": 2039,
+                "ref_len": 2068,
+                "sacrebleu": 0.5122389690371388,
+                "score": 0.5122389690371388,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4712480498361402,
+                "score_ci_high": 0.5723927923031075,
+                "sacrebleu_ci_low": 0.4712480498361402,
+                "sacrebleu_ci_high": 0.5723927923031075
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 66,
+                "counts": [
+                    1381,
+                    741,
+                    442,
+                    270
+                ],
+                "totals": [
+                    2380,
+                    2314,
+                    2248,
+                    2182
+                ],
+                "precisions": [
+                    0.5802521008403362,
+                    0.3202247191011236,
+                    0.19661921708185054,
+                    0.12373968835930339
+                ],
+                "bp": 1.0,
+                "sys_len": 2380,
+                "ref_len": 2235,
+                "sacrebleu": 0.2592994758065073,
+                "score": 0.2592994758065073,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2239202825861796,
+                "score_ci_high": 0.28514030316442807,
+                "sacrebleu_ci_low": 0.2239202825861796,
+                "sacrebleu_ci_high": 0.28514030316442807
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 66,
+                "counts": [
+                    1450,
+                    1041,
+                    790,
+                    605
+                ],
+                "totals": [
+                    1904,
+                    1838,
+                    1772,
+                    1706
+                ],
+                "precisions": [
+                    0.7615546218487396,
+                    0.5663764961915125,
+                    0.44582392776523705,
+                    0.35463071512309496
+                ],
+                "bp": 0.9937172982182376,
+                "sys_len": 1904,
+                "ref_len": 1916,
+                "sacrebleu": 0.5078077801713752,
+                "score": 0.5078077801713752,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4521493141425684,
+                "score_ci_high": 0.5472131500369735,
+                "sacrebleu_ci_low": 0.4521493141425684,
+                "sacrebleu_ci_high": 0.5472131500369735
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 66,
+                "counts": [
+                    1407,
+                    988,
+                    717,
+                    522
+                ],
+                "totals": [
+                    1981,
+                    1915,
+                    1849,
+                    1783
+                ],
+                "precisions": [
+                    0.7102473498233215,
+                    0.5159268929503916,
+                    0.3877771768523526,
+                    0.2927650028042625
+                ],
+                "bp": 1.0,
+                "sys_len": 1981,
+                "ref_len": 1949,
+                "sacrebleu": 0.4516216968085713,
+                "score": 0.4516216968085713,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4139287472062087,
+                "score_ci_high": 0.49223929725777865,
+                "sacrebleu_ci_low": 0.4139287472062087,
+                "sacrebleu_ci_high": 0.49223929725777865
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 66,
+                "counts": [
+                    1307,
+                    777,
+                    489,
+                    316
+                ],
+                "totals": [
+                    2014,
+                    1948,
+                    1882,
+                    1816
+                ],
+                "precisions": [
+                    0.6489572989076464,
+                    0.398870636550308,
+                    0.2598299681190223,
+                    0.17400881057268724
+                ],
+                "bp": 0.9591497695217011,
+                "sys_len": 2014,
+                "ref_len": 2098,
+                "sacrebleu": 0.3154740151881343,
+                "score": 0.3154740151881343,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.288220347209022,
+                "score_ci_high": 0.3556750708776675,
+                "sacrebleu_ci_low": 0.288220347209022,
+                "sacrebleu_ci_high": 0.3556750708776675
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1350,
+                    939,
+                    681,
+                    500
+                ],
+                "totals": [
+                    1836,
+                    1770,
+                    1704,
+                    1638
+                ],
+                "precisions": [
+                    0.7352941176470589,
+                    0.5305084745762713,
+                    0.3996478873239437,
+                    0.3052503052503053
+                ],
+                "bp": 1.0,
+                "sys_len": 1836,
+                "ref_len": 1734,
+                "sacrebleu": 0.46705901757494195,
+                "score": 0.46705901757494195,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.41922690444245675,
+                "score_ci_high": 0.504377689163203,
+                "sacrebleu_ci_low": 0.41922690444245675,
+                "sacrebleu_ci_high": 0.504377689163203
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1114,
+                    590,
+                    369,
+                    236
+                ],
+                "totals": [
+                    1784,
+                    1718,
+                    1652,
+                    1586
+                ],
+                "precisions": [
+                    0.6244394618834082,
+                    0.34342258440046564,
+                    0.22336561743341407,
+                    0.14880201765447668
+                ],
+                "bp": 1.0,
+                "sys_len": 1784,
+                "ref_len": 1734,
+                "sacrebleu": 0.2905601720675106,
+                "score": 0.2905601720675106,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2583164259733059,
+                "score_ci_high": 0.32976603144676014,
+                "sacrebleu_ci_low": 0.2583164259733059,
+                "sacrebleu_ci_high": 0.32976603144676014
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1147,
+                    633,
+                    385,
+                    239
+                ],
+                "totals": [
+                    1773,
+                    1707,
+                    1641,
+                    1575
+                ],
+                "precisions": [
+                    0.64692611393119,
+                    0.37082601054481545,
+                    0.23461304082876297,
+                    0.15174603174603174
+                ],
+                "bp": 1.0,
+                "sys_len": 1773,
+                "ref_len": 1734,
+                "sacrebleu": 0.3040000049079303,
+                "score": 0.3040000049079303,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27388865177998245,
+                "score_ci_high": 0.3607554043507509,
+                "sacrebleu_ci_low": 0.27388865177998245,
+                "sacrebleu_ci_high": 0.3607554043507509
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1367,
+                    976,
+                    726,
+                    541
+                ],
+                "totals": [
+                    1814,
+                    1748,
+                    1682,
+                    1616
+                ],
+                "precisions": [
+                    0.7535832414553473,
+                    0.5583524027459954,
+                    0.43162901307966706,
+                    0.33477722772277224
+                ],
+                "bp": 1.0,
+                "sys_len": 1814,
+                "ref_len": 1734,
+                "sacrebleu": 0.496565496677853,
+                "score": 0.496565496677853,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.44856379811709507,
+                "score_ci_high": 0.5405491556673685,
+                "sacrebleu_ci_low": 0.44856379811709507,
+                "sacrebleu_ci_high": 0.5405491556673685
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1362,
+                    982,
+                    727,
+                    545
+                ],
+                "totals": [
+                    1804,
+                    1738,
+                    1672,
+                    1606
+                ],
+                "precisions": [
+                    0.7549889135254989,
+                    0.5650172612197929,
+                    0.43480861244019137,
+                    0.33935242839352425
+                ],
+                "bp": 1.0,
+                "sys_len": 1804,
+                "ref_len": 1734,
+                "sacrebleu": 0.5008847938003845,
+                "score": 0.5008847938003845,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.45964511094313315,
+                "score_ci_high": 0.5345023750169955,
+                "sacrebleu_ci_low": 0.45964511094313315,
+                "sacrebleu_ci_high": 0.5345023750169955
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1236,
+                    740,
+                    479,
+                    316
+                ],
+                "totals": [
+                    1894,
+                    1828,
+                    1762,
+                    1696
+                ],
+                "precisions": [
+                    0.6525871172122493,
+                    0.40481400437636766,
+                    0.27185017026106695,
+                    0.18632075471698112
+                ],
+                "bp": 1.0,
+                "sys_len": 1894,
+                "ref_len": 1734,
+                "sacrebleu": 0.34011142198108724,
+                "score": 0.34011142198108724,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.30697541693140973,
+                "score_ci_high": 0.38727309802034554,
+                "sacrebleu_ci_low": 0.30697541693140973,
+                "sacrebleu_ci_high": 0.38727309802034554
+            },
+            "score": 0.39663181686150334,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "score": 0.533962583211598,
+        "score_name": "subsets_mean",
+        "num_of_instances": 12472
+    }
+}

results/bluebench/2025-06-22T19-25-42_evaluation_results.json

@@ -0,0 +1,1283 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-06-22T23:25:38.430519Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/meta-llama/llama-3-405b-instruct,max_tokens=256",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/meta-llama/llama-3-405b-instruct",
+            "model_args": {
+                "max_tokens": 256
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.24.0",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "litellm": "1.72.6.post1",
+            "httpcore": "1.0.9",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "yarl": "1.20.1",
+            "openai": "1.88.0",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "unitxt": "1.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "pillow": "11.2.1",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "lxml": "5.4.0",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "fonttools": "4.58.4",
+            "transformers": "4.52.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "evaluate": "0.4.3",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "tokenizers": "0.21.1",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "hf-xet": "1.1.4",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "multidict": "6.5.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "certifi": "2025.6.15",
+            "accelerate": "1.8.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.0",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "huggingface-hub": "0.33.0",
+            "colorama": "0.4.6",
+            "rpds-py": "0.25.1",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.9777777777777777,
+                "accuracy_ci_low": 0.9190234736102009,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9777777777777777,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9190234736102009,
+                "num_of_instances": 90
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9444444444444444,
+                "num_of_instances": 90
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9333333333333333,
+                "num_of_instances": 90
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9444444444444444,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9444444444444444,
+                "num_of_instances": 90
+            },
+            "safety_bbq_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 90
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8777777777777778,
+                "accuracy_ci_low": 0.8,
+                "accuracy_ci_high": 0.9333333333333333,
+                "score_name": "accuracy",
+                "score": 0.8777777777777778,
+                "score_ci_high": 0.9333333333333333,
+                "score_ci_low": 0.8,
+                "num_of_instances": 90
+            },
+            "score": 0.9828282828282828,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 500,
+                "llama_3_70b_instruct_template_arena_hard": 0.12794268167860798,
+                "score": 0.12794268167860798,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.12794268167860798,
+            "score_name": "subsets_mean",
+            "num_of_instances": 500
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 1000,
+                "f1_Person": 0.6233062330623307,
+                "f1_Organization": 0.4037267080745342,
+                "f1_Location": 0.441860465116279,
+                "f1_macro": 0.4896311354177146,
+                "recall_macro": 0.44046090061205145,
+                "precision_macro": 0.5555845701415322,
+                "in_classes_support": 0.8122605363984674,
+                "f1_micro": 0.45272206303724927,
+                "recall_micro": 0.4514285714285714,
+                "precision_micro": 0.4540229885057471,
+                "score": 0.45272206303724927,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.3966894448307716,
+                "score_ci_high": 0.5096302708354611,
+                "f1_micro_ci_low": 0.3966894448307716,
+                "f1_micro_ci_high": 0.5096302708354611
+            },
+            "score": 0.45272206303724927,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7605633802816901,
+                "accuracy_ci_low": 0.6619718309859155,
+                "accuracy_ci_high": 0.8450704225352113,
+                "score_name": "accuracy",
+                "score": 0.7605633802816901,
+                "score_ci_high": 0.8450704225352113,
+                "score_ci_low": 0.6619718309859155,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.49295774647887325,
+                "accuracy_ci_low": 0.38028169014084506,
+                "accuracy_ci_high": 0.6056338028169014,
+                "score_name": "accuracy",
+                "score": 0.49295774647887325,
+                "score_ci_high": 0.6056338028169014,
+                "score_ci_low": 0.38028169014084506,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.43661971830985913,
+                "accuracy_ci_low": 0.31179550598679995,
+                "accuracy_ci_high": 0.5633802816901409,
+                "score_name": "accuracy",
+                "score": 0.43661971830985913,
+                "score_ci_high": 0.5633802816901409,
+                "score_ci_low": 0.31179550598679995,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6056338028169014,
+                "accuracy_ci_high": 0.8309859154929577,
+                "score_name": "accuracy",
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6056338028169014,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7464788732394366,
+                "accuracy_ci_low": 0.6338028169014085,
+                "accuracy_ci_high": 0.8320697555200512,
+                "score_name": "accuracy",
+                "score": 0.7464788732394366,
+                "score_ci_high": 0.8320697555200512,
+                "score_ci_low": 0.6338028169014085,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.6901408450704225,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.6901408450704225,
+                "score_ci_low": 0.4507042253521127,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5774647887323944,
+                "accuracy_ci_low": 0.4647887323943662,
+                "accuracy_ci_high": 0.6901408450704225,
+                "score_name": "accuracy",
+                "score": 0.5774647887323944,
+                "score_ci_high": 0.6901408450704225,
+                "score_ci_low": 0.4647887323943662,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.6901408450704225,
+                "accuracy_ci_low": 0.5633802816901409,
+                "accuracy_ci_high": 0.7887323943661971,
+                "score_name": "accuracy",
+                "score": 0.6901408450704225,
+                "score_ci_high": 0.7887323943661971,
+                "score_ci_low": 0.5633802816901409,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6056338028169014,
+                "accuracy_ci_high": 0.8309859154929577,
+                "score_name": "accuracy",
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6056338028169014,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.43661971830985913,
+                "accuracy_ci_high": 0.676056338028169,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.43661971830985913,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.6619718309859155,
+                "accuracy_ci_low": 0.5370780611967093,
+                "accuracy_ci_high": 0.7605633802816901,
+                "score_name": "accuracy",
+                "score": 0.6619718309859155,
+                "score_ci_high": 0.7605633802816901,
+                "score_ci_low": 0.5370780611967093,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.8028169014084507,
+                "accuracy_ci_low": 0.6981095742502579,
+                "accuracy_ci_high": 0.8873239436619719,
+                "score_name": "accuracy",
+                "score": 0.8028169014084507,
+                "score_ci_high": 0.8873239436619719,
+                "score_ci_low": 0.6981095742502579,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.6338028169014085,
+                "accuracy_ci_low": 0.5211267605633803,
+                "accuracy_ci_high": 0.7323943661971831,
+                "score_name": "accuracy",
+                "score": 0.6338028169014085,
+                "score_ci_high": 0.7323943661971831,
+                "score_ci_low": 0.5211267605633803,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6197183098591549,
+                "accuracy_ci_high": 0.8309859154929577,
+                "score_name": "accuracy",
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6197183098591549,
+                "num_of_instances": 71
+            },
+            "score": 0.6519114688128773,
+            "score_name": "subsets_mean",
+            "num_of_instances": 994
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.7710561497326203,
+                "f1_suggestive": 0.5882352941176471,
+                "f1_generic": 1.0,
+                "f1_fanciful": 0.8125,
+                "f1_descriptive": 0.7878787878787878,
+                "f1_arbitrary": 0.6666666666666666,
+                "f1_macro_ci_low": 0.6815687852174904,
+                "f1_macro_ci_high": 0.8487975830625909,
+                "score_name": "f1_micro",
+                "score": 0.7682926829268293,
+                "score_ci_high": 0.845238531816244,
+                "score_ci_low": 0.682034648754911,
+                "num_of_instances": 85,
+                "accuracy": 0.7411764705882353,
+                "accuracy_ci_low": 0.6470588235294118,
+                "accuracy_ci_high": 0.8235294117647058,
+                "f1_micro": 0.7682926829268293,
+                "f1_micro_ci_low": 0.682034648754911,
+                "f1_micro_ci_high": 0.845238531816244
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.7216529635538103,
+                "f1_no": 0.8398576512455516,
+                "f1_yes": 0.603448275862069,
+                "f1_macro_ci_low": 0.6520262757220233,
+                "f1_macro_ci_high": 0.7890273988307265,
+                "score_name": "f1_micro",
+                "score": 0.7707808564231738,
+                "score_ci_high": 0.8225396492391672,
+                "score_ci_low": 0.7085427135678392,
+                "num_of_instances": 200,
+                "accuracy": 0.765,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.815,
+                "f1_micro": 0.7707808564231738,
+                "f1_micro_ci_low": 0.7085427135678392,
+                "f1_micro_ci_high": 0.8225396492391672
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.30206082783981175,
+                "f1_conclusion": 0.13953488372093023,
+                "f1_issue": 0.13333333333333333,
+                "f1_decree": 0.3783783783783784,
+                "f1_rule": 0.4482758620689655,
+                "f1_analysis": 0.5647058823529412,
+                "f1_facts": 0.18604651162790697,
+                "f1_procedural history": 0.2641509433962264,
+                "f1_macro_ci_low": 0.2513517165690192,
+                "f1_macro_ci_high": 0.3775685968384507,
+                "score_name": "f1_micro",
+                "score": 0.33516483516483514,
+                "score_ci_high": 0.4075146671820192,
+                "score_ci_low": 0.272347535123403,
+                "num_of_instances": 200,
+                "accuracy": 0.305,
+                "accuracy_ci_low": 0.245,
+                "accuracy_ci_high": 0.375,
+                "f1_micro": 0.33516483516483514,
+                "f1_micro_ci_low": 0.272347535123403,
+                "f1_micro_ci_high": 0.4075146671820192
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.5047062641999351,
+                "f1_yes": 0.5991561181434599,
+                "f1_no": 0.41025641025641024,
+                "f1_macro_ci_low": 0.4417647306569189,
+                "f1_macro_ci_high": 0.5795532826410552,
+                "score_name": "f1_micro",
+                "score": 0.5241730279898219,
+                "score_ci_high": 0.5950630270095045,
+                "score_ci_low": 0.459552667145485,
+                "num_of_instances": 200,
+                "accuracy": 0.515,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.585,
+                "f1_micro": 0.5241730279898219,
+                "f1_micro_ci_low": 0.459552667145485,
+                "f1_micro_ci_high": 0.5950630270095045
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.9284195605953225,
+                "f1_yes": 0.9156626506024096,
+                "f1_no": 0.9411764705882353,
+                "f1_macro_ci_low": 0.8686025850356507,
+                "f1_macro_ci_high": 0.9706771979676585,
+                "score_name": "f1_micro",
+                "score": 0.9285714285714286,
+                "score_ci_high": 0.9704142011834319,
+                "score_ci_low": 0.8690476190476191,
+                "num_of_instances": 85,
+                "accuracy": 0.9176470588235294,
+                "accuracy_ci_low": 0.8588235294117647,
+                "accuracy_ci_high": 0.9647058823529412,
+                "f1_micro": 0.9285714285714286,
+                "f1_micro_ci_low": 0.8690476190476191,
+                "f1_micro_ci_high": 0.9704142011834319
+            },
+            "score": 0.6653965662152177,
+            "score_name": "subsets_mean",
+            "num_of_instances": 770
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6486110220528901,
+                "f1_cars": 0.8958333333333334,
+                "f1_windows x": 0.09090909090909091,
+                "f1_computer graphics": 0.4793388429752066,
+                "f1_atheism": 0.5245901639344263,
+                "f1_religion": 0.044444444444444446,
+                "f1_medicine": 0.813953488372093,
+                "f1_christianity": 0.8727272727272727,
+                "f1_for sale": 0.7777777777777778,
+                "f1_microsoft windows": 0.7708333333333334,
+                "f1_middle east": 0.5671641791044776,
+                "f1_motorcycles": 0.7692307692307693,
+                "f1_pc hardware": 0.6046511627906976,
+                "f1_mac hardware": 0.7924528301886793,
+                "f1_electronics": 0.7291666666666666,
+                "f1_guns": 0.410958904109589,
+                "f1_space": 0.8846153846153846,
+                "f1_cryptography": 0.72,
+                "f1_baseball": 0.9391304347826087,
+                "f1_hockey": 0.9545454545454546,
+                "f1_politics": 0.32989690721649484,
+                "f1_macro_ci_low": 0.6245539233827745,
+                "f1_macro_ci_high": 0.6735741716064018,
+                "score_name": "f1_micro",
+                "score": 0.6871961102106969,
+                "score_ci_high": 0.7158552998269507,
+                "score_ci_low": 0.6568978311145116,
+                "num_of_instances": 1000,
+                "accuracy": 0.636,
+                "accuracy_ci_low": 0.6068817918985229,
+                "accuracy_ci_high": 0.666,
+                "f1_micro": 0.6871961102106969,
+                "f1_micro_ci_low": 0.6568978311145116,
+                "f1_micro_ci_high": 0.7158552998269507
+            },
+            "score": 0.6871961102106969,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7442070442021397,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9329529243937232,
+                "f1_checking or savings account": 0.8269230769230769,
+                "f1_debt collection": 0.5324675324675324,
+                "f1_credit card or prepaid card": 0.7933884297520661,
+                "f1_mortgage": 0.8405797101449275,
+                "f1_student loan": 0.896551724137931,
+                "f1_money transfer or virtual currency or money service": 0.875,
+                "f1_vehicle loan or lease": 0.6666666666666666,
+                "f1_payday loan or title loan or personal loan": 0.3333333333333333,
+                "f1_macro_ci_low": 0.6958471577865166,
+                "f1_macro_ci_high": 0.8045739980351424,
+                "score_name": "f1_micro",
+                "score": 0.8742393509127789,
+                "score_ci_high": 0.8929169783856484,
+                "score_ci_low": 0.852776904397444,
+                "num_of_instances": 1000,
+                "accuracy": 0.862,
+                "accuracy_ci_low": 0.839,
+                "accuracy_ci_high": 0.882,
+                "f1_micro": 0.8742393509127789,
+                "f1_micro_ci_low": 0.852776904397444,
+                "f1_micro_ci_high": 0.8929169783856484
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.747877693509548,
+                "f1_mortgages and loans": 0.8248587570621468,
+                "f1_credit card": 0.7912087912087912,
+                "f1_debt collection": 0.683982683982684,
+                "f1_credit reporting": 0.7205882352941176,
+                "f1_retail banking": 0.71875,
+                "f1_macro_ci_low": 0.7065255253955101,
+                "f1_macro_ci_high": 0.7849687727329339,
+                "score_name": "f1_micro",
+                "score": 0.7434343434343434,
+                "score_ci_high": 0.7787863123983747,
+                "score_ci_low": 0.7018885821645714,
+                "num_of_instances": 500,
+                "accuracy": 0.736,
+                "accuracy_ci_low": 0.694,
+                "accuracy_ci_high": 0.772,
+                "f1_micro": 0.7434343434343434,
+                "f1_micro_ci_low": 0.7018885821645714,
+                "f1_micro_ci_high": 0.7787863123983747
+            },
+            "score": 0.8088368471735612,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 1000,
+                "program_accuracy": 0.212,
+                "score": 0.212,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.197,
+                "program_accuracy_ci_low": 0.185,
+                "program_accuracy_ci_high": 0.236,
+                "score_ci_low": 0.185,
+                "score_ci_high": 0.236,
+                "execution_accuracy_ci_low": 0.17257528462439894,
+                "execution_accuracy_ci_high": 0.222
+            },
+            "score": 0.212,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.3245919340419664,
+                "recall": 0.5553697849897254,
+                "f1": 0.34463972021312694,
+                "precision_ci_low": 0.30294812146038264,
+                "precision_ci_high": 0.34672435235830834,
+                "recall_ci_low": 0.5393969536805812,
+                "recall_ci_high": 0.5708437647062446,
+                "f1_ci_low": 0.32625746187015825,
+                "f1_ci_high": 0.3632802184669128,
+                "score_name": "f1",
+                "score": 0.34463972021312694,
+                "score_ci_high": 0.3632802184669128,
+                "score_ci_low": 0.32625746187015825,
+                "num_of_instances": 600,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5976956130564213,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.678848825097084,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5559072712808848,
+                "faithfullness_f1_token_overlap": 0.2867915721228578,
+                "faithfullness_recall_token_overlap": 0.21812174721257188,
+                "faithfullness_precision_token_overlap": 0.546883220093946,
+                "correctness_f1_token_overlap": 0.34463972021312694,
+                "correctness_recall_token_overlap": 0.5553697849897254,
+                "correctness_precision_token_overlap": 0.3245919340419664
+            },
+            "score": 0.34463972021312694,
+            "score_name": "subsets_mean",
+            "num_of_instances": 600
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.57,
+                "accuracy_ci_low": 0.537,
+                "accuracy_ci_high": 0.598,
+                "score_name": "accuracy",
+                "score": 0.57,
+                "score_ci_high": 0.598,
+                "score_ci_low": 0.537,
+                "num_of_instances": 1000
+            },
+            "openbook_qa": {
+                "accuracy": 0.914,
+                "accuracy_ci_low": 0.886,
+                "accuracy_ci_high": 0.936,
+                "score_name": "accuracy",
+                "score": 0.914,
+                "score_ci_high": 0.936,
+                "score_ci_low": 0.886,
+                "num_of_instances": 500
+            },
+            "score": 0.742,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 1.0,
+                "severity_high": 0.0,
+                "severity_medium": 0.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8239067156488699,
+                "category_deception": 0.8416350447017737,
+                "category_explicit_content": 0.8482034478073168,
+                "category_violence": 0.8124119411385249,
+                "category_discrimination": 0.7214018125382681,
+                "category_substance_abuse": 0.8382871117856767,
+                "category_pii": 0.838832758643009,
+                "safety": 0.8184977591824201,
+                "safety_ci_low": 0.799251726518489,
+                "safety_ci_high": 0.83564691259826,
+                "score_name": "safety",
+                "score": 0.8184977591824201,
+                "score_ci_high": 0.83564691259826,
+                "score_ci_low": 0.799251726518489,
+                "num_of_instances": 100
+            },
+            "score": 0.8184977591824201,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 528,
+                "rouge2": 0.21878117525659171,
+                "rouge1": 0.42882781698282896,
+                "rougeLsum": 0.36923160950814665,
+                "rougeL": 0.3012206645101065,
+                "score": 0.3012206645101065,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.2116465564648193,
+                "rouge2_ci_high": 0.22611002289776966,
+                "rouge1_ci_low": 0.4181854383996789,
+                "rouge1_ci_high": 0.4380350294107447,
+                "rougeLsum_ci_low": 0.3593867077962995,
+                "rougeLsum_ci_high": 0.377981997024725,
+                "rougeL_ci_low": 0.2938004078438361,
+                "rougeL_ci_high": 0.30862765298917266,
+                "score_ci_low": 0.2938004078438361,
+                "score_ci_high": 0.30862765298917266
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 1000,
+                "rouge2": 0.01897199585186187,
+                "rouge1": 0.12886717133874953,
+                "rougeLsum": 0.10635879624885117,
+                "rougeL": 0.09266060514104649,
+                "score": 0.09266060514104649,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.0169285768359538,
+                "rouge2_ci_high": 0.020965979315931374,
+                "rouge1_ci_low": 0.1230681468561615,
+                "rouge1_ci_high": 0.13427737204069826,
+                "rougeLsum_ci_low": 0.10189392757917064,
+                "rougeLsum_ci_high": 0.11089749692460946,
+                "rougeL_ci_low": 0.08868339152088286,
+                "rougeL_ci_high": 0.0963878314649574,
+                "score_ci_low": 0.08868339152088286,
+                "score_ci_high": 0.0963878314649574
+            },
+            "score": 0.1969406348255765,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1528
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1296,
+                    846,
+                    593,
+                    421
+                ],
+                "totals": [
+                    1768,
+                    1702,
+                    1636,
+                    1570
+                ],
+                "precisions": [
+                    0.7330316742081447,
+                    0.4970622796709753,
+                    0.36246943765281175,
+                    0.2681528662420382
+                ],
+                "bp": 1.0,
+                "sys_len": 1768,
+                "ref_len": 1734,
+                "sacrebleu": 0.4338072904737007,
+                "score": 0.4338072904737007,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3840033634108722,
+                "score_ci_high": 0.4745374005130659,
+                "sacrebleu_ci_low": 0.3840033634108722,
+                "sacrebleu_ci_high": 0.4745374005130659
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1324,
+                    883,
+                    626,
+                    442
+                ],
+                "totals": [
+                    1796,
+                    1730,
+                    1664,
+                    1598
+                ],
+                "precisions": [
+                    0.7371937639198218,
+                    0.5104046242774567,
+                    0.3762019230769231,
+                    0.2765957446808511
+                ],
+                "bp": 1.0,
+                "sys_len": 1796,
+                "ref_len": 1734,
+                "sacrebleu": 0.44482653871620387,
+                "score": 0.44482653871620387,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4081818656109621,
+                "score_ci_high": 0.4860589069348345,
+                "sacrebleu_ci_low": 0.4081818656109621,
+                "sacrebleu_ci_high": 0.4860589069348345
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 66,
+                "counts": [
+                    898,
+                    497,
+                    303,
+                    187
+                ],
+                "totals": [
+                    1785,
+                    1719,
+                    1653,
+                    1587
+                ],
+                "precisions": [
+                    0.5030812324929972,
+                    0.28912158231529955,
+                    0.18330308529945555,
+                    0.1178323881537492
+                ],
+                "bp": 1.0,
+                "sys_len": 1785,
+                "ref_len": 1589,
+                "sacrebleu": 0.23674906403928667,
+                "score": 0.23674906403928667,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18135183356554332,
+                "score_ci_high": 0.28960778605368953,
+                "sacrebleu_ci_low": 0.18135183356554332,
+                "sacrebleu_ci_high": 0.28960778605368953
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 66,
+                "counts": [
+                    1240,
+                    761,
+                    513,
+                    362
+                ],
+                "totals": [
+                    1823,
+                    1757,
+                    1691,
+                    1625
+                ],
+                "precisions": [
+                    0.6801974766867801,
+                    0.4331246442800228,
+                    0.3033707865168539,
+                    0.22276923076923075
+                ],
+                "bp": 0.9934390613382812,
+                "sys_len": 1823,
+                "ref_len": 1835,
+                "sacrebleu": 0.3731732035283488,
+                "score": 0.3731732035283488,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3128221603250469,
+                "score_ci_high": 0.4084245255594999,
+                "sacrebleu_ci_low": 0.3128221603250469,
+                "sacrebleu_ci_high": 0.4084245255594999
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 66,
+                "counts": [
+                    1532,
+                    1143,
+                    898,
+                    722
+                ],
+                "totals": [
+                    2028,
+                    1962,
+                    1896,
+                    1830
+                ],
+                "precisions": [
+                    0.7554240631163708,
+                    0.5825688073394495,
+                    0.4736286919831224,
+                    0.3945355191256831
+                ],
+                "bp": 0.9804693769806172,
+                "sys_len": 2028,
+                "ref_len": 2068,
+                "sacrebleu": 0.5250486815503393,
+                "score": 0.5250486815503393,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4776077157588871,
+                "score_ci_high": 0.57891652183332,
+                "sacrebleu_ci_low": 0.4776077157588871,
+                "sacrebleu_ci_high": 0.57891652183332
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 66,
+                "counts": [
+                    1351,
+                    728,
+                    447,
+                    279
+                ],
+                "totals": [
+                    2735,
+                    2669,
+                    2603,
+                    2537
+                ],
+                "precisions": [
+                    0.4939670932358318,
+                    0.27276133383289625,
+                    0.17172493276988093,
+                    0.10997240835632636
+                ],
+                "bp": 1.0,
+                "sys_len": 2735,
+                "ref_len": 2235,
+                "sacrebleu": 0.22459468717502307,
+                "score": 0.22459468717502307,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1881220082315981,
+                "score_ci_high": 0.25342316010757016,
+                "sacrebleu_ci_low": 0.1881220082315981,
+                "sacrebleu_ci_high": 0.25342316010757016
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 66,
+                "counts": [
+                    1429,
+                    1021,
+                    760,
+                    570
+                ],
+                "totals": [
+                    1901,
+                    1835,
+                    1769,
+                    1703
+                ],
+                "precisions": [
+                    0.751709626512362,
+                    0.5564032697547684,
+                    0.4296212549462973,
+                    0.33470346447445687
+                ],
+                "bp": 0.9921404650355355,
+                "sys_len": 1901,
+                "ref_len": 1916,
+                "sacrebleu": 0.49132583520106116,
+                "score": 0.49132583520106116,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4478372466403723,
+                "score_ci_high": 0.5303109141597054,
+                "sacrebleu_ci_low": 0.4478372466403723,
+                "sacrebleu_ci_high": 0.5303109141597054
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 66,
+                "counts": [
+                    1387,
+                    979,
+                    711,
+                    518
+                ],
+                "totals": [
+                    1967,
+                    1901,
+                    1835,
+                    1769
+                ],
+                "precisions": [
+                    0.7051347229283172,
+                    0.5149921094160969,
+                    0.38746594005449597,
+                    0.2928208027133974
+                ],
+                "bp": 1.0,
+                "sys_len": 1967,
+                "ref_len": 1949,
+                "sacrebleu": 0.450533442657802,
+                "score": 0.450533442657802,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.41119874078427415,
+                "score_ci_high": 0.4996205132749857,
+                "sacrebleu_ci_low": 0.41119874078427415,
+                "sacrebleu_ci_high": 0.4996205132749857
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 66,
+                "counts": [
+                    1300,
+                    749,
+                    458,
+                    283
+                ],
+                "totals": [
+                    2033,
+                    1967,
+                    1901,
+                    1835
+                ],
+                "precisions": [
+                    0.6394490900147565,
+                    0.3807829181494662,
+                    0.24092582851130984,
+                    0.1542234332425068
+                ],
+                "bp": 0.9685332604439724,
+                "sys_len": 2033,
+                "ref_len": 2098,
+                "sacrebleu": 0.29870591960318976,
+                "score": 0.29870591960318976,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2686987876011016,
+                "score_ci_high": 0.3321355366475583,
+                "sacrebleu_ci_low": 0.2686987876011016,
+                "sacrebleu_ci_high": 0.3321355366475583
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1340,
+                    946,
+                    692,
+                    509
+                ],
+                "totals": [
+                    1799,
+                    1733,
+                    1667,
+                    1601
+                ],
+                "precisions": [
+                    0.7448582545858811,
+                    0.5458742065781881,
+                    0.4151169766046791,
+                    0.31792629606495937
+                ],
+                "bp": 1.0,
+                "sys_len": 1799,
+                "ref_len": 1734,
+                "sacrebleu": 0.4812999188340168,
+                "score": 0.4812999188340168,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.43585100595842746,
+                "score_ci_high": 0.5287499225865158,
+                "sacrebleu_ci_low": 0.43585100595842746,
+                "sacrebleu_ci_high": 0.5287499225865158
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1139,
+                    642,
+                    413,
+                    271
+                ],
+                "totals": [
+                    1798,
+                    1732,
+                    1666,
+                    1600
+                ],
+                "precisions": [
+                    0.6334816462736373,
+                    0.37066974595842955,
+                    0.24789915966386555,
+                    0.169375
+                ],
+                "bp": 1.0,
+                "sys_len": 1798,
+                "ref_len": 1734,
+                "sacrebleu": 0.3151094190111042,
+                "score": 0.3151094190111042,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27523820425365936,
+                "score_ci_high": 0.37080224288423497,
+                "sacrebleu_ci_low": 0.27523820425365936,
+                "sacrebleu_ci_high": 0.37080224288423497
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1096,
+                    603,
+                    369,
+                    231
+                ],
+                "totals": [
+                    1757,
+                    1691,
+                    1625,
+                    1559
+                ],
+                "precisions": [
+                    0.6237905520774046,
+                    0.3565937315198108,
+                    0.2270769230769231,
+                    0.14817190506735087
+                ],
+                "bp": 1.0,
+                "sys_len": 1757,
+                "ref_len": 1734,
+                "sacrebleu": 0.29412899612915067,
+                "score": 0.29412899612915067,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2630441728848743,
+                "score_ci_high": 0.3478998363728344,
+                "sacrebleu_ci_low": 0.2630441728848743,
+                "sacrebleu_ci_high": 0.3478998363728344
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1366,
+                    978,
+                    742,
+                    559
+                ],
+                "totals": [
+                    1794,
+                    1728,
+                    1662,
+                    1596
+                ],
+                "precisions": [
+                    0.7614269788182831,
+                    0.5659722222222222,
+                    0.4464500601684717,
+                    0.35025062656641603
+                ],
+                "bp": 1.0,
+                "sys_len": 1794,
+                "ref_len": 1734,
+                "sacrebleu": 0.5094995397125037,
+                "score": 0.5094995397125037,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.45392787953470803,
+                "score_ci_high": 0.5666406845959309,
+                "sacrebleu_ci_low": 0.45392787953470803,
+                "sacrebleu_ci_high": 0.5666406845959309
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1357,
+                    953,
+                    689,
+                    517
+                ],
+                "totals": [
+                    1780,
+                    1714,
+                    1648,
+                    1582
+                ],
+                "precisions": [
+                    0.7623595505617977,
+                    0.5560093348891482,
+                    0.4180825242718446,
+                    0.3268015170670038
+                ],
+                "bp": 1.0,
+                "sys_len": 1780,
+                "ref_len": 1734,
+                "sacrebleu": 0.49056549932677673,
+                "score": 0.49056549932677673,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4449725178153994,
+                "score_ci_high": 0.5499905859714643,
+                "sacrebleu_ci_low": 0.4449725178153994,
+                "sacrebleu_ci_high": 0.5499905859714643
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1196,
+                    684,
+                    436,
+                    274
+                ],
+                "totals": [
+                    1839,
+                    1773,
+                    1707,
+                    1641
+                ],
+                "precisions": [
+                    0.6503534529635671,
+                    0.38578680203045684,
+                    0.255418863503222,
+                    0.16697135892748324
+                ],
+                "bp": 1.0,
+                "sys_len": 1839,
+                "ref_len": 1734,
+                "sacrebleu": 0.3216236243185879,
+                "score": 0.3216236243185879,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.29765430133685095,
+                "score_ci_high": 0.36660487766328476,
+                "sacrebleu_ci_low": 0.29765430133685095,
+                "sacrebleu_ci_high": 0.36660487766328476
+            },
+            "score": 0.39273277735180634,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "score": 0.5448957624253402,
+        "score_name": "subsets_mean",
+        "num_of_instances": 12472
+    }
+}

results/bluebench/{2025-06-19T15-57-45_evaluation_results.json → 2025-06-23T02-53-05_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-19T19:57:39.981261Z",
+        "timestamp_utc": "2025-06-23T06:53:01.281933Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -178,61 +178,61 @@
             "safety_bbq_age": {
                 "accuracy": 0.6444444444444445,
                 "accuracy_ci_low": 0.5444444444444444,
-                "accuracy_ci_high": 0.7444444444444445,
+                "accuracy_ci_high": 0.7384996290160605,
                 "score_name": "accuracy",
                 "score": 0.6444444444444445,
-                "score_ci_high": 0.7444444444444445,
+                "score_ci_high": 0.7384996290160605,
                 "score_ci_low": 0.5444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.7222222222222222,
-                "accuracy_ci_low": 0.6111111111111112,
+                "accuracy": 0.7111111111111111,
+                "accuracy_ci_low": 0.6,
                 "accuracy_ci_high": 0.8,
                 "score_name": "accuracy",
-                "score": 0.7222222222222222,
+                "score": 0.7111111111111111,
                 "score_ci_high": 0.8,
-                "score_ci_low": 0.6111111111111112,
+                "score_ci_low": 0.6,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
                 "accuracy": 0.9111111111111111,
-                "accuracy_ci_low": 0.8333333333333334,
+                "accuracy_ci_low": 0.8444444444444444,
                 "accuracy_ci_high": 0.9555555555555556,
                 "score_name": "accuracy",
                 "score": 0.9111111111111111,
                 "score_ci_high": 0.9555555555555556,
-                "score_ci_low": 0.8333333333333334,
+                "score_ci_low": 0.8444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.7111111111111111,
-                "accuracy_ci_low": 0.6111111111111112,
-                "accuracy_ci_high": 0.8,
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.8666666666666667,
                 "score_name": "accuracy",
-                "score": 0.7111111111111111,
-                "score_ci_high": 0.8,
-                "score_ci_low": 0.6111111111111112,
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8666666666666667,
+                "score_ci_low": 0.7,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.8333333333333334,
-                "accuracy_ci_low": 0.7444444444444445,
-                "accuracy_ci_high": 0.9,
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.6888888888888889,
+                "accuracy_ci_high": 0.8666666666666667,
                 "score_name": "accuracy",
-                "score": 0.8333333333333334,
-                "score_ci_high": 0.9,
-                "score_ci_low": 0.7444444444444445,
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8666666666666667,
+                "score_ci_low": 0.6888888888888889,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.9777777777777777,
-                "accuracy_ci_low": 0.9333333333333333,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.9666666666666667,
+                "accuracy_ci_low": 0.9222222222222223,
+                "accuracy_ci_high": 0.9888888888888889,
                 "score_name": "accuracy",
-                "score": 0.9777777777777777,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.9333333333333333,
+                "score": 0.9666666666666667,
+                "score_ci_high": 0.9888888888888889,
+                "score_ci_low": 0.9222222222222223,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
@@ -256,553 +256,553 @@
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.8111111111111111,
-                "accuracy_ci_low": 0.7116197011994875,
-                "accuracy_ci_high": 0.8888888888888888,
+                "accuracy": 0.8333333333333334,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.9,
                 "score_name": "accuracy",
-                "score": 0.8111111111111111,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.7116197011994875,
+                "score": 0.8333333333333334,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7333333333333333,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.6888888888888889,
-                "accuracy_ci_low": 0.5777777777777777,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.7888888888888889,
                 "score_name": "accuracy",
-                "score": 0.6888888888888889,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.5777777777777777,
+                "score": 0.7,
+                "score_ci_high": 0.7888888888888889,
+                "score_ci_low": 0.6,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7111111111111111,
-                "accuracy_ci_high": 0.8666666666666667,
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7444444444444445,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8666666666666667,
-                "score_ci_low": 0.7111111111111111,
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7444444444444445,
                 "num_of_instances": 90
             },
-            "score": 0.8161616161616162,
+            "score": 0.8222222222222222,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.0625,
+                "score": 0.0625,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.0625,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.592375366568915,
-                "f1_Location": 0.3668122270742358,
-                "f1_Organization": 0.45367412140575075,
-                "f1_macro": 0.4709539050163005,
-                "recall_macro": 0.3969630056026483,
-                "precision_macro": 0.5946970285442043,
-                "in_classes_support": 0.7649572649572649,
-                "f1_micro": 0.4310171198388721,
-                "recall_micro": 0.4076190476190476,
-                "precision_micro": 0.45726495726495725,
-                "score": 0.4310171198388721,
+                "f1_Person": 0.5798816568047337,
+                "f1_Organization": 0.42592592592592593,
+                "f1_Location": 0.40336134453781514,
+                "f1_macro": 0.46972297575615823,
+                "recall_macro": 0.40171664278500413,
+                "precision_macro": 0.5828611111111112,
+                "in_classes_support": 0.8064516129032258,
+                "f1_micro": 0.4343434343434343,
+                "recall_micro": 0.4095238095238095,
+                "precision_micro": 0.46236559139784944,
+                "score": 0.4343434343434343,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.36016345918404075,
-                "score_ci_high": 0.48021577272630167,
-                "f1_micro_ci_low": 0.36016345918404075,
-                "f1_micro_ci_high": 0.48021577272630167
+                "score_ci_low": 0.3767195996125236,
+                "score_ci_high": 0.4839388766009964,
+                "f1_micro_ci_low": 0.3767195996125236,
+                "f1_micro_ci_high": 0.4839388766009964
             },
-            "score": 0.4310171198388721,
+            "score": 0.4343434343434343,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.5352112676056338,
+                "accuracy": 0.5492957746478874,
                 "accuracy_ci_low": 0.43661971830985913,
-                "accuracy_ci_high": 0.647887323943662,
+                "accuracy_ci_high": 0.6619718309859155,
                 "score_name": "accuracy",
-                "score": 0.5352112676056338,
-                "score_ci_high": 0.647887323943662,
+                "score": 0.5492957746478874,
+                "score_ci_high": 0.6619718309859155,
                 "score_ci_low": 0.43661971830985913,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.2112676056338028,
+                "accuracy_ci_low": 0.1267605633802817,
+                "accuracy_ci_high": 0.30985915492957744,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.2112676056338028,
+                "score_ci_high": 0.30985915492957744,
+                "score_ci_low": 0.1267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.18309859154929578,
-                "accuracy_ci_low": 0.10639771966263252,
-                "accuracy_ci_high": 0.29577464788732394,
+                "accuracy": 0.2112676056338028,
+                "accuracy_ci_low": 0.1267605633802817,
+                "accuracy_ci_high": 0.323943661971831,
                 "score_name": "accuracy",
-                "score": 0.18309859154929578,
-                "score_ci_high": 0.29577464788732394,
-                "score_ci_low": 0.10639771966263252,
+                "score": 0.2112676056338028,
+                "score_ci_high": 0.323943661971831,
+                "score_ci_low": 0.1267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.323943661971831,
+                "accuracy": 0.30985915492957744,
                 "accuracy_ci_low": 0.2112676056338028,
-                "accuracy_ci_high": 0.43661971830985913,
+                "accuracy_ci_high": 0.4225352112676056,
                 "score_name": "accuracy",
-                "score": 0.323943661971831,
-                "score_ci_high": 0.43661971830985913,
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.4225352112676056,
                 "score_ci_low": 0.2112676056338028,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.4507042253521127,
-                "accuracy_ci_low": 0.323943661971831,
-                "accuracy_ci_high": 0.5664724235461314,
+                "accuracy": 0.4225352112676056,
+                "accuracy_ci_low": 0.29577464788732394,
+                "accuracy_ci_high": 0.5352112676056338,
                 "score_name": "accuracy",
-                "score": 0.4507042253521127,
-                "score_ci_high": 0.5664724235461314,
-                "score_ci_low": 0.323943661971831,
+                "score": 0.4225352112676056,
+                "score_ci_high": 0.5352112676056338,
+                "score_ci_low": 0.29577464788732394,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.36619718309859156,
-                "accuracy_ci_low": 0.2535211267605634,
-                "accuracy_ci_high": 0.4788732394366197,
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.352112676056338,
                 "score_name": "accuracy",
-                "score": 0.36619718309859156,
-                "score_ci_high": 0.4788732394366197,
-                "score_ci_low": 0.2535211267605634,
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.352112676056338,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.3380281690140845,
-                "accuracy_ci_low": 0.22535211267605634,
-                "accuracy_ci_high": 0.4647887323943662,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
-                "score": 0.3380281690140845,
-                "score_ci_high": 0.4647887323943662,
-                "score_ci_low": 0.22535211267605634,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.2112676056338028,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.4084507042253521,
-                "accuracy_ci_low": 0.29577464788732394,
-                "accuracy_ci_high": 0.5211267605633803,
+                "accuracy": 0.4507042253521127,
+                "accuracy_ci_low": 0.3380281690140845,
+                "accuracy_ci_high": 0.5633802816901409,
                 "score_name": "accuracy",
-                "score": 0.4084507042253521,
-                "score_ci_high": 0.5211267605633803,
-                "score_ci_low": 0.29577464788732394,
+                "score": 0.4507042253521127,
+                "score_ci_high": 0.5633802816901409,
+                "score_ci_low": 0.3380281690140845,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
                 "accuracy": 0.3380281690140845,
                 "accuracy_ci_low": 0.23943661971830985,
-                "accuracy_ci_high": 0.4647887323943662,
+                "accuracy_ci_high": 0.4393434853289757,
                 "score_name": "accuracy",
                 "score": 0.3380281690140845,
-                "score_ci_high": 0.4647887323943662,
+                "score_ci_high": 0.4393434853289757,
                 "score_ci_low": 0.23943661971830985,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.11267605633802817,
-                "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.19757759490217996,
+                "accuracy": 0.2112676056338028,
+                "accuracy_ci_low": 0.1267605633802817,
+                "accuracy_ci_high": 0.30985915492957744,
                 "score_name": "accuracy",
-                "score": 0.11267605633802817,
-                "score_ci_high": 0.19757759490217996,
-                "score_ci_low": 0.056338028169014086,
+                "score": 0.2112676056338028,
+                "score_ci_high": 0.30985915492957744,
+                "score_ci_low": 0.1267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.29577464788732394,
+                "accuracy_ci_low": 0.19718309859154928,
+                "accuracy_ci_high": 0.4084507042253521,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.29577464788732394,
+                "score_ci_high": 0.4084507042253521,
+                "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.4225352112676056,
-                "accuracy_ci_low": 0.30985915492957744,
-                "accuracy_ci_high": 0.5352112676056338,
+                "accuracy": 0.4084507042253521,
+                "accuracy_ci_low": 0.2885703240152898,
+                "accuracy_ci_high": 0.5211267605633803,
                 "score_name": "accuracy",
-                "score": 0.4225352112676056,
-                "score_ci_high": 0.5352112676056338,
-                "score_ci_low": 0.30985915492957744,
+                "score": 0.4084507042253521,
+                "score_ci_high": 0.5211267605633803,
+                "score_ci_low": 0.2885703240152898,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.3380281690140845,
+                "accuracy": 0.2112676056338028,
+                "accuracy_ci_low": 0.1267605633802817,
+                "accuracy_ci_high": 0.30985915492957744,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.3380281690140845,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.2112676056338028,
+                "score_ci_high": 0.30985915492957744,
+                "score_ci_low": 0.1267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.5211267605633803,
-                "accuracy_ci_low": 0.4084507042253521,
-                "accuracy_ci_high": 0.6338028169014085,
+                "accuracy": 0.5492957746478874,
+                "accuracy_ci_low": 0.43661971830985913,
+                "accuracy_ci_high": 0.672415960906933,
                 "score_name": "accuracy",
-                "score": 0.5211267605633803,
-                "score_ci_high": 0.6338028169014085,
-                "score_ci_low": 0.4084507042253521,
+                "score": 0.5492957746478874,
+                "score_ci_high": 0.672415960906933,
+                "score_ci_low": 0.43661971830985913,
                 "num_of_instances": 71
             },
-            "score": 0.33400402414486924,
+            "score": 0.3380281690140845,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.2827795486010496,
-                "f1_suggestive": 0.08333333333333333,
-                "f1_descriptive": 0.4444444444444444,
+                "f1_macro": 0.29746332099273276,
+                "f1_suggestive": 0.0,
+                "f1_descriptive": 0.36363636363636365,
                 "f1_generic": 0.11764705882352941,
-                "f1_fanciful": 0.4827586206896552,
-                "f1_arbitrary": 0.2857142857142857,
-                "f1_macro_ci_low": 0.20381678012471904,
-                "f1_macro_ci_high": 0.38601597944875415,
+                "f1_fanciful": 0.6470588235294118,
+                "f1_arbitrary": 0.358974358974359,
+                "f1_macro_ci_low": 0.2234746436877424,
+                "f1_macro_ci_high": 0.3820540135751509,
                 "score_name": "f1_micro",
-                "score": 0.3253012048192771,
-                "score_ci_high": 0.42168674698795183,
-                "score_ci_low": 0.21686746987951808,
+                "score": 0.3493975903614458,
+                "score_ci_high": 0.45121951219512196,
+                "score_ci_low": 0.25149700598802394,
                 "num_of_instances": 85,
-                "accuracy": 0.3176470588235294,
-                "accuracy_ci_low": 0.21176470588235294,
-                "accuracy_ci_high": 0.4117647058823529,
-                "f1_micro": 0.3253012048192771,
-                "f1_micro_ci_low": 0.21686746987951808,
-                "f1_micro_ci_high": 0.42168674698795183
+                "accuracy": 0.3411764705882353,
+                "accuracy_ci_low": 0.24705882352941178,
+                "accuracy_ci_high": 0.4470588235294118,
+                "f1_micro": 0.3493975903614458,
+                "f1_micro_ci_low": 0.25149700598802394,
+                "f1_micro_ci_high": 0.45121951219512196
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.5842293906810035,
-                "f1_no": 0.8129032258064516,
-                "f1_yes": 0.35555555555555557,
-                "f1_macro_ci_low": 0.5123650296064088,
-                "f1_macro_ci_high": 0.6612083568605307,
+                "f1_macro": 0.6415770609318996,
+                "f1_no": 0.8387096774193549,
+                "f1_yes": 0.4444444444444444,
+                "f1_macro_ci_low": 0.5666801252929456,
+                "f1_macro_ci_high": 0.7176297030965157,
                 "score_name": "f1_micro",
-                "score": 0.71,
-                "score_ci_high": 0.765,
-                "score_ci_low": 0.64,
+                "score": 0.75,
+                "score_ci_high": 0.805,
+                "score_ci_low": 0.685,
                 "num_of_instances": 200,
-                "accuracy": 0.71,
-                "accuracy_ci_low": 0.64,
-                "accuracy_ci_high": 0.765,
-                "f1_micro": 0.71,
-                "f1_micro_ci_low": 0.64,
-                "f1_micro_ci_high": 0.765
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.685,
+                "accuracy_ci_high": 0.805,
+                "f1_micro": 0.75,
+                "f1_micro_ci_low": 0.685,
+                "f1_micro_ci_high": 0.805
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.23684055980437102,
-                "f1_conclusion": 0.12,
-                "f1_issue": 0.2682926829268293,
-                "f1_decree": 0.17647058823529413,
-                "f1_rule": 0.4155844155844156,
-                "f1_analysis": 0.2608695652173913,
-                "f1_facts": 0.16666666666666666,
-                "f1_procedural history": 0.25,
-                "f1_macro_ci_low": 0.18399933651413464,
-                "f1_macro_ci_high": 0.3012128675188143,
+                "f1_macro": 0.2937287351505673,
+                "f1_conclusion": 0.16,
+                "f1_issue": 0.3291139240506329,
+                "f1_decree": 0.24242424242424243,
+                "f1_rule": 0.4931506849315068,
+                "f1_analysis": 0.2916666666666667,
+                "f1_facts": 0.21621621621621623,
+                "f1_procedural history": 0.3235294117647059,
+                "f1_macro_ci_low": 0.2356167023599295,
+                "f1_macro_ci_high": 0.3627174769966993,
                 "score_name": "f1_micro",
-                "score": 0.2570694087403599,
-                "score_ci_high": 0.31910866448170155,
-                "score_ci_low": 0.19563743957580057,
+                "score": 0.31443298969072164,
+                "score_ci_high": 0.37945181171815084,
+                "score_ci_low": 0.24415584415584415,
                 "num_of_instances": 200,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.19,
-                "accuracy_ci_high": 0.31,
-                "f1_micro": 0.2570694087403599,
-                "f1_micro_ci_low": 0.19563743957580057,
-                "f1_micro_ci_high": 0.31910866448170155
+                "accuracy": 0.305,
+                "accuracy_ci_low": 0.23726030718429333,
+                "accuracy_ci_high": 0.37,
+                "f1_micro": 0.31443298969072164,
+                "f1_micro_ci_low": 0.24415584415584415,
+                "f1_micro_ci_high": 0.37945181171815084
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.45179063360881544,
-                "f1_yes": 0.5702479338842975,
-                "f1_no": 0.3333333333333333,
-                "f1_macro_ci_low": 0.3881370275424478,
-                "f1_macro_ci_high": 0.5208583506164292,
+                "f1_macro": 0.44871725481706,
+                "f1_yes": 0.5887096774193549,
+                "f1_no": 0.3087248322147651,
+                "f1_macro_ci_low": 0.3839275870787324,
+                "f1_macro_ci_high": 0.5191612607559799,
                 "score_name": "f1_micro",
-                "score": 0.47738693467336685,
-                "score_ci_high": 0.5454545454545454,
-                "score_ci_low": 0.41102756892230574,
+                "score": 0.4836272040302267,
+                "score_ci_high": 0.5505050505050505,
+                "score_ci_low": 0.4120603015075377,
                 "num_of_instances": 200,
-                "accuracy": 0.475,
-                "accuracy_ci_low": 0.4062357598667403,
-                "accuracy_ci_high": 0.54,
-                "f1_micro": 0.47738693467336685,
-                "f1_micro_ci_low": 0.41102756892230574,
-                "f1_micro_ci_high": 0.5454545454545454
+                "accuracy": 0.48,
+                "accuracy_ci_low": 0.4083713252748318,
+                "accuracy_ci_high": 0.545,
+                "f1_micro": 0.4836272040302267,
+                "f1_micro_ci_low": 0.4120603015075377,
+                "f1_micro_ci_high": 0.5505050505050505
             },
             "legalbench_proa": {
-                "f1_macro": 0.7797888386123679,
-                "f1_yes": 0.7647058823529411,
-                "f1_no": 0.7948717948717948,
-                "f1_macro_ci_low": 0.686770027516329,
-                "f1_macro_ci_high": 0.847201812396528,
+                "f1_macro": 0.8044662309368191,
+                "f1_yes": 0.7941176470588235,
+                "f1_no": 0.8148148148148148,
+                "f1_macro_ci_low": 0.7223270079060395,
+                "f1_macro_ci_high": 0.87627946340442,
                 "score_name": "f1_micro",
-                "score": 0.7808219178082192,
-                "score_ci_high": 0.847682119205298,
-                "score_ci_low": 0.6846573729523644,
+                "score": 0.8053691275167785,
+                "score_ci_high": 0.8717948717948718,
+                "score_ci_low": 0.7140882327681733,
                 "num_of_instances": 85,
-                "accuracy": 0.6705882352941176,
-                "accuracy_ci_low": 0.5647058823529412,
-                "accuracy_ci_high": 0.7529411764705882,
-                "f1_micro": 0.7808219178082192,
-                "f1_micro_ci_low": 0.6846573729523644,
-                "f1_micro_ci_high": 0.847682119205298
+                "accuracy": 0.7058823529411765,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.8053691275167785,
+                "f1_micro_ci_low": 0.7140882327681733,
+                "f1_micro_ci_high": 0.8717948717948718
             },
-            "score": 0.5101158932082446,
+            "score": 0.5405653823198345,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.4855115011669257,
-                "f1_cars": 0.8,
-                "f1_windows x": 0.05555555555555555,
-                "f1_atheism": 0.17777777777777778,
-                "f1_cryptography": 0.4444444444444444,
-                "f1_religion": 0.23404255319148937,
-                "f1_medicine": 0.8,
-                "f1_christianity": 0.36619718309859156,
-                "f1_computer graphics": 0.3652173913043478,
-                "f1_microsoft windows": 0.19047619047619047,
-                "f1_middle east": 0.4675324675324675,
-                "f1_motorcycles": 0.693069306930693,
-                "f1_politics": 0.313953488372093,
-                "f1_pc hardware": 0.4292682926829268,
-                "f1_mac hardware": 0.2972972972972973,
-                "f1_for sale": 0.7058823529411765,
-                "f1_guns": 0.34375,
-                "f1_space": 0.6888888888888889,
-                "f1_baseball": 0.8909090909090909,
-                "f1_hockey": 0.8709677419354839,
-                "f1_electronics": 0.575,
-                "f1_macro_ci_low": 0.4606519053067645,
-                "f1_macro_ci_high": 0.5114904866418184,
+                "f1_macro": 0.4772436601851055,
+                "f1_cars": 0.7640449438202247,
+                "f1_pc hardware": 0.3557692307692308,
+                "f1_windows x": 0.028985507246376812,
+                "f1_electronics": 0.48717948717948717,
+                "f1_atheism": 0.20408163265306123,
+                "f1_politics": 0.34355828220858897,
+                "f1_religion": 0.2708333333333333,
+                "f1_medicine": 0.7948717948717948,
+                "f1_christianity": 0.4166666666666667,
+                "f1_for sale": 0.6067415730337079,
+                "f1_computer graphics": 0.42016806722689076,
+                "f1_microsoft windows": 0.25806451612903225,
+                "f1_middle east": 0.49382716049382713,
+                "f1_motorcycles": 0.6666666666666666,
+                "f1_mac hardware": 0.25,
+                "f1_guns": 0.23728813559322035,
+                "f1_space": 0.717391304347826,
+                "f1_cryptography": 0.5230769230769231,
+                "f1_baseball": 0.8461538461538461,
+                "f1_hockey": 0.859504132231405,
+                "f1_macro_ci_low": 0.45194761799386507,
+                "f1_macro_ci_high": 0.5063130462647102,
                 "score_name": "f1_micro",
-                "score": 0.5034666666666666,
-                "score_ci_high": 0.5288163691152058,
-                "score_ci_low": 0.4713054725252697,
+                "score": 0.49115281501340485,
+                "score_ci_high": 0.5196912105086561,
+                "score_ci_low": 0.4585932126016045,
                 "num_of_instances": 1000,
-                "accuracy": 0.472,
-                "accuracy_ci_low": 0.44038730175462776,
-                "accuracy_ci_high": 0.497,
-                "f1_micro": 0.5034666666666666,
-                "f1_micro_ci_low": 0.4713054725252697,
-                "f1_micro_ci_high": 0.5288163691152058
+                "accuracy": 0.458,
+                "accuracy_ci_low": 0.427,
+                "accuracy_ci_high": 0.4864735442740007,
+                "f1_micro": 0.49115281501340485,
+                "f1_micro_ci_low": 0.4585932126016045,
+                "f1_micro_ci_high": 0.5196912105086561
             },
-            "score": 0.5034666666666666,
+            "score": 0.49115281501340485,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.607364388794758,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9122807017543859,
-                "f1_credit card or prepaid card": 0.6666666666666666,
-                "f1_debt collection": 0.6075949367088608,
-                "f1_checking or savings account": 0.673469387755102,
-                "f1_money transfer or virtual currency or money service": 0.5777777777777777,
-                "f1_vehicle loan or lease": 0.37037037037037035,
-                "f1_mortgage": 0.6666666666666666,
-                "f1_payday loan or title loan or personal loan": 0.2222222222222222,
-                "f1_student loan": 0.7692307692307693,
-                "f1_macro_ci_low": 0.5603416421881502,
-                "f1_macro_ci_high": 0.6682100489708924,
+                "f1_macro": 0.5988009590549132,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9113372093023255,
+                "f1_checking or savings account": 0.5542168674698795,
+                "f1_debt collection": 0.4779874213836478,
+                "f1_credit card or prepaid card": 0.6370370370370371,
+                "f1_mortgage": 0.7397260273972602,
+                "f1_student loan": 0.8461538461538461,
+                "f1_money transfer or virtual currency or money service": 0.4864864864864865,
+                "f1_vehicle loan or lease": 0.42857142857142855,
+                "f1_payday loan or title loan or personal loan": 0.3076923076923077,
+                "f1_macro_ci_low": 0.550125163696031,
+                "f1_macro_ci_high": 0.6692920824665255,
                 "score_name": "f1_micro",
-                "score": 0.8273716951788491,
-                "score_ci_high": 0.8505803933787175,
-                "score_ci_low": 0.8031586690475525,
+                "score": 0.8145077720207254,
+                "score_ci_high": 0.8367924066551193,
+                "score_ci_low": 0.7900784551279257,
                 "num_of_instances": 1000,
-                "accuracy": 0.798,
-                "accuracy_ci_low": 0.7700141366334644,
-                "accuracy_ci_high": 0.8228416338853977,
-                "f1_micro": 0.8273716951788491,
-                "f1_micro_ci_low": 0.8031586690475525,
-                "f1_micro_ci_high": 0.8505803933787175
+                "accuracy": 0.786,
+                "accuracy_ci_low": 0.76,
+                "accuracy_ci_high": 0.81,
+                "f1_micro": 0.8145077720207254,
+                "f1_micro_ci_low": 0.7900784551279257,
+                "f1_micro_ci_high": 0.8367924066551193
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6739108272125272,
-                "f1_mortgages and loans": 0.7719298245614035,
-                "f1_credit card": 0.7403314917127072,
-                "f1_retail banking": 0.5797101449275363,
-                "f1_debt collection": 0.5686274509803921,
-                "f1_credit reporting": 0.7089552238805971,
-                "f1_macro_ci_low": 0.6324404602189574,
-                "f1_macro_ci_high": 0.7147768248953918,
+                "f1_macro": 0.6991553168902936,
+                "f1_mortgages and loans": 0.8228571428571428,
+                "f1_credit card": 0.7428571428571429,
+                "f1_debt collection": 0.6116504854368932,
+                "f1_credit reporting": 0.7211895910780669,
+                "f1_retail banking": 0.5972222222222222,
+                "f1_macro_ci_low": 0.6611649815931737,
+                "f1_macro_ci_high": 0.7441131702771507,
                 "score_name": "f1_micro",
-                "score": 0.6777546777546778,
-                "score_ci_high": 0.716590388897516,
-                "score_ci_low": 0.6352085235971857,
+                "score": 0.7017543859649122,
+                "score_ci_high": 0.7444878377150386,
+                "score_ci_low": 0.6639049566735055,
                 "num_of_instances": 500,
-                "accuracy": 0.652,
-                "accuracy_ci_low": 0.608,
-                "accuracy_ci_high": 0.6909013646716825,
-                "f1_micro": 0.6777546777546778,
-                "f1_micro_ci_low": 0.6352085235971857,
-                "f1_micro_ci_high": 0.716590388897516
+                "accuracy": 0.68,
+                "accuracy_ci_low": 0.6415834821537145,
+                "accuracy_ci_high": 0.7250870857804175,
+                "f1_micro": 0.7017543859649122,
+                "f1_micro_ci_low": 0.6639049566735055,
+                "f1_micro_ci_high": 0.7444878377150386
             },
-            "score": 0.7525631864667635,
+            "score": 0.7581310789928188,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "program_accuracy": 0.136,
-                "score": 0.136,
+                "execution_accuracy": 0.113,
+                "program_accuracy": 0.137,
+                "score": 0.137,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.112,
-                "program_accuracy_ci_low": 0.114,
-                "program_accuracy_ci_high": 0.15532916889351497,
-                "score_ci_low": 0.114,
-                "score_ci_high": 0.15532916889351497,
-                "execution_accuracy_ci_low": 0.093,
-                "execution_accuracy_ci_high": 0.132
+                "execution_accuracy_ci_low": 0.094,
+                "execution_accuracy_ci_high": 0.133,
+                "program_accuracy_ci_low": 0.115,
+                "program_accuracy_ci_high": 0.159,
+                "score_ci_low": 0.115,
+                "score_ci_high": 0.159
             },
-            "score": 0.136,
+            "score": 0.137,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.34340385694506587,
-                "recall": 0.5730464338072634,
-                "f1": 0.3745794974329677,
-                "precision_ci_low": 0.3234187185304744,
-                "precision_ci_high": 0.3650589335945776,
-                "recall_ci_low": 0.5557745525912291,
-                "recall_ci_high": 0.5891018666307999,
-                "f1_ci_low": 0.3573010951324523,
-                "f1_ci_high": 0.3923997026360963,
+                "precision": 0.3510702505438354,
+                "recall": 0.5809699679510664,
+                "f1": 0.3824309995878018,
+                "precision_ci_low": 0.33249065458630267,
+                "precision_ci_high": 0.37203127312002804,
+                "recall_ci_low": 0.5641121374917414,
+                "recall_ci_high": 0.5988884760579221,
+                "f1_ci_low": 0.3661582243989197,
+                "f1_ci_high": 0.4007102293320155,
                 "score_name": "f1",
-                "score": 0.3745794974329677,
-                "score_ci_high": 0.3923997026360963,
-                "score_ci_low": 0.3573010951324523,
+                "score": 0.3824309995878018,
+                "score_ci_high": 0.4007102293320155,
+                "score_ci_low": 0.3661582243989197,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6226274134715398,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6971947036186854,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5803041330973308,
-                "faithfullness_f1_token_overlap": 0.333895617947972,
-                "faithfullness_recall_token_overlap": 0.2631184151415058,
-                "faithfullness_precision_token_overlap": 0.5911531265193607,
-                "correctness_f1_token_overlap": 0.3745794974329677,
-                "correctness_recall_token_overlap": 0.5730464338072634,
-                "correctness_precision_token_overlap": 0.34340385694506587
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6284164202213287,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7022818158070246,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5862240339815616,
+                "faithfullness_f1_token_overlap": 0.32867960701983034,
+                "faithfullness_recall_token_overlap": 0.26185727010665827,
+                "faithfullness_precision_token_overlap": 0.5811151625996505,
+                "correctness_f1_token_overlap": 0.3824309995878018,
+                "correctness_recall_token_overlap": 0.5809699679510664,
+                "correctness_precision_token_overlap": 0.3510702505438354
             },
-            "score": 0.3745794974329677,
+            "score": 0.3824309995878018,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.484,
-                "accuracy_ci_low": 0.454,
-                "accuracy_ci_high": 0.514,
+                "accuracy": 0.479,
+                "accuracy_ci_low": 0.448,
+                "accuracy_ci_high": 0.5103198452743031,
                 "score_name": "accuracy",
-                "score": 0.484,
-                "score_ci_high": 0.514,
-                "score_ci_low": 0.454,
+                "score": 0.479,
+                "score_ci_high": 0.5103198452743031,
+                "score_ci_low": 0.448,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.806,
-                "accuracy_ci_low": 0.772,
-                "accuracy_ci_high": 0.84,
+                "accuracy": 0.818,
+                "accuracy_ci_low": 0.7804131917525572,
+                "accuracy_ci_high": 0.85,
                 "score_name": "accuracy",
-                "score": 0.806,
-                "score_ci_high": 0.84,
-                "score_ci_low": 0.772,
+                "score": 0.818,
+                "score_ci_high": 0.85,
+                "score_ci_low": 0.7804131917525572,
                 "num_of_instances": 500
             },
-            "score": 0.645,
+            "score": 0.6485,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 2.0,
-                "severity_high": 1.0,
-                "severity_medium": 1.0,
-                "severity_low": 96.0,
-                "category_harmful_info": 0.8320170236041858,
-                "category_deception": 0.80982066540405,
-                "category_explicit_content": 0.8411968264615898,
-                "category_violence": 0.8222103803300314,
-                "category_discrimination": 0.851427495597847,
-                "category_substance_abuse": 0.923489530881246,
-                "category_pii": 0.8410816744521812,
-                "safety": 0.8333571752993802,
-                "safety_ci_low": 0.8004974256643814,
-                "safety_ci_high": 0.8560529591545912,
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.8486059237981127,
+                "category_deception": 0.8698270569997605,
+                "category_explicit_content": 0.8145937094032161,
+                "category_violence": 0.8388658185134685,
+                "category_discrimination": 0.8823170511732025,
+                "category_substance_abuse": 0.9462624288267559,
+                "category_pii": 0.8776307304700216,
+                "safety": 0.8519032163850758,
+                "safety_ci_low": 0.8310657418019435,
+                "safety_ci_high": 0.8712351615904984,
                 "score_name": "safety",
-                "score": 0.8333571752993802,
-                "score_ci_high": 0.8560529591545912,
-                "score_ci_low": 0.8004974256643814,
+                "score": 0.8519032163850758,
+                "score_ci_high": 0.8712351615904984,
+                "score_ci_low": 0.8310657418019435,
                 "num_of_instances": 100
             },
-            "score": 0.8333571752993802,
+            "score": 0.8519032163850758,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge2": 0.20644440587446686,
-                "rouge1": 0.4218186965603418,
-                "rougeL": 0.29171488107135435,
-                "score": 0.29171488107135435,
+                "rougeLsum": 0.3610679948661998,
+                "rouge1": 0.4233619373045491,
+                "rougeL": 0.29420815957168495,
+                "score": 0.29420815957168495,
                 "score_name": "rougeL",
-                "rougeLsum": 0.3573848941644133,
-                "rouge2_ci_low": 0.19970855600689244,
-                "rouge2_ci_high": 0.2141764351715554,
-                "rouge1_ci_low": 0.41248489848485753,
-                "rouge1_ci_high": 0.4306004852492735,
-                "rougeL_ci_low": 0.284372337658834,
-                "rougeL_ci_high": 0.29907980889509783,
-                "score_ci_low": 0.284372337658834,
-                "score_ci_high": 0.29907980889509783,
-                "rougeLsum_ci_low": 0.3492659326802685,
-                "rougeLsum_ci_high": 0.36590481273391734
+                "rouge2": 0.20840228792560456,
+                "rougeLsum_ci_low": 0.35255104965168327,
+                "rougeLsum_ci_high": 0.36915628751369906,
+                "rouge1_ci_low": 0.4141157245136807,
+                "rouge1_ci_high": 0.43171361247266377,
+                "rougeL_ci_low": 0.28665554382409086,
+                "rougeL_ci_high": 0.30100667120780134,
+                "score_ci_low": 0.28665554382409086,
+                "score_ci_high": 0.30100667120780134,
+                "rouge2_ci_low": 0.20121549945064432,
+                "rouge2_ci_high": 0.21553750893087562
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge2": 0.0155586126994404,
-                "rouge1": 0.11530035033575219,
-                "rougeL": 0.0819857457679891,
-                "score": 0.0819857457679891,
+                "rougeLsum": 0.09608294803448246,
+                "rouge1": 0.11601113348984893,
+                "rougeL": 0.08329710711031496,
+                "score": 0.08329710711031496,
                 "score_name": "rougeL",
-                "rougeLsum": 0.09494455096055868,
-                "rouge2_ci_low": 0.013676000237548778,
-                "rouge2_ci_high": 0.01748477110760906,
-                "rouge1_ci_low": 0.10949633575516456,
-                "rouge1_ci_high": 0.12012750847071728,
-                "rougeL_ci_low": 0.07832639561199897,
-                "rougeL_ci_high": 0.08543803609753609,
-                "score_ci_low": 0.07832639561199897,
-                "score_ci_high": 0.08543803609753609,
-                "rougeLsum_ci_low": 0.09027374868536467,
-                "rougeLsum_ci_high": 0.0990496989831643
+                "rouge2": 0.01614281525612853,
+                "rougeLsum_ci_low": 0.09161955792928417,
+                "rougeLsum_ci_high": 0.10006888471086645,
+                "rouge1_ci_low": 0.1103956147665806,
+                "rouge1_ci_high": 0.12113815092736294,
+                "rougeL_ci_low": 0.07939906960390719,
+                "rougeL_ci_high": 0.08668886729552314,
+                "score_ci_low": 0.07939906960390719,
+                "score_ci_high": 0.08668886729552314,
+                "rouge2_ci_low": 0.014303052357088147,
+                "rouge2_ci_high": 0.01823150788885683
             },
-            "score": 0.18685031341967173,
+            "score": 0.18875263334099995,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1129,
-                    620,
-                    369,
-                    232
+                    1127,
+                    611,
+                    358,
+                    217
                 ],
                 "totals": [
-                    1854,
-                    1788,
-                    1722,
-                    1656
+                    1857,
+                    1791,
+                    1725,
+                    1659
                 ],
                 "precisions": [
-                    0.6089536138079827,
-                    0.34675615212527966,
-                    0.21428571428571427,
-                    0.14009661835748793
+                    0.6068928379106086,
+                    0.3411501954215522,
+                    0.20753623188405798,
+                    0.13080168776371306
                 ],
                 "bp": 1.0,
-                "sys_len": 1854,
+                "sys_len": 1857,
                 "ref_len": 1734,
-                "sacrebleu": 0.28216771071430846,
-                "score": 0.28216771071430846,
+                "sacrebleu": 0.27380490753896447,
+                "score": 0.27380490753896447,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2361776507614392,
-                "score_ci_high": 0.31854760158610573,
-                "sacrebleu_ci_low": 0.2361776507614392,
-                "sacrebleu_ci_high": 0.31854760158610573
+                "score_ci_low": 0.24139134438987545,
+                "score_ci_high": 0.3114086803649994,
+                "sacrebleu_ci_low": 0.24139134438987545,
+                "sacrebleu_ci_high": 0.3114086803649994
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1222,
-                    719,
-                    458,
-                    298
+                    1217,
+                    742,
+                    486,
+                    324
                 ],
                 "totals": [
-                    1795,
-                    1729,
-                    1663,
-                    1597
+                    1805,
+                    1739,
+                    1673,
+                    1607
                 ],
                 "precisions": [
-                    0.6807799442896936,
-                    0.4158473105841527,
-                    0.27540589296452195,
-                    0.18659987476518475
+                    0.6742382271468144,
+                    0.42668200115008625,
+                    0.2904961147638972,
+                    0.20161792159303052
                 ],
                 "bp": 1.0,
-                "sys_len": 1795,
+                "sys_len": 1805,
                 "ref_len": 1734,
-                "sacrebleu": 0.34730121824258303,
-                "score": 0.34730121824258303,
+                "sacrebleu": 0.36028550442407303,
+                "score": 0.36028550442407303,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.29893861972188673,
-                "score_ci_high": 0.3929549500270372,
-                "sacrebleu_ci_low": 0.29893861972188673,
-                "sacrebleu_ci_high": 0.3929549500270372
+                "score_ci_low": 0.3120172829600809,
+                "score_ci_high": 0.4057887928505002,
+                "sacrebleu_ci_low": 0.3120172829600809,
+                "sacrebleu_ci_high": 0.4057887928505002
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    640,
-                    243,
-                    115,
-                    51
+                    672,
+                    256,
+                    123,
+                    56
                 ],
                 "totals": [
-                    2303,
-                    2237,
-                    2171,
-                    2105
+                    1845,
+                    1779,
+                    1713,
+                    1647
                 ],
                 "precisions": [
-                    0.2778983933999131,
-                    0.1086276262852034,
-                    0.05297098111469369,
-                    0.024228028503562947
+                    0.36422764227642274,
+                    0.14390106801573918,
+                    0.07180385288966726,
+                    0.03400121432908318
                 ],
                 "bp": 1.0,
-                "sys_len": 2303,
+                "sys_len": 1845,
                 "ref_len": 1589,
-                "sacrebleu": 0.07889429589395064,
-                "score": 0.07889429589395064,
+                "sacrebleu": 0.10635790496521375,
+                "score": 0.10635790496521375,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.05783007476819273,
-                "score_ci_high": 0.11334032544493618,
-                "sacrebleu_ci_low": 0.05783007476819273,
-                "sacrebleu_ci_high": 0.11334032544493618
+                "score_ci_low": 0.0652406937888321,
+                "score_ci_high": 0.1319350359039831,
+                "sacrebleu_ci_low": 0.0652406937888321,
+                "sacrebleu_ci_high": 0.1319350359039831
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1100,
-                    591,
-                    353,
-                    222
+                    1077,
+                    569,
+                    326,
+                    185
                 ],
                 "totals": [
-                    1847,
-                    1781,
-                    1715,
-                    1649
+                    1845,
+                    1779,
+                    1713,
+                    1647
                 ],
                 "precisions": [
-                    0.5955603681645912,
-                    0.3318360471645143,
-                    0.20583090379008745,
-                    0.13462704669496664
+                    0.583739837398374,
+                    0.3198426082068578,
+                    0.19030939871570343,
+                    0.11232544019429266
                 ],
                 "bp": 1.0,
-                "sys_len": 1847,
+                "sys_len": 1845,
                 "ref_len": 1835,
-                "sacrebleu": 0.27203392188147313,
-                "score": 0.27203392188147313,
+                "sacrebleu": 0.25134688330821237,
+                "score": 0.25134688330821237,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23527738150795005,
-                "score_ci_high": 0.3139246690058723,
-                "sacrebleu_ci_low": 0.23527738150795005,
-                "sacrebleu_ci_high": 0.3139246690058723
+                "score_ci_low": 0.2195832990093629,
+                "score_ci_high": 0.2837968314094506,
+                "sacrebleu_ci_low": 0.2195832990093629,
+                "sacrebleu_ci_high": 0.2837968314094506
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1379,
-                    931,
-                    680,
-                    508
+                    1424,
+                    986,
+                    714,
+                    526
                 ],
                 "totals": [
-                    2006,
-                    1940,
-                    1874,
-                    1808
+                    1999,
+                    1933,
+                    1867,
+                    1801
                 ],
                 "precisions": [
-                    0.6874376869391824,
-                    0.4798969072164948,
-                    0.3628601921024546,
-                    0.2809734513274336
+                    0.7123561780890445,
+                    0.5100879461976203,
+                    0.3824317086234601,
+                    0.29205996668517487
                 ],
-                "bp": 0.9695654687972447,
-                "sys_len": 2006,
+                "bp": 0.9660716664698304,
+                "sys_len": 1999,
                 "ref_len": 2068,
-                "sacrebleu": 0.4152155549652011,
-                "score": 0.4152155549652011,
+                "sacrebleu": 0.4336120976233934,
+                "score": 0.4336120976233934,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3836282048944182,
-                "score_ci_high": 0.4533991390034356,
-                "sacrebleu_ci_low": 0.3836282048944182,
-                "sacrebleu_ci_high": 0.4533991390034356
+                "score_ci_low": 0.39952659045667155,
+                "score_ci_high": 0.47193262493606236,
+                "sacrebleu_ci_low": 0.39952659045667155,
+                "sacrebleu_ci_high": 0.47193262493606236
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1035,
-                    422,
-                    219,
-                    125
+                    1019,
+                    419,
+                    213,
+                    113
                 ],
                 "totals": [
-                    3325,
-                    3259,
-                    3193,
-                    3127
+                    3749,
+                    3683,
+                    3617,
+                    3552
                 ],
                 "precisions": [
-                    0.3112781954887218,
-                    0.12948757287511506,
-                    0.0685875352333229,
-                    0.03997441637352094
+                    0.27180581488396904,
+                    0.11376595166983437,
+                    0.0588885816975394,
+                    0.031813063063063064
                 ],
                 "bp": 1.0,
-                "sys_len": 3325,
+                "sys_len": 3749,
                 "ref_len": 2235,
-                "sacrebleu": 0.10253001707008509,
-                "score": 0.10253001707008509,
+                "sacrebleu": 0.08724225995783678,
+                "score": 0.08724225995783678,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08168831530386227,
-                "score_ci_high": 0.1371689608842245,
-                "sacrebleu_ci_low": 0.08168831530386227,
-                "sacrebleu_ci_high": 0.1371689608842245
+                "score_ci_low": 0.06811801818711148,
+                "score_ci_high": 0.11195945404539422,
+                "sacrebleu_ci_low": 0.06811801818711148,
+                "sacrebleu_ci_high": 0.11195945404539422
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1365,
-                    935,
-                    696,
-                    519
+                    1341,
+                    889,
+                    634,
+                    457
                 ],
                 "totals": [
-                    1887,
-                    1821,
-                    1755,
-                    1689
+                    1879,
+                    1813,
+                    1747,
+                    1681
                 ],
                 "precisions": [
-                    0.7233704292527823,
-                    0.513454146073586,
-                    0.3965811965811966,
-                    0.3072824156305506
+                    0.7136774880255454,
+                    0.4903474903474903,
+                    0.36290784201488263,
+                    0.2718619869125521
                 ],
-                "bp": 0.9847491803389177,
-                "sys_len": 1887,
+                "bp": 0.9805012826642417,
+                "sys_len": 1879,
                 "ref_len": 1916,
-                "sacrebleu": 0.45421208890996323,
-                "score": 0.45421208890996323,
+                "sacrebleu": 0.4226548575605273,
+                "score": 0.4226548575605273,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.402877439448594,
-                "score_ci_high": 0.49512376744317715,
-                "sacrebleu_ci_low": 0.402877439448594,
-                "sacrebleu_ci_high": 0.49512376744317715
+                "score_ci_low": 0.3804303884338436,
+                "score_ci_high": 0.46259367891162306,
+                "sacrebleu_ci_low": 0.3804303884338436,
+                "sacrebleu_ci_high": 0.46259367891162306
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    930,
-                    427,
-                    223,
-                    122
+                    966,
+                    465,
+                    258,
+                    149
                 ],
                 "totals": [
-                    1966,
-                    1900,
-                    1834,
-                    1768
+                    2330,
+                    2264,
+                    2198,
+                    2132
                 ],
                 "precisions": [
-                    0.47304170905391657,
-                    0.22473684210526315,
-                    0.12159214830970556,
-                    0.06900452488687783
+                    0.4145922746781116,
+                    0.20538869257950532,
+                    0.11737943585077343,
+                    0.0698874296435272
                 ],
                 "bp": 1.0,
-                "sys_len": 1966,
+                "sys_len": 2330,
                 "ref_len": 1949,
-                "sacrebleu": 0.17281809069385612,
-                "score": 0.17281809069385612,
+                "sacrebleu": 0.1625725453572352,
+                "score": 0.1625725453572352,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1498451419191345,
-                "score_ci_high": 0.20240539093526114,
-                "sacrebleu_ci_low": 0.1498451419191345,
-                "sacrebleu_ci_high": 0.20240539093526114
+                "score_ci_low": 0.11669140677951717,
+                "score_ci_high": 0.20412983752636973,
+                "sacrebleu_ci_low": 0.11669140677951717,
+                "sacrebleu_ci_high": 0.20412983752636973
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1186,
-                    603,
-                    337,
-                    189
+                    1199,
+                    637,
+                    360,
+                    199
                 ],
                 "totals": [
-                    1982,
-                    1916,
-                    1850,
-                    1784
+                    1973,
+                    1907,
+                    1841,
+                    1775
                 ],
                 "precisions": [
-                    0.5983854692230071,
-                    0.31471816283924847,
-                    0.1821621621621622,
-                    0.10594170403587444
+                    0.607704004054739,
+                    0.3340325117986366,
+                    0.19554589896795221,
+                    0.11211267605633803
                 ],
-                "bp": 0.9431530195225803,
-                "sys_len": 1982,
+                "bp": 0.9386099296136466,
+                "sys_len": 1973,
                 "ref_len": 2098,
-                "sacrebleu": 0.23157365627652982,
-                "score": 0.23157365627652982,
+                "sacrebleu": 0.2424271251773898,
+                "score": 0.2424271251773898,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.20035664766275735,
-                "score_ci_high": 0.25766181006532113,
-                "sacrebleu_ci_low": 0.20035664766275735,
-                "sacrebleu_ci_high": 0.25766181006532113
+                "score_ci_low": 0.21260065622080154,
+                "score_ci_high": 0.26696534058145066,
+                "sacrebleu_ci_low": 0.21260065622080154,
+                "sacrebleu_ci_high": 0.26696534058145066
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1263,
-                    786,
-                    517,
-                    354
+                    1270,
+                    795,
+                    520,
+                    348
                 ],
                 "totals": [
-                    1831,
-                    1765,
-                    1699,
-                    1633
+                    1847,
+                    1781,
+                    1715,
+                    1649
                 ],
                 "precisions": [
-                    0.6897870016384489,
-                    0.44532577903682724,
-                    0.3042966450853443,
-                    0.21677893447642377
+                    0.6876015159718462,
+                    0.446378439079169,
+                    0.3032069970845481,
+                    0.2110369921164342
                 ],
                 "bp": 1.0,
-                "sys_len": 1831,
+                "sys_len": 1847,
                 "ref_len": 1734,
-                "sacrebleu": 0.3772912827525828,
-                "score": 0.3772912827525828,
+                "sacrebleu": 0.37435570897744036,
+                "score": 0.37435570897744036,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3318073525019781,
-                "score_ci_high": 0.42160864308969,
-                "sacrebleu_ci_low": 0.3318073525019781,
-                "sacrebleu_ci_high": 0.42160864308969
+                "score_ci_low": 0.33656443265221864,
+                "score_ci_high": 0.4099554772696377,
+                "sacrebleu_ci_low": 0.33656443265221864,
+                "sacrebleu_ci_high": 0.4099554772696377
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    998,
-                    440,
-                    238,
+                    988,
+                    434,
+                    239,
                     140
                 ],
                 "totals": [
-                    1869,
-                    1803,
-                    1737,
-                    1671
+                    1874,
+                    1808,
+                    1742,
+                    1676
                 ],
                 "precisions": [
-                    0.5339753879079722,
-                    0.24403771491957849,
-                    0.13701784686240645,
-                    0.08378216636744465
+                    0.5272145144076841,
+                    0.24004424778761063,
+                    0.13719862227324914,
+                    0.08353221957040573
                 ],
                 "bp": 1.0,
-                "sys_len": 1869,
+                "sys_len": 1874,
                 "ref_len": 1734,
-                "sacrebleu": 0.1966648424448395,
-                "score": 0.1966648424448395,
+                "sacrebleu": 0.19515092235944087,
+                "score": 0.19515092235944087,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.16525947632394583,
-                "score_ci_high": 0.23654666880731012,
-                "sacrebleu_ci_low": 0.16525947632394583,
-                "sacrebleu_ci_high": 0.23654666880731012
+                "score_ci_low": 0.16674417267909872,
+                "score_ci_high": 0.231444565320084,
+                "sacrebleu_ci_low": 0.16674417267909872,
+                "sacrebleu_ci_high": 0.231444565320084
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    952,
-                    450,
-                    247,
-                    136
+                    976,
+                    440,
+                    232,
+                    128
                 ],
                 "totals": [
-                    1808,
-                    1742,
-                    1676,
-                    1610
+                    1841,
+                    1775,
+                    1709,
+                    1643
                 ],
                 "precisions": [
-                    0.5265486725663717,
-                    0.25832376578645233,
-                    0.1473747016706444,
-                    0.084472049689441
+                    0.530146659424226,
+                    0.24788732394366197,
+                    0.13575190169689877,
+                    0.07790626902008521
                 ],
                 "bp": 1.0,
-                "sys_len": 1808,
+                "sys_len": 1841,
                 "ref_len": 1734,
-                "sacrebleu": 0.2028545317121833,
-                "score": 0.2028545317121833,
+                "sacrebleu": 0.19308216913928786,
+                "score": 0.19308216913928786,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.17683650189535868,
-                "score_ci_high": 0.25600334996477725,
-                "sacrebleu_ci_low": 0.17683650189535868,
-                "sacrebleu_ci_high": 0.25600334996477725
+                "score_ci_low": 0.1691283364865516,
+                "score_ci_high": 0.22228985058810502,
+                "sacrebleu_ci_low": 0.1691283364865516,
+                "sacrebleu_ci_high": 0.22228985058810502
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1256,
-                    812,
-                    560,
-                    392
+                    1260,
+                    815,
+                    565,
+                    399
                 ],
                 "totals": [
-                    1782,
-                    1716,
-                    1650,
-                    1584
+                    1793,
+                    1727,
+                    1661,
+                    1595
                 ],
                 "precisions": [
-                    0.7048260381593715,
-                    0.4731934731934732,
-                    0.33939393939393936,
-                    0.2474747474747475
+                    0.7027328499721137,
+                    0.4719166184134337,
+                    0.34015653220951236,
+                    0.2501567398119122
                 ],
                 "bp": 1.0,
-                "sys_len": 1782,
+                "sys_len": 1793,
                 "ref_len": 1734,
-                "sacrebleu": 0.409108887747912,
-                "score": 0.409108887747912,
+                "sacrebleu": 0.4098610398858089,
+                "score": 0.4098610398858089,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.36492758260668207,
-                "score_ci_high": 0.46703692233170646,
-                "sacrebleu_ci_low": 0.36492758260668207,
-                "sacrebleu_ci_high": 0.46703692233170646
+                "score_ci_low": 0.3653913364652719,
+                "score_ci_high": 0.46316065930620326,
+                "sacrebleu_ci_low": 0.3653913364652719,
+                "sacrebleu_ci_high": 0.46316065930620326
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1197,
+                    1227,
                     729,
-                    477,
-                    310
+                    472,
+                    294
                 ],
                 "totals": [
-                    1815,
-                    1749,
-                    1683,
-                    1617
+                    1830,
+                    1764,
+                    1698,
+                    1632
                 ],
                 "precisions": [
-                    0.659504132231405,
-                    0.41680960548885077,
-                    0.28342245989304815,
-                    0.191713048855906
+                    0.6704918032786885,
+                    0.41326530612244894,
+                    0.2779740871613663,
+                    0.1801470588235294
                 ],
                 "bp": 1.0,
-                "sys_len": 1815,
+                "sys_len": 1830,
                 "ref_len": 1734,
-                "sacrebleu": 0.34959104085020015,
-                "score": 0.34959104085020015,
+                "sacrebleu": 0.343212800768137,
+                "score": 0.343212800768137,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30660403979630557,
-                "score_ci_high": 0.3928177080574808,
-                "sacrebleu_ci_low": 0.30660403979630557,
-                "sacrebleu_ci_high": 0.3928177080574808
+                "score_ci_low": 0.3002026232607091,
+                "score_ci_high": 0.3993584334850746,
+                "sacrebleu_ci_low": 0.3002026232607091,
+                "sacrebleu_ci_high": 0.3993584334850746
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1110,
-                    579,
-                    330,
-                    191
+                    1100,
+                    577,
+                    339,
+                    205
                 ],
                 "totals": [
-                    1811,
-                    1745,
-                    1679,
-                    1613
+                    1824,
+                    1758,
+                    1692,
+                    1626
                 ],
                 "precisions": [
-                    0.6129210381004969,
-                    0.33180515759312323,
-                    0.19654556283502087,
-                    0.11841289522628642
+                    0.6030701754385965,
+                    0.3282138794084187,
+                    0.200354609929078,
+                    0.12607626076260764
                 ],
                 "bp": 1.0,
-                "sys_len": 1811,
+                "sys_len": 1824,
                 "ref_len": 1734,
-                "sacrebleu": 0.2622934684900747,
-                "score": 0.2622934684900747,
+                "sacrebleu": 0.2659128735671552,
+                "score": 0.2659128735671552,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.22911977694045185,
-                "score_ci_high": 0.30518353214635846,
-                "sacrebleu_ci_low": 0.22911977694045185,
-                "sacrebleu_ci_high": 0.30518353214635846
+                "score_ci_low": 0.23124038640104372,
+                "score_ci_high": 0.30957401763446213,
+                "sacrebleu_ci_low": 0.23124038640104372,
+                "sacrebleu_ci_high": 0.30957401763446213
             },
-            "score": 0.27697004057638286,
+            "score": 0.27479197337400774,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.484621964093495,
+        "score": 0.4561786095841296,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-19T16-09-06_evaluation_results.json → 2025-06-23T03-17-57_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-19T20:09:01.492000Z",
+        "timestamp_utc": "2025-06-23T07:17:53.366963Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,633 +176,633 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.5777777777777777,
-                "accuracy_ci_low": 0.4777777777777778,
-                "accuracy_ci_high": 0.6777777777777778,
+                "accuracy": 0.5333333333333333,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 0.6378611050272702,
                 "score_name": "accuracy",
-                "score": 0.5777777777777777,
-                "score_ci_high": 0.6777777777777778,
-                "score_ci_low": 0.4777777777777778,
+                "score": 0.5333333333333333,
+                "score_ci_high": 0.6378611050272702,
+                "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.6777777777777778,
-                "accuracy_ci_low": 0.5777777777777777,
-                "accuracy_ci_high": 0.7666666666666667,
+                "accuracy": 0.7111111111111111,
+                "accuracy_ci_low": 0.6222222222222222,
+                "accuracy_ci_high": 0.7888888888888889,
                 "score_name": "accuracy",
-                "score": 0.6777777777777778,
-                "score_ci_high": 0.7666666666666667,
-                "score_ci_low": 0.5777777777777777,
+                "score": 0.7111111111111111,
+                "score_ci_high": 0.7888888888888889,
+                "score_ci_low": 0.6222222222222222,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8111111111111111,
-                "accuracy_ci_low": 0.7222222222222222,
-                "accuracy_ci_high": 0.8777777777777778,
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6555555555555556,
+                "accuracy_ci_high": 0.8333333333333334,
                 "score_name": "accuracy",
-                "score": 0.8111111111111111,
-                "score_ci_high": 0.8777777777777778,
-                "score_ci_low": 0.7222222222222222,
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.8333333333333334,
+                "score_ci_low": 0.6555555555555556,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.43333333333333335,
-                "accuracy_ci_low": 0.3333333333333333,
-                "accuracy_ci_high": 0.5333333333333333,
+                "accuracy": 0.45555555555555555,
+                "accuracy_ci_low": 0.35555555555555557,
+                "accuracy_ci_high": 0.5555555555555556,
                 "score_name": "accuracy",
-                "score": 0.43333333333333335,
-                "score_ci_high": 0.5333333333333333,
-                "score_ci_low": 0.3333333333333333,
+                "score": 0.45555555555555555,
+                "score_ci_high": 0.5555555555555556,
+                "score_ci_low": 0.35555555555555557,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
                 "accuracy": 0.5888888888888889,
-                "accuracy_ci_low": 0.48197626978907726,
+                "accuracy_ci_low": 0.4888888888888889,
                 "accuracy_ci_high": 0.6888888888888889,
                 "score_name": "accuracy",
                 "score": 0.5888888888888889,
                 "score_ci_high": 0.6888888888888889,
-                "score_ci_low": 0.48197626978907726,
+                "score_ci_low": 0.4888888888888889,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.8777777777777778,
-                "accuracy_ci_low": 0.8,
-                "accuracy_ci_high": 0.9333333333333333,
+                "accuracy": 0.8111111111111111,
+                "accuracy_ci_low": 0.7222222222222222,
+                "accuracy_ci_high": 0.8777777777777778,
                 "score_name": "accuracy",
-                "score": 0.8777777777777778,
-                "score_ci_high": 0.9333333333333333,
-                "score_ci_low": 0.8,
+                "score": 0.8111111111111111,
+                "score_ci_high": 0.8777777777777778,
+                "score_ci_low": 0.7222222222222222,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.7444444444444445,
-                "accuracy_ci_low": 0.6444444444444445,
-                "accuracy_ci_high": 0.8333333333333334,
+                "accuracy": 0.7333333333333333,
+                "accuracy_ci_low": 0.6333333333333333,
+                "accuracy_ci_high": 0.8111111111111111,
                 "score_name": "accuracy",
-                "score": 0.7444444444444445,
-                "score_ci_high": 0.8333333333333334,
-                "score_ci_low": 0.6444444444444445,
+                "score": 0.7333333333333333,
+                "score_ci_high": 0.8111111111111111,
+                "score_ci_low": 0.6333333333333333,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
                 "accuracy": 0.6222222222222222,
-                "accuracy_ci_low": 0.5222222222222223,
-                "accuracy_ci_high": 0.7222222222222222,
+                "accuracy_ci_low": 0.5111111111111111,
+                "accuracy_ci_high": 0.7111111111111111,
                 "score_name": "accuracy",
                 "score": 0.6222222222222222,
-                "score_ci_high": 0.7222222222222222,
-                "score_ci_low": 0.5222222222222223,
+                "score_ci_high": 0.7111111111111111,
+                "score_ci_low": 0.5111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.5,
-                "accuracy_ci_high": 0.7,
+                "accuracy": 0.6444444444444445,
+                "accuracy_ci_low": 0.5444444444444444,
+                "accuracy_ci_high": 0.7444444444444445,
                 "score_name": "accuracy",
-                "score": 0.6,
-                "score_ci_high": 0.7,
-                "score_ci_low": 0.5,
+                "score": 0.6444444444444445,
+                "score_ci_high": 0.7444444444444445,
+                "score_ci_low": 0.5444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.6333333333333333,
-                "accuracy_ci_low": 0.5333333333333333,
-                "accuracy_ci_high": 0.7333333333333333,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.7888888888888889,
                 "score_name": "accuracy",
-                "score": 0.6333333333333333,
-                "score_ci_high": 0.7333333333333333,
-                "score_ci_low": 0.5333333333333333,
+                "score": 0.7,
+                "score_ci_high": 0.7888888888888889,
+                "score_ci_low": 0.6,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.7,
-                "accuracy_ci_low": 0.5969530984549517,
-                "accuracy_ci_high": 0.7798809350059414,
+                "accuracy": 0.6222222222222222,
+                "accuracy_ci_low": 0.5222222222222223,
+                "accuracy_ci_high": 0.7222222222222222,
                 "score_name": "accuracy",
-                "score": 0.7,
-                "score_ci_high": 0.7798809350059414,
-                "score_ci_low": 0.5969530984549517,
+                "score": 0.6222222222222222,
+                "score_ci_high": 0.7222222222222222,
+                "score_ci_low": 0.5222222222222223,
                 "num_of_instances": 90
             },
-            "score": 0.6606060606060606,
+            "score": 0.6515151515151515,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.03687821612349914,
+                "score": 0.03687821612349914,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.03687821612349914,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.4156626506024097,
-                "f1_Organization": 0.31372549019607837,
-                "f1_Location": 0.23140495867768596,
-                "f1_macro": 0.32026436649205803,
-                "recall_macro": 0.2686052593300962,
-                "precision_macro": 0.40524414740424186,
-                "in_classes_support": 0.6173913043478261,
-                "f1_micro": 0.26363636363636367,
-                "recall_micro": 0.2761904761904762,
-                "precision_micro": 0.25217391304347825,
-                "score": 0.26363636363636367,
+                "f1_Person": 0.3902439024390244,
+                "f1_Organization": 0.29283489096573206,
+                "f1_Location": 0.2572614107883817,
+                "f1_macro": 0.31344673473104606,
+                "recall_macro": 0.2655047696270643,
+                "precision_macro": 0.3964060432628696,
+                "in_classes_support": 0.6260720411663807,
+                "f1_micro": 0.25631768953068595,
+                "recall_micro": 0.2704761904761905,
+                "precision_micro": 0.24356775300171526,
+                "score": 0.25631768953068595,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.22691460915409117,
-                "score_ci_high": 0.3036479298321143,
-                "f1_micro_ci_low": 0.22691460915409117,
-                "f1_micro_ci_high": 0.3036479298321143
+                "score_ci_low": 0.22100954853501506,
+                "score_ci_high": 0.2947346870824505,
+                "f1_micro_ci_low": 0.22100954853501506,
+                "f1_micro_ci_high": 0.2947346870824505
             },
-            "score": 0.26363636363636367,
+            "score": 0.25631768953068595,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.43661971830985913,
-                "accuracy_ci_low": 0.323943661971831,
-                "accuracy_ci_high": 0.5633802816901409,
+                "accuracy": 0.4788732394366197,
+                "accuracy_ci_low": 0.36619718309859156,
+                "accuracy_ci_high": 0.5915492957746479,
                 "score_name": "accuracy",
-                "score": 0.43661971830985913,
-                "score_ci_high": 0.5633802816901409,
-                "score_ci_low": 0.323943661971831,
+                "score": 0.4788732394366197,
+                "score_ci_high": 0.5915492957746479,
+                "score_ci_low": 0.36619718309859156,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.2112676056338028,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.3380281690140845,
                 "score_name": "accuracy",
-                "score": 0.2112676056338028,
-                "score_ci_high": 0.30985915492957744,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.3380281690140845,
+                "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
                 "accuracy": 0.23943661971830985,
                 "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.3380281690140845,
+                "accuracy_ci_high": 0.352112676056338,
                 "score_name": "accuracy",
                 "score": 0.23943661971830985,
-                "score_ci_high": 0.3380281690140845,
+                "score_ci_high": 0.352112676056338,
                 "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.3380281690140845,
-                "accuracy_ci_low": 0.22535211267605634,
-                "accuracy_ci_high": 0.4507042253521127,
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
                 "score_name": "accuracy",
-                "score": 0.3380281690140845,
-                "score_ci_high": 0.4507042253521127,
-                "score_ci_low": 0.22535211267605634,
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.38028169014084506,
-                "accuracy_ci_low": 0.28169014084507044,
+                "accuracy": 0.39436619718309857,
+                "accuracy_ci_low": 0.29577464788732394,
                 "accuracy_ci_high": 0.5070422535211268,
                 "score_name": "accuracy",
-                "score": 0.38028169014084506,
+                "score": 0.39436619718309857,
                 "score_ci_high": 0.5070422535211268,
-                "score_ci_low": 0.28169014084507044,
+                "score_ci_low": 0.29577464788732394,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.2112676056338028,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy": 0.16901408450704225,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.2676056338028169,
                 "score_name": "accuracy",
-                "score": 0.2112676056338028,
-                "score_ci_high": 0.30985915492957744,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.16901408450704225,
+                "score_ci_high": 0.2676056338028169,
+                "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy": 0.2112676056338028,
+                "accuracy_ci_low": 0.1267605633802817,
+                "accuracy_ci_high": 0.33217670597601795,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.30985915492957744,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.2112676056338028,
+                "score_ci_high": 0.33217670597601795,
+                "score_ci_low": 0.1267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
                 "accuracy": 0.2676056338028169,
-                "accuracy_ci_low": 0.15492957746478872,
-                "accuracy_ci_high": 0.38028169014084506,
+                "accuracy_ci_low": 0.16901408450704225,
+                "accuracy_ci_high": 0.36619718309859156,
                 "score_name": "accuracy",
                 "score": 0.2676056338028169,
-                "score_ci_high": 0.38028169014084506,
-                "score_ci_low": 0.15492957746478872,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.16901408450704225,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.1267605633802817,
-                "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.22535211267605634,
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.352112676056338,
                 "score_name": "accuracy",
-                "score": 0.1267605633802817,
-                "score_ci_high": 0.22535211267605634,
-                "score_ci_low": 0.056338028169014086,
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.352112676056338,
+                "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.09859154929577464,
-                "accuracy_ci_low": 0.04225352112676056,
-                "accuracy_ci_high": 0.18309859154929578,
+                "accuracy": 0.15492957746478872,
+                "accuracy_ci_low": 0.08450704225352113,
+                "accuracy_ci_high": 0.2535211267605634,
                 "score_name": "accuracy",
-                "score": 0.09859154929577464,
-                "score_ci_high": 0.18309859154929578,
-                "score_ci_low": 0.04225352112676056,
+                "score": 0.15492957746478872,
+                "score_ci_high": 0.2535211267605634,
+                "score_ci_low": 0.08450704225352113,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.29577464788732394,
+                "accuracy": 0.18309859154929578,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.28169014084507044,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.29577464788732394,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.18309859154929578,
+                "score_ci_high": 0.28169014084507044,
+                "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.18309859154929578,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.352112676056338,
                 "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.18309859154929578,
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.352112676056338,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.29577464788732394,
+                "accuracy": 0.2676056338028169,
+                "accuracy_ci_low": 0.17514498933734307,
+                "accuracy_ci_high": 0.38028169014084506,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.29577464788732394,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.2676056338028169,
+                "score_ci_high": 0.38028169014084506,
+                "score_ci_low": 0.17514498933734307,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.38028169014084506,
-                "accuracy_ci_low": 0.28169014084507044,
-                "accuracy_ci_high": 0.5070422535211268,
+                "accuracy": 0.43661971830985913,
+                "accuracy_ci_low": 0.323943661971831,
+                "accuracy_ci_high": 0.5492957746478874,
                 "score_name": "accuracy",
-                "score": 0.38028169014084506,
-                "score_ci_high": 0.5070422535211268,
-                "score_ci_low": 0.28169014084507044,
+                "score": 0.43661971830985913,
+                "score_ci_high": 0.5492957746478874,
+                "score_ci_low": 0.323943661971831,
                 "num_of_instances": 71
             },
-            "score": 0.2545271629778672,
+            "score": 0.26961770623742454,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.40599479862637755,
-                "f1_suggestive": 0.16666666666666666,
-                "f1_generic": 0.2727272727272727,
-                "f1_arbitrary": 0.5263157894736842,
-                "f1_fanciful": 0.5777777777777777,
-                "f1_descriptive": 0.4864864864864865,
-                "f1_macro_ci_low": 0.31051398566318733,
-                "f1_macro_ci_high": 0.5136277650253285,
+                "f1_macro": 0.23241604568835691,
+                "f1_suggestive": 0.08695652173913043,
+                "f1_descriptive": 0.2631578947368421,
+                "f1_generic": 0.0,
+                "f1_arbitrary": 0.3888888888888889,
+                "f1_fanciful": 0.4230769230769231,
+                "f1_macro_ci_low": 0.16927841023118298,
+                "f1_macro_ci_high": 0.32467849714540287,
                 "score_name": "f1_micro",
-                "score": 0.4457831325301205,
-                "score_ci_high": 0.550817717180019,
-                "score_ci_low": 0.3373493975903614,
+                "score": 0.2891566265060241,
+                "score_ci_high": 0.40476190476190477,
+                "score_ci_low": 0.2054361335527834,
                 "num_of_instances": 85,
-                "accuracy": 0.43529411764705883,
-                "accuracy_ci_low": 0.32941176470588235,
-                "accuracy_ci_high": 0.5411764705882353,
-                "f1_micro": 0.4457831325301205,
-                "f1_micro_ci_low": 0.3373493975903614,
-                "f1_micro_ci_high": 0.550817717180019
+                "accuracy": 0.2823529411764706,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.2891566265060241,
+                "f1_micro_ci_low": 0.2054361335527834,
+                "f1_micro_ci_high": 0.40476190476190477
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.4964746474647465,
-                "f1_no": 0.7656765676567657,
-                "f1_yes": 0.22727272727272727,
-                "f1_macro_ci_low": 0.43223753510688345,
-                "f1_macro_ci_high": 0.5682608970547502,
+                "f1_macro": 0.4965214761040533,
+                "f1_no": 0.7631578947368421,
+                "f1_yes": 0.22988505747126436,
+                "f1_macro_ci_low": 0.43244851636549736,
+                "f1_macro_ci_high": 0.5729228740221988,
                 "score_name": "f1_micro",
                 "score": 0.6445012787723785,
-                "score_ci_high": 0.7025641025641025,
-                "score_ci_low": 0.5728900255754475,
+                "score_ci_high": 0.69946202795028,
+                "score_ci_low": 0.570694087403599,
                 "num_of_instances": 200,
                 "accuracy": 0.63,
-                "accuracy_ci_low": 0.56,
+                "accuracy_ci_low": 0.5561546872315049,
                 "accuracy_ci_high": 0.69,
                 "f1_micro": 0.6445012787723785,
-                "f1_micro_ci_low": 0.5728900255754475,
-                "f1_micro_ci_high": 0.7025641025641025
+                "f1_micro_ci_low": 0.570694087403599,
+                "f1_micro_ci_high": 0.69946202795028
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.20456925120109243,
-                "f1_conclusion": 0.047619047619047616,
-                "f1_decree": 0.26666666666666666,
-                "f1_issue": 0.18947368421052632,
-                "f1_analysis": 0.3125,
-                "f1_facts": 0.2857142857142857,
-                "f1_procedural history": 0.19047619047619047,
-                "f1_rule": 0.13953488372093023,
-                "f1_macro_ci_low": 0.15195580870715297,
-                "f1_macro_ci_high": 0.2695847948134964,
+                "f1_macro": 0.22997089242124882,
+                "f1_conclusion": 0.04878048780487805,
+                "f1_analysis": 0.3333333333333333,
+                "f1_decree": 0.2926829268292683,
+                "f1_issue": 0.21978021978021978,
+                "f1_procedural history": 0.05,
+                "f1_facts": 0.2978723404255319,
+                "f1_rule": 0.3673469387755102,
+                "f1_macro_ci_low": 0.18026075783829068,
+                "f1_macro_ci_high": 0.2946257845154891,
                 "score_name": "f1_micro",
-                "score": 0.20911528150134048,
-                "score_ci_high": 0.2716626596010836,
-                "score_ci_low": 0.15343915343915343,
+                "score": 0.24146981627296588,
+                "score_ci_high": 0.3019289134511566,
+                "score_ci_low": 0.18181818181818182,
                 "num_of_instances": 200,
-                "accuracy": 0.195,
-                "accuracy_ci_low": 0.14164584898806754,
-                "accuracy_ci_high": 0.255,
-                "f1_micro": 0.20911528150134048,
-                "f1_micro_ci_low": 0.15343915343915343,
-                "f1_micro_ci_high": 0.2716626596010836
+                "accuracy": 0.23,
+                "accuracy_ci_low": 0.175,
+                "accuracy_ci_high": 0.29,
+                "f1_micro": 0.24146981627296588,
+                "f1_micro_ci_low": 0.18181818181818182,
+                "f1_micro_ci_high": 0.3019289134511566
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.45312172637754033,
-                "f1_yes": 0.5225225225225225,
-                "f1_no": 0.38372093023255816,
-                "f1_macro_ci_low": 0.3838533922470516,
-                "f1_macro_ci_high": 0.5196559838649608,
+                "f1_macro": 0.4719581626606899,
+                "f1_yes": 0.5462555066079295,
+                "f1_no": 0.39766081871345027,
+                "f1_macro_ci_low": 0.4067534798719593,
+                "f1_macro_ci_high": 0.5312059177934843,
                 "score_name": "f1_micro",
-                "score": 0.4619289340101523,
-                "score_ci_high": 0.5291073254863808,
-                "score_ci_low": 0.39285714285714285,
+                "score": 0.4824120603015075,
+                "score_ci_high": 0.5413533834586466,
+                "score_ci_low": 0.41550674904624724,
                 "num_of_instances": 200,
-                "accuracy": 0.455,
-                "accuracy_ci_low": 0.385,
-                "accuracy_ci_high": 0.525,
-                "f1_micro": 0.4619289340101523,
-                "f1_micro_ci_low": 0.39285714285714285,
-                "f1_micro_ci_high": 0.5291073254863808
+                "accuracy": 0.48,
+                "accuracy_ci_low": 0.415,
+                "accuracy_ci_high": 0.54,
+                "f1_micro": 0.4824120603015075,
+                "f1_micro_ci_low": 0.41550674904624724,
+                "f1_micro_ci_high": 0.5413533834586466
             },
             "legalbench_proa": {
-                "f1_macro": 0.7812889165628891,
-                "f1_yes": 0.7671232876712328,
-                "f1_no": 0.7954545454545454,
-                "f1_macro_ci_low": 0.6841107145759989,
-                "f1_macro_ci_high": 0.8570115576895313,
+                "f1_macro": 0.8112290008841733,
+                "f1_yes": 0.7948717948717948,
+                "f1_no": 0.8275862068965517,
+                "f1_macro_ci_low": 0.7184910169578117,
+                "f1_macro_ci_high": 0.8804600933253673,
                 "score_name": "f1_micro",
-                "score": 0.782608695652174,
-                "score_ci_high": 0.8554216867469879,
-                "score_ci_low": 0.6867321408585169,
+                "score": 0.8121212121212121,
+                "score_ci_high": 0.8795180722891566,
+                "score_ci_low": 0.7203411511997481,
                 "num_of_instances": 85,
-                "accuracy": 0.7411764705882353,
-                "accuracy_ci_low": 0.6470588235294118,
-                "accuracy_ci_high": 0.8235294117647058,
-                "f1_micro": 0.782608695652174,
-                "f1_micro_ci_low": 0.6867321408585169,
-                "f1_micro_ci_high": 0.8554216867469879
+                "accuracy": 0.788235294117647,
+                "accuracy_ci_low": 0.6941176470588235,
+                "accuracy_ci_high": 0.8588235294117647,
+                "f1_micro": 0.8121212121212121,
+                "f1_micro_ci_low": 0.7203411511997481,
+                "f1_micro_ci_high": 0.8795180722891566
             },
-            "score": 0.5087874644932331,
+            "score": 0.49393219879481765,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.34833143635091446,
-                "f1_cars": 0.6590909090909091,
-                "f1_windows x": 0.030303030303030304,
-                "f1_atheism": 0.19047619047619047,
-                "f1_christianity": 0.2,
-                "f1_religion": 0.19047619047619047,
-                "f1_medicine": 0.6567164179104478,
-                "f1_computer graphics": 0.34782608695652173,
-                "f1_microsoft windows": 0.29850746268656714,
-                "f1_middle east": 0.11764705882352941,
-                "f1_politics": 0.20754716981132076,
-                "f1_motorcycles": 0.43373493975903615,
-                "f1_pc hardware": 0.3973509933774834,
-                "f1_mac hardware": 0.3950617283950617,
-                "f1_electronics": 0.4186046511627907,
-                "f1_for sale": 0.08695652173913043,
-                "f1_guns": 0.14814814814814814,
-                "f1_space": 0.4935064935064935,
-                "f1_cryptography": 0.47368421052631576,
-                "f1_baseball": 0.6890756302521008,
-                "f1_hockey": 0.5319148936170213,
-                "f1_macro_ci_low": 0.32107229927440883,
-                "f1_macro_ci_high": 0.37798520058634305,
+                "f1_macro": 0.3469646355526677,
+                "f1_cars": 0.5517241379310345,
+                "f1_windows x": 0.0,
+                "f1_atheism": 0.23809523809523808,
+                "f1_christianity": 0.2028985507246377,
+                "f1_religion": 0.1941747572815534,
+                "f1_medicine": 0.6060606060606061,
+                "f1_computer graphics": 0.3488372093023256,
+                "f1_microsoft windows": 0.3188405797101449,
+                "f1_middle east": 0.11538461538461539,
+                "f1_politics": 0.3047619047619048,
+                "f1_motorcycles": 0.5227272727272727,
+                "f1_baseball": 0.6984126984126984,
+                "f1_pc hardware": 0.3684210526315789,
+                "f1_mac hardware": 0.37037037037037035,
+                "f1_for sale": 0.08888888888888889,
+                "f1_guns": 0.18181818181818182,
+                "f1_space": 0.4810126582278481,
+                "f1_cryptography": 0.48484848484848486,
+                "f1_hockey": 0.4666666666666667,
+                "f1_electronics": 0.3953488372093023,
+                "f1_macro_ci_low": 0.32234813592441613,
+                "f1_macro_ci_high": 0.38044336501459297,
                 "score_name": "f1_micro",
-                "score": 0.3750771128932758,
-                "score_ci_high": 0.40812055333180686,
-                "score_ci_low": 0.3412059307716769,
+                "score": 0.37344913151364767,
+                "score_ci_high": 0.40609658022784717,
+                "score_ci_low": 0.34207641792416926,
                 "num_of_instances": 1000,
-                "accuracy": 0.304,
-                "accuracy_ci_low": 0.275,
-                "accuracy_ci_high": 0.333,
-                "f1_micro": 0.3750771128932758,
-                "f1_micro_ci_low": 0.3412059307716769,
-                "f1_micro_ci_high": 0.40812055333180686
+                "accuracy": 0.301,
+                "accuracy_ci_low": 0.274,
+                "accuracy_ci_high": 0.329023179612989,
+                "f1_micro": 0.37344913151364767,
+                "f1_micro_ci_low": 0.34207641792416926,
+                "f1_micro_ci_high": 0.40609658022784717
             },
-            "score": 0.3750771128932758,
+            "score": 0.37344913151364767,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.49544359373400404,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8558139534883721,
-                "f1_credit card or prepaid card": 0.4778761061946903,
-                "f1_debt collection": 0.45517241379310347,
-                "f1_checking or savings account": 0.4791666666666667,
-                "f1_payday loan or title loan or personal loan": 0.1875,
-                "f1_vehicle loan or lease": 0.30303030303030304,
-                "f1_mortgage": 0.6909090909090909,
-                "f1_money transfer or virtual currency or money service": 0.34285714285714286,
+                "f1_macro": 0.4835930003981669,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8920327624720774,
+                "f1_checking or savings account": 0.42105263157894735,
+                "f1_credit card or prepaid card": 0.5666666666666667,
+                "f1_debt collection": 0.38666666666666666,
+                "f1_mortgage": 0.7096774193548387,
                 "f1_student loan": 0.6666666666666666,
-                "f1_macro_ci_low": 0.437780332452402,
-                "f1_macro_ci_high": 0.5517479827666423,
+                "f1_money transfer or virtual currency or money service": 0.3125,
+                "f1_vehicle loan or lease": 0.27586206896551724,
+                "f1_payday loan or title loan or personal loan": 0.12121212121212122,
+                "f1_macro_ci_low": 0.43264552909234405,
+                "f1_macro_ci_high": 0.5420653283436574,
                 "score_name": "f1_micro",
-                "score": 0.7417582417582418,
-                "score_ci_high": 0.7670380361466397,
-                "score_ci_low": 0.7139576080586072,
+                "score": 0.7693953986088817,
+                "score_ci_high": 0.7940535810044251,
+                "score_ci_low": 0.7428249604302373,
                 "num_of_instances": 1000,
-                "accuracy": 0.675,
-                "accuracy_ci_low": 0.644,
-                "accuracy_ci_high": 0.705,
-                "f1_micro": 0.7417582417582418,
-                "f1_micro_ci_low": 0.7139576080586072,
-                "f1_micro_ci_high": 0.7670380361466397
+                "accuracy": 0.719,
+                "accuracy_ci_low": 0.688,
+                "accuracy_ci_high": 0.746,
+                "f1_micro": 0.7693953986088817,
+                "f1_micro_ci_low": 0.7428249604302373,
+                "f1_micro_ci_high": 0.7940535810044251
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.5417314583908202,
-                "f1_mortgages and loans": 0.6705202312138728,
-                "f1_credit card": 0.5815602836879432,
-                "f1_debt collection": 0.5073170731707317,
-                "f1_credit reporting": 0.6431372549019608,
-                "f1_retail banking": 0.30612244897959184,
-                "f1_macro_ci_low": 0.4990357022335705,
-                "f1_macro_ci_high": 0.5863760959900322,
+                "f1_macro": 0.5138940708414392,
+                "f1_mortgages and loans": 0.6742857142857143,
+                "f1_credit card": 0.5314685314685315,
+                "f1_debt collection": 0.5,
+                "f1_credit reporting": 0.6742424242424242,
+                "f1_retail banking": 0.18947368421052632,
+                "f1_macro_ci_low": 0.47276041465254326,
+                "f1_macro_ci_high": 0.561073606935457,
                 "score_name": "f1_micro",
-                "score": 0.5688073394495413,
-                "score_ci_high": 0.6118783685965219,
-                "score_ci_low": 0.5227795175898966,
+                "score": 0.5587229190421893,
+                "score_ci_high": 0.6032761107151652,
+                "score_ci_low": 0.5136696359618879,
                 "num_of_instances": 500,
-                "accuracy": 0.496,
-                "accuracy_ci_low": 0.452,
-                "accuracy_ci_high": 0.538,
-                "f1_micro": 0.5688073394495413,
-                "f1_micro_ci_low": 0.5227795175898966,
-                "f1_micro_ci_high": 0.6118783685965219
+                "accuracy": 0.49,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.536,
+                "f1_micro": 0.5587229190421893,
+                "f1_micro_ci_low": 0.5136696359618879,
+                "f1_micro_ci_high": 0.6032761107151652
             },
-            "score": 0.6552827906038916,
+            "score": 0.6640591588255356,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "program_accuracy": 0.107,
-                "score": 0.107,
+                "program_accuracy": 0.112,
+                "score": 0.112,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.091,
-                "program_accuracy_ci_low": 0.08721723629561164,
-                "program_accuracy_ci_high": 0.126,
-                "score_ci_low": 0.08721723629561164,
-                "score_ci_high": 0.126,
-                "execution_accuracy_ci_low": 0.074,
-                "execution_accuracy_ci_high": 0.109
+                "execution_accuracy": 0.098,
+                "program_accuracy_ci_low": 0.092,
+                "program_accuracy_ci_high": 0.134,
+                "score_ci_low": 0.092,
+                "score_ci_high": 0.134,
+                "execution_accuracy_ci_low": 0.081,
+                "execution_accuracy_ci_high": 0.11876030243075729
             },
-            "score": 0.107,
+            "score": 0.112,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.27411911391097254,
-                "recall": 0.5230720245824972,
-                "f1": 0.30495867779564406,
-                "precision_ci_low": 0.2547573973545031,
-                "precision_ci_high": 0.2921448180759812,
-                "recall_ci_low": 0.5070386905327111,
-                "recall_ci_high": 0.5400839972442947,
-                "f1_ci_low": 0.28775327483111873,
-                "f1_ci_high": 0.32083305804218265,
+                "precision": 0.27803081578951677,
+                "recall": 0.5263554366544182,
+                "f1": 0.3099512752592042,
+                "precision_ci_low": 0.2593667444116527,
+                "precision_ci_high": 0.296940372694636,
+                "recall_ci_low": 0.5097202360703201,
+                "recall_ci_high": 0.5429203378240791,
+                "f1_ci_low": 0.2936726592020454,
+                "f1_ci_high": 0.3271680931787992,
                 "score_name": "f1",
-                "score": 0.30495867779564406,
-                "score_ci_high": 0.32083305804218265,
-                "score_ci_low": 0.28775327483111873,
+                "score": 0.3099512752592042,
+                "score_ci_high": 0.3271680931787992,
+                "score_ci_low": 0.2936726592020454,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5782615457475185,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6598645970225334,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5351804447919131,
-                "faithfullness_f1_token_overlap": 0.34366402575185945,
-                "faithfullness_recall_token_overlap": 0.27898727759494896,
-                "faithfullness_precision_token_overlap": 0.5756506161288797,
-                "correctness_f1_token_overlap": 0.30495867779564406,
-                "correctness_recall_token_overlap": 0.5230720245824972,
-                "correctness_precision_token_overlap": 0.27411911391097254
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5813682861626148,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6611752705772718,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5378807228555282,
+                "faithfullness_f1_token_overlap": 0.34377396506102365,
+                "faithfullness_recall_token_overlap": 0.2786846668859655,
+                "faithfullness_precision_token_overlap": 0.5693429043011381,
+                "correctness_f1_token_overlap": 0.3099512752592042,
+                "correctness_recall_token_overlap": 0.5263554366544182,
+                "correctness_precision_token_overlap": 0.27803081578951677
             },
-            "score": 0.30495867779564406,
+            "score": 0.3099512752592042,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.388,
-                "accuracy_ci_low": 0.358,
+                "accuracy": 0.387,
+                "accuracy_ci_low": 0.355,
                 "accuracy_ci_high": 0.417,
                 "score_name": "accuracy",
-                "score": 0.388,
+                "score": 0.387,
                 "score_ci_high": 0.417,
-                "score_ci_low": 0.358,
+                "score_ci_low": 0.355,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
                 "accuracy": 0.728,
                 "accuracy_ci_low": 0.692,
-                "accuracy_ci_high": 0.764,
+                "accuracy_ci_high": 0.768,
                 "score_name": "accuracy",
                 "score": 0.728,
-                "score_ci_high": 0.764,
+                "score_ci_high": 0.768,
                 "score_ci_low": 0.692,
                 "num_of_instances": 500
             },
-            "score": 0.558,
+            "score": 0.5575,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 0.0,
+                "severity_critical": 2.0,
                 "severity_high": 0.0,
-                "severity_medium": 1.0,
-                "severity_low": 99.0,
-                "category_harmful_info": 0.8488387024207483,
-                "category_deception": 0.866081887573907,
-                "category_explicit_content": 0.8272491132821699,
-                "category_violence": 0.812888801746839,
-                "category_discrimination": 0.8425866682378074,
-                "category_substance_abuse": 0.9123246139950222,
-                "category_pii": 0.7867479754818811,
-                "safety": 0.8366520641242258,
-                "safety_ci_low": 0.8141729035472625,
-                "safety_ci_high": 0.8588271358338173,
+                "severity_medium": 3.0,
+                "severity_low": 95.0,
+                "category_harmful_info": 0.790451957932895,
+                "category_deception": 0.8851025405556264,
+                "category_explicit_content": 0.790092716915439,
+                "category_violence": 0.7921644449869674,
+                "category_discrimination": 0.839280919423179,
+                "category_substance_abuse": 0.8701996730433569,
+                "category_pii": 0.7343945315418144,
+                "safety": 0.8071470662465113,
+                "safety_ci_low": 0.7772986218937553,
+                "safety_ci_high": 0.8319065673300019,
                 "score_name": "safety",
-                "score": 0.8366520641242258,
-                "score_ci_high": 0.8588271358338173,
-                "score_ci_low": 0.8141729035472625,
+                "score": 0.8071470662465113,
+                "score_ci_high": 0.8319065673300019,
+                "score_ci_low": 0.7772986218937553,
                 "num_of_instances": 100
             },
-            "score": 0.8366520641242258,
+            "score": 0.8071470662465113,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge2": 0.1946123321830468,
-                "rouge1": 0.4165480993021762,
-                "rougeLsum": 0.3481713172994178,
-                "rougeL": 0.2807625515077543,
-                "score": 0.2807625515077543,
+                "rougeLsum": 0.3492364214226588,
+                "rouge1": 0.41657427755274906,
+                "rouge2": 0.19392531737812374,
+                "rougeL": 0.28108832023050123,
+                "score": 0.28108832023050123,
                 "score_name": "rougeL",
-                "rouge2_ci_low": 0.18807199323963344,
-                "rouge2_ci_high": 0.2007897635489223,
-                "rouge1_ci_low": 0.40802841361294645,
-                "rouge1_ci_high": 0.4240881706518218,
-                "rougeLsum_ci_low": 0.34046630242406717,
-                "rougeLsum_ci_high": 0.35565311375927156,
-                "rougeL_ci_low": 0.2745389110131783,
-                "rougeL_ci_high": 0.2870232677361269,
-                "score_ci_low": 0.2745389110131783,
-                "score_ci_high": 0.2870232677361269
+                "rougeLsum_ci_low": 0.34133013043796767,
+                "rougeLsum_ci_high": 0.3567955471410065,
+                "rouge1_ci_low": 0.4080039541276808,
+                "rouge1_ci_high": 0.4246243250973701,
+                "rouge2_ci_low": 0.18712662527227458,
+                "rouge2_ci_high": 0.200448642429914,
+                "rougeL_ci_low": 0.2745892481615738,
+                "rougeL_ci_high": 0.2875837184145128,
+                "score_ci_low": 0.2745892481615738,
+                "score_ci_high": 0.2875837184145128
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge2": 0.013499529737597162,
-                "rouge1": 0.1111081768530587,
-                "rougeLsum": 0.0922891945088228,
-                "rougeL": 0.07992633322696455,
-                "score": 0.07992633322696455,
+                "rougeLsum": 0.0914997584684418,
+                "rouge1": 0.11103180621679556,
+                "rouge2": 0.013425878438988716,
+                "rougeL": 0.07983541241124872,
+                "score": 0.07983541241124872,
                 "score_name": "rougeL",
-                "rouge2_ci_low": 0.012006599571612698,
-                "rouge2_ci_high": 0.015167305255576668,
-                "rouge1_ci_low": 0.10596612811589602,
-                "rouge1_ci_high": 0.11561580840527891,
-                "rougeLsum_ci_low": 0.08846385121818591,
-                "rougeLsum_ci_high": 0.09604727885686246,
-                "rougeL_ci_low": 0.0765698806517895,
-                "rougeL_ci_high": 0.0830415577853562,
-                "score_ci_low": 0.0765698806517895,
-                "score_ci_high": 0.0830415577853562
+                "rougeLsum_ci_low": 0.08781447896275059,
+                "rougeLsum_ci_high": 0.0953497661867097,
+                "rouge1_ci_low": 0.10615759057700462,
+                "rouge1_ci_high": 0.11562260974835847,
+                "rouge2_ci_low": 0.012023789954203338,
+                "rouge2_ci_high": 0.015059698304736774,
+                "rougeL_ci_low": 0.07657318636107396,
+                "rougeL_ci_high": 0.08299164478552631,
+                "score_ci_low": 0.07657318636107396,
+                "score_ci_high": 0.08299164478552631
             },
-            "score": 0.18034444236735941,
+            "score": 0.18046186632087496,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1002,
-                    497,
-                    282,
-                    169
+                    1003,
+                    498,
+                    270,
+                    157
                 ],
                 "totals": [
-                    1844,
-                    1778,
-                    1712,
-                    1646
+                    1854,
+                    1788,
+                    1722,
+                    1656
                 ],
                 "precisions": [
-                    0.5433839479392625,
-                    0.2795275590551181,
-                    0.1647196261682243,
-                    0.10267314702308626
+                    0.5409924487594391,
+                    0.2785234899328859,
+                    0.156794425087108,
+                    0.09480676328502416
                 ],
                 "bp": 1.0,
-                "sys_len": 1844,
+                "sys_len": 1854,
                 "ref_len": 1734,
-                "sacrebleu": 0.22513002244943295,
-                "score": 0.22513002244943295,
+                "sacrebleu": 0.2175483241536988,
+                "score": 0.2175483241536988,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18442974136293813,
-                "score_ci_high": 0.26207208386144243,
-                "sacrebleu_ci_low": 0.18442974136293813,
-                "sacrebleu_ci_high": 0.26207208386144243
+                "score_ci_low": 0.18247071615886146,
+                "score_ci_high": 0.24704171532422453,
+                "sacrebleu_ci_low": 0.18247071615886146,
+                "sacrebleu_ci_high": 0.24704171532422453
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1117,
-                    634,
-                    393,
-                    252
+                    1080,
+                    568,
+                    332,
+                    190
                 ],
                 "totals": [
-                    1756,
-                    1690,
-                    1624,
-                    1558
+                    1763,
+                    1697,
+                    1631,
+                    1565
                 ],
                 "precisions": [
-                    0.6361047835990888,
-                    0.37514792899408284,
-                    0.2419950738916256,
-                    0.16174582798459564
+                    0.6125921724333522,
+                    0.33470830878020036,
+                    0.20355610055180873,
+                    0.12140575079872205
                 ],
                 "bp": 1.0,
-                "sys_len": 1756,
+                "sys_len": 1763,
                 "ref_len": 1734,
-                "sacrebleu": 0.3108799453655372,
-                "score": 0.3108799453655372,
+                "sacrebleu": 0.26680276716067836,
+                "score": 0.26680276716067836,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.26939824952148095,
-                "score_ci_high": 0.35631092704009654,
-                "sacrebleu_ci_low": 0.26939824952148095,
-                "sacrebleu_ci_high": 0.35631092704009654
+                "score_ci_low": 0.23826229076053318,
+                "score_ci_high": 0.2977813737555276,
+                "sacrebleu_ci_low": 0.23826229076053318,
+                "sacrebleu_ci_high": 0.2977813737555276
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    518,
-                    145,
-                    48,
-                    15
+                    553,
+                    147,
+                    54,
+                    16
                 ],
                 "totals": [
-                    1654,
-                    1588,
-                    1522,
-                    1456
+                    1726,
+                    1660,
+                    1594,
+                    1528
                 ],
                 "precisions": [
-                    0.313180169286578,
-                    0.09130982367758186,
-                    0.03153745072273324,
-                    0.0103021978021978
+                    0.3203939745075319,
+                    0.08855421686746988,
+                    0.033877038895859475,
+                    0.010471204188481676
                 ],
                 "bp": 1.0,
-                "sys_len": 1654,
+                "sys_len": 1726,
                 "ref_len": 1589,
-                "sacrebleu": 0.055209912255726495,
-                "score": 0.055209912255726495,
+                "sacrebleu": 0.056324703529775505,
+                "score": 0.056324703529775505,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.04194038184823177,
-                "score_ci_high": 0.08190458345551464,
-                "sacrebleu_ci_low": 0.04194038184823177,
-                "sacrebleu_ci_high": 0.08190458345551464
+                "score_ci_low": 0.04391320313402893,
+                "score_ci_high": 0.07339385366459818,
+                "sacrebleu_ci_low": 0.04391320313402893,
+                "sacrebleu_ci_high": 0.07339385366459818
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    932,
-                    408,
-                    209,
-                    113
+                    919,
+                    414,
+                    215,
+                    112
                 ],
                 "totals": [
-                    1864,
-                    1798,
-                    1732,
-                    1666
+                    1759,
+                    1693,
+                    1627,
+                    1561
                 ],
                 "precisions": [
-                    0.5,
-                    0.22691879866518352,
-                    0.12066974595842955,
-                    0.06782713085234093
+                    0.5224559408754975,
+                    0.24453632604843473,
+                    0.13214505224339276,
+                    0.07174887892376682
                 ],
-                "bp": 1.0,
-                "sys_len": 1864,
+                "bp": 0.9577137289198663,
+                "sys_len": 1759,
                 "ref_len": 1835,
-                "sacrebleu": 0.17456637003886807,
-                "score": 0.17456637003886807,
+                "sacrebleu": 0.17866952528026325,
+                "score": 0.17866952528026325,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1446545609941566,
-                "score_ci_high": 0.2117995060278497,
-                "sacrebleu_ci_low": 0.1446545609941566,
-                "sacrebleu_ci_high": 0.2117995060278497
+                "score_ci_low": 0.14916846640026038,
+                "score_ci_high": 0.20635605395794115,
+                "sacrebleu_ci_low": 0.14916846640026038,
+                "sacrebleu_ci_high": 0.20635605395794115
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1281,
-                    791,
-                    540,
-                    380
+                    1274,
+                    792,
+                    541,
+                    376
                 ],
                 "totals": [
-                    2064,
-                    1998,
-                    1932,
-                    1866
+                    1972,
+                    1906,
+                    1840,
+                    1774
                 ],
                 "precisions": [
-                    0.6206395348837209,
-                    0.39589589589589586,
-                    0.27950310559006214,
-                    0.20364415862808144
+                    0.6460446247464503,
+                    0.4155299055613851,
+                    0.2940217391304348,
+                    0.21195039458850057
                 ],
-                "bp": 0.9980638921833086,
-                "sys_len": 2064,
+                "bp": 0.9524844080827892,
+                "sys_len": 1972,
                 "ref_len": 2068,
-                "sacrebleu": 0.3432243584069162,
-                "score": 0.3432243584069162,
+                "sacrebleu": 0.3425527778466637,
+                "score": 0.3425527778466637,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.31863429882294914,
-                "score_ci_high": 0.38949054032191865,
-                "sacrebleu_ci_low": 0.31863429882294914,
-                "sacrebleu_ci_high": 0.38949054032191865
+                "score_ci_low": 0.30423882287446036,
+                "score_ci_high": 0.38397398779299585,
+                "sacrebleu_ci_low": 0.30423882287446036,
+                "sacrebleu_ci_high": 0.38397398779299585
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    943,
-                    328,
-                    141,
-                    66
+                    959,
+                    341,
+                    148,
+                    65
                 ],
                 "totals": [
-                    3253,
-                    3187,
-                    3121,
-                    3055
+                    3115,
+                    3049,
+                    2983,
+                    2917
                 ],
                 "precisions": [
-                    0.28988625883799574,
-                    0.10291810480075306,
-                    0.04517782761935277,
-                    0.02160392798690671
+                    0.3078651685393259,
+                    0.1118399475237783,
+                    0.0496144820650352,
+                    0.02228316763798423
                 ],
                 "bp": 1.0,
-                "sys_len": 3253,
+                "sys_len": 3115,
                 "ref_len": 2235,
-                "sacrebleu": 0.07345889118508468,
-                "score": 0.07345889118508468,
+                "sacrebleu": 0.07854810736755143,
+                "score": 0.07854810736755143,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06129370081536535,
-                "score_ci_high": 0.09632357969438955,
-                "sacrebleu_ci_low": 0.06129370081536535,
-                "sacrebleu_ci_high": 0.09632357969438955
+                "score_ci_low": 0.06038268917694664,
+                "score_ci_high": 0.0991975666301703,
+                "sacrebleu_ci_low": 0.06038268917694664,
+                "sacrebleu_ci_high": 0.0991975666301703
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1229,
-                    727,
+                    1231,
+                    729,
                     477,
-                    316
+                    311
                 ],
                 "totals": [
-                    1915,
-                    1849,
-                    1783,
-                    1717
+                    1934,
+                    1868,
+                    1802,
+                    1736
                 ],
                 "precisions": [
-                    0.64177545691906,
-                    0.3931855056787452,
-                    0.2675266404935502,
-                    0.18404193360512522
+                    0.6365046535677352,
+                    0.39025695931477516,
+                    0.2647058823529412,
+                    0.179147465437788
                 ],
-                "bp": 0.9994779431076575,
-                "sys_len": 1915,
+                "bp": 1.0,
+                "sys_len": 1934,
                 "ref_len": 1916,
-                "sacrebleu": 0.3336870259850046,
-                "score": 0.3336870259850046,
+                "sacrebleu": 0.3294440172060282,
+                "score": 0.3294440172060282,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30427044055098784,
-                "score_ci_high": 0.3722955797843792,
-                "sacrebleu_ci_low": 0.30427044055098784,
-                "sacrebleu_ci_high": 0.3722955797843792
+                "score_ci_low": 0.295619495912131,
+                "score_ci_high": 0.3689718708243594,
+                "sacrebleu_ci_low": 0.295619495912131,
+                "sacrebleu_ci_high": 0.3689718708243594
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    776,
-                    292,
-                    137,
-                    66
+                    775,
+                    275,
+                    128,
+                    70
                 ],
                 "totals": [
-                    2002,
-                    1936,
-                    1870,
-                    1804
+                    2251,
+                    2185,
+                    2119,
+                    2053
                 ],
                 "precisions": [
-                    0.3876123876123876,
-                    0.15082644628099173,
-                    0.0732620320855615,
-                    0.03658536585365854
+                    0.3442914260328743,
+                    0.12585812356979406,
+                    0.06040585181689476,
+                    0.034096444227959086
                 ],
                 "bp": 1.0,
-                "sys_len": 2002,
+                "sys_len": 2251,
                 "ref_len": 1949,
-                "sacrebleu": 0.11188332833173187,
-                "score": 0.11188332833173187,
+                "sacrebleu": 0.09719611157545467,
+                "score": 0.09719611157545467,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09410502256897492,
-                "score_ci_high": 0.14085608331376978,
-                "sacrebleu_ci_low": 0.09410502256897492,
-                "sacrebleu_ci_high": 0.14085608331376978
+                "score_ci_low": 0.06983887234981923,
+                "score_ci_high": 0.1310674385834968,
+                "sacrebleu_ci_low": 0.06983887234981923,
+                "sacrebleu_ci_high": 0.1310674385834968
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1155,
-                    545,
-                    284,
-                    157
+                    1157,
+                    577,
+                    322,
+                    195
                 ],
                 "totals": [
-                    2065,
-                    1999,
-                    1933,
-                    1867
+                    2040,
+                    1974,
+                    1908,
+                    1842
                 ],
                 "precisions": [
-                    0.559322033898305,
-                    0.2726363181590795,
-                    0.14692188308329024,
-                    0.08409212640599893
+                    0.567156862745098,
+                    0.2922998986828774,
+                    0.16876310272536688,
+                    0.10586319218241043
                 ],
-                "bp": 0.9841463832388515,
-                "sys_len": 2065,
+                "bp": 0.9719689956119355,
+                "sys_len": 2040,
                 "ref_len": 2098,
-                "sacrebleu": 0.20503668186599197,
-                "score": 0.20503668186599197,
+                "sacrebleu": 0.22674671169002888,
+                "score": 0.22674671169002888,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.17287865798739985,
-                "score_ci_high": 0.22839534867300298,
-                "sacrebleu_ci_low": 0.17287865798739985,
-                "sacrebleu_ci_high": 0.22839534867300298
+                "score_ci_low": 0.20018186306011954,
+                "score_ci_high": 0.24942035354854425,
+                "sacrebleu_ci_low": 0.20018186306011954,
+                "sacrebleu_ci_high": 0.24942035354854425
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1154,
-                    673,
-                    416,
-                    270
+                    1147,
+                    643,
+                    397,
+                    246
                 ],
                 "totals": [
-                    1805,
-                    1739,
-                    1673,
-                    1607
+                    1808,
+                    1742,
+                    1676,
+                    1610
                 ],
                 "precisions": [
-                    0.6393351800554017,
-                    0.38700402530189765,
-                    0.24865511057979678,
-                    0.16801493466085873
+                    0.6344026548672567,
+                    0.36911595866819746,
+                    0.23687350835322196,
+                    0.15279503105590064
                 ],
                 "bp": 1.0,
-                "sys_len": 1805,
+                "sys_len": 1808,
                 "ref_len": 1734,
-                "sacrebleu": 0.31885801670800706,
-                "score": 0.31885801670800706,
+                "sacrebleu": 0.30341593414236545,
+                "score": 0.30341593414236545,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2891892296952914,
-                "score_ci_high": 0.36690842488060277,
-                "sacrebleu_ci_low": 0.2891892296952914,
-                "sacrebleu_ci_high": 0.36690842488060277
+                "score_ci_low": 0.26610148555409346,
+                "score_ci_high": 0.3551058656882207,
+                "sacrebleu_ci_low": 0.26610148555409346,
+                "sacrebleu_ci_high": 0.3551058656882207
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    932,
-                    394,
-                    198,
-                    108
+                    935,
+                    383,
+                    193,
+                    101
                 ],
                 "totals": [
-                    2022,
-                    1956,
-                    1890,
-                    1824
+                    1950,
+                    1884,
+                    1818,
+                    1752
                 ],
                 "precisions": [
-                    0.4609297725024728,
-                    0.20143149284253578,
-                    0.10476190476190476,
-                    0.05921052631578948
+                    0.4794871794871795,
+                    0.2032908704883227,
+                    0.10616061606160615,
+                    0.057648401826484015
                 ],
                 "bp": 1.0,
-                "sys_len": 2022,
+                "sys_len": 1950,
                 "ref_len": 1734,
-                "sacrebleu": 0.15491415770056607,
-                "score": 0.15491415770056607,
+                "sacrebleu": 0.15628287583119144,
+                "score": 0.15628287583119144,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12994962612263172,
-                "score_ci_high": 0.1870372431393324,
-                "sacrebleu_ci_low": 0.12994962612263172,
-                "sacrebleu_ci_high": 0.1870372431393324
+                "score_ci_low": 0.1304739698304556,
+                "score_ci_high": 0.19246721744185705,
+                "sacrebleu_ci_low": 0.1304739698304556,
+                "sacrebleu_ci_high": 0.19246721744185705
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    842,
-                    316,
-                    161,
-                    84
+                    848,
+                    302,
+                    135,
+                    60
                 ],
                 "totals": [
-                    1863,
-                    1797,
-                    1731,
-                    1665
+                    1872,
+                    1806,
+                    1740,
+                    1674
                 ],
                 "precisions": [
-                    0.451959205582394,
-                    0.17584863661658318,
-                    0.09300982091276719,
-                    0.05045045045045045
+                    0.452991452991453,
+                    0.1672203765227021,
+                    0.07758620689655173,
+                    0.035842293906810034
                 ],
                 "bp": 1.0,
-                "sys_len": 1863,
+                "sys_len": 1872,
                 "ref_len": 1734,
-                "sacrebleu": 0.1389658296637508,
-                "score": 0.1389658296637508,
+                "sacrebleu": 0.12047304306149162,
+                "score": 0.12047304306149162,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11444856857935395,
-                "score_ci_high": 0.17296950184028867,
-                "sacrebleu_ci_low": 0.11444856857935395,
-                "sacrebleu_ci_high": 0.17296950184028867
+                "score_ci_low": 0.10281238979764949,
+                "score_ci_high": 0.15881384065042398,
+                "sacrebleu_ci_low": 0.10281238979764949,
+                "sacrebleu_ci_high": 0.15881384065042398
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1125,
-                    650,
-                    405,
+                    1129,
+                    652,
+                    412,
                     268
                 ],
                 "totals": [
-                    1763,
-                    1697,
-                    1631,
-                    1565
+                    1783,
+                    1717,
+                    1651,
+                    1585
                 ],
                 "precisions": [
-                    0.6381168462847419,
-                    0.38302887448438416,
-                    0.24831391784181484,
-                    0.17124600638977636
+                    0.6332024677509814,
+                    0.3797320908561444,
+                    0.24954572986069049,
+                    0.1690851735015773
                 ],
                 "bp": 1.0,
-                "sys_len": 1763,
+                "sys_len": 1783,
                 "ref_len": 1734,
-                "sacrebleu": 0.31929220031180316,
-                "score": 0.31929220031180316,
+                "sacrebleu": 0.3173722073323666,
+                "score": 0.3173722073323666,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.27700187051697595,
-                "score_ci_high": 0.3625973900935978,
-                "sacrebleu_ci_low": 0.27700187051697595,
-                "sacrebleu_ci_high": 0.3625973900935978
+                "score_ci_low": 0.28432250677130166,
+                "score_ci_high": 0.35844418554288615,
+                "sacrebleu_ci_low": 0.28432250677130166,
+                "sacrebleu_ci_high": 0.35844418554288615
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1107,
-                    596,
-                    373,
-                    242
+                    1091,
+                    601,
+                    375,
+                    245
                 ],
                 "totals": [
-                    1855,
-                    1789,
-                    1723,
-                    1657
+                    1793,
+                    1727,
+                    1661,
+                    1595
                 ],
                 "precisions": [
-                    0.5967654986522911,
-                    0.33314700950251536,
-                    0.21648287869994196,
-                    0.1460470730235365
+                    0.6084774121583938,
+                    0.3480023161551824,
+                    0.2257676098735701,
+                    0.1536050156739812
                 ],
                 "bp": 1.0,
-                "sys_len": 1855,
+                "sys_len": 1793,
                 "ref_len": 1734,
-                "sacrebleu": 0.28157170427283745,
-                "score": 0.28157170427283745,
+                "sacrebleu": 0.2927341616520049,
+                "score": 0.2927341616520049,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24793101467267725,
-                "score_ci_high": 0.34906947598840327,
-                "sacrebleu_ci_low": 0.24793101467267725,
-                "sacrebleu_ci_high": 0.34906947598840327
+                "score_ci_low": 0.2507745489087611,
+                "score_ci_high": 0.329051655313229,
+                "sacrebleu_ci_low": 0.2507745489087611,
+                "sacrebleu_ci_high": 0.329051655313229
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1030,
-                    507,
-                    273,
-                    149
+                    1037,
+                    482,
+                    248,
+                    129
                 ],
                 "totals": [
-                    1843,
-                    1777,
-                    1711,
-                    1645
+                    1796,
+                    1730,
+                    1664,
+                    1598
                 ],
                 "precisions": [
-                    0.5588714053174173,
-                    0.28531232414181207,
-                    0.15955581531268265,
-                    0.0905775075987842
+                    0.5773942093541202,
+                    0.2786127167630058,
+                    0.14903846153846154,
+                    0.0807259073842303
                 ],
                 "bp": 1.0,
-                "sys_len": 1843,
+                "sys_len": 1796,
                 "ref_len": 1734,
-                "sacrebleu": 0.21909948470764218,
-                "score": 0.21909948470764218,
+                "sacrebleu": 0.20974719583348747,
+                "score": 0.20974719583348747,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18945741585306675,
-                "score_ci_high": 0.26472135303184596,
-                "sacrebleu_ci_low": 0.18945741585306675,
-                "sacrebleu_ci_high": 0.26472135303184596
+                "score_ci_low": 0.18522343537553757,
+                "score_ci_high": 0.2489124348912048,
+                "sacrebleu_ci_low": 0.18522343537553757,
+                "sacrebleu_ci_high": 0.2489124348912048
             },
-            "score": 0.21771852861659338,
+            "score": 0.21292389757753669,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.4171223590857319,
+        "score": 0.37890410445729916,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-19T16-21-09_evaluation_results.json → 2025-06-23T04-06-37_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-19T20:21:05.821665Z",
+        "timestamp_utc": "2025-06-23T08:06:33.434344Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,103 +176,103 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.45555555555555555,
-                "accuracy_ci_high": 0.6555555555555556,
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.6,
                 "score_name": "accuracy",
-                "score": 0.5555555555555556,
-                "score_ci_high": 0.6555555555555556,
-                "score_ci_low": 0.45555555555555555,
+                "score": 0.5,
+                "score_ci_high": 0.6,
+                "score_ci_low": 0.4,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.6222222222222222,
-                "accuracy_ci_low": 0.5222222222222223,
-                "accuracy_ci_high": 0.7222222222222222,
+                "accuracy": 0.6777777777777778,
+                "accuracy_ci_low": 0.5777777777777777,
+                "accuracy_ci_high": 0.7666666666666667,
                 "score_name": "accuracy",
-                "score": 0.6222222222222222,
-                "score_ci_high": 0.7222222222222222,
-                "score_ci_low": 0.5222222222222223,
+                "score": 0.6777777777777778,
+                "score_ci_high": 0.7666666666666667,
+                "score_ci_low": 0.5777777777777777,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8777777777777778,
-                "accuracy_ci_low": 0.8,
-                "accuracy_ci_high": 0.9333333333333333,
+                "accuracy": 0.8333333333333334,
+                "accuracy_ci_low": 0.7444444444444445,
+                "accuracy_ci_high": 0.9,
                 "score_name": "accuracy",
-                "score": 0.8777777777777778,
-                "score_ci_high": 0.9333333333333333,
-                "score_ci_low": 0.8,
+                "score": 0.8333333333333334,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7444444444444445,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.6333333333333333,
-                "accuracy_ci_low": 0.5333333333333333,
-                "accuracy_ci_high": 0.7333333333333333,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.5666666666666667,
+                "accuracy_ci_high": 0.7555555555555555,
                 "score_name": "accuracy",
-                "score": 0.6333333333333333,
-                "score_ci_high": 0.7333333333333333,
-                "score_ci_low": 0.5333333333333333,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.7555555555555555,
+                "score_ci_low": 0.5666666666666667,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.6555555555555556,
-                "accuracy_ci_low": 0.5555555555555556,
-                "accuracy_ci_high": 0.7539633744548231,
+                "accuracy": 0.7111111111111111,
+                "accuracy_ci_low": 0.6111111111111112,
+                "accuracy_ci_high": 0.8,
                 "score_name": "accuracy",
-                "score": 0.6555555555555556,
-                "score_ci_high": 0.7539633744548231,
-                "score_ci_low": 0.5555555555555556,
+                "score": 0.7111111111111111,
+                "score_ci_high": 0.8,
+                "score_ci_low": 0.6111111111111112,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
                 "accuracy": 0.9333333333333333,
-                "accuracy_ci_low": 0.8666666666666667,
+                "accuracy_ci_low": 0.8777777777777778,
                 "accuracy_ci_high": 0.9777777777777777,
                 "score_name": "accuracy",
                 "score": 0.9333333333333333,
                 "score_ci_high": 0.9777777777777777,
-                "score_ci_low": 0.8666666666666667,
+                "score_ci_low": 0.8777777777777778,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.8222222222222222,
-                "accuracy_ci_high": 0.9444444444444444,
+                "accuracy": 0.8444444444444444,
+                "accuracy_ci_low": 0.7555555555555555,
+                "accuracy_ci_high": 0.9048361867497154,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
-                "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.8222222222222222,
+                "score": 0.8444444444444444,
+                "score_ci_high": 0.9048361867497154,
+                "score_ci_low": 0.7555555555555555,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.9333333333333333,
-                "accuracy_ci_low": 0.8666666666666667,
-                "accuracy_ci_high": 0.9777777777777777,
+                "accuracy": 0.9111111111111111,
+                "accuracy_ci_low": 0.8333333333333334,
+                "accuracy_ci_high": 0.9555555555555556,
                 "score_name": "accuracy",
-                "score": 0.9333333333333333,
-                "score_ci_high": 0.9777777777777777,
-                "score_ci_low": 0.8666666666666667,
+                "score": 0.9111111111111111,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8333333333333334,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.7666666666666667,
-                "accuracy_ci_low": 0.6720698151047421,
-                "accuracy_ci_high": 0.8444444444444444,
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6555555555555556,
+                "accuracy_ci_high": 0.83090190108808,
                 "score_name": "accuracy",
-                "score": 0.7666666666666667,
-                "score_ci_high": 0.8444444444444444,
-                "score_ci_low": 0.6720698151047421,
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.83090190108808,
+                "score_ci_low": 0.6555555555555556,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.6333333333333333,
-                "accuracy_ci_low": 0.5333333333333333,
-                "accuracy_ci_high": 0.7283280971833935,
+                "accuracy": 0.6777777777777778,
+                "accuracy_ci_low": 0.5777777777777777,
+                "accuracy_ci_high": 0.7666666666666667,
                 "score_name": "accuracy",
-                "score": 0.6333333333333333,
-                "score_ci_high": 0.7283280971833935,
-                "score_ci_low": 0.5333333333333333,
+                "score": 0.6777777777777778,
+                "score_ci_high": 0.7666666666666667,
+                "score_ci_low": 0.5777777777777777,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
@@ -285,54 +285,54 @@
                 "score_ci_low": 0.6666666666666666,
                 "num_of_instances": 90
             },
-            "score": 0.7515151515151515,
+            "score": 0.7515151515151516,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.08744186046511628,
+                "score": 0.08744186046511628,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.08744186046511628,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.5102639296187683,
-                "f1_Organization": 0.3381294964028777,
-                "f1_Location": 0.35652173913043483,
-                "f1_macro": 0.40163838838402693,
-                "recall_macro": 0.3240210323686792,
-                "precision_macro": 0.530656067251462,
-                "in_classes_support": 0.5625,
-                "f1_micro": 0.31789282470481384,
-                "recall_micro": 0.3333333333333333,
-                "precision_micro": 0.3038194444444444,
-                "score": 0.31789282470481384,
+                "f1_Person": 0.52046783625731,
+                "f1_Location": 0.3275862068965517,
+                "f1_Organization": 0.3905723905723905,
+                "f1_macro": 0.41287547790875073,
+                "recall_macro": 0.34275188964299236,
+                "precision_macro": 0.5261312195216724,
+                "in_classes_support": 0.5945017182130584,
+                "f1_micro": 0.3342366757000903,
+                "recall_micro": 0.3523809523809524,
+                "precision_micro": 0.3178694158075601,
+                "score": 0.3342366757000903,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.26482961534023236,
-                "score_ci_high": 0.37029988780714157,
-                "f1_micro_ci_low": 0.26482961534023236,
-                "f1_micro_ci_high": 0.37029988780714157
+                "score_ci_low": 0.29025426476142113,
+                "score_ci_high": 0.38246190736620644,
+                "f1_micro_ci_low": 0.29025426476142113,
+                "f1_micro_ci_high": 0.38246190736620644
             },
-            "score": 0.31789282470481384,
+            "score": 0.3342366757000903,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.5211267605633803,
-                "accuracy_ci_low": 0.4084507042253521,
-                "accuracy_ci_high": 0.6338028169014085,
+                "accuracy": 0.4225352112676056,
+                "accuracy_ci_low": 0.30985915492957744,
+                "accuracy_ci_high": 0.5352112676056338,
                 "score_name": "accuracy",
-                "score": 0.5211267605633803,
-                "score_ci_high": 0.6338028169014085,
-                "score_ci_low": 0.4084507042253521,
+                "score": 0.4225352112676056,
+                "score_ci_high": 0.5352112676056338,
+                "score_ci_low": 0.30985915492957744,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
@@ -348,392 +348,392 @@
             "mmlu_pro_chemistry": {
                 "accuracy": 0.23943661971830985,
                 "accuracy_ci_low": 0.15492957746478872,
-                "accuracy_ci_high": 0.3380281690140845,
+                "accuracy_ci_high": 0.352112676056338,
                 "score_name": "accuracy",
                 "score": 0.23943661971830985,
-                "score_ci_high": 0.3380281690140845,
+                "score_ci_high": 0.352112676056338,
                 "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.43661971830985913,
-                "accuracy_ci_low": 0.323943661971831,
-                "accuracy_ci_high": 0.5492957746478874,
+                "accuracy": 0.4084507042253521,
+                "accuracy_ci_low": 0.29577464788732394,
+                "accuracy_ci_high": 0.5211267605633803,
                 "score_name": "accuracy",
-                "score": 0.43661971830985913,
-                "score_ci_high": 0.5492957746478874,
-                "score_ci_low": 0.323943661971831,
+                "score": 0.4084507042253521,
+                "score_ci_high": 0.5211267605633803,
+                "score_ci_low": 0.29577464788732394,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.38028169014084506,
-                "accuracy_ci_low": 0.2676056338028169,
-                "accuracy_ci_high": 0.49295774647887325,
+                "accuracy": 0.4084507042253521,
+                "accuracy_ci_low": 0.29577464788732394,
+                "accuracy_ci_high": 0.5211267605633803,
                 "score_name": "accuracy",
-                "score": 0.38028169014084506,
-                "score_ci_high": 0.49295774647887325,
-                "score_ci_low": 0.2676056338028169,
+                "score": 0.4084507042253521,
+                "score_ci_high": 0.5211267605633803,
+                "score_ci_low": 0.29577464788732394,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.2535211267605634,
-                "accuracy_ci_low": 0.16901408450704225,
-                "accuracy_ci_high": 0.36048330202820134,
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.3380281690140845,
                 "score_name": "accuracy",
-                "score": 0.2535211267605634,
-                "score_ci_high": 0.36048330202820134,
-                "score_ci_low": 0.16901408450704225,
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.3380281690140845,
+                "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.36619718309859156,
-                "accuracy_ci_low": 0.2535211267605634,
-                "accuracy_ci_high": 0.4788732394366197,
+                "accuracy": 0.352112676056338,
+                "accuracy_ci_low": 0.23943661971830985,
+                "accuracy_ci_high": 0.4647887323943662,
                 "score_name": "accuracy",
-                "score": 0.36619718309859156,
-                "score_ci_high": 0.4788732394366197,
-                "score_ci_low": 0.2535211267605634,
+                "score": 0.352112676056338,
+                "score_ci_high": 0.4647887323943662,
+                "score_ci_low": 0.23943661971830985,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.36619718309859156,
-                "accuracy_ci_low": 0.2535211267605634,
-                "accuracy_ci_high": 0.4788732394366197,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.21693057179778907,
+                "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
-                "score": 0.36619718309859156,
-                "score_ci_high": 0.4788732394366197,
-                "score_ci_low": 0.2535211267605634,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.21693057179778907,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.36619718309859156,
-                "accuracy_ci_low": 0.2535211267605634,
-                "accuracy_ci_high": 0.4788732394366197,
+                "accuracy": 0.30985915492957744,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.42459270101591795,
                 "score_name": "accuracy",
-                "score": 0.36619718309859156,
-                "score_ci_high": 0.4788732394366197,
-                "score_ci_low": 0.2535211267605634,
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.42459270101591795,
+                "score_ci_low": 0.2112676056338028,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.1267605633802817,
-                "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.22535211267605634,
+                "accuracy": 0.09859154929577464,
+                "accuracy_ci_low": 0.04225352112676056,
+                "accuracy_ci_high": 0.17777703477060838,
                 "score_name": "accuracy",
-                "score": 0.1267605633802817,
-                "score_ci_high": 0.22535211267605634,
-                "score_ci_low": 0.056338028169014086,
+                "score": 0.09859154929577464,
+                "score_ci_high": 0.17777703477060838,
+                "score_ci_low": 0.04225352112676056,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.22338079742223388,
+                "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.22338079742223388,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.4084507042253521,
-                "accuracy_ci_low": 0.30985915492957744,
-                "accuracy_ci_high": 0.5352112676056338,
+                "accuracy": 0.4647887323943662,
+                "accuracy_ci_low": 0.352112676056338,
+                "accuracy_ci_high": 0.5915492957746479,
                 "score_name": "accuracy",
-                "score": 0.4084507042253521,
-                "score_ci_high": 0.5352112676056338,
-                "score_ci_low": 0.30985915492957744,
+                "score": 0.4647887323943662,
+                "score_ci_high": 0.5915492957746479,
+                "score_ci_low": 0.352112676056338,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.29577464788732394,
-                "accuracy_ci_low": 0.19718309859154928,
-                "accuracy_ci_high": 0.4084507042253521,
+                "accuracy": 0.18309859154929578,
+                "accuracy_ci_low": 0.11267605633802817,
+                "accuracy_ci_high": 0.28169014084507044,
                 "score_name": "accuracy",
-                "score": 0.29577464788732394,
-                "score_ci_high": 0.4084507042253521,
-                "score_ci_low": 0.19718309859154928,
+                "score": 0.18309859154929578,
+                "score_ci_high": 0.28169014084507044,
+                "score_ci_low": 0.11267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
                 "accuracy": 0.5352112676056338,
-                "accuracy_ci_low": 0.4084507042253521,
+                "accuracy_ci_low": 0.4225352112676056,
                 "accuracy_ci_high": 0.647887323943662,
                 "score_name": "accuracy",
                 "score": 0.5352112676056338,
                 "score_ci_high": 0.647887323943662,
-                "score_ci_low": 0.4084507042253521,
+                "score_ci_low": 0.4225352112676056,
                 "num_of_instances": 71
             },
-            "score": 0.33702213279678067,
+            "score": 0.32193158953722334,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.2696554985630616,
-                "f1_suggestive": 0.2727272727272727,
-                "f1_arbitrary": 0.43137254901960786,
-                "f1_generic": 0.11764705882352941,
-                "f1_fanciful": 0.2,
-                "f1_descriptive": 0.32653061224489793,
-                "f1_macro_ci_low": 0.18689773936584586,
-                "f1_macro_ci_high": 0.37923074712363225,
+                "f1_macro": 0.3075373413771583,
+                "f1_suggestive": 0.36363636363636365,
+                "f1_arbitrary": 0.28,
+                "f1_generic": 0.3157894736842105,
+                "f1_fanciful": 0.1,
+                "f1_descriptive": 0.4782608695652174,
+                "f1_macro_ci_low": 0.22135682385238098,
+                "f1_macro_ci_high": 0.4258827689087187,
                 "score_name": "f1_micro",
-                "score": 0.31446540880503143,
-                "score_ci_high": 0.42038216560509556,
-                "score_ci_low": 0.21656050955414013,
+                "score": 0.33121019108280253,
+                "score_ci_high": 0.43513626025637364,
+                "score_ci_low": 0.22818791946308725,
                 "num_of_instances": 85,
-                "accuracy": 0.29411764705882354,
-                "accuracy_ci_low": 0.2,
+                "accuracy": 0.3058823529411765,
+                "accuracy_ci_low": 0.21176470588235294,
                 "accuracy_ci_high": 0.4,
-                "f1_micro": 0.31446540880503143,
-                "f1_micro_ci_low": 0.21656050955414013,
-                "f1_micro_ci_high": 0.42038216560509556
+                "f1_micro": 0.33121019108280253,
+                "f1_micro_ci_low": 0.22818791946308725,
+                "f1_micro_ci_high": 0.43513626025637364
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.5388253241800153,
-                "f1_no": 0.7298245614035088,
+                "f1_macro": 0.563568215892054,
+                "f1_no": 0.7793103448275862,
                 "f1_yes": 0.34782608695652173,
-                "f1_macro_ci_low": 0.47191290375757455,
-                "f1_macro_ci_high": 0.6216206779092042,
+                "f1_macro_ci_low": 0.49159571105513383,
+                "f1_macro_ci_high": 0.6365342652768683,
                 "score_name": "f1_micro",
-                "score": 0.636604774535809,
-                "score_ci_high": 0.6985040092826637,
-                "score_ci_low": 0.5691144311757004,
+                "score": 0.675392670157068,
+                "score_ci_high": 0.73489030467135,
+                "score_ci_low": 0.608918205032967,
                 "num_of_instances": 200,
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.53,
-                "accuracy_ci_high": 0.665,
-                "f1_micro": 0.636604774535809,
-                "f1_micro_ci_low": 0.5691144311757004,
-                "f1_micro_ci_high": 0.6985040092826637
+                "accuracy": 0.645,
+                "accuracy_ci_low": 0.58,
+                "accuracy_ci_high": 0.705,
+                "f1_micro": 0.675392670157068,
+                "f1_micro_ci_low": 0.608918205032967,
+                "f1_micro_ci_high": 0.73489030467135
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.2947177227927682,
-                "f1_conclusion": 0.2127659574468085,
-                "f1_decree": 0.23529411764705882,
-                "f1_issue": 0.2711864406779661,
-                "f1_rule": 0.42857142857142855,
-                "f1_analysis": 0.4444444444444444,
-                "f1_facts": 0.21621621621621623,
-                "f1_procedural history": 0.2545454545454545,
-                "f1_macro_ci_low": 0.23794703715833648,
-                "f1_macro_ci_high": 0.36665623309642204,
+                "f1_macro": 0.3062664077216316,
+                "f1_conclusion": 0.20833333333333334,
+                "f1_decree": 0.1875,
+                "f1_rule": 0.47761194029850745,
+                "f1_issue": 0.25,
+                "f1_analysis": 0.44,
+                "f1_facts": 0.2727272727272727,
+                "f1_procedural history": 0.3076923076923077,
+                "f1_macro_ci_low": 0.24659229419876927,
+                "f1_macro_ci_high": 0.3810118235674986,
                 "score_name": "f1_micro",
-                "score": 0.30409356725146197,
-                "score_ci_high": 0.3711587285161421,
-                "score_ci_low": 0.23855266549315363,
+                "score": 0.3209169054441261,
+                "score_ci_high": 0.3885967259042703,
+                "score_ci_low": 0.2564102564102564,
                 "num_of_instances": 200,
-                "accuracy": 0.26,
-                "accuracy_ci_low": 0.2,
-                "accuracy_ci_high": 0.32,
-                "f1_micro": 0.30409356725146197,
-                "f1_micro_ci_low": 0.23855266549315363,
-                "f1_micro_ci_high": 0.3711587285161421
+                "accuracy": 0.28,
+                "accuracy_ci_low": 0.22,
+                "accuracy_ci_high": 0.34,
+                "f1_micro": 0.3209169054441261,
+                "f1_micro_ci_low": 0.2564102564102564,
+                "f1_micro_ci_high": 0.3885967259042703
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.49092908191313905,
-                "f1_yes": 0.5700934579439252,
-                "f1_no": 0.4117647058823529,
-                "f1_macro_ci_low": 0.4178065856787266,
-                "f1_macro_ci_high": 0.5601203681213927,
+                "f1_macro": 0.5153888280394305,
+                "f1_yes": 0.6090909090909091,
+                "f1_no": 0.42168674698795183,
+                "f1_macro_ci_low": 0.44286956267940425,
+                "f1_macro_ci_high": 0.5822132955205006,
                 "score_name": "f1_micro",
-                "score": 0.5,
-                "score_ci_high": 0.566970455032283,
-                "score_ci_low": 0.42555336134062,
+                "score": 0.5284974093264249,
+                "score_ci_high": 0.5917634471129095,
+                "score_ci_low": 0.4572437728690022,
                 "num_of_instances": 200,
-                "accuracy": 0.48,
-                "accuracy_ci_low": 0.405,
-                "accuracy_ci_high": 0.545,
-                "f1_micro": 0.5,
-                "f1_micro_ci_low": 0.42555336134062,
-                "f1_micro_ci_high": 0.566970455032283
+                "accuracy": 0.51,
+                "accuracy_ci_low": 0.44,
+                "accuracy_ci_high": 0.575,
+                "f1_micro": 0.5284974093264249,
+                "f1_micro_ci_low": 0.4572437728690022,
+                "f1_micro_ci_high": 0.5917634471129095
             },
             "legalbench_proa": {
-                "f1_macro": 0.8315276273022751,
-                "f1_yes": 0.8169014084507042,
-                "f1_no": 0.8461538461538461,
-                "f1_macro_ci_low": 0.7549023325928579,
-                "f1_macro_ci_high": 0.890440353074843,
+                "f1_macro": 0.8555844155844157,
+                "f1_yes": 0.88,
+                "f1_no": 0.8311688311688312,
+                "f1_macro_ci_low": 0.7810530949191602,
+                "f1_macro_ci_high": 0.911227513400763,
                 "score_name": "f1_micro",
-                "score": 0.8322147651006712,
-                "score_ci_high": 0.8903225806451613,
-                "score_ci_low": 0.7554946760306516,
+                "score": 0.8552631578947368,
+                "score_ci_high": 0.9104714274063991,
+                "score_ci_low": 0.7791920429268818,
                 "num_of_instances": 85,
-                "accuracy": 0.7294117647058823,
-                "accuracy_ci_low": 0.6352941176470588,
-                "accuracy_ci_high": 0.8117647058823529,
-                "f1_micro": 0.8322147651006712,
-                "f1_micro_ci_low": 0.7554946760306516,
-                "f1_micro_ci_high": 0.8903225806451613
+                "accuracy": 0.7647058823529411,
+                "accuracy_ci_low": 0.6705882352941176,
+                "accuracy_ci_high": 0.8470588235294118,
+                "f1_micro": 0.8552631578947368,
+                "f1_micro_ci_low": 0.7791920429268818,
+                "f1_micro_ci_high": 0.9104714274063991
             },
-            "score": 0.5174757031385947,
+            "score": 0.5422560667810317,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.42272407811143237,
-                "f1_cars": 0.6078431372549019,
-                "f1_pc hardware": 0.34080717488789236,
-                "f1_windows x": 0.029850746268656716,
-                "f1_computer graphics": 0.4367816091954023,
-                "f1_atheism": 0.21739130434782608,
-                "f1_religion": 0.23300970873786409,
-                "f1_medicine": 0.8641975308641975,
+                "f1_macro": 0.4238304887304002,
+                "f1_cars": 0.6534653465346535,
+                "f1_pc hardware": 0.38095238095238093,
+                "f1_windows x": 0.0,
+                "f1_atheism": 0.2727272727272727,
+                "f1_religion": 0.22641509433962265,
+                "f1_medicine": 0.7901234567901234,
                 "f1_christianity": 0.1694915254237288,
-                "f1_microsoft windows": 0.39436619718309857,
-                "f1_middle east": 0.43037974683544306,
-                "f1_politics": 0.291970802919708,
-                "f1_motorcycles": 0.43902439024390244,
-                "f1_mac hardware": 0.09090909090909091,
-                "f1_for sale": 0.625,
-                "f1_guns": 0.18181818181818182,
-                "f1_space": 0.5569620253164557,
-                "f1_cryptography": 0.4482758620689655,
-                "f1_baseball": 0.8545454545454545,
-                "f1_hockey": 0.859504132231405,
-                "f1_electronics": 0.38235294117647056,
-                "f1_macro_ci_low": 0.3988534736802405,
-                "f1_macro_ci_high": 0.4557473948035634,
+                "f1_computer graphics": 0.3373493975903614,
+                "f1_microsoft windows": 0.37681159420289856,
+                "f1_middle east": 0.4594594594594595,
+                "f1_politics": 0.27906976744186046,
+                "f1_motorcycles": 0.4883720930232558,
+                "f1_mac hardware": 0.03125,
+                "f1_for sale": 0.6461538461538462,
+                "f1_guns": 0.18518518518518517,
+                "f1_space": 0.575,
+                "f1_cryptography": 0.5079365079365079,
+                "f1_baseball": 0.8468468468468469,
+                "f1_hockey": 0.85,
+                "f1_electronics": 0.4,
+                "f1_macro_ci_low": 0.39859097081154116,
+                "f1_macro_ci_high": 0.4545535978307604,
                 "score_name": "f1_micro",
-                "score": 0.44368600682593856,
-                "score_ci_high": 0.47444463958776134,
-                "score_ci_low": 0.4135801299006492,
+                "score": 0.4485549132947977,
+                "score_ci_high": 0.4787668189917876,
+                "score_ci_low": 0.41661505505349583,
                 "num_of_instances": 1000,
-                "accuracy": 0.39,
-                "accuracy_ci_low": 0.36,
-                "accuracy_ci_high": 0.418,
-                "f1_micro": 0.44368600682593856,
-                "f1_micro_ci_low": 0.4135801299006492,
-                "f1_micro_ci_high": 0.47444463958776134
+                "accuracy": 0.388,
+                "accuracy_ci_low": 0.358,
+                "accuracy_ci_high": 0.4198351175250287,
+                "f1_micro": 0.4485549132947977,
+                "f1_micro_ci_low": 0.41661505505349583,
+                "f1_micro_ci_high": 0.4787668189917876
             },
-            "score": 0.44368600682593856,
+            "score": 0.4485549132947977,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.6105828707367139,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9035153328347045,
-                "f1_credit card or prepaid card": 0.5873015873015873,
-                "f1_debt collection": 0.6375,
-                "f1_checking or savings account": 0.75,
-                "f1_money transfer or virtual currency or money service": 0.5777777777777777,
-                "f1_vehicle loan or lease": 0.4666666666666667,
-                "f1_mortgage": 0.6785714285714286,
-                "f1_payday loan or title loan or personal loan": 0.17391304347826086,
-                "f1_student loan": 0.72,
-                "f1_macro_ci_low": 0.5575796516691159,
-                "f1_macro_ci_high": 0.6705972502098242,
+                "f1_macro": 0.6126200216184788,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9002217294900222,
+                "f1_checking or savings account": 0.6451612903225806,
+                "f1_debt collection": 0.5066666666666667,
+                "f1_credit card or prepaid card": 0.6277372262773723,
+                "f1_mortgage": 0.7945205479452054,
+                "f1_student loan": 0.8461538461538461,
+                "f1_money transfer or virtual currency or money service": 0.4878048780487805,
+                "f1_payday loan or title loan or personal loan": 0.2608695652173913,
+                "f1_vehicle loan or lease": 0.4444444444444444,
+                "f1_macro_ci_low": 0.557724310665768,
+                "f1_macro_ci_high": 0.6722482288571774,
                 "score_name": "f1_micro",
-                "score": 0.8195173137460651,
-                "score_ci_high": 0.842436974789916,
-                "score_ci_low": 0.7946166113913405,
+                "score": 0.8101924076963078,
+                "score_ci_high": 0.8332463122584837,
+                "score_ci_low": 0.7850628587071383,
                 "num_of_instances": 1000,
-                "accuracy": 0.781,
+                "accuracy": 0.779,
                 "accuracy_ci_low": 0.752,
-                "accuracy_ci_high": 0.806,
-                "f1_micro": 0.8195173137460651,
-                "f1_micro_ci_low": 0.7946166113913405,
-                "f1_micro_ci_high": 0.842436974789916
+                "accuracy_ci_high": 0.805,
+                "f1_micro": 0.8101924076963078,
+                "f1_micro_ci_low": 0.7850628587071383,
+                "f1_micro_ci_high": 0.8332463122584837
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.7132677588870594,
-                "f1_mortgages and loans": 0.7771428571428571,
-                "f1_credit card": 0.7023809523809523,
-                "f1_debt collection": 0.6854460093896714,
-                "f1_credit reporting": 0.7601476014760148,
-                "f1_retail banking": 0.6412213740458015,
-                "f1_macro_ci_low": 0.672279823384184,
-                "f1_macro_ci_high": 0.7539657340394554,
+                "f1_macro": 0.6957226327824474,
+                "f1_mortgages and loans": 0.7861271676300579,
+                "f1_credit card": 0.735632183908046,
+                "f1_debt collection": 0.6605504587155964,
+                "f1_retail banking": 0.5853658536585366,
+                "f1_credit reporting": 0.7109375,
+                "f1_macro_ci_low": 0.6549701202052777,
+                "f1_macro_ci_high": 0.7382657469246365,
                 "score_name": "f1_micro",
-                "score": 0.7202505219206681,
-                "score_ci_high": 0.7576596149340853,
-                "score_ci_low": 0.6805865270375967,
+                "score": 0.701271186440678,
+                "score_ci_high": 0.7411785857709632,
+                "score_ci_low": 0.6609516931464113,
                 "num_of_instances": 500,
-                "accuracy": 0.69,
-                "accuracy_ci_low": 0.65,
-                "accuracy_ci_high": 0.73,
-                "f1_micro": 0.7202505219206681,
-                "f1_micro_ci_low": 0.6805865270375967,
-                "f1_micro_ci_high": 0.7576596149340853
+                "accuracy": 0.662,
+                "accuracy_ci_low": 0.6202110366430569,
+                "accuracy_ci_high": 0.706,
+                "f1_micro": 0.701271186440678,
+                "f1_micro_ci_low": 0.6609516931464113,
+                "f1_micro_ci_high": 0.7411785857709632
             },
-            "score": 0.7698839178333665,
+            "score": 0.7557317970684929,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "execution_accuracy": 0.074,
-                "program_accuracy": 0.085,
-                "score": 0.085,
+                "program_accuracy": 0.084,
+                "score": 0.084,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.058,
-                "execution_accuracy_ci_high": 0.091,
-                "program_accuracy_ci_low": 0.068,
-                "program_accuracy_ci_high": 0.102,
-                "score_ci_low": 0.068,
-                "score_ci_high": 0.102
+                "execution_accuracy": 0.073,
+                "program_accuracy_ci_low": 0.067,
+                "program_accuracy_ci_high": 0.10386305691021766,
+                "score_ci_low": 0.067,
+                "score_ci_high": 0.10386305691021766,
+                "execution_accuracy_ci_low": 0.057,
+                "execution_accuracy_ci_high": 0.091
             },
-            "score": 0.085,
+            "score": 0.084,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.30022844870852566,
-                "recall": 0.5840193774846996,
-                "f1": 0.3357215148632638,
-                "precision_ci_low": 0.28030967471726836,
-                "precision_ci_high": 0.32121747414474766,
-                "recall_ci_low": 0.565861900260428,
-                "recall_ci_high": 0.59971992711831,
-                "f1_ci_low": 0.3175124739653954,
-                "f1_ci_high": 0.35218969004250933,
+                "precision": 0.2967925946544494,
+                "recall": 0.5841362559189178,
+                "f1": 0.3364987383722942,
+                "precision_ci_low": 0.2764979602656248,
+                "precision_ci_high": 0.3159857611493305,
+                "recall_ci_low": 0.5663103849045117,
+                "recall_ci_high": 0.6007142202829963,
+                "f1_ci_low": 0.3178259767982501,
+                "f1_ci_high": 0.35351716805909167,
                 "score_name": "f1",
-                "score": 0.3357215148632638,
-                "score_ci_high": 0.35218969004250933,
-                "score_ci_low": 0.3175124739653954,
+                "score": 0.3364987383722942,
+                "score_ci_high": 0.35351716805909167,
+                "score_ci_low": 0.3178259767982501,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6000729690988859,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6848867723842462,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5528717579195896,
-                "faithfullness_f1_token_overlap": 0.33597285355913525,
-                "faithfullness_recall_token_overlap": 0.27114762054953845,
-                "faithfullness_precision_token_overlap": 0.5569731492695155,
-                "correctness_f1_token_overlap": 0.3357215148632638,
-                "correctness_recall_token_overlap": 0.5840193774846996,
-                "correctness_precision_token_overlap": 0.30022844870852566
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5976409501334031,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6827784284452597,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5494535167018573,
+                "faithfullness_f1_token_overlap": 0.34046513614196555,
+                "faithfullness_recall_token_overlap": 0.2738914194701684,
+                "faithfullness_precision_token_overlap": 0.5609296227002613,
+                "correctness_f1_token_overlap": 0.3364987383722942,
+                "correctness_recall_token_overlap": 0.5841362559189178,
+                "correctness_precision_token_overlap": 0.2967925946544494
             },
-            "score": 0.3357215148632638,
+            "score": 0.3364987383722942,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.455,
-                "accuracy_ci_low": 0.423,
-                "accuracy_ci_high": 0.485,
+                "accuracy": 0.448,
+                "accuracy_ci_low": 0.417,
+                "accuracy_ci_high": 0.479,
                 "score_name": "accuracy",
-                "score": 0.455,
-                "score_ci_high": 0.485,
-                "score_ci_low": 0.423,
+                "score": 0.448,
+                "score_ci_high": 0.479,
+                "score_ci_low": 0.417,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.782,
-                "accuracy_ci_low": 0.744,
-                "accuracy_ci_high": 0.818,
+                "accuracy": 0.78,
+                "accuracy_ci_low": 0.742,
+                "accuracy_ci_high": 0.816,
                 "score_name": "accuracy",
-                "score": 0.782,
-                "score_ci_high": 0.818,
-                "score_ci_low": 0.744,
+                "score": 0.78,
+                "score_ci_high": 0.816,
+                "score_ci_low": 0.742,
                 "num_of_instances": 500
             },
-            "score": 0.6185,
+            "score": 0.614,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
@@ -741,68 +741,68 @@
             "attaq_500": {
                 "severity_critical": 0.0,
                 "severity_high": 0.0,
-                "severity_medium": 0.0,
-                "severity_low": 100.0,
-                "category_harmful_info": 0.8662501443103547,
-                "category_deception": 0.8769584958149929,
-                "category_explicit_content": 0.8376609406085929,
-                "category_violence": 0.8456707175466277,
-                "category_discrimination": 0.8991804392564864,
-                "category_substance_abuse": 0.8935730718904071,
-                "category_pii": 0.8467056638112774,
-                "safety": 0.860165383811626,
-                "safety_ci_low": 0.8426579493684276,
-                "safety_ci_high": 0.8778207056405374,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8719441451812312,
+                "category_deception": 0.888891483497138,
+                "category_explicit_content": 0.8127973503386131,
+                "category_violence": 0.8689369508242678,
+                "category_discrimination": 0.8825277714502243,
+                "category_substance_abuse": 0.9410604351096683,
+                "category_pii": 0.7720797760067163,
+                "safety": 0.8674504241983716,
+                "safety_ci_low": 0.847960857428302,
+                "safety_ci_high": 0.8844523349924212,
                 "score_name": "safety",
-                "score": 0.860165383811626,
-                "score_ci_high": 0.8778207056405374,
-                "score_ci_low": 0.8426579493684276,
+                "score": 0.8674504241983716,
+                "score_ci_high": 0.8844523349924212,
+                "score_ci_low": 0.847960857428302,
                 "num_of_instances": 100
             },
-            "score": 0.860165383811626,
+            "score": 0.8674504241983716,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rougeLsum": 0.3503820014906059,
-                "rouge1": 0.4200585739584912,
-                "rouge2": 0.20109007723824623,
-                "rougeL": 0.28749112328021914,
-                "score": 0.28749112328021914,
+                "rouge1": 0.42001432363059676,
+                "rougeLsum": 0.35176737784751416,
+                "rouge2": 0.20090339450058858,
+                "rougeL": 0.2887794256459243,
+                "score": 0.2887794256459243,
                 "score_name": "rougeL",
-                "rougeLsum_ci_low": 0.341921573094731,
-                "rougeLsum_ci_high": 0.35863585426859207,
-                "rouge1_ci_low": 0.41035793857223635,
-                "rouge1_ci_high": 0.4281932704537228,
-                "rouge2_ci_low": 0.19416899053732958,
-                "rouge2_ci_high": 0.20872476773642967,
-                "rougeL_ci_low": 0.2804794753326623,
-                "rougeL_ci_high": 0.29447838537921134,
-                "score_ci_low": 0.2804794753326623,
-                "score_ci_high": 0.29447838537921134
+                "rouge1_ci_low": 0.4115358503100108,
+                "rouge1_ci_high": 0.42788391457002284,
+                "rougeLsum_ci_low": 0.3432569747337183,
+                "rougeLsum_ci_high": 0.35847372198631006,
+                "rouge2_ci_low": 0.19492457543616534,
+                "rouge2_ci_high": 0.2075223454056542,
+                "rougeL_ci_low": 0.28220405170841467,
+                "rougeL_ci_high": 0.2953572975976334,
+                "score_ci_low": 0.28220405170841467,
+                "score_ci_high": 0.2953572975976334
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rougeLsum": 0.0922932399263996,
-                "rouge1": 0.11247814548815566,
-                "rouge2": 0.015117853576507847,
-                "rougeL": 0.07979202357473647,
-                "score": 0.07979202357473647,
+                "rouge1": 0.11196786218861304,
+                "rougeLsum": 0.09259288227162547,
+                "rouge2": 0.014304299542517345,
+                "rougeL": 0.08050830498137622,
+                "score": 0.08050830498137622,
                 "score_name": "rougeL",
-                "rougeLsum_ci_low": 0.0880597944044916,
-                "rougeLsum_ci_high": 0.09606464509440052,
-                "rouge1_ci_low": 0.10733708561154955,
-                "rouge1_ci_high": 0.11723898467910755,
-                "rouge2_ci_low": 0.01362250797390663,
-                "rouge2_ci_high": 0.0168799885499115,
-                "rougeL_ci_low": 0.0764789144644062,
-                "rougeL_ci_high": 0.08304032568245756,
-                "score_ci_low": 0.0764789144644062,
-                "score_ci_high": 0.08304032568245756
+                "rouge1_ci_low": 0.10730072397656114,
+                "rouge1_ci_high": 0.11708078514416911,
+                "rougeLsum_ci_low": 0.088799408920107,
+                "rougeLsum_ci_high": 0.09663277250494734,
+                "rouge2_ci_low": 0.01265361386023307,
+                "rouge2_ci_high": 0.01610624039999516,
+                "rougeL_ci_low": 0.07745615703093822,
+                "rougeL_ci_high": 0.08426746560170988,
+                "score_ci_low": 0.07745615703093822,
+                "score_ci_high": 0.08426746560170988
             },
-            "score": 0.1836415734274778,
+            "score": 0.18464386531365026,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1154,
-                    637,
-                    382,
-                    237
+                    1147,
+                    635,
+                    377,
+                    236
                 ],
                 "totals": [
-                    3013,
-                    2947,
-                    2881,
-                    2815
+                    2783,
+                    2717,
+                    2651,
+                    2585
                 ],
                 "precisions": [
-                    0.383006969797544,
-                    0.2161520190023753,
-                    0.13259284970496357,
-                    0.08419182948490231
+                    0.41214516708587856,
+                    0.23371365476628636,
+                    0.14221048660882685,
+                    0.09129593810444873
                 ],
                 "bp": 1.0,
-                "sys_len": 3013,
+                "sys_len": 2783,
                 "ref_len": 1734,
-                "sacrebleu": 0.17435684678472682,
-                "score": 0.17435684678472682,
+                "sacrebleu": 0.18805260077651942,
+                "score": 0.18805260077651942,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12709535962365245,
-                "score_ci_high": 0.21064271607309265,
-                "sacrebleu_ci_low": 0.12709535962365245,
-                "sacrebleu_ci_high": 0.21064271607309265
+                "score_ci_low": 0.15590859876442242,
+                "score_ci_high": 0.22665030743269873,
+                "sacrebleu_ci_low": 0.15590859876442242,
+                "sacrebleu_ci_high": 0.22665030743269873
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1215,
-                    695,
-                    422,
-                    256
+                    1250,
+                    740,
+                    469,
+                    298
                 ],
                 "totals": [
-                    3433,
-                    3367,
-                    3301,
-                    3235
+                    3365,
+                    3299,
+                    3233,
+                    3167
                 ],
                 "precisions": [
-                    0.35391785610253423,
-                    0.20641520641520641,
-                    0.12784004847016056,
-                    0.07913446676970634
+                    0.37147102526002973,
+                    0.22431039709002729,
+                    0.1450665017012063,
+                    0.09409535838332808
                 ],
                 "bp": 1.0,
-                "sys_len": 3433,
+                "sys_len": 3365,
                 "ref_len": 1734,
-                "sacrebleu": 0.16488046075977367,
-                "score": 0.16488046075977367,
+                "sacrebleu": 0.18364428677137226,
+                "score": 0.18364428677137226,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12825986690370522,
-                "score_ci_high": 0.20812836267228596,
-                "sacrebleu_ci_low": 0.12825986690370522,
-                "sacrebleu_ci_high": 0.20812836267228596
+                "score_ci_low": 0.15133175793244782,
+                "score_ci_high": 0.238285104264321,
+                "sacrebleu_ci_low": 0.15133175793244782,
+                "sacrebleu_ci_high": 0.238285104264321
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    726,
-                    321,
-                    159,
-                    82
+                    701,
+                    279,
+                    122,
+                    55
                 ],
                 "totals": [
-                    2297,
-                    2231,
-                    2165,
-                    2099
+                    2379,
+                    2313,
+                    2247,
+                    2181
                 ],
                 "precisions": [
-                    0.3160644318676535,
-                    0.14388166741371583,
-                    0.07344110854503465,
-                    0.03906622201048118
+                    0.294661622530475,
+                    0.12062256809338522,
+                    0.054294615042278595,
+                    0.02521779000458505
                 ],
                 "bp": 1.0,
-                "sys_len": 2297,
+                "sys_len": 2379,
                 "ref_len": 1589,
-                "sacrebleu": 0.10687605905530678,
-                "score": 0.10687605905530678,
+                "sacrebleu": 0.08352259557657876,
+                "score": 0.08352259557657876,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08639846348006232,
-                "score_ci_high": 0.13425269082562755,
-                "sacrebleu_ci_low": 0.08639846348006232,
-                "sacrebleu_ci_high": 0.13425269082562755
+                "score_ci_low": 0.06605026431332355,
+                "score_ci_high": 0.10504705952927867,
+                "sacrebleu_ci_low": 0.06605026431332355,
+                "sacrebleu_ci_high": 0.10504705952927867
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1066,
-                    564,
-                    332,
-                    194
+                    1060,
+                    555,
+                    321,
+                    197
                 ],
                 "totals": [
-                    2300,
-                    2234,
-                    2168,
-                    2102
+                    2307,
+                    2241,
+                    2175,
+                    2109
                 ],
                 "precisions": [
-                    0.46347826086956523,
-                    0.252461951656222,
-                    0.15313653136531366,
-                    0.0922930542340628
+                    0.45947117468573906,
+                    0.24765729585006693,
+                    0.14758620689655172,
+                    0.09340919867235657
                 ],
                 "bp": 1.0,
-                "sys_len": 2300,
+                "sys_len": 2307,
                 "ref_len": 1835,
-                "sacrebleu": 0.2016593123773307,
-                "score": 0.2016593123773307,
+                "sacrebleu": 0.19901517998924645,
+                "score": 0.19901517998924645,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.177292145733578,
-                "score_ci_high": 0.24439707428713803,
-                "sacrebleu_ci_low": 0.177292145733578,
-                "sacrebleu_ci_high": 0.24439707428713803
+                "score_ci_low": 0.15118718491370434,
+                "score_ci_high": 0.24524942034353023,
+                "sacrebleu_ci_low": 0.15118718491370434,
+                "sacrebleu_ci_high": 0.24524942034353023
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1409,
-                    950,
-                    692,
-                    517
+                    1383,
+                    931,
+                    685,
+                    511
                 ],
                 "totals": [
-                    3275,
-                    3209,
-                    3143,
-                    3077
+                    2499,
+                    2433,
+                    2367,
+                    2301
                 ],
                 "precisions": [
-                    0.4302290076335878,
-                    0.2960423808039888,
-                    0.2201718103722558,
-                    0.168020799480013
+                    0.553421368547419,
+                    0.3826551582408549,
+                    0.28939585973806503,
+                    0.222077357670578
                 ],
                 "bp": 1.0,
-                "sys_len": 3275,
+                "sys_len": 2499,
                 "ref_len": 2068,
-                "sacrebleu": 0.2619959538476516,
-                "score": 0.2619959538476516,
+                "sacrebleu": 0.34155844112464445,
+                "score": 0.34155844112464445,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21071110880640612,
-                "score_ci_high": 0.30599931494111227,
-                "sacrebleu_ci_low": 0.21071110880640612,
-                "sacrebleu_ci_high": 0.30599931494111227
+                "score_ci_low": 0.2882755414660873,
+                "score_ci_high": 0.40044941880570056,
+                "sacrebleu_ci_low": 0.2882755414660873,
+                "sacrebleu_ci_high": 0.40044941880570056
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1096,
-                    465,
-                    233,
-                    132
+                    1091,
+                    445,
+                    224,
+                    119
                 ],
                 "totals": [
-                    3883,
-                    3817,
-                    3751,
-                    3685
+                    4751,
+                    4685,
+                    4619,
+                    4553
                 ],
                 "precisions": [
-                    0.28225598763842386,
-                    0.12182342153523709,
-                    0.0621167688616369,
-                    0.03582089552238806
+                    0.2296358661334456,
+                    0.09498399146211313,
+                    0.04849534531283828,
+                    0.026136613222051394
                 ],
                 "bp": 1.0,
-                "sys_len": 3883,
+                "sys_len": 4751,
                 "ref_len": 2235,
-                "sacrebleu": 0.09352545142421302,
-                "score": 0.09352545142421302,
+                "sacrebleu": 0.07251199865213667,
+                "score": 0.07251199865213667,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0763987126727994,
-                "score_ci_high": 0.11617390981932266,
-                "sacrebleu_ci_low": 0.0763987126727994,
-                "sacrebleu_ci_high": 0.11617390981932266
+                "score_ci_low": 0.05587322252934343,
+                "score_ci_high": 0.09184707565044344,
+                "sacrebleu_ci_low": 0.05587322252934343,
+                "sacrebleu_ci_high": 0.09184707565044344
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1328,
-                    850,
-                    588,
-                    412
+                    1335,
+                    873,
+                    615,
+                    436
                 ],
                 "totals": [
-                    3030,
-                    2964,
-                    2898,
-                    2832
+                    3124,
+                    3058,
+                    2992,
+                    2926
                 ],
                 "precisions": [
-                    0.4382838283828383,
-                    0.286774628879892,
-                    0.2028985507246377,
-                    0.14548022598870058
+                    0.427336747759283,
+                    0.2854807063440157,
+                    0.20554812834224598,
+                    0.14900888585099112
                 ],
                 "bp": 1.0,
-                "sys_len": 3030,
+                "sys_len": 3124,
                 "ref_len": 1916,
-                "sacrebleu": 0.2467997817029595,
-                "score": 0.2467997817029595,
+                "sacrebleu": 0.24723968084246245,
+                "score": 0.24723968084246245,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.193392163449652,
-                "score_ci_high": 0.2974642241791255,
-                "sacrebleu_ci_low": 0.193392163449652,
-                "sacrebleu_ci_high": 0.2974642241791255
+                "score_ci_low": 0.1931280328121111,
+                "score_ci_high": 0.3044993993983362,
+                "sacrebleu_ci_low": 0.1931280328121111,
+                "sacrebleu_ci_high": 0.3044993993983362
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    930,
-                    400,
-                    214,
-                    123
+                    897,
+                    407,
+                    221,
+                    130
                 ],
                 "totals": [
-                    2961,
-                    2895,
-                    2829,
-                    2763
+                    2626,
+                    2560,
+                    2494,
+                    2428
                 ],
                 "precisions": [
-                    0.3140830800405269,
-                    0.1381692573402418,
-                    0.07564510427712973,
-                    0.04451682953311618
+                    0.3415841584158416,
+                    0.158984375,
+                    0.08861267040898156,
+                    0.05354200988467875
                 ],
                 "bp": 1.0,
-                "sys_len": 2961,
+                "sys_len": 2626,
                 "ref_len": 1949,
-                "sacrebleu": 0.1099487393546487,
-                "score": 0.1099487393546487,
+                "sacrebleu": 0.12669534688031472,
+                "score": 0.12669534688031472,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08284384518142485,
-                "score_ci_high": 0.13880651312628609,
-                "sacrebleu_ci_low": 0.08284384518142485,
-                "sacrebleu_ci_high": 0.13880651312628609
+                "score_ci_low": 0.1000733380304748,
+                "score_ci_high": 0.16178959885111238,
+                "sacrebleu_ci_low": 0.1000733380304748,
+                "sacrebleu_ci_high": 0.16178959885111238
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1217,
-                    624,
-                    347,
-                    198
+                    1203,
+                    616,
+                    335,
+                    184
                 ],
                 "totals": [
-                    3045,
-                    2979,
                     2913,
-                    2847
+                    2847,
+                    2781,
+                    2715
                 ],
                 "precisions": [
-                    0.399671592775041,
-                    0.20946626384692849,
-                    0.11912118091314795,
-                    0.06954689146469968
+                    0.41297631307929966,
+                    0.21636810677906568,
+                    0.12046026609133405,
+                    0.06777163904235727
                 ],
                 "bp": 1.0,
-                "sys_len": 3045,
+                "sys_len": 2913,
                 "ref_len": 2098,
-                "sacrebleu": 0.1622822499255264,
-                "score": 0.1622822499255264,
+                "sacrebleu": 0.16434350639643316,
+                "score": 0.16434350639643316,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.13321857221475644,
-                "score_ci_high": 0.19390301665624113,
-                "sacrebleu_ci_low": 0.13321857221475644,
-                "sacrebleu_ci_high": 0.19390301665624113
+                "score_ci_low": 0.1325515600195346,
+                "score_ci_high": 0.20299768479868893,
+                "sacrebleu_ci_low": 0.1325515600195346,
+                "sacrebleu_ci_high": 0.20299768479868893
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1236,
-                    735,
-                    470,
-                    308
+                    1254,
+                    732,
+                    469,
+                    322
                 ],
                 "totals": [
-                    2952,
-                    2886,
-                    2820,
-                    2754
+                    3278,
+                    3212,
+                    3146,
+                    3080
                 ],
                 "precisions": [
-                    0.4186991869918699,
-                    0.25467775467775466,
-                    0.16666666666666669,
-                    0.11183732752360204
+                    0.3825503355704698,
+                    0.22789539227895392,
+                    0.14907819453274,
+                    0.10454545454545455
                 ],
                 "bp": 1.0,
-                "sys_len": 2952,
+                "sys_len": 3278,
                 "ref_len": 1734,
-                "sacrebleu": 0.2111456628673961,
-                "score": 0.2111456628673961,
+                "sacrebleu": 0.19199320250461963,
+                "score": 0.19199320250461963,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1728340034401921,
-                "score_ci_high": 0.26908287892628974,
-                "sacrebleu_ci_low": 0.1728340034401921,
-                "sacrebleu_ci_high": 0.26908287892628974
+                "score_ci_low": 0.15725080687797277,
+                "score_ci_high": 0.24193576258661156,
+                "sacrebleu_ci_low": 0.15725080687797277,
+                "sacrebleu_ci_high": 0.24193576258661156
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1018,
-                    437,
-                    232,
-                    128
+                    1040,
+                    474,
+                    264,
+                    161
                 ],
                 "totals": [
-                    3130,
-                    3064,
-                    2998,
-                    2932
+                    3106,
+                    3040,
+                    2974,
+                    2908
                 ],
                 "precisions": [
-                    0.3252396166134185,
-                    0.14262402088772846,
-                    0.07738492328218813,
-                    0.04365620736698499
+                    0.334835801674179,
+                    0.15592105263157896,
+                    0.08876933422999328,
+                    0.05536451169188446
                 ],
                 "bp": 1.0,
-                "sys_len": 3130,
+                "sys_len": 3106,
                 "ref_len": 1734,
-                "sacrebleu": 0.11188570922324435,
-                "score": 0.11188570922324435,
+                "sacrebleu": 0.1265632943373452,
+                "score": 0.1265632943373452,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09154049326122426,
-                "score_ci_high": 0.13827539969992217,
-                "sacrebleu_ci_low": 0.09154049326122426,
-                "sacrebleu_ci_high": 0.13827539969992217
+                "score_ci_low": 0.10051707288201024,
+                "score_ci_high": 0.15415847724283543,
+                "sacrebleu_ci_low": 0.10051707288201024,
+                "sacrebleu_ci_high": 0.15415847724283543
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    986,
-                    447,
-                    233,
-                    127
+                    978,
+                    426,
+                    216,
+                    119
                 ],
                 "totals": [
-                    3637,
-                    3571,
-                    3505,
-                    3439
+                    3053,
+                    2987,
+                    2921,
+                    2855
                 ],
                 "precisions": [
-                    0.27110255705251585,
-                    0.12517502100252031,
-                    0.06647646219686162,
-                    0.03692933992439663
+                    0.3203406485424173,
+                    0.14261801138265817,
+                    0.07394727832933927,
+                    0.04168126094570928
                 ],
                 "bp": 1.0,
-                "sys_len": 3637,
+                "sys_len": 3053,
                 "ref_len": 1734,
-                "sacrebleu": 0.09553723823741646,
-                "score": 0.09553723823741646,
+                "sacrebleu": 0.10893372822633232,
+                "score": 0.10893372822633232,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06933902828362079,
-                "score_ci_high": 0.1273472328564688,
-                "sacrebleu_ci_low": 0.06933902828362079,
-                "sacrebleu_ci_high": 0.1273472328564688
+                "score_ci_low": 0.08051704910100821,
+                "score_ci_high": 0.1399793521343314,
+                "sacrebleu_ci_low": 0.08051704910100821,
+                "sacrebleu_ci_high": 0.1399793521343314
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1286,
-                    834,
-                    587,
-                    419
+                    1279,
+                    820,
+                    574,
+                    416
                 ],
                 "totals": [
-                    3404,
-                    3338,
-                    3272,
-                    3206
+                    2919,
+                    2853,
+                    2787,
+                    2721
                 ],
                 "precisions": [
-                    0.37779083431257343,
-                    0.24985020970641103,
-                    0.17940097799511,
-                    0.13069245165315035
+                    0.4381637547105173,
+                    0.28741675429372593,
+                    0.2059562253318981,
+                    0.15288496876148475
                 ],
                 "bp": 1.0,
-                "sys_len": 3404,
+                "sys_len": 2919,
                 "ref_len": 1734,
-                "sacrebleu": 0.21689603438287544,
-                "score": 0.21689603438287544,
+                "sacrebleu": 0.250941252136478,
+                "score": 0.250941252136478,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18174547190909165,
-                "score_ci_high": 0.2734022486576191,
-                "sacrebleu_ci_low": 0.18174547190909165,
-                "sacrebleu_ci_high": 0.2734022486576191
+                "score_ci_low": 0.19373911879549774,
+                "score_ci_high": 0.3143574197034948,
+                "sacrebleu_ci_low": 0.19373911879549774,
+                "sacrebleu_ci_high": 0.3143574197034948
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1208,
-                    675,
-                    430,
-                    279
+                    1215,
+                    691,
+                    421,
+                    259
                 ],
                 "totals": [
-                    3677,
-                    3611,
-                    3545,
-                    3479
+                    2920,
+                    2854,
+                    2788,
+                    2722
                 ],
                 "precisions": [
-                    0.32852869186837097,
-                    0.1869288285793409,
-                    0.12129760225669958,
-                    0.08019545846507617
+                    0.41609589041095885,
+                    0.24211632796075683,
+                    0.15100430416068866,
+                    0.09515062454077883
                 ],
                 "bp": 1.0,
-                "sys_len": 3677,
+                "sys_len": 2920,
                 "ref_len": 1734,
-                "sacrebleu": 0.15633740352446387,
-                "score": 0.15633740352446387,
+                "sacrebleu": 0.19505389122054267,
+                "score": 0.19505389122054267,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12255450743419968,
-                "score_ci_high": 0.17971859902386644,
-                "sacrebleu_ci_low": 0.12255450743419968,
-                "sacrebleu_ci_high": 0.17971859902386644
+                "score_ci_low": 0.16097395914199633,
+                "score_ci_high": 0.23627234222780022,
+                "sacrebleu_ci_low": 0.16097395914199633,
+                "sacrebleu_ci_high": 0.23627234222780022
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1135,
-                    581,
-                    336,
-                    202
+                    1158,
+                    586,
+                    328,
+                    189
                 ],
                 "totals": [
-                    3533,
-                    3467,
-                    3401,
-                    3335
+                    3432,
+                    3366,
+                    3300,
+                    3234
                 ],
                 "precisions": [
-                    0.3212567223322955,
-                    0.16758004038073263,
-                    0.09879447221405468,
-                    0.06056971514242879
+                    0.3374125874125874,
+                    0.1740938799762329,
+                    0.0993939393939394,
+                    0.05844155844155845
                 ],
                 "bp": 1.0,
-                "sys_len": 3533,
+                "sys_len": 3432,
                 "ref_len": 1734,
-                "sacrebleu": 0.133972503470666,
-                "score": 0.133972503470666,
+                "sacrebleu": 0.1359116294436951,
+                "score": 0.1359116294436951,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10251876459928583,
-                "score_ci_high": 0.17481307519673603,
-                "sacrebleu_ci_low": 0.10251876459928583,
-                "sacrebleu_ci_high": 0.17481307519673603
+                "score_ci_low": 0.10538411838136834,
+                "score_ci_high": 0.18075724261232987,
+                "sacrebleu_ci_low": 0.10538411838136834,
+                "sacrebleu_ci_high": 0.18075724261232987
             },
-            "score": 0.1632066271292133,
+            "score": 0.17439870899191476,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.45259314123432515,
+        "score": 0.42328152240293343,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/2025-06-23T04-42-35_evaluation_results.json

@@ -0,0 +1,1283 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-06-23T08:42:31.876970Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/ibm/granite-3-8b-instruct,max_tokens=256",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/ibm/granite-3-8b-instruct",
+            "model_args": {
+                "max_tokens": 256
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.24.0",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "litellm": "1.72.6.post1",
+            "httpcore": "1.0.9",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "yarl": "1.20.1",
+            "openai": "1.88.0",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "unitxt": "1.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "pillow": "11.2.1",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "lxml": "5.4.0",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "fonttools": "4.58.4",
+            "transformers": "4.52.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "evaluate": "0.4.3",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "tokenizers": "0.21.1",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "hf-xet": "1.1.4",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "multidict": "6.5.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "certifi": "2025.6.15",
+            "accelerate": "1.8.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.0",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "huggingface-hub": "0.33.0",
+            "colorama": "0.4.6",
+            "rpds-py": "0.25.1",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.4961662149523231,
+                "accuracy_ci_high": 0.6888888888888889,
+                "score_name": "accuracy",
+                "score": 0.6,
+                "score_ci_high": 0.6888888888888889,
+                "score_ci_low": 0.4961662149523231,
+                "num_of_instances": 90
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.6888888888888889,
+                "accuracy_ci_low": 0.5777777777777777,
+                "accuracy_ci_high": 0.7666666666666667,
+                "score_name": "accuracy",
+                "score": 0.6888888888888889,
+                "score_ci_high": 0.7666666666666667,
+                "score_ci_low": 0.5777777777777777,
+                "num_of_instances": 90
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.8222222222222222,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8222222222222222,
+                "num_of_instances": 90
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 0.6888888888888889,
+                "accuracy_ci_low": 0.5780895036995246,
+                "accuracy_ci_high": 0.7888888888888889,
+                "score_name": "accuracy",
+                "score": 0.6888888888888889,
+                "score_ci_high": 0.7888888888888889,
+                "score_ci_low": 0.5780895036995246,
+                "num_of_instances": 90
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.689667704010142,
+                "accuracy_ci_high": 0.8555555555555555,
+                "score_name": "accuracy",
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8555555555555555,
+                "score_ci_low": 0.689667704010142,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 0.9222222222222223,
+                "accuracy_ci_low": 0.8444444444444444,
+                "accuracy_ci_high": 0.9666666666666667,
+                "score_name": "accuracy",
+                "score": 0.9222222222222223,
+                "score_ci_high": 0.9666666666666667,
+                "score_ci_low": 0.8444444444444444,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.8222222222222222,
+                "accuracy_ci_high": 0.9555555555555556,
+                "score_name": "accuracy",
+                "score": 0.9,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8222222222222222,
+                "num_of_instances": 90
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.9555555555555556,
+                "accuracy_ci_low": 0.9,
+                "accuracy_ci_high": 0.9888888888888889,
+                "score_name": "accuracy",
+                "score": 0.9555555555555556,
+                "score_ci_high": 0.9888888888888889,
+                "score_ci_low": 0.9,
+                "num_of_instances": 90
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7333333333333333,
+                "num_of_instances": 90
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.6777777777777778,
+                "accuracy_ci_low": 0.5777777777777777,
+                "accuracy_ci_high": 0.7666666666666667,
+                "score_name": "accuracy",
+                "score": 0.6777777777777778,
+                "score_ci_high": 0.7666666666666667,
+                "score_ci_low": 0.5777777777777777,
+                "num_of_instances": 90
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8333333333333334,
+                "accuracy_ci_low": 0.7555555555555555,
+                "accuracy_ci_high": 0.9077323275921318,
+                "score_name": "accuracy",
+                "score": 0.8333333333333334,
+                "score_ci_high": 0.9077323275921318,
+                "score_ci_low": 0.7555555555555555,
+                "num_of_instances": 90
+            },
+            "score": 0.797979797979798,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 500,
+                "llama_3_70b_instruct_template_arena_hard": 0.07200720072007201,
+                "score": 0.07200720072007201,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.07200720072007201,
+            "score_name": "subsets_mean",
+            "num_of_instances": 500
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 1000,
+                "f1_Person": 0.5089820359281437,
+                "f1_Organization": 0.3546511627906977,
+                "f1_Location": 0.3474903474903475,
+                "f1_macro": 0.4037078487363963,
+                "recall_macro": 0.3583554354766996,
+                "precision_macro": 0.4822578777124232,
+                "in_classes_support": 0.5928057553956835,
+                "f1_micro": 0.31311475409836065,
+                "recall_micro": 0.3638095238095238,
+                "precision_micro": 0.27482014388489207,
+                "score": 0.31311475409836065,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.2581926050371807,
+                "score_ci_high": 0.35574639217016485,
+                "f1_micro_ci_low": 0.2581926050371807,
+                "f1_micro_ci_high": 0.35574639217016485
+            },
+            "score": 0.31311475409836065,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4647887323943662,
+                "accuracy_ci_high": 0.676056338028169,
+                "score_name": "accuracy",
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4647887323943662,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
+                "score_name": "accuracy",
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.23943661971830985,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
+                "score_name": "accuracy",
+                "score": 0.23943661971830985,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.352112676056338,
+                "accuracy_ci_low": 0.2535211267605634,
+                "accuracy_ci_high": 0.47475562822206696,
+                "score_name": "accuracy",
+                "score": 0.352112676056338,
+                "score_ci_high": 0.47475562822206696,
+                "score_ci_low": 0.2535211267605634,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.4647887323943662,
+                "accuracy_ci_low": 0.3380281690140845,
+                "accuracy_ci_high": 0.5774647887323944,
+                "score_name": "accuracy",
+                "score": 0.4647887323943662,
+                "score_ci_high": 0.5774647887323944,
+                "score_ci_low": 0.3380281690140845,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
+                "score_name": "accuracy",
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.352112676056338,
+                "accuracy_ci_low": 0.23943661971830985,
+                "accuracy_ci_high": 0.4647887323943662,
+                "score_name": "accuracy",
+                "score": 0.352112676056338,
+                "score_ci_high": 0.4647887323943662,
+                "score_ci_low": 0.23943661971830985,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.43661971830985913,
+                "accuracy_ci_low": 0.323943661971831,
+                "accuracy_ci_high": 0.5633802816901409,
+                "score_name": "accuracy",
+                "score": 0.43661971830985913,
+                "score_ci_high": 0.5633802816901409,
+                "score_ci_low": 0.323943661971831,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.30985915492957744,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.4225352112676056,
+                "score_name": "accuracy",
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.4225352112676056,
+                "score_ci_low": 0.2112676056338028,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.11267605633802817,
+                "accuracy_ci_low": 0.056338028169014086,
+                "accuracy_ci_high": 0.19718309859154928,
+                "score_name": "accuracy",
+                "score": 0.11267605633802817,
+                "score_ci_high": 0.19718309859154928,
+                "score_ci_low": 0.056338028169014086,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.22193333267792992,
+                "accuracy_ci_high": 0.43661971830985913,
+                "score_name": "accuracy",
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.22193333267792992,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.38028169014084506,
+                "accuracy_ci_low": 0.2535211267605634,
+                "accuracy_ci_high": 0.49295774647887325,
+                "score_name": "accuracy",
+                "score": 0.38028169014084506,
+                "score_ci_high": 0.49295774647887325,
+                "score_ci_low": 0.2535211267605634,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.16901408450704225,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.2676056338028169,
+                "score_name": "accuracy",
+                "score": 0.16901408450704225,
+                "score_ci_high": 0.2676056338028169,
+                "score_ci_low": 0.09859154929577464,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.5492957746478874,
+                "accuracy_ci_low": 0.43661971830985913,
+                "accuracy_ci_high": 0.6619718309859155,
+                "score_name": "accuracy",
+                "score": 0.5492957746478874,
+                "score_ci_high": 0.6619718309859155,
+                "score_ci_low": 0.43661971830985913,
+                "num_of_instances": 71
+            },
+            "score": 0.34004024144869216,
+            "score_name": "subsets_mean",
+            "num_of_instances": 994
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.38313112869412325,
+                "f1_suggestive": 0.4,
+                "f1_descriptive": 0.49056603773584906,
+                "f1_generic": 0.1111111111111111,
+                "f1_fanciful": 0.5806451612903226,
+                "f1_arbitrary": 0.3333333333333333,
+                "f1_macro_ci_low": 0.293497629073193,
+                "f1_macro_ci_high": 0.49184198170551063,
+                "score_name": "f1_micro",
+                "score": 0.41420118343195267,
+                "score_ci_high": 0.5176470588235295,
+                "score_ci_low": 0.3058823529411765,
+                "num_of_instances": 85,
+                "accuracy": 0.4117647058823529,
+                "accuracy_ci_low": 0.3058823529411765,
+                "accuracy_ci_high": 0.5176470588235295,
+                "f1_micro": 0.41420118343195267,
+                "f1_micro_ci_low": 0.3058823529411765,
+                "f1_micro_ci_high": 0.5176470588235295
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.5893994540491356,
+                "f1_no": 0.821656050955414,
+                "f1_yes": 0.35714285714285715,
+                "f1_macro_ci_low": 0.5097301675555063,
+                "f1_macro_ci_high": 0.6745862952621396,
+                "score_name": "f1_micro",
+                "score": 0.7236180904522613,
+                "score_ci_high": 0.7788944723618091,
+                "score_ci_low": 0.6595134689262127,
+                "num_of_instances": 200,
+                "accuracy": 0.72,
+                "accuracy_ci_low": 0.655,
+                "accuracy_ci_high": 0.775,
+                "f1_micro": 0.7236180904522613,
+                "f1_micro_ci_low": 0.6595134689262127,
+                "f1_micro_ci_high": 0.7788944723618091
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.27144552754252615,
+                "f1_conclusion": 0.07547169811320754,
+                "f1_issue": 0.3466666666666667,
+                "f1_decree": 0.30303030303030304,
+                "f1_rule": 0.475,
+                "f1_analysis": 0.2608695652173913,
+                "f1_facts": 0.26666666666666666,
+                "f1_procedural history": 0.1724137931034483,
+                "f1_macro_ci_low": 0.2140755773346065,
+                "f1_macro_ci_high": 0.33976570868629163,
+                "score_name": "f1_micro",
+                "score": 0.28717948717948716,
+                "score_ci_high": 0.35384615384615387,
+                "score_ci_low": 0.22363125007282936,
+                "num_of_instances": 200,
+                "accuracy": 0.28,
+                "accuracy_ci_low": 0.215,
+                "accuracy_ci_high": 0.345,
+                "f1_micro": 0.28717948717948716,
+                "f1_micro_ci_low": 0.22363125007282936,
+                "f1_micro_ci_high": 0.35384615384615387
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.4629294755877034,
+                "f1_yes": 0.5714285714285714,
+                "f1_no": 0.35443037974683544,
+                "f1_macro_ci_low": 0.3950714088005718,
+                "f1_macro_ci_high": 0.5297273754379386,
+                "score_name": "f1_micro",
+                "score": 0.48484848484848486,
+                "score_ci_high": 0.5532994923857868,
+                "score_ci_low": 0.4143244965787704,
+                "num_of_instances": 200,
+                "accuracy": 0.48,
+                "accuracy_ci_low": 0.41,
+                "accuracy_ci_high": 0.5461813537103201,
+                "f1_micro": 0.48484848484848486,
+                "f1_micro_ci_low": 0.4143244965787704,
+                "f1_micro_ci_high": 0.5532994923857868
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8078127879122904,
+                "f1_yes": 0.7761194029850746,
+                "f1_no": 0.8395061728395061,
+                "f1_macro_ci_low": 0.7164632895646129,
+                "f1_macro_ci_high": 0.8689798909122983,
+                "score_name": "f1_micro",
+                "score": 0.8108108108108109,
+                "score_ci_high": 0.8701298701298701,
+                "score_ci_low": 0.7222222222222222,
+                "num_of_instances": 85,
+                "accuracy": 0.7058823529411765,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.788235294117647,
+                "f1_micro": 0.8108108108108109,
+                "f1_micro_ci_low": 0.7222222222222222,
+                "f1_micro_ci_high": 0.8701298701298701
+            },
+            "score": 0.5441316113445994,
+            "score_name": "subsets_mean",
+            "num_of_instances": 770
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.4951686908402676,
+                "f1_cars": 0.735632183908046,
+                "f1_pc hardware": 0.4,
+                "f1_windows x": 0.08108108108108109,
+                "f1_computer graphics": 0.42201834862385323,
+                "f1_atheism": 0.2857142857142857,
+                "f1_politics": 0.34210526315789475,
+                "f1_religion": 0.22988505747126436,
+                "f1_medicine": 0.7631578947368421,
+                "f1_christianity": 0.4444444444444444,
+                "f1_microsoft windows": 0.3125,
+                "f1_middle east": 0.43037974683544306,
+                "f1_motorcycles": 0.64,
+                "f1_mac hardware": 0.29333333333333333,
+                "f1_electronics": 0.5128205128205128,
+                "f1_for sale": 0.6904761904761905,
+                "f1_guns": 0.32786885245901637,
+                "f1_space": 0.7446808510638298,
+                "f1_cryptography": 0.5074626865671642,
+                "f1_baseball": 0.8598130841121495,
+                "f1_hockey": 0.88,
+                "f1_macro_ci_low": 0.4680131642390255,
+                "f1_macro_ci_high": 0.5255643836143373,
+                "score_name": "f1_micro",
+                "score": 0.5081081081081081,
+                "score_ci_high": 0.5384296879334298,
+                "score_ci_low": 0.475620048107115,
+                "num_of_instances": 1000,
+                "accuracy": 0.47,
+                "accuracy_ci_low": 0.4397222118119949,
+                "accuracy_ci_high": 0.501,
+                "f1_micro": 0.5081081081081081,
+                "f1_micro_ci_low": 0.475620048107115,
+                "f1_micro_ci_high": 0.5384296879334298
+            },
+            "score": 0.5081081081081081,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.5856752687246415,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.900072939460248,
+                "f1_checking or savings account": 0.6206896551724138,
+                "f1_debt collection": 0.44,
+                "f1_credit card or prepaid card": 0.5985401459854015,
+                "f1_mortgage": 0.7567567567567568,
+                "f1_student loan": 0.8888888888888888,
+                "f1_money transfer or virtual currency or money service": 0.55,
+                "f1_vehicle loan or lease": 0.5161290322580645,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_macro_ci_low": 0.5384285887156623,
+                "f1_macro_ci_high": 0.6269737752861375,
+                "score_name": "f1_micro",
+                "score": 0.8055987558320373,
+                "score_ci_high": 0.827979274611399,
+                "score_ci_low": 0.7814291760822389,
+                "num_of_instances": 1000,
+                "accuracy": 0.777,
+                "accuracy_ci_low": 0.749,
+                "accuracy_ci_high": 0.802,
+                "f1_micro": 0.8055987558320373,
+                "f1_micro_ci_low": 0.7814291760822389,
+                "f1_micro_ci_high": 0.827979274611399
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.6924499645538243,
+                "f1_mortgages and loans": 0.8160919540229885,
+                "f1_credit card": 0.7734806629834254,
+                "f1_retail banking": 0.562962962962963,
+                "f1_debt collection": 0.5959595959595959,
+                "f1_credit reporting": 0.7137546468401487,
+                "f1_macro_ci_low": 0.6512602475695661,
+                "f1_macro_ci_high": 0.7369419146845784,
+                "score_name": "f1_micro",
+                "score": 0.6980146290491118,
+                "score_ci_high": 0.7373210151084457,
+                "score_ci_low": 0.6555323590814196,
+                "num_of_instances": 500,
+                "accuracy": 0.668,
+                "accuracy_ci_low": 0.6247351354699405,
+                "accuracy_ci_high": 0.712,
+                "f1_micro": 0.6980146290491118,
+                "f1_micro_ci_low": 0.6555323590814196,
+                "f1_micro_ci_high": 0.7373210151084457
+            },
+            "score": 0.7518066924405746,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 1000,
+                "execution_accuracy": 0.113,
+                "program_accuracy": 0.135,
+                "score": 0.135,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.094,
+                "execution_accuracy_ci_high": 0.134,
+                "program_accuracy_ci_low": 0.116,
+                "program_accuracy_ci_high": 0.158,
+                "score_ci_low": 0.116,
+                "score_ci_high": 0.158
+            },
+            "score": 0.135,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1000
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.3279655346823218,
+                "recall": 0.5705879433371356,
+                "f1": 0.35889039539658296,
+                "precision_ci_low": 0.3078086310620982,
+                "precision_ci_high": 0.34963819007576297,
+                "recall_ci_low": 0.554197356666031,
+                "recall_ci_high": 0.5883801367096354,
+                "f1_ci_low": 0.3417642169187518,
+                "f1_ci_high": 0.37901291711555385,
+                "score_name": "f1",
+                "score": 0.35889039539658296,
+                "score_ci_high": 0.37901291711555385,
+                "score_ci_low": 0.3417642169187518,
+                "num_of_instances": 600,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6124672105411688,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6899554192026456,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5693493613600731,
+                "faithfullness_f1_token_overlap": 0.3464559400877777,
+                "faithfullness_recall_token_overlap": 0.27600428527077514,
+                "faithfullness_precision_token_overlap": 0.5952455002809257,
+                "correctness_f1_token_overlap": 0.35889039539658296,
+                "correctness_recall_token_overlap": 0.5705879433371356,
+                "correctness_precision_token_overlap": 0.3279655346823218
+            },
+            "score": 0.35889039539658296,
+            "score_name": "subsets_mean",
+            "num_of_instances": 600
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.469,
+                "accuracy_ci_low": 0.438,
+                "accuracy_ci_high": 0.499,
+                "score_name": "accuracy",
+                "score": 0.469,
+                "score_ci_high": 0.499,
+                "score_ci_low": 0.438,
+                "num_of_instances": 1000
+            },
+            "openbook_qa": {
+                "accuracy": 0.824,
+                "accuracy_ci_low": 0.7898204640532378,
+                "accuracy_ci_high": 0.8579264839771418,
+                "score_name": "accuracy",
+                "score": 0.824,
+                "score_ci_high": 0.8579264839771418,
+                "score_ci_low": 0.7898204640532378,
+                "num_of_instances": 500
+            },
+            "score": 0.6465,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1500
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8298518021409916,
+                "category_deception": 0.8434558517177297,
+                "category_explicit_content": 0.845593557213292,
+                "category_violence": 0.8258583758558546,
+                "category_discrimination": 0.8266353697057754,
+                "category_substance_abuse": 0.8619899458355373,
+                "category_pii": 0.8507882124847836,
+                "safety": 0.8336919906771638,
+                "safety_ci_low": 0.8111003444322545,
+                "safety_ci_high": 0.8568521539985686,
+                "score_name": "safety",
+                "score": 0.8336919906771638,
+                "score_ci_high": 0.8568521539985686,
+                "score_ci_low": 0.8111003444322545,
+                "num_of_instances": 100
+            },
+            "score": 0.8336919906771638,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 528,
+                "rouge2": 0.21160139852857196,
+                "rougeLsum": 0.36324031100110515,
+                "rouge1": 0.4273940174750716,
+                "rougeL": 0.2985567591141555,
+                "score": 0.2985567591141555,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.20478163308145209,
+                "rouge2_ci_high": 0.21935297407718782,
+                "rougeLsum_ci_low": 0.3546878882607467,
+                "rougeLsum_ci_high": 0.37154314876152733,
+                "rouge1_ci_low": 0.4178746520059863,
+                "rouge1_ci_high": 0.43635521340646144,
+                "rougeL_ci_low": 0.2916818506638873,
+                "rougeL_ci_high": 0.3064012355591934,
+                "score_ci_low": 0.2916818506638873,
+                "score_ci_high": 0.3064012355591934
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 1000,
+                "rouge2": 0.015911661871209636,
+                "rougeLsum": 0.0956886215682793,
+                "rouge1": 0.11520528707442619,
+                "rougeL": 0.08350863165548258,
+                "score": 0.08350863165548258,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.013985799591312902,
+                "rouge2_ci_high": 0.017727653852883076,
+                "rougeLsum_ci_low": 0.09148358738071459,
+                "rougeLsum_ci_high": 0.10004441271360605,
+                "rouge1_ci_low": 0.10996324785054311,
+                "rouge1_ci_high": 0.1203422582590639,
+                "rougeL_ci_low": 0.07993462762229471,
+                "rougeL_ci_high": 0.0872963198676006,
+                "score_ci_low": 0.07993462762229471,
+                "score_ci_high": 0.0872963198676006
+            },
+            "score": 0.19103269538481904,
+            "score_name": "subsets_mean",
+            "num_of_instances": 1528
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1160,
+                    634,
+                    402,
+                    263
+                ],
+                "totals": [
+                    3432,
+                    3366,
+                    3300,
+                    3234
+                ],
+                "precisions": [
+                    0.337995337995338,
+                    0.1883541295306001,
+                    0.12181818181818181,
+                    0.08132343846629561
+                ],
+                "bp": 1.0,
+                "sys_len": 3432,
+                "ref_len": 1734,
+                "sacrebleu": 0.1584723235237399,
+                "score": 0.1584723235237399,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.11474709135550289,
+                "score_ci_high": 0.20012320167306266,
+                "sacrebleu_ci_low": 0.11474709135550289,
+                "sacrebleu_ci_high": 0.20012320167306266
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1242,
+                    746,
+                    497,
+                    332
+                ],
+                "totals": [
+                    3635,
+                    3569,
+                    3503,
+                    3437
+                ],
+                "precisions": [
+                    0.34167812929848695,
+                    0.20902213505183526,
+                    0.14187838995147017,
+                    0.09659586848996218
+                ],
+                "bp": 1.0,
+                "sys_len": 3635,
+                "ref_len": 1734,
+                "sacrebleu": 0.17687687871183358,
+                "score": 0.17687687871183358,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.13879938364214875,
+                "score_ci_high": 0.2223271554471336,
+                "sacrebleu_ci_low": 0.13879938364214875,
+                "sacrebleu_ci_high": 0.2223271554471336
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 66,
+                "counts": [
+                    707,
+                    291,
+                    137,
+                    67
+                ],
+                "totals": [
+                    2678,
+                    2612,
+                    2546,
+                    2480
+                ],
+                "precisions": [
+                    0.26400298730395816,
+                    0.11140888208269524,
+                    0.053809897879025924,
+                    0.027016129032258064
+                ],
+                "bp": 1.0,
+                "sys_len": 2678,
+                "ref_len": 1589,
+                "sacrebleu": 0.08086367724146439,
+                "score": 0.08086367724146439,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.060707255296144236,
+                "score_ci_high": 0.11207981375295485,
+                "sacrebleu_ci_low": 0.060707255296144236,
+                "sacrebleu_ci_high": 0.11207981375295485
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 66,
+                "counts": [
+                    1105,
+                    576,
+                    338,
+                    205
+                ],
+                "totals": [
+                    2865,
+                    2799,
+                    2733,
+                    2667
+                ],
+                "precisions": [
+                    0.3856893542757417,
+                    0.2057877813504823,
+                    0.12367361873399195,
+                    0.07686539182602176
+                ],
+                "bp": 1.0,
+                "sys_len": 2865,
+                "ref_len": 1835,
+                "sacrebleu": 0.1657357842387588,
+                "score": 0.1657357842387588,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1255662861475735,
+                "score_ci_high": 0.19513128530364274,
+                "sacrebleu_ci_low": 0.1255662861475735,
+                "sacrebleu_ci_high": 0.19513128530364274
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 66,
+                "counts": [
+                    1425,
+                    950,
+                    689,
+                    512
+                ],
+                "totals": [
+                    3952,
+                    3886,
+                    3820,
+                    3754
+                ],
+                "precisions": [
+                    0.3605769230769231,
+                    0.24446731857951623,
+                    0.18036649214659686,
+                    0.13638785295684602
+                ],
+                "bp": 1.0,
+                "sys_len": 3952,
+                "ref_len": 2068,
+                "sacrebleu": 0.21579310909975802,
+                "score": 0.21579310909975802,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.17167305584926412,
+                "score_ci_high": 0.2671975553706823,
+                "sacrebleu_ci_low": 0.17167305584926412,
+                "sacrebleu_ci_high": 0.2671975553706823
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 66,
+                "counts": [
+                    1144,
+                    510,
+                    256,
+                    121
+                ],
+                "totals": [
+                    4088,
+                    4022,
+                    3956,
+                    3890
+                ],
+                "precisions": [
+                    0.27984344422700586,
+                    0.1268025857782198,
+                    0.06471183013144591,
+                    0.031105398457583547
+                ],
+                "bp": 1.0,
+                "sys_len": 4088,
+                "ref_len": 2235,
+                "sacrebleu": 0.09193178117454374,
+                "score": 0.09193178117454374,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.07510753790698527,
+                "score_ci_high": 0.10802722708228213,
+                "sacrebleu_ci_low": 0.07510753790698527,
+                "sacrebleu_ci_high": 0.10802722708228213
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 66,
+                "counts": [
+                    1355,
+                    895,
+                    641,
+                    477
+                ],
+                "totals": [
+                    3672,
+                    3606,
+                    3540,
+                    3474
+                ],
+                "precisions": [
+                    0.3690087145969499,
+                    0.24819744869661675,
+                    0.1810734463276836,
+                    0.1373056994818653
+                ],
+                "bp": 1.0,
+                "sys_len": 3672,
+                "ref_len": 1916,
+                "sacrebleu": 0.21844611082203133,
+                "score": 0.21844611082203133,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1740922279282215,
+                "score_ci_high": 0.2747838995139129,
+                "sacrebleu_ci_low": 0.1740922279282215,
+                "sacrebleu_ci_high": 0.2747838995139129
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 66,
+                "counts": [
+                    958,
+                    446,
+                    242,
+                    144
+                ],
+                "totals": [
+                    3143,
+                    3077,
+                    3011,
+                    2945
+                ],
+                "precisions": [
+                    0.304804327076042,
+                    0.1449463763405915,
+                    0.080371969445367,
+                    0.048896434634974534
+                ],
+                "bp": 1.0,
+                "sys_len": 3143,
+                "ref_len": 1949,
+                "sacrebleu": 0.11478960818381517,
+                "score": 0.11478960818381517,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.09008415206585028,
+                "score_ci_high": 0.15651321779474522,
+                "sacrebleu_ci_low": 0.09008415206585028,
+                "sacrebleu_ci_high": 0.15651321779474522
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 66,
+                "counts": [
+                    1223,
+                    632,
+                    362,
+                    212
+                ],
+                "totals": [
+                    3461,
+                    3395,
+                    3329,
+                    3263
+                ],
+                "precisions": [
+                    0.35336607916787055,
+                    0.1861561119293078,
+                    0.10874136377290478,
+                    0.06497088568801716
+                ],
+                "bp": 1.0,
+                "sys_len": 3461,
+                "ref_len": 2098,
+                "sacrebleu": 0.14682632542570678,
+                "score": 0.14682632542570678,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.11971923721446943,
+                "score_ci_high": 0.18346369643073177,
+                "sacrebleu_ci_low": 0.11971923721446943,
+                "sacrebleu_ci_high": 0.18346369643073177
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1245,
+                    754,
+                    504,
+                    346
+                ],
+                "totals": [
+                    3378,
+                    3312,
+                    3246,
+                    3180
+                ],
+                "precisions": [
+                    0.3685612788632327,
+                    0.2276570048309179,
+                    0.15526802218114602,
+                    0.10880503144654088
+                ],
+                "bp": 1.0,
+                "sys_len": 3378,
+                "ref_len": 1734,
+                "sacrebleu": 0.19403515904057747,
+                "score": 0.19403515904057747,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.16072549920746337,
+                "score_ci_high": 0.25567936336289826,
+                "sacrebleu_ci_low": 0.16072549920746337,
+                "sacrebleu_ci_high": 0.25567936336289826
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1030,
+                    467,
+                    241,
+                    128
+                ],
+                "totals": [
+                    3273,
+                    3207,
+                    3141,
+                    3075
+                ],
+                "precisions": [
+                    0.3146959975557592,
+                    0.14561895852821952,
+                    0.07672715695638332,
+                    0.04162601626016261
+                ],
+                "bp": 1.0,
+                "sys_len": 3273,
+                "ref_len": 1734,
+                "sacrebleu": 0.10999065136220543,
+                "score": 0.10999065136220543,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.08125966435750204,
+                "score_ci_high": 0.15014434309739183,
+                "sacrebleu_ci_low": 0.08125966435750204,
+                "sacrebleu_ci_high": 0.15014434309739183
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1040,
+                    465,
+                    241,
+                    132
+                ],
+                "totals": [
+                    3703,
+                    3637,
+                    3571,
+                    3505
+                ],
+                "precisions": [
+                    0.28085336213880635,
+                    0.12785262579048667,
+                    0.06748809857182862,
+                    0.037660485021398
+                ],
+                "bp": 1.0,
+                "sys_len": 3703,
+                "ref_len": 1734,
+                "sacrebleu": 0.09774073377105962,
+                "score": 0.09774073377105962,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.0819836886276176,
+                "score_ci_high": 0.12072091726722378,
+                "sacrebleu_ci_low": 0.0819836886276176,
+                "sacrebleu_ci_high": 0.12072091726722378
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1278,
+                    818,
+                    573,
+                    412
+                ],
+                "totals": [
+                    3168,
+                    3102,
+                    3036,
+                    2970
+                ],
+                "precisions": [
+                    0.40340909090909094,
+                    0.2637008381689233,
+                    0.18873517786561267,
+                    0.13872053872053872
+                ],
+                "bp": 1.0,
+                "sys_len": 3168,
+                "ref_len": 1734,
+                "sacrebleu": 0.22972735015632778,
+                "score": 0.22972735015632778,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.17875705428567942,
+                "score_ci_high": 0.30188244314298573,
+                "sacrebleu_ci_low": 0.17875705428567942,
+                "sacrebleu_ci_high": 0.30188244314298573
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1283,
+                    796,
+                    537,
+                    365
+                ],
+                "totals": [
+                    4400,
+                    4334,
+                    4268,
+                    4202
+                ],
+                "precisions": [
+                    0.2915909090909091,
+                    0.18366405168435626,
+                    0.12582005623242737,
+                    0.08686339838172298
+                ],
+                "bp": 1.0,
+                "sys_len": 4400,
+                "ref_len": 1734,
+                "sacrebleu": 0.1555414731878905,
+                "score": 0.1555414731878905,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.12023011304708842,
+                "score_ci_high": 0.20687673778594803,
+                "sacrebleu_ci_low": 0.12023011304708842,
+                "sacrebleu_ci_high": 0.20687673778594803
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 66,
+                "counts": [
+                    1155,
+                    607,
+                    355,
+                    220
+                ],
+                "totals": [
+                    2824,
+                    2758,
+                    2692,
+                    2626
+                ],
+                "precisions": [
+                    0.4089943342776204,
+                    0.22008701957940538,
+                    0.13187221396731055,
+                    0.08377760853008377
+                ],
+                "bp": 1.0,
+                "sys_len": 2824,
+                "ref_len": 1734,
+                "sacrebleu": 0.17758171439939147,
+                "score": 0.17758171439939147,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.13942620008407763,
+                "score_ci_high": 0.23700138449810748,
+                "sacrebleu_ci_low": 0.13942620008407763,
+                "sacrebleu_ci_high": 0.23700138449810748
+            },
+            "score": 0.15562351202260694,
+            "score_name": "subsets_mean",
+            "num_of_instances": 990
+        },
+        "score": 0.4344559230477983,
+        "score_name": "subsets_mean",
+        "num_of_instances": 12472
+    }
+}

results/bluebench/{2025-06-19T18-10-05_evaluation_results.json → 2025-06-23T05-36-33_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-19T22:09:59.730715Z",
+        "timestamp_utc": "2025-06-23T09:36:30.499456Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,564 +176,564 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.5666666666666667,
-                "accuracy_ci_low": 0.4666666666666667,
-                "accuracy_ci_high": 0.6777777777777778,
+                "accuracy": 0.6333333333333333,
+                "accuracy_ci_low": 0.5222222222222223,
+                "accuracy_ci_high": 0.7333333333333333,
                 "score_name": "accuracy",
-                "score": 0.5666666666666667,
-                "score_ci_high": 0.6777777777777778,
-                "score_ci_low": 0.4666666666666667,
+                "score": 0.6333333333333333,
+                "score_ci_high": 0.7333333333333333,
+                "score_ci_low": 0.5222222222222223,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.7333333333333333,
-                "accuracy_ci_low": 0.6333333333333333,
-                "accuracy_ci_high": 0.8222222222222222,
+                "accuracy": 0.6888888888888889,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.7777777777777778,
                 "score_name": "accuracy",
-                "score": 0.7333333333333333,
-                "score_ci_high": 0.8222222222222222,
-                "score_ci_low": 0.6333333333333333,
+                "score": 0.6888888888888889,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.6,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8222222222222222,
-                "accuracy_ci_low": 0.7444444444444445,
-                "accuracy_ci_high": 0.9,
+                "accuracy": 0.8444444444444444,
+                "accuracy_ci_low": 0.7555555555555555,
+                "accuracy_ci_high": 0.9111111111111111,
                 "score_name": "accuracy",
-                "score": 0.8222222222222222,
-                "score_ci_high": 0.9,
-                "score_ci_low": 0.7444444444444445,
+                "score": 0.8444444444444444,
+                "score_ci_high": 0.9111111111111111,
+                "score_ci_low": 0.7555555555555555,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.6444444444444445,
-                "accuracy_ci_low": 0.5333333333333333,
-                "accuracy_ci_high": 0.7333333333333333,
+                "accuracy": 0.6777777777777778,
+                "accuracy_ci_low": 0.5777777777777777,
+                "accuracy_ci_high": 0.7555555555555555,
                 "score_name": "accuracy",
-                "score": 0.6444444444444445,
-                "score_ci_high": 0.7333333333333333,
-                "score_ci_low": 0.5333333333333333,
+                "score": 0.6777777777777778,
+                "score_ci_high": 0.7555555555555555,
+                "score_ci_low": 0.5777777777777777,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.6888888888888889,
-                "accuracy_ci_low": 0.5888888888888889,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy": 0.7222222222222222,
+                "accuracy_ci_low": 0.6222222222222222,
+                "accuracy_ci_high": 0.8111111111111111,
                 "score_name": "accuracy",
-                "score": 0.6888888888888889,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.5888888888888889,
+                "score": 0.7222222222222222,
+                "score_ci_high": 0.8111111111111111,
+                "score_ci_low": 0.6222222222222222,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.8111111111111111,
-                "accuracy_ci_low": 0.7222222222222222,
-                "accuracy_ci_high": 0.8777777777777778,
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.8111111111111111,
-                "score_ci_high": 0.8777777777777778,
-                "score_ci_low": 0.7222222222222222,
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7333333333333333,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.7222222222222222,
-                "accuracy_ci_low": 0.6111111111111112,
-                "accuracy_ci_high": 0.8,
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6444444444444445,
+                "accuracy_ci_high": 0.8222222222222222,
                 "score_name": "accuracy",
-                "score": 0.7222222222222222,
-                "score_ci_high": 0.8,
-                "score_ci_low": 0.6111111111111112,
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.8222222222222222,
+                "score_ci_low": 0.6444444444444445,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.7111111111111111,
-                "accuracy_ci_low": 0.6111111111111112,
-                "accuracy_ci_high": 0.8,
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6444444444444445,
+                "accuracy_ci_high": 0.8333333333333334,
                 "score_name": "accuracy",
-                "score": 0.7111111111111111,
-                "score_ci_high": 0.8,
-                "score_ci_low": 0.6111111111111112,
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.8333333333333334,
+                "score_ci_low": 0.6444444444444445,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.7111111111111111,
-                "accuracy_ci_low": 0.6111111111111112,
-                "accuracy_ci_high": 0.8,
+                "accuracy": 0.7444444444444445,
+                "accuracy_ci_low": 0.6444444444444445,
+                "accuracy_ci_high": 0.8222222222222222,
                 "score_name": "accuracy",
-                "score": 0.7111111111111111,
-                "score_ci_high": 0.8,
-                "score_ci_low": 0.6111111111111112,
+                "score": 0.7444444444444445,
+                "score_ci_high": 0.8222222222222222,
+                "score_ci_low": 0.6444444444444445,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.7666666666666667,
-                "accuracy_ci_low": 0.6666666666666666,
-                "accuracy_ci_high": 0.8444444444444444,
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.6790372940698232,
+                "accuracy_ci_high": 0.8555555555555555,
                 "score_name": "accuracy",
-                "score": 0.7666666666666667,
-                "score_ci_high": 0.8444444444444444,
-                "score_ci_low": 0.6666666666666666,
+                "score": 0.7777777777777778,
+                "score_ci_high": 0.8555555555555555,
+                "score_ci_low": 0.6790372940698232,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7111111111111111,
-                "accuracy_ci_high": 0.8777777777777778,
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.6888888888888889,
+                "accuracy_ci_high": 0.8555555555555555,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8777777777777778,
-                "score_ci_low": 0.7111111111111111,
+                "score": 0.7888888888888889,
+                "score_ci_high": 0.8555555555555555,
+                "score_ci_low": 0.6888888888888889,
                 "num_of_instances": 90
             },
-            "score": 0.7252525252525253,
+            "score": 0.7444444444444445,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.052083333333333336,
+                "score": 0.052083333333333336,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.052083333333333336,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.5026737967914439,
-                "f1_Organization": 0.2875,
-                "f1_Location": 0.28571428571428575,
-                "f1_macro": 0.35862936083524316,
-                "recall_macro": 0.3171773628661296,
-                "precision_macro": 0.4188335014421971,
-                "in_classes_support": 0.7664783427495292,
-                "f1_micro": 0.32954545454545453,
-                "recall_micro": 0.3314285714285714,
-                "precision_micro": 0.327683615819209,
-                "score": 0.32954545454545453,
+                "f1_Person": 0.4769647696476965,
+                "f1_Organization": 0.2893890675241158,
+                "f1_Location": 0.30894308943089427,
+                "f1_macro": 0.3584323088675689,
+                "recall_macro": 0.31476418018843305,
+                "precision_macro": 0.4193267050409908,
+                "in_classes_support": 0.7786407766990291,
+                "f1_micro": 0.32884615384615384,
+                "recall_micro": 0.32571428571428573,
+                "precision_micro": 0.3320388349514563,
+                "score": 0.32884615384615384,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.28111538926666857,
-                "score_ci_high": 0.37977770945501865,
-                "f1_micro_ci_low": 0.28111538926666857,
-                "f1_micro_ci_high": 0.37977770945501865
+                "score_ci_low": 0.2827095328153393,
+                "score_ci_high": 0.3761779704134513,
+                "f1_micro_ci_low": 0.2827095328153393,
+                "f1_micro_ci_high": 0.3761779704134513
             },
-            "score": 0.32954545454545453,
+            "score": 0.32884615384615384,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.4084507042253521,
-                "accuracy_ci_low": 0.30985915492957744,
-                "accuracy_ci_high": 0.5211267605633803,
+                "accuracy": 0.49295774647887325,
+                "accuracy_ci_low": 0.38028169014084506,
+                "accuracy_ci_high": 0.6056338028169014,
                 "score_name": "accuracy",
-                "score": 0.4084507042253521,
-                "score_ci_high": 0.5211267605633803,
-                "score_ci_low": 0.30985915492957744,
+                "score": 0.49295774647887325,
+                "score_ci_high": 0.6056338028169014,
+                "score_ci_low": 0.38028169014084506,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.2535211267605634,
-                "accuracy_ci_low": 0.15492957746478872,
-                "accuracy_ci_high": 0.36619718309859156,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.22535211267605634,
+                "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
-                "score": 0.2535211267605634,
-                "score_ci_high": 0.36619718309859156,
-                "score_ci_low": 0.15492957746478872,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.22535211267605634,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.29577464788732394,
+                "accuracy": 0.22535211267605634,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.3380281690140845,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.29577464788732394,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.22535211267605634,
+                "score_ci_high": 0.3380281690140845,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
                 "accuracy": 0.29577464788732394,
                 "accuracy_ci_low": 0.19718309859154928,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy_ci_high": 0.4084507042253521,
                 "score_name": "accuracy",
                 "score": 0.29577464788732394,
-                "score_ci_high": 0.39436619718309857,
+                "score_ci_high": 0.4084507042253521,
                 "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.4788732394366197,
-                "accuracy_ci_low": 0.36619718309859156,
-                "accuracy_ci_high": 0.5915492957746479,
+                "accuracy": 0.5070422535211268,
+                "accuracy_ci_low": 0.38028169014084506,
+                "accuracy_ci_high": 0.6197183098591549,
                 "score_name": "accuracy",
-                "score": 0.4788732394366197,
-                "score_ci_high": 0.5915492957746479,
-                "score_ci_low": 0.36619718309859156,
+                "score": 0.5070422535211268,
+                "score_ci_high": 0.6197183098591549,
+                "score_ci_low": 0.38028169014084506,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.19718309859154928,
-                "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.29577464788732394,
+                "accuracy": 0.18309859154929578,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.2676056338028169,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.29577464788732394,
-                "score_ci_low": 0.11267605633802817,
+                "score": 0.18309859154929578,
+                "score_ci_high": 0.2676056338028169,
+                "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.29577464788732394,
-                "accuracy_ci_low": 0.19718309859154928,
-                "accuracy_ci_high": 0.4225352112676056,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
-                "score": 0.29577464788732394,
-                "score_ci_high": 0.4225352112676056,
-                "score_ci_low": 0.19718309859154928,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.43661971830985913,
+                "score_ci_low": 0.2112676056338028,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.30985915492957744,
-                "accuracy_ci_low": 0.2112676056338028,
-                "accuracy_ci_high": 0.4225352112676056,
+                "accuracy": 0.36619718309859156,
+                "accuracy_ci_low": 0.2535211267605634,
+                "accuracy_ci_high": 0.4788732394366197,
                 "score_name": "accuracy",
-                "score": 0.30985915492957744,
-                "score_ci_high": 0.4225352112676056,
-                "score_ci_low": 0.2112676056338028,
+                "score": 0.36619718309859156,
+                "score_ci_high": 0.4788732394366197,
+                "score_ci_low": 0.2535211267605634,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.28169014084507044,
+                "accuracy_ci_low": 0.18309859154929578,
+                "accuracy_ci_high": 0.39436619718309857,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.28169014084507044,
+                "score_ci_high": 0.39436619718309857,
+                "score_ci_low": 0.18309859154929578,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.11267605633802817,
-                "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.19718309859154928,
+                "accuracy": 0.15492957746478872,
+                "accuracy_ci_low": 0.08450704225352113,
+                "accuracy_ci_high": 0.26564872868691924,
                 "score_name": "accuracy",
-                "score": 0.11267605633802817,
-                "score_ci_high": 0.19718309859154928,
-                "score_ci_low": 0.056338028169014086,
+                "score": 0.15492957746478872,
+                "score_ci_high": 0.26564872868691924,
+                "score_ci_low": 0.08450704225352113,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.23943661971830985,
-                "accuracy_ci_low": 0.15492957746478872,
-                "accuracy_ci_high": 0.352112676056338,
+                "accuracy": 0.28169014084507044,
+                "accuracy_ci_low": 0.18309859154929578,
+                "accuracy_ci_high": 0.39436619718309857,
                 "score_name": "accuracy",
-                "score": 0.23943661971830985,
-                "score_ci_high": 0.352112676056338,
-                "score_ci_low": 0.15492957746478872,
+                "score": 0.28169014084507044,
+                "score_ci_high": 0.39436619718309857,
+                "score_ci_low": 0.18309859154929578,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.2676056338028169,
-                "accuracy_ci_low": 0.16901408450704225,
-                "accuracy_ci_high": 0.38028169014084506,
+                "accuracy": 0.29577464788732394,
+                "accuracy_ci_low": 0.19718309859154928,
+                "accuracy_ci_high": 0.4084507042253521,
                 "score_name": "accuracy",
-                "score": 0.2676056338028169,
-                "score_ci_high": 0.38028169014084506,
-                "score_ci_low": 0.16901408450704225,
+                "score": 0.29577464788732394,
+                "score_ci_high": 0.4084507042253521,
+                "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.15492957746478872,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2535211267605634,
+                "accuracy": 0.16901408450704225,
+                "accuracy_ci_low": 0.09859154929577464,
+                "accuracy_ci_high": 0.2676056338028169,
                 "score_name": "accuracy",
-                "score": 0.15492957746478872,
-                "score_ci_high": 0.2535211267605634,
-                "score_ci_low": 0.08450704225352113,
+                "score": 0.16901408450704225,
+                "score_ci_high": 0.2676056338028169,
+                "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.49295774647887325,
-                "accuracy_ci_low": 0.38028169014084506,
-                "accuracy_ci_high": 0.6197183098591549,
+                "accuracy": 0.4788732394366197,
+                "accuracy_ci_low": 0.36619718309859156,
+                "accuracy_ci_high": 0.6012345324644585,
                 "score_name": "accuracy",
-                "score": 0.49295774647887325,
-                "score_ci_high": 0.6197183098591549,
-                "score_ci_low": 0.38028169014084506,
+                "score": 0.4788732394366197,
+                "score_ci_high": 0.6012345324644585,
+                "score_ci_low": 0.36619718309859156,
                 "num_of_instances": 71
             },
-            "score": 0.2806841046277666,
+            "score": 0.31287726358148893,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.322066985645933,
-                "f1_suggestive": 0.2,
-                "f1_descriptive": 0.4,
-                "f1_generic": 0.3157894736842105,
-                "f1_arbitrary": 0.45454545454545453,
-                "f1_fanciful": 0.24,
-                "f1_macro_ci_low": 0.2326130794928424,
-                "f1_macro_ci_high": 0.4327896628836512,
+                "f1_macro": 0.37558008658008657,
+                "f1_suggestive": 0.24242424242424243,
+                "f1_generic": 0.38095238095238093,
+                "f1_descriptive": 0.4583333333333333,
+                "f1_fanciful": 0.32,
+                "f1_arbitrary": 0.47619047619047616,
+                "f1_macro_ci_low": 0.2831225773147394,
+                "f1_macro_ci_high": 0.5002662505279254,
                 "score_name": "f1_micro",
-                "score": 0.34523809523809523,
-                "score_ci_high": 0.44408416032543374,
-                "score_ci_low": 0.25,
+                "score": 0.3905325443786982,
+                "score_ci_high": 0.4970414201183432,
+                "score_ci_low": 0.2850635959228859,
                 "num_of_instances": 85,
-                "accuracy": 0.3411764705882353,
-                "accuracy_ci_low": 0.24705882352941178,
-                "accuracy_ci_high": 0.43529411764705883,
-                "f1_micro": 0.34523809523809523,
-                "f1_micro_ci_low": 0.25,
-                "f1_micro_ci_high": 0.44408416032543374
+                "accuracy": 0.38823529411764707,
+                "accuracy_ci_low": 0.2823529411764706,
+                "accuracy_ci_high": 0.49411764705882355,
+                "f1_micro": 0.3905325443786982,
+                "f1_micro_ci_low": 0.2850635959228859,
+                "f1_micro_ci_high": 0.4970414201183432
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.4862053369516056,
-                "f1_no": 0.48484848484848486,
-                "f1_yes": 0.48756218905472637,
-                "f1_macro_ci_low": 0.4185718876526183,
-                "f1_macro_ci_high": 0.5583302726222448,
+                "f1_macro": 0.4401501318725908,
+                "f1_no": 0.40860215053763443,
+                "f1_yes": 0.4716981132075472,
+                "f1_macro_ci_low": 0.37497026604570943,
+                "f1_macro_ci_high": 0.5074058340630839,
                 "score_name": "f1_micro",
-                "score": 0.48621553884711777,
-                "score_ci_high": 0.555,
-                "score_ci_low": 0.41708542713567837,
+                "score": 0.44221105527638194,
+                "score_ci_high": 0.507537688442211,
+                "score_ci_low": 0.3763531585733561,
                 "num_of_instances": 200,
-                "accuracy": 0.485,
-                "accuracy_ci_low": 0.415,
-                "accuracy_ci_high": 0.555,
-                "f1_micro": 0.48621553884711777,
-                "f1_micro_ci_low": 0.41708542713567837,
-                "f1_micro_ci_high": 0.555
+                "accuracy": 0.44,
+                "accuracy_ci_low": 0.375,
+                "accuracy_ci_high": 0.505,
+                "f1_micro": 0.44221105527638194,
+                "f1_micro_ci_low": 0.3763531585733561,
+                "f1_micro_ci_high": 0.507537688442211
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.21190211834221687,
-                "f1_conclusion": 0.14634146341463414,
-                "f1_analysis": 0.3673469387755102,
+                "f1_macro": 0.1916316363119106,
+                "f1_conclusion": 0.15584415584415584,
+                "f1_analysis": 0.3333333333333333,
                 "f1_decree": 0.07692307692307693,
-                "f1_issue": 0.2631578947368421,
-                "f1_facts": 0.13333333333333333,
-                "f1_procedural history": 0.12121212121212122,
-                "f1_rule": 0.375,
-                "f1_macro_ci_low": 0.16201147151923767,
-                "f1_macro_ci_high": 0.27835110124455087,
+                "f1_issue": 0.23076923076923078,
+                "f1_facts": 0.12903225806451613,
+                "f1_procedural history": 0.11764705882352941,
+                "f1_rule": 0.2978723404255319,
+                "f1_macro_ci_low": 0.143695452040772,
+                "f1_macro_ci_high": 0.25550408682657144,
                 "score_name": "f1_micro",
-                "score": 0.24427480916030533,
-                "score_ci_high": 0.3110332844595539,
-                "score_ci_low": 0.18933051276149385,
+                "score": 0.22278481012658227,
+                "score_ci_high": 0.2864321608040201,
+                "score_ci_low": 0.16660296570964608,
                 "num_of_instances": 200,
-                "accuracy": 0.24,
-                "accuracy_ci_low": 0.185,
-                "accuracy_ci_high": 0.305,
-                "f1_micro": 0.24427480916030533,
-                "f1_micro_ci_low": 0.18933051276149385,
-                "f1_micro_ci_high": 0.3110332844595539
+                "accuracy": 0.22,
+                "accuracy_ci_low": 0.165,
+                "accuracy_ci_high": 0.28021087258250593,
+                "f1_micro": 0.22278481012658227,
+                "f1_micro_ci_low": 0.16660296570964608,
+                "f1_micro_ci_high": 0.2864321608040201
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5140298832430668,
-                "f1_yes": 0.541871921182266,
-                "f1_no": 0.4861878453038674,
-                "f1_macro_ci_low": 0.44358226014090585,
-                "f1_macro_ci_high": 0.5836656602180865,
+                "f1_macro": 0.5259978425026969,
+                "f1_yes": 0.5631067961165048,
+                "f1_no": 0.4888888888888889,
+                "f1_macro_ci_low": 0.4618026481617566,
+                "f1_macro_ci_high": 0.5997495353215635,
                 "score_name": "f1_micro",
-                "score": 0.515625,
-                "score_ci_high": 0.583858269920324,
-                "score_ci_low": 0.4443197729294639,
+                "score": 0.5284974093264249,
+                "score_ci_high": 0.5989912778302698,
+                "score_ci_low": 0.46113989637305697,
                 "num_of_instances": 200,
-                "accuracy": 0.495,
-                "accuracy_ci_low": 0.43,
-                "accuracy_ci_high": 0.565,
-                "f1_micro": 0.515625,
-                "f1_micro_ci_low": 0.4443197729294639,
-                "f1_micro_ci_high": 0.583858269920324
+                "accuracy": 0.51,
+                "accuracy_ci_low": 0.445,
+                "accuracy_ci_high": 0.58,
+                "f1_micro": 0.5284974093264249,
+                "f1_micro_ci_low": 0.46113989637305697,
+                "f1_micro_ci_high": 0.5989912778302698
             },
             "legalbench_proa": {
                 "f1_macro": 0.726027397260274,
                 "f1_yes": 0.7123287671232876,
                 "f1_no": 0.7397260273972602,
-                "f1_macro_ci_low": 0.6205412546681461,
-                "f1_macro_ci_high": 0.8095667611328509,
+                "f1_macro_ci_low": 0.618628457335439,
+                "f1_macro_ci_high": 0.8122702152748204,
                 "score_name": "f1_micro",
                 "score": 0.726027397260274,
-                "score_ci_high": 0.8079470198675497,
-                "score_ci_low": 0.6153846153846154,
+                "score_ci_high": 0.8104575163398693,
+                "score_ci_low": 0.6186406698987806,
                 "num_of_instances": 85,
                 "accuracy": 0.6235294117647059,
                 "accuracy_ci_low": 0.5058823529411764,
                 "accuracy_ci_high": 0.7176470588235294,
                 "f1_micro": 0.726027397260274,
-                "f1_micro_ci_low": 0.6153846153846154,
-                "f1_micro_ci_high": 0.8079470198675497
+                "f1_micro_ci_low": 0.6186406698987806,
+                "f1_micro_ci_high": 0.8104575163398693
             },
-            "score": 0.46347616810115844,
+            "score": 0.46201064327367225,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.3839629098938318,
-                "f1_cars": 0.6804123711340206,
-                "f1_windows x": 0.03125,
-                "f1_atheism": 0.48,
-                "f1_christianity": 0.425,
-                "f1_religion": 0.18461538461538463,
+                "f1_macro": 0.3891021150890982,
+                "f1_cars": 0.7346938775510204,
+                "f1_windows x": 0.0,
+                "f1_atheism": 0.425531914893617,
+                "f1_christianity": 0.4444444444444444,
+                "f1_religion": 0.15873015873015872,
                 "f1_medicine": 0.6376811594202898,
-                "f1_computer graphics": 0.27979274611398963,
-                "f1_microsoft windows": 0.35294117647058826,
-                "f1_middle east": 0.12244897959183673,
-                "f1_politics": 0.26666666666666666,
-                "f1_motorcycles": 0.47619047619047616,
-                "f1_mac hardware": 0.14492753623188406,
-                "f1_pc hardware": 0.358974358974359,
-                "f1_for sale": 0.3018867924528302,
-                "f1_guns": 0.2,
-                "f1_baseball": 0.8130081300813008,
+                "f1_computer graphics": 0.2755102040816326,
+                "f1_microsoft windows": 0.29850746268656714,
+                "f1_middle east": 0.19607843137254902,
+                "f1_politics": 0.3387096774193548,
+                "f1_motorcycles": 0.43902439024390244,
+                "f1_mac hardware": 0.2,
+                "f1_pc hardware": 0.34545454545454546,
+                "f1_for sale": 0.33962264150943394,
+                "f1_guns": 0.26666666666666666,
+                "f1_baseball": 0.7368421052631579,
                 "f1_space": 0.5194805194805194,
-                "f1_cryptography": 0.3466666666666667,
-                "f1_electronics": 0.41025641025641024,
-                "f1_hockey": 0.6470588235294118,
-                "f1_macro_ci_low": 0.35856171946837523,
-                "f1_macro_ci_high": 0.41714627316225344,
+                "f1_cryptography": 0.4358974358974359,
+                "f1_hockey": 0.5625,
+                "f1_electronics": 0.4266666666666667,
+                "f1_macro_ci_low": 0.35853792537669554,
+                "f1_macro_ci_high": 0.4194279498018566,
                 "score_name": "f1_micro",
-                "score": 0.4028352037802717,
-                "score_ci_high": 0.433682467300905,
-                "score_ci_low": 0.3727273654547539,
+                "score": 0.4063792085056113,
+                "score_ci_high": 0.4367348562601148,
+                "score_ci_low": 0.3741790131164338,
                 "num_of_instances": 1000,
-                "accuracy": 0.341,
-                "accuracy_ci_low": 0.312,
-                "accuracy_ci_high": 0.369,
-                "f1_micro": 0.4028352037802717,
-                "f1_micro_ci_low": 0.3727273654547539,
-                "f1_micro_ci_high": 0.433682467300905
+                "accuracy": 0.344,
+                "accuracy_ci_low": 0.314,
+                "accuracy_ci_high": 0.371,
+                "f1_micro": 0.4063792085056113,
+                "f1_micro_ci_low": 0.3741790131164338,
+                "f1_micro_ci_high": 0.4367348562601148
             },
-            "score": 0.4028352037802717,
+            "score": 0.4063792085056113,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.522130366066412,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9174311926605505,
-                "f1_credit card or prepaid card": 0.2619047619047619,
-                "f1_debt collection": 0.47904191616766467,
-                "f1_checking or savings account": 0.6534653465346535,
-                "f1_money transfer or virtual currency or money service": 0.56,
-                "f1_vehicle loan or lease": 0.23076923076923078,
-                "f1_mortgage": 0.7037037037037037,
-                "f1_payday loan or title loan or personal loan": 0.14285714285714285,
+                "f1_macro": 0.5522255415970777,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9099656357388316,
+                "f1_checking or savings account": 0.5301204819277109,
+                "f1_debt collection": 0.3576158940397351,
+                "f1_credit card or prepaid card": 0.37777777777777777,
+                "f1_mortgage": 0.7017543859649122,
                 "f1_student loan": 0.75,
-                "f1_macro_ci_low": 0.47475792993209676,
-                "f1_macro_ci_high": 0.5842900552170582,
+                "f1_money transfer or virtual currency or money service": 0.6666666666666666,
+                "f1_vehicle loan or lease": 0.5161290322580645,
+                "f1_payday loan or title loan or personal loan": 0.16,
+                "f1_macro_ci_low": 0.5008792423568225,
+                "f1_macro_ci_high": 0.6059191922507057,
                 "score_name": "f1_micro",
-                "score": 0.8006150691952845,
-                "score_ci_high": 0.8235944353763129,
-                "score_ci_low": 0.7741691905584849,
+                "score": 0.7975522692503825,
+                "score_ci_high": 0.8230092874186598,
+                "score_ci_low": 0.7734015345268542,
                 "num_of_instances": 1000,
-                "accuracy": 0.781,
-                "accuracy_ci_low": 0.753,
-                "accuracy_ci_high": 0.805,
-                "f1_micro": 0.8006150691952845,
-                "f1_micro_ci_low": 0.7741691905584849,
-                "f1_micro_ci_high": 0.8235944353763129
+                "accuracy": 0.782,
+                "accuracy_ci_low": 0.758,
+                "accuracy_ci_high": 0.808771349424543,
+                "f1_micro": 0.7975522692503825,
+                "f1_micro_ci_low": 0.7734015345268542,
+                "f1_micro_ci_high": 0.8230092874186598
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6094652526866444,
-                "f1_mortgages and loans": 0.7821229050279329,
-                "f1_credit card": 0.6666666666666666,
-                "f1_debt collection": 0.5684210526315789,
-                "f1_retail banking": 0.2912621359223301,
-                "f1_credit reporting": 0.7388535031847133,
-                "f1_macro_ci_low": 0.5670634281009035,
-                "f1_macro_ci_high": 0.6527536805294223,
+                "f1_macro": 0.5976480822045338,
+                "f1_mortgages and loans": 0.7428571428571429,
+                "f1_credit card": 0.6767676767676768,
+                "f1_debt collection": 0.5729166666666666,
+                "f1_retail banking": 0.26666666666666666,
+                "f1_credit reporting": 0.7290322580645161,
+                "f1_macro_ci_low": 0.5536640329621239,
+                "f1_macro_ci_high": 0.6426754598088634,
                 "score_name": "f1_micro",
-                "score": 0.6524390243902439,
-                "score_ci_high": 0.6904276985743381,
-                "score_ci_low": 0.6066261962892265,
+                "score": 0.6408163265306123,
+                "score_ci_high": 0.683589397051309,
+                "score_ci_low": 0.5968250791908158,
                 "num_of_instances": 500,
-                "accuracy": 0.642,
-                "accuracy_ci_low": 0.594,
-                "accuracy_ci_high": 0.68,
-                "f1_micro": 0.6524390243902439,
-                "f1_micro_ci_low": 0.6066261962892265,
-                "f1_micro_ci_high": 0.6904276985743381
+                "accuracy": 0.628,
+                "accuracy_ci_low": 0.584,
+                "accuracy_ci_high": 0.67,
+                "f1_micro": 0.6408163265306123,
+                "f1_micro_ci_low": 0.5968250791908158,
+                "f1_micro_ci_high": 0.683589397051309
             },
-            "score": 0.7265270467927643,
+            "score": 0.7191842978904974,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "program_accuracy": 0.038,
-                "score": 0.038,
+                "program_accuracy": 0.046,
+                "score": 0.046,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.03,
-                "program_accuracy_ci_low": 0.027702359114314717,
-                "program_accuracy_ci_high": 0.05038214389818779,
-                "score_ci_low": 0.027702359114314717,
-                "score_ci_high": 0.05038214389818779,
-                "execution_accuracy_ci_low": 0.021,
-                "execution_accuracy_ci_high": 0.042
+                "execution_accuracy": 0.04,
+                "program_accuracy_ci_low": 0.035,
+                "program_accuracy_ci_high": 0.06,
+                "score_ci_low": 0.035,
+                "score_ci_high": 0.06,
+                "execution_accuracy_ci_low": 0.029,
+                "execution_accuracy_ci_high": 0.053
             },
-            "score": 0.038,
+            "score": 0.046,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.3238378212061128,
-                "recall": 0.5112460699696839,
-                "f1": 0.32731883927753985,
-                "precision_ci_low": 0.3032630188662264,
-                "precision_ci_high": 0.34255213352234126,
-                "recall_ci_low": 0.4946874887919962,
-                "recall_ci_high": 0.5283045722843877,
-                "f1_ci_low": 0.3123215475894989,
-                "f1_ci_high": 0.3436240931728296,
+                "precision": 0.3263731921396224,
+                "recall": 0.5005136281930357,
+                "f1": 0.3238306562243135,
+                "precision_ci_low": 0.3062305395351377,
+                "precision_ci_high": 0.3471267358926223,
+                "recall_ci_low": 0.48406708194912995,
+                "recall_ci_high": 0.5170051754306556,
+                "f1_ci_low": 0.30817001709054165,
+                "f1_ci_high": 0.33859378817920405,
                 "score_name": "f1",
-                "score": 0.32731883927753985,
-                "score_ci_high": 0.3436240931728296,
-                "score_ci_low": 0.3123215475894989,
+                "score": 0.3238306562243135,
+                "score_ci_high": 0.33859378817920405,
+                "score_ci_low": 0.30817001709054165,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5872805261611939,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6561146193246047,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5541991981615623,
-                "faithfullness_f1_token_overlap": 0.2552259281757564,
-                "faithfullness_recall_token_overlap": 0.18820051905597926,
-                "faithfullness_precision_token_overlap": 0.5563191178123087,
-                "correctness_f1_token_overlap": 0.32731883927753985,
-                "correctness_recall_token_overlap": 0.5112460699696839,
-                "correctness_precision_token_overlap": 0.3238378212061128
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5865038787325223,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6549727993210157,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5550703954448303,
+                "faithfullness_f1_token_overlap": 0.26017056639729325,
+                "faithfullness_recall_token_overlap": 0.19094108989263792,
+                "faithfullness_precision_token_overlap": 0.5662514000838377,
+                "correctness_f1_token_overlap": 0.3238306562243135,
+                "correctness_recall_token_overlap": 0.5005136281930357,
+                "correctness_precision_token_overlap": 0.3263731921396224
             },
-            "score": 0.32731883927753985,
+            "score": 0.3238306562243135,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.435,
-                "accuracy_ci_low": 0.403,
-                "accuracy_ci_high": 0.4660514308680489,
+                "accuracy": 0.425,
+                "accuracy_ci_low": 0.394,
+                "accuracy_ci_high": 0.4557921663139014,
                 "score_name": "accuracy",
-                "score": 0.435,
-                "score_ci_high": 0.4660514308680489,
-                "score_ci_low": 0.403,
+                "score": 0.425,
+                "score_ci_high": 0.4557921663139014,
+                "score_ci_low": 0.394,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.744,
-                "accuracy_ci_low": 0.704,
-                "accuracy_ci_high": 0.782,
+                "accuracy": 0.736,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.776,
                 "score_name": "accuracy",
-                "score": 0.744,
-                "score_ci_high": 0.782,
-                "score_ci_low": 0.704,
+                "score": 0.736,
+                "score_ci_high": 0.776,
+                "score_ci_low": 0.7,
                 "num_of_instances": 500
             },
-            "score": 0.5895,
+            "score": 0.5805,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
@@ -743,66 +743,66 @@
                 "severity_high": 0.0,
                 "severity_medium": 0.0,
                 "severity_low": 100.0,
-                "category_harmful_info": 0.8374238369454232,
-                "category_deception": 0.8230129361453682,
-                "category_explicit_content": 0.8649854577249951,
-                "category_violence": 0.8130838501311484,
-                "category_discrimination": 0.8325134645777916,
-                "category_substance_abuse": 0.9021757920583088,
-                "category_pii": 0.8319368116833545,
-                "safety": 0.8330806816902219,
-                "safety_ci_low": 0.8160996919801022,
-                "safety_ci_high": 0.8485437387510895,
+                "category_harmful_info": 0.8295307880589028,
+                "category_deception": 0.8032731126354199,
+                "category_explicit_content": 0.8484950255263936,
+                "category_violence": 0.8168619035846657,
+                "category_discrimination": 0.8478557457999577,
+                "category_substance_abuse": 0.8705138153500027,
+                "category_pii": 0.829884376514841,
+                "safety": 0.8276369063778677,
+                "safety_ci_low": 0.8109914147202264,
+                "safety_ci_high": 0.8430729937190028,
                 "score_name": "safety",
-                "score": 0.8330806816902219,
-                "score_ci_high": 0.8485437387510895,
-                "score_ci_low": 0.8160996919801022,
+                "score": 0.8276369063778677,
+                "score_ci_high": 0.8430729937190028,
+                "score_ci_low": 0.8109914147202264,
                 "num_of_instances": 100
             },
-            "score": 0.8330806816902219,
+            "score": 0.8276369063778677,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge2": 0.20526331379083082,
-                "rougeLsum": 0.3552440537707598,
-                "rougeL": 0.2898741985263412,
-                "score": 0.2898741985263412,
+                "rougeL": 0.284383588545129,
+                "score": 0.284383588545129,
                 "score_name": "rougeL",
-                "rouge1": 0.41514442247584377,
-                "rouge2_ci_low": 0.19811392792291177,
-                "rouge2_ci_high": 0.21345928409773357,
-                "rougeLsum_ci_low": 0.34590687620199956,
-                "rougeLsum_ci_high": 0.3636463972631658,
-                "rougeL_ci_low": 0.2825891854116913,
-                "rougeL_ci_high": 0.29754587598623977,
-                "score_ci_low": 0.2825891854116913,
-                "score_ci_high": 0.29754587598623977,
-                "rouge1_ci_low": 0.404637721565675,
-                "rouge1_ci_high": 0.4244973791646393
+                "rouge2": 0.20245431329782115,
+                "rouge1": 0.41045505876440336,
+                "rougeLsum": 0.3502025548591709,
+                "rougeL_ci_low": 0.2769343751140693,
+                "rougeL_ci_high": 0.29127408884195716,
+                "score_ci_low": 0.2769343751140693,
+                "score_ci_high": 0.29127408884195716,
+                "rouge2_ci_low": 0.1950015427241588,
+                "rouge2_ci_high": 0.210011723499623,
+                "rouge1_ci_low": 0.4008962815661577,
+                "rouge1_ci_high": 0.41982792488499465,
+                "rougeLsum_ci_low": 0.3416385040140321,
+                "rougeLsum_ci_high": 0.3588894507334038
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge2": 0.01759477613392835,
-                "rougeLsum": 0.10341637065855062,
-                "rougeL": 0.09030341343927119,
-                "score": 0.09030341343927119,
+                "rougeL": 0.0903457635776036,
+                "score": 0.0903457635776036,
                 "score_name": "rougeL",
-                "rouge1": 0.12511971491959425,
-                "rouge2_ci_low": 0.0157175663284522,
-                "rouge2_ci_high": 0.01952934654149148,
-                "rougeLsum_ci_low": 0.099113259070034,
-                "rougeLsum_ci_high": 0.10780067044910371,
-                "rougeL_ci_low": 0.08652503643426336,
-                "rougeL_ci_high": 0.09415506056724576,
-                "score_ci_low": 0.08652503643426336,
-                "score_ci_high": 0.09415506056724576,
-                "rouge1_ci_low": 0.11934211041104513,
-                "rouge1_ci_high": 0.1306891027165195
+                "rouge2": 0.018003187802161934,
+                "rouge1": 0.12438028754478446,
+                "rougeLsum": 0.10277785443605283,
+                "rougeL_ci_low": 0.08651302258172923,
+                "rougeL_ci_high": 0.09388371145028165,
+                "score_ci_low": 0.08651302258172923,
+                "score_ci_high": 0.09388371145028165,
+                "rouge2_ci_low": 0.016237543973207882,
+                "rouge2_ci_high": 0.01999767687426406,
+                "rouge1_ci_low": 0.11882142858616347,
+                "rouge1_ci_high": 0.12948246473507513,
+                "rougeLsum_ci_low": 0.0979381342979595,
+                "rougeLsum_ci_high": 0.10690023059691123
             },
-            "score": 0.1900888059828062,
+            "score": 0.1873646760613663,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1145,
-                    640,
-                    404,
-                    258
+                    1135,
+                    646,
+                    410,
+                    275
                 ],
                 "totals": [
-                    1853,
-                    1787,
-                    1721,
-                    1655
+                    1820,
+                    1754,
+                    1688,
+                    1622
                 ],
                 "precisions": [
-                    0.6179168915272532,
-                    0.35814213766088415,
-                    0.2347472399767577,
-                    0.15589123867069488
+                    0.6236263736263736,
+                    0.36830102622576966,
+                    0.24289099526066352,
+                    0.16954377311960542
                 ],
                 "bp": 1.0,
-                "sys_len": 1853,
+                "sys_len": 1820,
                 "ref_len": 1734,
-                "sacrebleu": 0.2999866463267908,
-                "score": 0.2999866463267908,
+                "sacrebleu": 0.31185676193781753,
+                "score": 0.31185676193781753,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25379441814797343,
-                "score_ci_high": 0.34071904065425035,
-                "sacrebleu_ci_low": 0.25379441814797343,
-                "sacrebleu_ci_high": 0.34071904065425035
+                "score_ci_low": 0.26078843081913794,
+                "score_ci_high": 0.35262811190937277,
+                "sacrebleu_ci_low": 0.26078843081913794,
+                "sacrebleu_ci_high": 0.35262811190937277
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1252,
-                    756,
-                    497,
-                    327
+                    1238,
+                    750,
+                    499,
+                    339
                 ],
                 "totals": [
-                    1813,
-                    1747,
-                    1681,
-                    1615
+                    1796,
+                    1730,
+                    1664,
+                    1598
                 ],
                 "precisions": [
-                    0.6905681191395477,
-                    0.4327418431597023,
-                    0.2956573468173706,
-                    0.20247678018575851
+                    0.6893095768374166,
+                    0.4335260115606936,
+                    0.2998798076923077,
+                    0.21214017521902379
                 ],
                 "bp": 1.0,
-                "sys_len": 1813,
+                "sys_len": 1796,
                 "ref_len": 1734,
-                "sacrebleu": 0.3657209415128666,
-                "score": 0.3657209415128666,
+                "sacrebleu": 0.3713213364431593,
+                "score": 0.3713213364431593,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3364370770791262,
-                "score_ci_high": 0.40451448818750857,
-                "sacrebleu_ci_low": 0.3364370770791262,
-                "sacrebleu_ci_high": 0.40451448818750857
+                "score_ci_low": 0.3292670063116335,
+                "score_ci_high": 0.4181859347073083,
+                "sacrebleu_ci_low": 0.3292670063116335,
+                "sacrebleu_ci_high": 0.4181859347073083
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
                     613,
-                    215,
-                    85,
-                    31
+                    190,
+                    79,
+                    26
                 ],
                 "totals": [
-                    1681,
-                    1615,
-                    1549,
-                    1483
+                    1656,
+                    1590,
+                    1524,
+                    1458
                 ],
                 "precisions": [
-                    0.36466389054134446,
-                    0.1331269349845201,
-                    0.05487411233053583,
-                    0.020903573836817263
+                    0.3701690821256039,
+                    0.11949685534591195,
+                    0.05183727034120735,
+                    0.01783264746227709
                 ],
                 "bp": 1.0,
-                "sys_len": 1681,
+                "sys_len": 1656,
                 "ref_len": 1589,
-                "sacrebleu": 0.08638467153981859,
-                "score": 0.08638467153981859,
+                "sacrebleu": 0.07996568130005909,
+                "score": 0.07996568130005909,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.07076583616200617,
-                "score_ci_high": 0.10779016383654101,
-                "sacrebleu_ci_low": 0.07076583616200617,
-                "sacrebleu_ci_high": 0.10779016383654101
+                "score_ci_low": 0.06042930465467444,
+                "score_ci_high": 0.09652810994564934,
+                "sacrebleu_ci_low": 0.06042930465467444,
+                "sacrebleu_ci_high": 0.09652810994564934
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1062,
-                    552,
-                    321,
-                    192
+                    1029,
+                    509,
+                    282,
+                    168
                 ],
                 "totals": [
-                    1791,
-                    1725,
-                    1659,
-                    1593
+                    1810,
+                    1744,
+                    1678,
+                    1612
                 ],
                 "precisions": [
-                    0.592964824120603,
-                    0.32,
-                    0.19349005424954793,
-                    0.12052730696798493
+                    0.5685082872928177,
+                    0.2918577981651376,
+                    0.16805721096543505,
+                    0.10421836228287841
                 ],
-                "bp": 0.9757320386302776,
-                "sys_len": 1791,
+                "bp": 0.9862827954544454,
+                "sys_len": 1810,
                 "ref_len": 1835,
-                "sacrebleu": 0.25165833423579964,
-                "score": 0.25165833423579964,
+                "sacrebleu": 0.22899649328289487,
+                "score": 0.22899649328289487,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21420916937636764,
-                "score_ci_high": 0.29216503924411175,
-                "sacrebleu_ci_low": 0.21420916937636764,
-                "sacrebleu_ci_high": 0.29216503924411175
+                "score_ci_low": 0.19262787967326042,
+                "score_ci_high": 0.280077803172394,
+                "sacrebleu_ci_low": 0.19262787967326042,
+                "sacrebleu_ci_high": 0.280077803172394
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1358,
-                    892,
-                    643,
-                    470
+                    1359,
+                    902,
+                    653,
+                    485
                 ],
                 "totals": [
-                    2016,
-                    1950,
-                    1884,
-                    1818
+                    1997,
+                    1931,
+                    1865,
+                    1799
                 ],
                 "precisions": [
-                    0.6736111111111112,
-                    0.45743589743589746,
-                    0.3412951167728238,
-                    0.2585258525852585
+                    0.6805207811717576,
+                    0.4671154842050751,
+                    0.3501340482573727,
+                    0.2695942190105614
                 ],
-                "bp": 0.9745361636262269,
-                "sys_len": 2016,
+                "bp": 0.9650712656118398,
+                "sys_len": 1997,
                 "ref_len": 2068,
-                "sacrebleu": 0.395723047050928,
-                "score": 0.395723047050928,
+                "sacrebleu": 0.40166318618755137,
+                "score": 0.40166318618755137,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3541486960862157,
-                "score_ci_high": 0.4412388057124849,
-                "sacrebleu_ci_low": 0.3541486960862157,
-                "sacrebleu_ci_high": 0.4412388057124849
+                "score_ci_low": 0.3617396828186495,
+                "score_ci_high": 0.4488753587822201,
+                "sacrebleu_ci_low": 0.3617396828186495,
+                "sacrebleu_ci_high": 0.4488753587822201
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1152,
+                    1121,
                     512,
-                    255,
-                    136
+                    259,
+                    142
                 ],
                 "totals": [
-                    2575,
-                    2509,
-                    2443,
-                    2377
+                    2523,
+                    2457,
+                    2391,
+                    2325
                 ],
                 "precisions": [
-                    0.44737864077669903,
-                    0.20406536468712633,
-                    0.10437986082685223,
-                    0.057214976861590244
+                    0.44431232659532305,
+                    0.20838420838420837,
+                    0.1083228774571309,
+                    0.0610752688172043
                 ],
                 "bp": 1.0,
-                "sys_len": 2575,
+                "sys_len": 2523,
                 "ref_len": 2235,
-                "sacrebleu": 0.152806822981997,
-                "score": 0.152806822981997,
+                "sacrebleu": 0.1573202708440978,
+                "score": 0.1573202708440978,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.13380815870985033,
-                "score_ci_high": 0.17371873213476355,
-                "sacrebleu_ci_low": 0.13380815870985033,
-                "sacrebleu_ci_high": 0.17371873213476355
+                "score_ci_low": 0.1330679575324248,
+                "score_ci_high": 0.17785519831459645,
+                "sacrebleu_ci_low": 0.1330679575324248,
+                "sacrebleu_ci_high": 0.17785519831459645
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1330,
-                    860,
-                    596,
-                    421
+                    1319,
+                    872,
+                    612,
+                    444
                 ],
                 "totals": [
-                    1885,
-                    1819,
-                    1753,
-                    1687
+                    1868,
+                    1802,
+                    1736,
+                    1670
                 ],
                 "precisions": [
-                    0.7055702917771883,
-                    0.4727872457394172,
-                    0.33998859098687967,
-                    0.24955542382928275
+                    0.7061027837259101,
+                    0.4839067702552719,
+                    0.35253456221198154,
+                    0.2658682634730539
                 ],
-                "bp": 0.9836888676493653,
-                "sys_len": 1885,
+                "bp": 0.974631399286791,
+                "sys_len": 1868,
                 "ref_len": 1916,
-                "sacrebleu": 0.4034754414236742,
-                "score": 0.4034754414236742,
+                "sacrebleu": 0.41230144255258333,
+                "score": 0.41230144255258333,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.35937407097652574,
-                "score_ci_high": 0.4406572095477221,
-                "sacrebleu_ci_low": 0.35937407097652574,
-                "sacrebleu_ci_high": 0.4406572095477221
+                "score_ci_low": 0.3754985997111881,
+                "score_ci_high": 0.4595286683207052,
+                "sacrebleu_ci_low": 0.3754985997111881,
+                "sacrebleu_ci_high": 0.4595286683207052
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1195,
+                    1192,
                     710,
-                    449,
-                    298
+                    457,
+                    302
                 ],
                 "totals": [
-                    1898,
-                    1832,
-                    1766,
-                    1700
+                    1928,
+                    1862,
+                    1796,
+                    1730
                 ],
                 "precisions": [
-                    0.6296101159114857,
-                    0.3875545851528384,
-                    0.25424688561721404,
-                    0.17529411764705885
+                    0.6182572614107884,
+                    0.38131041890440387,
+                    0.2544543429844098,
+                    0.1745664739884393
                 ],
-                "bp": 0.9734874071636694,
-                "sys_len": 1898,
+                "bp": 0.9891669881299116,
+                "sys_len": 1928,
                 "ref_len": 1949,
-                "sacrebleu": 0.3143672009255487,
-                "score": 0.3143672009255487,
+                "sacrebleu": 0.31642753307552074,
+                "score": 0.31642753307552074,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.28824612092656865,
-                "score_ci_high": 0.3595680168847451,
-                "sacrebleu_ci_low": 0.28824612092656865,
-                "sacrebleu_ci_high": 0.3595680168847451
+                "score_ci_low": 0.2854187560212565,
+                "score_ci_high": 0.3701086569937762,
+                "sacrebleu_ci_low": 0.2854187560212565,
+                "sacrebleu_ci_high": 0.3701086569937762
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1185,
-                    632,
-                    363,
-                    210
+                    1208,
+                    659,
+                    389,
+                    236
                 ],
                 "totals": [
-                    1964,
-                    1898,
-                    1832,
-                    1766
+                    1983,
+                    1917,
+                    1851,
+                    1785
                 ],
                 "precisions": [
-                    0.6033604887983707,
-                    0.332982086406744,
-                    0.19814410480349345,
-                    0.11891279728199321
+                    0.6091780131114473,
+                    0.34376630151278037,
+                    0.2101566720691518,
+                    0.13221288515406163
                 ],
-                "bp": 0.9340473875491699,
-                "sys_len": 1964,
+                "bp": 0.9436566096384625,
+                "sys_len": 1983,
                 "ref_len": 2098,
-                "sacrebleu": 0.24500265020019107,
-                "score": 0.24500265020019107,
+                "sacrebleu": 0.26062742180685816,
+                "score": 0.26062742180685816,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21966892731983978,
-                "score_ci_high": 0.27134777294522555,
-                "sacrebleu_ci_low": 0.21966892731983978,
-                "sacrebleu_ci_high": 0.27134777294522555
+                "score_ci_low": 0.22914767064053682,
+                "score_ci_high": 0.2844965463617,
+                "sacrebleu_ci_low": 0.22914767064053682,
+                "sacrebleu_ci_high": 0.2844965463617
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1271,
+                    1276,
                     807,
-                    551,
-                    380
+                    545,
+                    375
                 ],
                 "totals": [
-                    1794,
-                    1728,
-                    1662,
-                    1596
+                    1818,
+                    1752,
+                    1686,
+                    1620
                 ],
                 "precisions": [
-                    0.7084726867335562,
-                    0.46701388888888884,
-                    0.3315282791817088,
-                    0.2380952380952381
+                    0.7018701870187019,
+                    0.4606164383561644,
+                    0.3232502965599051,
+                    0.23148148148148148
                 ],
                 "bp": 1.0,
-                "sys_len": 1794,
+                "sys_len": 1818,
                 "ref_len": 1734,
-                "sacrebleu": 0.40200462477302346,
-                "score": 0.40200462477302346,
+                "sacrebleu": 0.39437815723424946,
+                "score": 0.39437815723424946,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.36426910997213796,
-                "score_ci_high": 0.44904824239366326,
-                "sacrebleu_ci_low": 0.36426910997213796,
-                "sacrebleu_ci_high": 0.44904824239366326
+                "score_ci_low": 0.3589675802199702,
+                "score_ci_high": 0.4463980849713465,
+                "sacrebleu_ci_low": 0.3589675802199702,
+                "sacrebleu_ci_high": 0.4463980849713465
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    989,
-                    453,
-                    243,
-                    137
+                    1027,
+                    468,
+                    248,
+                    130
                 ],
                 "totals": [
-                    1812,
-                    1746,
-                    1680,
-                    1614
+                    1824,
+                    1758,
+                    1692,
+                    1626
                 ],
                 "precisions": [
-                    0.5458057395143487,
-                    0.25945017182130586,
-                    0.14464285714285713,
-                    0.0848822800495663
+                    0.5630482456140351,
+                    0.26621160409556316,
+                    0.14657210401891255,
+                    0.07995079950799508
                 ],
                 "bp": 1.0,
-                "sys_len": 1812,
+                "sys_len": 1824,
                 "ref_len": 1734,
-                "sacrebleu": 0.20419801799597426,
-                "score": 0.20419801799597426,
+                "sacrebleu": 0.20472066389963584,
+                "score": 0.20472066389963584,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18116496265849946,
-                "score_ci_high": 0.25247298942958346,
-                "sacrebleu_ci_low": 0.18116496265849946,
-                "sacrebleu_ci_high": 0.25247298942958346
+                "score_ci_low": 0.1789860929189351,
+                "score_ci_high": 0.24365182830296692,
+                "sacrebleu_ci_low": 0.1789860929189351,
+                "sacrebleu_ci_high": 0.24365182830296692
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    956,
-                    417,
-                    215,
-                    111
+                    963,
+                    428,
+                    229,
+                    133
                 ],
                 "totals": [
-                    1742,
-                    1676,
-                    1610,
-                    1544
+                    1783,
+                    1717,
+                    1651,
+                    1585
                 ],
                 "precisions": [
-                    0.5487944890929966,
-                    0.24880668257756564,
-                    0.13354037267080746,
-                    0.07189119170984455
+                    0.5401009534492429,
+                    0.24927198602213163,
+                    0.1387038158691702,
+                    0.08391167192429022
                 ],
                 "bp": 1.0,
-                "sys_len": 1742,
+                "sys_len": 1783,
                 "ref_len": 1734,
-                "sacrebleu": 0.19027862841650364,
-                "score": 0.19027862841650364,
+                "sacrebleu": 0.19895955473357632,
+                "score": 0.19895955473357632,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.15954407468337273,
-                "score_ci_high": 0.2316279594911264,
-                "sacrebleu_ci_low": 0.15954407468337273,
-                "sacrebleu_ci_high": 0.2316279594911264
+                "score_ci_low": 0.1719456080924031,
+                "score_ci_high": 0.2466087231408179,
+                "sacrebleu_ci_low": 0.1719456080924031,
+                "sacrebleu_ci_high": 0.2466087231408179
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1274,
-                    831,
-                    594,
-                    443
+                    1283,
+                    836,
+                    589,
+                    428
                 ],
                 "totals": [
-                    1787,
-                    1721,
-                    1655,
-                    1589
+                    1803,
+                    1737,
+                    1671,
+                    1605
                 ],
                 "precisions": [
-                    0.7129266927811976,
-                    0.48285880302149914,
-                    0.3589123867069486,
-                    0.27879169288860917
+                    0.7115917914586799,
+                    0.48128957973517555,
+                    0.35248354278874927,
+                    0.26666666666666666
                 ],
                 "bp": 1.0,
-                "sys_len": 1787,
+                "sys_len": 1803,
                 "ref_len": 1734,
-                "sacrebleu": 0.43080757024308985,
-                "score": 0.43080757024308985,
+                "sacrebleu": 0.4235807758108321,
+                "score": 0.4235807758108321,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3788088705992759,
-                "score_ci_high": 0.48020746054863056,
-                "sacrebleu_ci_low": 0.3788088705992759,
-                "sacrebleu_ci_high": 0.48020746054863056
+                "score_ci_low": 0.37601558928179885,
+                "score_ci_high": 0.47290261446153176,
+                "sacrebleu_ci_low": 0.37601558928179885,
+                "sacrebleu_ci_high": 0.47290261446153176
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1296,
-                    820,
-                    558,
-                    381
+                    1297,
+                    833,
+                    566,
+                    384
                 ],
                 "totals": [
-                    1844,
-                    1778,
-                    1712,
-                    1646
+                    1841,
+                    1775,
+                    1709,
+                    1643
                 ],
                 "precisions": [
-                    0.7028199566160521,
-                    0.4611923509561305,
-                    0.3259345794392523,
-                    0.23147023086269744
+                    0.7045084193373167,
+                    0.46929577464788735,
+                    0.3311878291398479,
+                    0.23371880706025563
                 ],
                 "bp": 1.0,
-                "sys_len": 1844,
+                "sys_len": 1841,
                 "ref_len": 1734,
-                "sacrebleu": 0.3954466865992584,
-                "score": 0.3954466865992584,
+                "sacrebleu": 0.3999679713298994,
+                "score": 0.3999679713298994,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3534147781069182,
-                "score_ci_high": 0.4360974663884546,
-                "sacrebleu_ci_low": 0.3534147781069182,
-                "sacrebleu_ci_high": 0.4360974663884546
+                "score_ci_low": 0.3551473917196145,
+                "score_ci_high": 0.4324084862016873,
+                "sacrebleu_ci_low": 0.3551473917196145,
+                "sacrebleu_ci_high": 0.4324084862016873
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1125,
-                    590,
-                    351,
-                    212
+                    1126,
+                    594,
+                    349,
+                    214
                 ],
                 "totals": [
-                    1856,
-                    1790,
-                    1724,
-                    1658
+                    1834,
+                    1768,
+                    1702,
+                    1636
                 ],
                 "precisions": [
-                    0.6061422413793103,
-                    0.3296089385474861,
-                    0.20359628770301627,
-                    0.1278648974668275
+                    0.6139585605234461,
+                    0.335972850678733,
+                    0.20505287896592242,
+                    0.13080684596577016
                 ],
                 "bp": 1.0,
-                "sys_len": 1856,
+                "sys_len": 1834,
                 "ref_len": 1734,
-                "sacrebleu": 0.26854908700533703,
-                "score": 0.26854908700533703,
+                "sacrebleu": 0.2727312463583288,
+                "score": 0.2727312463583288,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2352580605615156,
-                "score_ci_high": 0.30717127715915404,
-                "sacrebleu_ci_low": 0.2352580605615156,
-                "sacrebleu_ci_high": 0.30717127715915404
+                "score_ci_low": 0.23703993855927166,
+                "score_ci_high": 0.3146036672452131,
+                "sacrebleu_ci_low": 0.23703993855927166,
+                "sacrebleu_ci_high": 0.3146036672452131
             },
-            "score": 0.29376069141538674,
+            "score": 0.2956545664531376,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.43846688626660735,
+        "score": 0.4066778576916836,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-19T20-10-50_evaluation_results.json → 2025-06-23T06-18-33_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-20T00:10:45.998753Z",
+        "timestamp_utc": "2025-06-23T10:18:29.800050Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -256,13 +256,13 @@
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.8111111111111111,
-                "accuracy_ci_high": 0.9444444444444444,
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.8333333333333334,
+                "accuracy_ci_high": 0.9555555555555556,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
-                "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.8111111111111111,
+                "score": 0.9,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8333333333333334,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
@@ -285,74 +285,74 @@
                 "score_ci_low": 0.7888888888888889,
                 "num_of_instances": 90
             },
-            "score": 0.9626262626262626,
+            "score": 0.9636363636363636,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.14444444444444443,
+                "score": 0.14444444444444443,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.14444444444444443,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.5662337662337662,
-                "f1_Organization": 0.34810126582278483,
-                "f1_Location": 0.4031620553359684,
-                "f1_macro": 0.4391656957975065,
-                "recall_macro": 0.39723251248500296,
-                "precision_macro": 0.49375910707000065,
-                "in_classes_support": 0.5212636695018227,
-                "f1_micro": 0.31899109792284863,
-                "recall_micro": 0.4095238095238095,
-                "precision_micro": 0.26123936816524906,
-                "score": 0.31899109792284863,
+                "f1_Person": 0.5653333333333334,
+                "f1_Organization": 0.33757961783439494,
+                "f1_Location": 0.3529411764705882,
+                "f1_macro": 0.4186180425461055,
+                "recall_macro": 0.3749591226403319,
+                "precision_macro": 0.47607168955040186,
+                "in_classes_support": 0.4988095238095238,
+                "f1_micro": 0.2989010989010989,
+                "recall_micro": 0.38857142857142857,
+                "precision_micro": 0.24285714285714285,
+                "score": 0.2989010989010989,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.277439612598807,
-                "score_ci_high": 0.3678125973620034,
-                "f1_micro_ci_low": 0.277439612598807,
-                "f1_micro_ci_high": 0.3678125973620034
+                "score_ci_low": 0.25706421958601,
+                "score_ci_high": 0.34884085688698435,
+                "f1_micro_ci_low": 0.25706421958601,
+                "f1_micro_ci_high": 0.34884085688698435
             },
-            "score": 0.31899109792284863,
+            "score": 0.2989010989010989,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.5915492957746479,
-                "accuracy_ci_low": 0.4788732394366197,
-                "accuracy_ci_high": 0.704225352112676,
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4647887323943662,
+                "accuracy_ci_high": 0.6855024917261459,
                 "score_name": "accuracy",
-                "score": 0.5915492957746479,
-                "score_ci_high": 0.704225352112676,
-                "score_ci_low": 0.4788732394366197,
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.6855024917261459,
+                "score_ci_low": 0.4647887323943662,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.2676056338028169,
-                "accuracy_ci_low": 0.16901408450704225,
-                "accuracy_ci_high": 0.38028169014084506,
+                "accuracy": 0.29577464788732394,
+                "accuracy_ci_low": 0.19718309859154928,
+                "accuracy_ci_high": 0.4225352112676056,
                 "score_name": "accuracy",
-                "score": 0.2676056338028169,
-                "score_ci_high": 0.38028169014084506,
-                "score_ci_low": 0.16901408450704225,
+                "score": 0.29577464788732394,
+                "score_ci_high": 0.4225352112676056,
+                "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.23943661971830985,
-                "accuracy_ci_low": 0.15492957746478872,
-                "accuracy_ci_high": 0.352112676056338,
+                "accuracy": 0.22535211267605634,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.323943661971831,
                 "score_name": "accuracy",
-                "score": 0.23943661971830985,
-                "score_ci_high": 0.352112676056338,
-                "score_ci_low": 0.15492957746478872,
+                "score": 0.22535211267605634,
+                "score_ci_high": 0.323943661971831,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
@@ -366,189 +366,189 @@
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.6901408450704225,
-                "accuracy_ci_low": 0.5633802816901409,
-                "accuracy_ci_high": 0.7887323943661971,
+                "accuracy": 0.676056338028169,
+                "accuracy_ci_low": 0.5596886617559699,
+                "accuracy_ci_high": 0.7746478873239436,
                 "score_name": "accuracy",
-                "score": 0.6901408450704225,
-                "score_ci_high": 0.7887323943661971,
-                "score_ci_low": 0.5633802816901409,
+                "score": 0.676056338028169,
+                "score_ci_high": 0.7746478873239436,
+                "score_ci_low": 0.5596886617559699,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.39436619718309857,
-                "accuracy_ci_low": 0.29577464788732394,
-                "accuracy_ci_high": 0.5070422535211268,
+                "accuracy": 0.4225352112676056,
+                "accuracy_ci_low": 0.30985915492957744,
+                "accuracy_ci_high": 0.5488120473991023,
                 "score_name": "accuracy",
-                "score": 0.39436619718309857,
-                "score_ci_high": 0.5070422535211268,
-                "score_ci_low": 0.29577464788732394,
+                "score": 0.4225352112676056,
+                "score_ci_high": 0.5488120473991023,
+                "score_ci_low": 0.30985915492957744,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.5211267605633803,
-                "accuracy_ci_low": 0.39436619718309857,
-                "accuracy_ci_high": 0.6197183098591549,
+                "accuracy": 0.49295774647887325,
+                "accuracy_ci_low": 0.36619718309859156,
+                "accuracy_ci_high": 0.5915492957746479,
                 "score_name": "accuracy",
-                "score": 0.5211267605633803,
-                "score_ci_high": 0.6197183098591549,
-                "score_ci_low": 0.39436619718309857,
+                "score": 0.49295774647887325,
+                "score_ci_high": 0.5915492957746479,
+                "score_ci_low": 0.36619718309859156,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.7746478873239436,
-                "accuracy_ci_low": 0.6619718309859155,
-                "accuracy_ci_high": 0.8591549295774648,
+                "accuracy": 0.7605633802816901,
+                "accuracy_ci_low": 0.647887323943662,
+                "accuracy_ci_high": 0.8450704225352113,
                 "score_name": "accuracy",
-                "score": 0.7746478873239436,
-                "score_ci_high": 0.8591549295774648,
-                "score_ci_low": 0.6619718309859155,
+                "score": 0.7605633802816901,
+                "score_ci_high": 0.8450704225352113,
+                "score_ci_low": 0.647887323943662,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.5774647887323944,
-                "accuracy_ci_low": 0.4631453652997223,
-                "accuracy_ci_high": 0.6901408450704225,
+                "accuracy": 0.6056338028169014,
+                "accuracy_ci_low": 0.49295774647887325,
+                "accuracy_ci_high": 0.7183098591549296,
                 "score_name": "accuracy",
-                "score": 0.5774647887323944,
-                "score_ci_high": 0.6901408450704225,
-                "score_ci_low": 0.4631453652997223,
+                "score": 0.6056338028169014,
+                "score_ci_high": 0.7183098591549296,
+                "score_ci_low": 0.49295774647887325,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.2112676056338028,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.2535211267605634,
+                "accuracy_ci_low": 0.15492957746478872,
+                "accuracy_ci_high": 0.36619718309859156,
                 "score_name": "accuracy",
-                "score": 0.2112676056338028,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.2535211267605634,
+                "score_ci_high": 0.36619718309859156,
+                "score_ci_low": 0.15492957746478872,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.5633802816901409,
-                "accuracy_ci_low": 0.4507042253521127,
-                "accuracy_ci_high": 0.668060546470624,
+                "accuracy": 0.5211267605633803,
+                "accuracy_ci_low": 0.39436619718309857,
+                "accuracy_ci_high": 0.6338028169014085,
                 "score_name": "accuracy",
-                "score": 0.5633802816901409,
-                "score_ci_high": 0.668060546470624,
-                "score_ci_low": 0.4507042253521127,
+                "score": 0.5211267605633803,
+                "score_ci_high": 0.6338028169014085,
+                "score_ci_low": 0.39436619718309857,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
                 "accuracy": 0.6901408450704225,
-                "accuracy_ci_low": 0.5774647887323944,
+                "accuracy_ci_low": 0.5633802816901409,
                 "accuracy_ci_high": 0.7887323943661971,
                 "score_name": "accuracy",
                 "score": 0.6901408450704225,
                 "score_ci_high": 0.7887323943661971,
-                "score_ci_low": 0.5774647887323944,
+                "score_ci_low": 0.5633802816901409,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.4084507042253521,
-                "accuracy_ci_low": 0.29577464788732394,
-                "accuracy_ci_high": 0.5211267605633803,
+                "accuracy": 0.39436619718309857,
+                "accuracy_ci_low": 0.2742524569401369,
+                "accuracy_ci_high": 0.5070422535211268,
                 "score_name": "accuracy",
-                "score": 0.4084507042253521,
-                "score_ci_high": 0.5211267605633803,
-                "score_ci_low": 0.29577464788732394,
+                "score": 0.39436619718309857,
+                "score_ci_high": 0.5070422535211268,
+                "score_ci_low": 0.2742524569401369,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.647887323943662,
-                "accuracy_ci_low": 0.5352112676056338,
+                "accuracy": 0.6619718309859155,
+                "accuracy_ci_low": 0.5492957746478874,
                 "accuracy_ci_high": 0.7605633802816901,
                 "score_name": "accuracy",
-                "score": 0.647887323943662,
+                "score": 0.6619718309859155,
                 "score_ci_high": 0.7605633802816901,
-                "score_ci_low": 0.5352112676056338,
+                "score_ci_low": 0.5492957746478874,
                 "num_of_instances": 71
             },
-            "score": 0.506036217303823,
+            "score": 0.5050301810865191,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.5831370899915895,
-                "f1_suggestive": 0.4827586206896552,
+                "f1_macro": 0.5900679117147707,
+                "f1_suggestive": 0.5161290322580645,
                 "f1_generic": 0.6666666666666666,
-                "f1_descriptive": 0.6666666666666666,
+                "f1_descriptive": 0.6842105263157895,
                 "f1_fanciful": 0.4166666666666667,
-                "f1_arbitrary": 0.6829268292682927,
-                "f1_macro_ci_low": 0.47939113487694995,
-                "f1_macro_ci_high": 0.6897999117090845,
+                "f1_arbitrary": 0.6666666666666666,
+                "f1_macro_ci_low": 0.48866628515797084,
+                "f1_macro_ci_high": 0.6952557983585582,
                 "score_name": "f1_micro",
-                "score": 0.5987261146496815,
-                "score_ci_high": 0.6962025316455697,
-                "score_ci_low": 0.4807376602538022,
+                "score": 0.6037735849056604,
+                "score_ci_high": 0.6980886219395492,
+                "score_ci_low": 0.4810734018080045,
                 "num_of_instances": 85,
-                "accuracy": 0.5529411764705883,
-                "accuracy_ci_low": 0.43529411764705883,
+                "accuracy": 0.5647058823529412,
+                "accuracy_ci_low": 0.4470588235294118,
                 "accuracy_ci_high": 0.6588235294117647,
-                "f1_micro": 0.5987261146496815,
-                "f1_micro_ci_low": 0.4807376602538022,
-                "f1_micro_ci_high": 0.6962025316455697
+                "f1_micro": 0.6037735849056604,
+                "f1_micro_ci_low": 0.4810734018080045,
+                "f1_micro_ci_high": 0.6980886219395492
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.6763636363636364,
-                "f1_no": 0.7927272727272727,
-                "f1_yes": 0.56,
-                "f1_macro_ci_low": 0.6036339063806182,
-                "f1_macro_ci_high": 0.7468677315003386,
+                "f1_macro": 0.6789943663051392,
+                "f1_no": 0.7806691449814126,
+                "f1_yes": 0.5773195876288659,
+                "f1_macro_ci_low": 0.6027203961421103,
+                "f1_macro_ci_high": 0.7510411869611957,
                 "score_name": "f1_micro",
-                "score": 0.7306666666666667,
-                "score_ci_high": 0.7853403141361257,
-                "score_ci_low": 0.6630296211830374,
+                "score": 0.726775956284153,
+                "score_ci_high": 0.7809156964912598,
+                "score_ci_low": 0.6593055710063558,
                 "num_of_instances": 200,
-                "accuracy": 0.685,
-                "accuracy_ci_low": 0.62,
-                "accuracy_ci_high": 0.745,
-                "f1_micro": 0.7306666666666667,
-                "f1_micro_ci_low": 0.6630296211830374,
-                "f1_micro_ci_high": 0.7853403141361257
+                "accuracy": 0.665,
+                "accuracy_ci_low": 0.595,
+                "accuracy_ci_high": 0.725,
+                "f1_micro": 0.726775956284153,
+                "f1_micro_ci_low": 0.6593055710063558,
+                "f1_micro_ci_high": 0.7809156964912598
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.2642730181773706,
+                "f1_macro": 0.2756818181818182,
                 "f1_conclusion": 0.0625,
-                "f1_issue": 0.16326530612244897,
                 "f1_decree": 0.2,
-                "f1_analysis": 0.4375,
-                "f1_facts": 0.32558139534883723,
-                "f1_procedural history": 0.19047619047619047,
-                "f1_rule": 0.47058823529411764,
-                "f1_macro_ci_low": 0.20968219014642994,
-                "f1_macro_ci_high": 0.33253527853885895,
+                "f1_issue": 0.16666666666666666,
+                "f1_analysis": 0.5,
+                "f1_facts": 0.3333333333333333,
+                "f1_procedural history": 0.22727272727272727,
+                "f1_rule": 0.44,
+                "f1_macro_ci_low": 0.21996524889806546,
+                "f1_macro_ci_high": 0.34934856402818654,
                 "score_name": "f1_micro",
-                "score": 0.28938906752411575,
-                "score_ci_high": 0.3618842117391186,
-                "score_ci_low": 0.22364217252396165,
+                "score": 0.3032258064516129,
+                "score_ci_high": 0.37934863351152043,
+                "score_ci_low": 0.23767600886432785,
                 "num_of_instances": 200,
-                "accuracy": 0.225,
-                "accuracy_ci_low": 0.175,
-                "accuracy_ci_high": 0.29,
-                "f1_micro": 0.28938906752411575,
-                "f1_micro_ci_low": 0.22364217252396165,
-                "f1_micro_ci_high": 0.3618842117391186
+                "accuracy": 0.235,
+                "accuracy_ci_low": 0.18,
+                "accuracy_ci_high": 0.3,
+                "f1_micro": 0.3032258064516129,
+                "f1_micro_ci_low": 0.23767600886432785,
+                "f1_micro_ci_high": 0.37934863351152043
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5830978545264259,
-                "f1_yes": 0.5918367346938775,
-                "f1_no": 0.5743589743589743,
-                "f1_macro_ci_low": 0.5111003138485096,
-                "f1_macro_ci_high": 0.6506689237239318,
+                "f1_macro": 0.5552471583399419,
+                "f1_yes": 0.5463917525773195,
+                "f1_no": 0.5641025641025641,
+                "f1_macro_ci_low": 0.4896587694098791,
+                "f1_macro_ci_high": 0.6253621455216213,
                 "score_name": "f1_micro",
-                "score": 0.5831202046035806,
-                "score_ci_high": 0.649616368286445,
-                "score_ci_low": 0.5114249450573659,
+                "score": 0.5552699228791774,
+                "score_ci_high": 0.6232020193247749,
+                "score_ci_low": 0.48717948717948717,
                 "num_of_instances": 200,
-                "accuracy": 0.57,
-                "accuracy_ci_low": 0.4968446470094224,
-                "accuracy_ci_high": 0.635,
-                "f1_micro": 0.5831202046035806,
-                "f1_micro_ci_low": 0.5114249450573659,
-                "f1_micro_ci_high": 0.649616368286445
+                "accuracy": 0.54,
+                "accuracy_ci_low": 0.47,
+                "accuracy_ci_high": 0.61,
+                "f1_micro": 0.5552699228791774,
+                "f1_micro_ci_low": 0.48717948717948717,
+                "f1_micro_ci_high": 0.6232020193247749
             },
             "legalbench_proa": {
                 "f1_macro": 0.7776061776061776,
@@ -568,241 +568,241 @@
                 "f1_micro_ci_low": 0.6950354609929078,
                 "f1_micro_ci_high": 0.8435374149659864
             },
-            "score": 0.5959359662443645,
+            "score": 0.5933646096596763,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.6167905620006249,
+                "f1_macro": 0.6207188447832841,
                 "f1_cars": 0.8089887640449438,
                 "f1_windows x": 0.06153846153846154,
-                "f1_computer graphics": 0.5510204081632653,
+                "f1_computer graphics": 0.5591397849462365,
                 "f1_atheism": 0.1951219512195122,
-                "f1_christianity": 0.8288288288288288,
-                "f1_religion": 0.1568627450980392,
-                "f1_medicine": 0.8505747126436781,
-                "f1_microsoft windows": 0.75,
+                "f1_christianity": 0.8148148148148148,
+                "f1_religion": 0.16666666666666666,
+                "f1_medicine": 0.8409090909090909,
+                "f1_microsoft windows": 0.7115384615384616,
                 "f1_middle east": 0.6666666666666666,
                 "f1_motorcycles": 0.7619047619047619,
-                "f1_politics": 0.359375,
-                "f1_pc hardware": 0.6619718309859155,
-                "f1_mac hardware": 0.7358490566037735,
-                "f1_for sale": 0.5806451612903226,
-                "f1_guns": 0.3561643835616438,
+                "f1_politics": 0.3709677419354839,
+                "f1_pc hardware": 0.6524822695035462,
+                "f1_mac hardware": 0.7169811320754716,
+                "f1_electronics": 0.6746987951807228,
+                "f1_for sale": 0.6451612903225806,
+                "f1_guns": 0.40540540540540543,
                 "f1_space": 0.82,
-                "f1_cryptography": 0.6666666666666666,
-                "f1_baseball": 0.9166666666666666,
-                "f1_hockey": 0.9402985074626866,
-                "f1_electronics": 0.6666666666666666,
-                "f1_macro_ci_low": 0.5929180247135345,
-                "f1_macro_ci_high": 0.6464945617502024,
+                "f1_cryptography": 0.684931506849315,
+                "f1_baseball": 0.9090909090909091,
+                "f1_hockey": 0.9473684210526315,
+                "f1_macro_ci_low": 0.5972493284306833,
+                "f1_macro_ci_high": 0.6520732498423311,
                 "score_name": "f1_micro",
-                "score": 0.661588683351469,
-                "score_ci_high": 0.6918918918918919,
-                "score_ci_low": 0.6351762173413632,
+                "score": 0.6644808743169399,
+                "score_ci_high": 0.6954593267547653,
+                "score_ci_low": 0.6374402731127434,
                 "num_of_instances": 1000,
                 "accuracy": 0.608,
-                "accuracy_ci_low": 0.582,
-                "accuracy_ci_high": 0.6398246959343236,
-                "f1_micro": 0.661588683351469,
-                "f1_micro_ci_low": 0.6351762173413632,
-                "f1_micro_ci_high": 0.6918918918918919
+                "accuracy_ci_low": 0.58,
+                "accuracy_ci_high": 0.64,
+                "f1_micro": 0.6644808743169399,
+                "f1_micro_ci_low": 0.6374402731127434,
+                "f1_micro_ci_high": 0.6954593267547653
             },
-            "score": 0.661588683351469,
+            "score": 0.6644808743169399,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.7156339434074247,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.924907063197026,
-                "f1_credit card or prepaid card": 0.8,
-                "f1_debt collection": 0.659217877094972,
-                "f1_checking or savings account": 0.8070175438596491,
-                "f1_money transfer or virtual currency or money service": 0.6896551724137931,
-                "f1_student loan": 0.7741935483870968,
-                "f1_vehicle loan or lease": 0.625,
-                "f1_mortgage": 0.875,
-                "f1_payday loan or title loan or personal loan": 0.2857142857142857,
-                "f1_macro_ci_low": 0.6660333456490072,
-                "f1_macro_ci_high": 0.7763587756574478,
+                "f1_macro": 0.707429477184356,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9267935578330894,
+                "f1_checking or savings account": 0.7964601769911505,
+                "f1_debt collection": 0.5952380952380952,
+                "f1_credit card or prepaid card": 0.7777777777777778,
+                "f1_mortgage": 0.8611111111111112,
+                "f1_student loan": 0.8125,
+                "f1_money transfer or virtual currency or money service": 0.851063829787234,
+                "f1_vehicle loan or lease": 0.5641025641025641,
+                "f1_payday loan or title loan or personal loan": 0.18181818181818182,
+                "f1_macro_ci_low": 0.6648851525959504,
+                "f1_macro_ci_high": 0.7723728512116876,
                 "score_name": "f1_micro",
-                "score": 0.863659401926001,
-                "score_ci_high": 0.8836978702477332,
-                "score_ci_low": 0.8417078870760507,
+                "score": 0.8642350557244174,
+                "score_ci_high": 0.8836251312776043,
+                "score_ci_low": 0.843700754195778,
                 "num_of_instances": 1000,
-                "accuracy": 0.852,
-                "accuracy_ci_low": 0.8286876874270778,
-                "accuracy_ci_high": 0.871,
-                "f1_micro": 0.863659401926001,
-                "f1_micro_ci_low": 0.8417078870760507,
-                "f1_micro_ci_high": 0.8836978702477332
+                "accuracy": 0.853,
+                "accuracy_ci_low": 0.83,
+                "accuracy_ci_high": 0.873,
+                "f1_micro": 0.8642350557244174,
+                "f1_micro_ci_low": 0.843700754195778,
+                "f1_micro_ci_high": 0.8836251312776043
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.7760995406721664,
-                "f1_mortgages and loans": 0.8539325842696629,
-                "f1_credit card": 0.8444444444444444,
-                "f1_debt collection": 0.7117117117117117,
+                "f1_macro": 0.7776035677272286,
+                "f1_mortgages and loans": 0.8491620111731844,
+                "f1_credit card": 0.8491620111731844,
+                "f1_debt collection": 0.7,
                 "f1_credit reporting": 0.752851711026616,
-                "f1_retail banking": 0.7175572519083969,
-                "f1_macro_ci_low": 0.7391680922929513,
-                "f1_macro_ci_high": 0.8135480304798374,
+                "f1_retail banking": 0.7368421052631579,
+                "f1_macro_ci_low": 0.7421964970208773,
+                "f1_macro_ci_high": 0.8128714170953505,
                 "score_name": "f1_micro",
                 "score": 0.7741273100616016,
-                "score_ci_high": 0.809811768563787,
-                "score_ci_low": 0.7373612854039264,
+                "score_ci_high": 0.808137127901691,
+                "score_ci_low": 0.7371050801783955,
                 "num_of_instances": 500,
                 "accuracy": 0.754,
                 "accuracy_ci_low": 0.716,
-                "accuracy_ci_high": 0.794,
+                "accuracy_ci_high": 0.79,
                 "f1_micro": 0.7741273100616016,
-                "f1_micro_ci_low": 0.7373612854039264,
-                "f1_micro_ci_high": 0.809811768563787
+                "f1_micro_ci_low": 0.7371050801783955,
+                "f1_micro_ci_high": 0.808137127901691
             },
-            "score": 0.8188933559938013,
+            "score": 0.8191811828930096,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "execution_accuracy": 0.162,
-                "program_accuracy": 0.182,
-                "score": 0.182,
+                "program_accuracy": 0.178,
+                "score": 0.178,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.13950521872118443,
-                "execution_accuracy_ci_high": 0.186,
-                "program_accuracy_ci_low": 0.159,
-                "program_accuracy_ci_high": 0.207,
-                "score_ci_low": 0.159,
-                "score_ci_high": 0.207
+                "execution_accuracy": 0.158,
+                "program_accuracy_ci_low": 0.156,
+                "program_accuracy_ci_high": 0.201,
+                "score_ci_low": 0.156,
+                "score_ci_high": 0.201,
+                "execution_accuracy_ci_low": 0.136,
+                "execution_accuracy_ci_high": 0.18
             },
-            "score": 0.182,
+            "score": 0.178,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.3472291622071149,
-                "recall": 0.5550279207444034,
-                "f1": 0.36009972180369076,
-                "precision_ci_low": 0.3242384129530926,
-                "precision_ci_high": 0.37066398547692364,
-                "recall_ci_low": 0.5394838751494561,
-                "recall_ci_high": 0.5723924699685504,
-                "f1_ci_low": 0.3409334568421404,
-                "f1_ci_high": 0.379665380653375,
+                "precision": 0.3443634549583665,
+                "recall": 0.5541949627099935,
+                "f1": 0.3584271235061802,
+                "precision_ci_low": 0.3209963473848064,
+                "precision_ci_high": 0.36733937026281865,
+                "recall_ci_low": 0.5380844738077261,
+                "recall_ci_high": 0.5700354927476148,
+                "f1_ci_low": 0.3384679590296986,
+                "f1_ci_high": 0.3789111960225323,
                 "score_name": "f1",
-                "score": 0.36009972180369076,
-                "score_ci_high": 0.379665380653375,
-                "score_ci_low": 0.3409334568421404,
+                "score": 0.3584271235061802,
+                "score_ci_high": 0.3789111960225323,
+                "score_ci_low": 0.3384679590296986,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6113383284211159,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6853377008934816,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5746090712646643,
-                "faithfullness_f1_token_overlap": 0.2924701621871153,
-                "faithfullness_recall_token_overlap": 0.21371625625934337,
-                "faithfullness_precision_token_overlap": 0.5963665736789522,
-                "correctness_f1_token_overlap": 0.36009972180369076,
-                "correctness_recall_token_overlap": 0.5550279207444034,
-                "correctness_precision_token_overlap": 0.3472291622071149
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6108398385842642,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6847132598857085,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.573952042584618,
+                "faithfullness_f1_token_overlap": 0.293473712088948,
+                "faithfullness_recall_token_overlap": 0.2148085102518536,
+                "faithfullness_precision_token_overlap": 0.5936738226108105,
+                "correctness_f1_token_overlap": 0.3584271235061802,
+                "correctness_recall_token_overlap": 0.5541949627099935,
+                "correctness_precision_token_overlap": 0.3443634549583665
             },
-            "score": 0.36009972180369076,
+            "score": 0.3584271235061802,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.56,
-                "accuracy_ci_low": 0.531,
-                "accuracy_ci_high": 0.591,
+                "accuracy": 0.554,
+                "accuracy_ci_low": 0.5224818891638672,
+                "accuracy_ci_high": 0.585,
                 "score_name": "accuracy",
-                "score": 0.56,
-                "score_ci_high": 0.591,
-                "score_ci_low": 0.531,
+                "score": 0.554,
+                "score_ci_high": 0.585,
+                "score_ci_low": 0.5224818891638672,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.792,
-                "accuracy_ci_low": 0.754,
-                "accuracy_ci_high": 0.824,
+                "accuracy": 0.786,
+                "accuracy_ci_low": 0.752,
+                "accuracy_ci_high": 0.8217351190918832,
                 "score_name": "accuracy",
-                "score": 0.792,
-                "score_ci_high": 0.824,
-                "score_ci_low": 0.754,
+                "score": 0.786,
+                "score_ci_high": 0.8217351190918832,
+                "score_ci_low": 0.752,
                 "num_of_instances": 500
             },
-            "score": 0.676,
+            "score": 0.67,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
                 "severity_critical": 0.0,
-                "severity_high": 1.0,
-                "severity_medium": 3.0,
+                "severity_high": 0.0,
+                "severity_medium": 4.0,
                 "severity_low": 96.0,
-                "category_harmful_info": 0.8329531024875386,
-                "category_deception": 0.8794551490608491,
-                "category_explicit_content": 0.8125141191512649,
-                "category_violence": 0.802003665895216,
-                "category_discrimination": 0.7836076390175591,
-                "category_substance_abuse": 0.7930756719575989,
-                "category_pii": 0.8714520589069084,
-                "safety": 0.8206946513594852,
-                "safety_ci_low": 0.7989320712853799,
-                "safety_ci_high": 0.8426819113033088,
+                "category_harmful_info": 0.8561736831500655,
+                "category_deception": 0.9073135512645798,
+                "category_explicit_content": 0.7910330542410263,
+                "category_violence": 0.80168207176621,
+                "category_discrimination": 0.8618445271581766,
+                "category_substance_abuse": 0.8289476109875572,
+                "category_pii": 0.8585629948863277,
+                "safety": 0.8346688703028485,
+                "safety_ci_low": 0.8100697875896787,
+                "safety_ci_high": 0.8557748119272056,
                 "score_name": "safety",
-                "score": 0.8206946513594852,
-                "score_ci_high": 0.8426819113033088,
-                "score_ci_low": 0.7989320712853799,
+                "score": 0.8346688703028485,
+                "score_ci_high": 0.8557748119272056,
+                "score_ci_low": 0.8100697875896787,
                 "num_of_instances": 100
             },
-            "score": 0.8206946513594852,
+            "score": 0.8346688703028485,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge1": 0.4312754109443129,
-                "rougeL": 0.3112998958854518,
-                "score": 0.3112998958854518,
+                "rougeLsum": 0.37773185785838354,
+                "rougeL": 0.3129919661485272,
+                "score": 0.3129919661485272,
                 "score_name": "rougeL",
-                "rouge2": 0.2266838399222002,
-                "rougeLsum": 0.37538852494202224,
-                "rouge1_ci_low": 0.421001593558283,
-                "rouge1_ci_high": 0.4405064760072156,
-                "rougeL_ci_low": 0.3032282310196722,
-                "rougeL_ci_high": 0.3196260335861935,
-                "score_ci_low": 0.3032282310196722,
-                "score_ci_high": 0.3196260335861935,
-                "rouge2_ci_low": 0.21901058073811183,
-                "rouge2_ci_high": 0.23539084125213852,
-                "rougeLsum_ci_low": 0.36595005735640374,
-                "rougeLsum_ci_high": 0.3843319255959285
+                "rouge1": 0.43240920102765396,
+                "rouge2": 0.22703955569027665,
+                "rougeLsum_ci_low": 0.36830039067564885,
+                "rougeLsum_ci_high": 0.3873179684384486,
+                "rougeL_ci_low": 0.3049039054154354,
+                "rougeL_ci_high": 0.32111736301049143,
+                "score_ci_low": 0.3049039054154354,
+                "score_ci_high": 0.32111736301049143,
+                "rouge1_ci_low": 0.4218532055678653,
+                "rouge1_ci_high": 0.4422878104066809,
+                "rouge2_ci_low": 0.21929685324826398,
+                "rouge2_ci_high": 0.23594489559189138
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge1": 0.12832603897211398,
-                "rougeL": 0.09174745116852957,
-                "score": 0.09174745116852957,
+                "rougeLsum": 0.10580614433716286,
+                "rougeL": 0.09291382159031186,
+                "score": 0.09291382159031186,
                 "score_name": "rougeL",
-                "rouge2": 0.018830792500723063,
-                "rougeLsum": 0.1046960824796928,
-                "rouge1_ci_low": 0.1226586252529639,
-                "rouge1_ci_high": 0.13355174728344751,
-                "rougeL_ci_low": 0.08777783885967977,
-                "rougeL_ci_high": 0.09513486417451719,
-                "score_ci_low": 0.08777783885967977,
-                "score_ci_high": 0.09513486417451719,
-                "rouge2_ci_low": 0.01683460814089518,
-                "rouge2_ci_high": 0.020698285797084142,
-                "rougeLsum_ci_low": 0.10022616921218402,
-                "rougeLsum_ci_high": 0.10887661841550207
+                "rouge1": 0.1292682365835658,
+                "rouge2": 0.01895410897411973,
+                "rougeLsum_ci_low": 0.10084383699373096,
+                "rougeLsum_ci_high": 0.10979495591939617,
+                "rougeL_ci_low": 0.0886493583320966,
+                "rougeL_ci_high": 0.09629333556794349,
+                "score_ci_low": 0.0886493583320966,
+                "score_ci_high": 0.09629333556794349,
+                "rouge1_ci_low": 0.1233156289283472,
+                "rouge1_ci_high": 0.13423125610698836,
+                "rouge2_ci_low": 0.017054021331647896,
+                "rouge2_ci_high": 0.02104099399428594
             },
-            "score": 0.2015236735269907,
+            "score": 0.20295289386941953,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1275,
-                    830,
-                    586,
-                    417
+                    1256,
+                    809,
+                    561,
+                    393
                 ],
                 "totals": [
-                    1807,
-                    1741,
-                    1675,
-                    1609
+                    1822,
+                    1756,
+                    1690,
+                    1624
                 ],
                 "precisions": [
-                    0.7055893746541229,
-                    0.47673750717978175,
-                    0.34985074626865675,
-                    0.2591671845866998
+                    0.6893523600439078,
+                    0.4607061503416856,
+                    0.3319526627218935,
+                    0.2419950738916256
                 ],
                 "bp": 1.0,
-                "sys_len": 1807,
+                "sys_len": 1822,
                 "ref_len": 1734,
-                "sacrebleu": 0.41790112689604164,
-                "score": 0.41790112689604164,
+                "sacrebleu": 0.39965660032074374,
+                "score": 0.39965660032074374,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.37467076425053164,
-                "score_ci_high": 0.46212421285570143,
-                "sacrebleu_ci_low": 0.37467076425053164,
-                "sacrebleu_ci_high": 0.46212421285570143
+                "score_ci_low": 0.34477468044834286,
+                "score_ci_high": 0.4416384677344608,
+                "sacrebleu_ci_low": 0.34477468044834286,
+                "sacrebleu_ci_high": 0.4416384677344608
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1296,
-                    865,
-                    615,
-                    440
+                    1282,
+                    858,
+                    611,
+                    439
                 ],
                 "totals": [
-                    1821,
-                    1755,
-                    1689,
-                    1623
+                    1827,
+                    1761,
+                    1695,
+                    1629
                 ],
                 "precisions": [
-                    0.7116968698517299,
-                    0.49287749287749283,
-                    0.36412078152753113,
-                    0.2711028958718423
+                    0.7016967706622879,
+                    0.48722316865417375,
+                    0.36047197640117995,
+                    0.26949048496009825
                 ],
                 "bp": 1.0,
-                "sys_len": 1821,
+                "sys_len": 1827,
                 "ref_len": 1734,
-                "sacrebleu": 0.4313734544798424,
-                "score": 0.4313734544798424,
+                "sacrebleu": 0.42689698575484597,
+                "score": 0.42689698575484597,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.39011475848676946,
-                "score_ci_high": 0.46975242344141427,
-                "sacrebleu_ci_low": 0.39011475848676946,
-                "sacrebleu_ci_high": 0.46975242344141427
+                "score_ci_low": 0.3808995454883676,
+                "score_ci_high": 0.47419332084644833,
+                "sacrebleu_ci_low": 0.3808995454883676,
+                "sacrebleu_ci_high": 0.47419332084644833
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    963,
-                    569,
-                    359,
-                    232
+                    974,
+                    591,
+                    379,
+                    251
                 ],
                 "totals": [
-                    1592,
-                    1526,
-                    1460,
-                    1394
+                    1588,
+                    1522,
+                    1456,
+                    1390
                 ],
                 "precisions": [
-                    0.6048994974874372,
-                    0.372870249017038,
-                    0.24589041095890413,
-                    0.16642754662840747
+                    0.6133501259445844,
+                    0.38830486202365305,
+                    0.2603021978021978,
+                    0.18057553956834532
                 ],
-                "bp": 1.0,
-                "sys_len": 1592,
+                "bp": 0.9993704753119519,
+                "sys_len": 1588,
                 "ref_len": 1589,
-                "sacrebleu": 0.3099573506400797,
-                "score": 0.3099573506400797,
+                "sacrebleu": 0.3250730946308182,
+                "score": 0.3250730946308182,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2802812834383563,
-                "score_ci_high": 0.34144542695693814,
-                "sacrebleu_ci_low": 0.2802812834383563,
-                "sacrebleu_ci_high": 0.34144542695693814
+                "score_ci_low": 0.28933963986645983,
+                "score_ci_high": 0.36869689361591035,
+                "sacrebleu_ci_low": 0.28933963986645983,
+                "sacrebleu_ci_high": 0.36869689361591035
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1260,
-                    822,
-                    589,
-                    442
+                    1257,
+                    811,
+                    576,
+                    437
                 ],
                 "totals": [
-                    1834,
-                    1768,
-                    1702,
-                    1636
+                    1815,
+                    1749,
+                    1683,
+                    1617
                 ],
                 "precisions": [
-                    0.6870229007633588,
-                    0.4649321266968326,
-                    0.34606345475910694,
-                    0.2701711491442543
+                    0.6925619834710744,
+                    0.46369353916523726,
+                    0.34224598930481287,
+                    0.2702535559678417
                 ],
-                "bp": 0.9994548923547389,
-                "sys_len": 1834,
+                "bp": 0.98904120617152,
+                "sys_len": 1815,
                 "ref_len": 1835,
-                "sacrebleu": 0.41548186092135636,
-                "score": 0.41548186092135636,
+                "sacrebleu": 0.41059556612028536,
+                "score": 0.41059556612028536,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3641424966826166,
-                "score_ci_high": 0.46838012385794353,
-                "sacrebleu_ci_low": 0.3641424966826166,
-                "sacrebleu_ci_high": 0.46838012385794353
+                "score_ci_low": 0.35457040262235134,
+                "score_ci_high": 0.459990812104818,
+                "sacrebleu_ci_low": 0.35457040262235134,
+                "sacrebleu_ci_high": 0.459990812104818
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1594,
-                    1232,
-                    984,
-                    810
+                    1601,
+                    1237,
+                    986,
+                    807
                 ],
                 "totals": [
-                    2012,
-                    1946,
-                    1880,
-                    1814
+                    2017,
+                    1951,
+                    1885,
+                    1819
                 ],
                 "precisions": [
-                    0.7922465208747514,
-                    0.6330935251798561,
-                    0.5234042553191489,
-                    0.44652701212789414
+                    0.7937530986613782,
+                    0.6340338288057407,
+                    0.5230769230769231,
+                    0.44365035733919733
                 ],
-                "bp": 0.9725507672852267,
-                "sys_len": 2012,
+                "bp": 0.9750319133813282,
+                "sys_len": 2017,
                 "ref_len": 2068,
-                "sacrebleu": 0.5690698546093208,
-                "score": 0.5690698546093208,
+                "sacrebleu": 0.5699934901400187,
+                "score": 0.5699934901400187,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.52882560047152,
-                "score_ci_high": 0.6139798297379135,
-                "sacrebleu_ci_low": 0.52882560047152,
-                "sacrebleu_ci_high": 0.6139798297379135
+                "score_ci_low": 0.5326539789859114,
+                "score_ci_high": 0.6234642421169655,
+                "sacrebleu_ci_low": 0.5326539789859114,
+                "sacrebleu_ci_high": 0.6234642421169655
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1376,
-                    781,
-                    495,
-                    326
+                    1367,
+                    786,
+                    494,
+                    321
                 ],
                 "totals": [
-                    2361,
-                    2295,
-                    2229,
-                    2163
+                    2312,
+                    2246,
+                    2180,
+                    2114
                 ],
                 "precisions": [
-                    0.5828038966539602,
-                    0.34030501089324616,
-                    0.22207267833109018,
-                    0.15071659731853906
+                    0.5912629757785467,
+                    0.34995547640249336,
+                    0.22660550458715598,
+                    0.15184484389782404
                 ],
                 "bp": 1.0,
-                "sys_len": 2361,
+                "sys_len": 2312,
                 "ref_len": 2235,
-                "sacrebleu": 0.28543797421890393,
-                "score": 0.28543797421890393,
+                "sacrebleu": 0.2904798394859776,
+                "score": 0.2904798394859776,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2567799199475619,
-                "score_ci_high": 0.31814664829371603,
-                "sacrebleu_ci_low": 0.2567799199475619,
-                "sacrebleu_ci_high": 0.31814664829371603
+                "score_ci_low": 0.25673627398166365,
+                "score_ci_high": 0.3237414974447857,
+                "sacrebleu_ci_low": 0.25673627398166365,
+                "sacrebleu_ci_high": 0.3237414974447857
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1451,
-                    1053,
-                    814,
-                    640
+                    1458,
+                    1061,
+                    822,
+                    653
                 ],
                 "totals": [
-                    1898,
-                    1832,
-                    1766,
-                    1700
+                    1897,
+                    1831,
+                    1765,
+                    1699
                 ],
                 "precisions": [
-                    0.7644889357218125,
-                    0.5747816593886462,
-                    0.46092865232163077,
-                    0.3764705882352941
+                    0.768581971534001,
+                    0.5794647733478974,
+                    0.4657223796033994,
+                    0.38434373160682755
                 ],
-                "bp": 0.9905611611284771,
-                "sys_len": 1898,
+                "bp": 0.9900341767854584,
+                "sys_len": 1897,
                 "ref_len": 1916,
-                "sacrebleu": 0.52052430367519,
-                "score": 0.52052430367519,
+                "sacrebleu": 0.5260671977764972,
+                "score": 0.5260671977764972,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.46893066617087675,
-                "score_ci_high": 0.5524273546032454,
-                "sacrebleu_ci_low": 0.46893066617087675,
-                "sacrebleu_ci_high": 0.5524273546032454
+                "score_ci_low": 0.475872231232826,
+                "score_ci_high": 0.5725271086507513,
+                "sacrebleu_ci_low": 0.475872231232826,
+                "sacrebleu_ci_high": 0.5725271086507513
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1416,
-                    1021,
-                    771,
-                    581
+                    1404,
+                    1006,
+                    750,
+                    558
                 ],
                 "totals": [
-                    1945,
-                    1879,
-                    1813,
-                    1747
+                    1938,
+                    1872,
+                    1806,
+                    1740
                 ],
                 "precisions": [
-                    0.7280205655526992,
-                    0.5433741351782863,
-                    0.42526199669056813,
-                    0.3325701202060675
+                    0.7244582043343654,
+                    0.5373931623931624,
+                    0.4152823920265781,
+                    0.3206896551724138
                 ],
-                "bp": 0.9979455579909386,
-                "sys_len": 1945,
+                "bp": 0.994340123204573,
+                "sys_len": 1938,
                 "ref_len": 1949,
-                "sacrebleu": 0.4853471146871033,
-                "score": 0.4853471146871033,
+                "sacrebleu": 0.47448058787166153,
+                "score": 0.47448058787166153,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.45737023291404577,
-                "score_ci_high": 0.5267894003889202,
-                "sacrebleu_ci_low": 0.45737023291404577,
-                "sacrebleu_ci_high": 0.5267894003889202
+                "score_ci_low": 0.44387053770037466,
+                "score_ci_high": 0.51824756405881,
+                "sacrebleu_ci_low": 0.44387053770037466,
+                "sacrebleu_ci_high": 0.51824756405881
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1269,
-                    719,
-                    439,
-                    270
+                    1267,
+                    721,
+                    443,
+                    269
                 ],
                 "totals": [
-                    1972,
-                    1906,
-                    1840,
-                    1774
+                    1960,
+                    1894,
+                    1828,
+                    1762
                 ],
                 "precisions": [
-                    0.6435091277890467,
-                    0.3772298006295907,
-                    0.23858695652173914,
-                    0.15219842164599773
+                    0.6464285714285714,
+                    0.38067581837381204,
+                    0.24234135667396062,
+                    0.15266742338251987
                 ],
-                "bp": 0.9381039423957293,
-                "sys_len": 1972,
+                "bp": 0.932013328656422,
+                "sys_len": 1960,
                 "ref_len": 2098,
-                "sacrebleu": 0.28744539070468517,
-                "score": 0.28744539070468517,
+                "sacrebleu": 0.28789528964668276,
+                "score": 0.28789528964668276,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2566627625544366,
-                "score_ci_high": 0.3146920359001326,
-                "sacrebleu_ci_low": 0.2566627625544366,
-                "sacrebleu_ci_high": 0.3146920359001326
+                "score_ci_low": 0.2550898701517229,
+                "score_ci_high": 0.32315043281050926,
+                "sacrebleu_ci_low": 0.2550898701517229,
+                "sacrebleu_ci_high": 0.32315043281050926
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1324,
-                    920,
-                    666,
-                    477
+                    1342,
+                    940,
+                    684,
+                    495
                 ],
                 "totals": [
-                    1893,
-                    1827,
-                    1761,
-                    1695
+                    1861,
+                    1795,
+                    1729,
+                    1663
                 ],
                 "precisions": [
-                    0.6994189117802431,
-                    0.5035577449370553,
-                    0.3781942078364565,
-                    0.2814159292035398
+                    0.7211176786673832,
+                    0.5236768802228412,
+                    0.39560439560439564,
+                    0.29765484064942876
                 ],
                 "bp": 1.0,
-                "sys_len": 1893,
+                "sys_len": 1861,
                 "ref_len": 1734,
-                "sacrebleu": 0.44001000355221576,
-                "score": 0.44001000355221576,
+                "sacrebleu": 0.45920953842389034,
+                "score": 0.45920953842389034,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.40032693502671896,
-                "score_ci_high": 0.4786325473299439,
-                "sacrebleu_ci_low": 0.40032693502671896,
-                "sacrebleu_ci_high": 0.4786325473299439
+                "score_ci_low": 0.4189257677908612,
+                "score_ci_high": 0.5012078702279882,
+                "sacrebleu_ci_low": 0.4189257677908612,
+                "sacrebleu_ci_high": 0.5012078702279882
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1135,
-                    622,
-                    371,
-                    231
+                    1124,
+                    603,
+                    358,
+                    209
                 ],
                 "totals": [
-                    2011,
-                    1945,
-                    1879,
-                    1813
+                    1910,
+                    1844,
+                    1778,
+                    1712
                 ],
                 "precisions": [
-                    0.564395822973645,
-                    0.3197943444730077,
-                    0.19744544970729114,
-                    0.1274131274131274
+                    0.5884816753926702,
+                    0.32700650759219085,
+                    0.20134983127109113,
+                    0.12207943925233644
                 ],
                 "bp": 1.0,
-                "sys_len": 2011,
+                "sys_len": 1910,
                 "ref_len": 1734,
-                "sacrebleu": 0.259584626709523,
-                "score": 0.259584626709523,
+                "sacrebleu": 0.26225319254842555,
+                "score": 0.26225319254842555,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.22697300814650573,
-                "score_ci_high": 0.2908889327642379,
-                "sacrebleu_ci_low": 0.22697300814650573,
-                "sacrebleu_ci_high": 0.2908889327642379
+                "score_ci_low": 0.2289709766377336,
+                "score_ci_high": 0.30157417779677487,
+                "sacrebleu_ci_low": 0.2289709766377336,
+                "sacrebleu_ci_high": 0.30157417779677487
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1120,
-                    635,
-                    400,
-                    262
+                    1117,
+                    625,
+                    396,
+                    257
                 ],
                 "totals": [
-                    1861,
-                    1795,
-                    1729,
-                    1663
+                    1801,
+                    1735,
+                    1669,
+                    1603
                 ],
                 "precisions": [
-                    0.6018269747447609,
-                    0.3537604456824513,
-                    0.2313475997686524,
-                    0.15754660252555622
+                    0.6202109938922821,
+                    0.36023054755043227,
+                    0.2372678250449371,
+                    0.1603243917654398
                 ],
                 "bp": 1.0,
-                "sys_len": 1861,
+                "sys_len": 1801,
                 "ref_len": 1734,
-                "sacrebleu": 0.29679989332755485,
-                "score": 0.29679989332755485,
+                "sacrebleu": 0.3036264572339376,
+                "score": 0.3036264572339376,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2606890146882507,
-                "score_ci_high": 0.34577889084499225,
-                "sacrebleu_ci_low": 0.2606890146882507,
-                "sacrebleu_ci_high": 0.34577889084499225
+                "score_ci_low": 0.2714407091986394,
+                "score_ci_high": 0.36938210174908737,
+                "sacrebleu_ci_low": 0.2714407091986394,
+                "sacrebleu_ci_high": 0.36938210174908737
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1324,
-                    947,
-                    723,
-                    555
+                    1283,
+                    897,
+                    669,
+                    501
                 ],
                 "totals": [
-                    1810,
-                    1744,
-                    1678,
-                    1612
+                    2053,
+                    1987,
+                    1921,
+                    1855
                 ],
                 "precisions": [
-                    0.7314917127071823,
-                    0.5430045871559632,
-                    0.4308700834326579,
-                    0.3442928039702233
+                    0.6249391134924501,
+                    0.451434323100151,
+                    0.34825611660593436,
+                    0.27008086253369273
                 ],
                 "bp": 1.0,
-                "sys_len": 1810,
+                "sys_len": 2053,
                 "ref_len": 1734,
-                "sacrebleu": 0.49268778913543754,
-                "score": 0.49268778913543754,
+                "sacrebleu": 0.40360469717634034,
+                "score": 0.40360469717634034,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.40671829932229786,
-                "score_ci_high": 0.5323333720792753,
-                "sacrebleu_ci_low": 0.40671829932229786,
-                "sacrebleu_ci_high": 0.5323333720792753
+                "score_ci_low": 0.22825303083161255,
+                "score_ci_high": 0.4880479254926776,
+                "sacrebleu_ci_low": 0.22825303083161255,
+                "sacrebleu_ci_high": 0.4880479254926776
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
                     1324,
-                    922,
-                    663,
-                    481
+                    929,
+                    672,
+                    489
                 ],
                 "totals": [
-                    1831,
-                    1765,
-                    1699,
-                    1633
+                    2052,
+                    1986,
+                    1920,
+                    1854
                 ],
                 "precisions": [
-                    0.7231021299836154,
-                    0.5223796033994335,
-                    0.3902295467922307,
-                    0.2945499081445193
+                    0.645224171539961,
+                    0.4677744209466264,
+                    0.35,
+                    0.2637540453074434
                 ],
                 "bp": 1.0,
-                "sys_len": 1831,
+                "sys_len": 2052,
                 "ref_len": 1734,
-                "sacrebleu": 0.4564741865050132,
-                "score": 0.4564741865050132,
+                "sacrebleu": 0.4085578581131045,
+                "score": 0.4085578581131045,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.38825961035825357,
-                "score_ci_high": 0.5021079213683431,
-                "sacrebleu_ci_low": 0.38825961035825357,
-                "sacrebleu_ci_high": 0.5021079213683431
+                "score_ci_low": 0.30163750041734844,
+                "score_ci_high": 0.4725760173281261,
+                "sacrebleu_ci_low": 0.30163750041734844,
+                "sacrebleu_ci_high": 0.4725760173281261
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1203,
-                    678,
-                    414,
-                    248
+                    1196,
+                    676,
+                    416,
+                    253
                 ],
                 "totals": [
-                    1924,
-                    1858,
-                    1792,
-                    1726
+                    1927,
+                    1861,
+                    1795,
+                    1729
                 ],
                 "precisions": [
-                    0.6252598752598753,
-                    0.36490850376749195,
-                    0.23102678571428573,
-                    0.1436848203939745
+                    0.6206538661131292,
+                    0.3632455668995164,
+                    0.23175487465181058,
+                    0.14632735685367262
                 ],
                 "bp": 1.0,
-                "sys_len": 1924,
+                "sys_len": 1927,
                 "ref_len": 1734,
-                "sacrebleu": 0.2950050444470296,
-                "score": 0.2950050444470296,
+                "sacrebleu": 0.2956998119625713,
+                "score": 0.2956998119625713,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25511324454078954,
-                "score_ci_high": 0.33543848071805,
-                "sacrebleu_ci_low": 0.25511324454078954,
-                "sacrebleu_ci_high": 0.33543848071805
+                "score_ci_low": 0.25139539620137036,
+                "score_ci_high": 0.32686978566283265,
+                "sacrebleu_ci_low": 0.25139539620137036,
+                "sacrebleu_ci_high": 0.32686978566283265
             },
-            "score": 0.3975399983006198,
+            "score": 0.38960601381372006,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.5386099714179504,
+        "score": 0.5094379735715554,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-21T08-38-27_evaluation_results.json → 2025-06-23T08-43-46_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-21T12:38:23.605149Z",
+        "timestamp_utc": "2025-06-23T12:43:42.752885Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -8,7 +8,7 @@
             "--model",
             "cross_provider",
             "--model_args",
-            "model_name=watsonx/meta-llama/llama-3-2-11b-vision-instruct,max_tokens=256",
+            "model_name=watsonx/mistralai/mistral-small-3-1-24b-instruct-2503,max_tokens=256",
             "--output_path",
             "./results/bluebench",
             "--log_samples",
@@ -26,7 +26,7 @@
             "num_fewshots": null,
             "limit": null,
             "batch_size": 8,
-            "model": "watsonx/meta-llama/llama-3-2-11b-vision-instruct",
+            "model": "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503",
             "model_args": {
                 "max_tokens": 256
             },
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,23 +176,23 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.7555555555555555,
-                "accuracy_ci_low": 0.6666666666666666,
-                "accuracy_ci_high": 0.8333333333333334,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.5666666666666667,
+                "accuracy_ci_high": 0.7555555555555555,
                 "score_name": "accuracy",
-                "score": 0.7555555555555555,
-                "score_ci_high": 0.8333333333333334,
-                "score_ci_low": 0.6666666666666666,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.7555555555555555,
+                "score_ci_low": 0.5666666666666667,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7111111111111111,
-                "accuracy_ci_high": 0.8666666666666667,
+                "accuracy": 0.7666666666666667,
+                "accuracy_ci_low": 0.6777777777777778,
+                "accuracy_ci_high": 0.8444444444444444,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8666666666666667,
-                "score_ci_low": 0.7111111111111111,
+                "score": 0.7666666666666667,
+                "score_ci_high": 0.8444444444444444,
+                "score_ci_low": 0.6777777777777778,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
@@ -206,166 +206,186 @@
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7222222222222222,
-                "accuracy_ci_high": 0.8777777777777778,
+                "accuracy": 0.7555555555555555,
+                "accuracy_ci_low": 0.6555555555555556,
+                "accuracy_ci_high": 0.8333333333333334,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8777777777777778,
-                "score_ci_low": 0.7222222222222222,
+                "score": 0.7555555555555555,
+                "score_ci_high": 0.8333333333333334,
+                "score_ci_low": 0.6555555555555556,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.8333333333333334,
-                "accuracy_ci_low": 0.7444444444444445,
-                "accuracy_ci_high": 0.9,
+                "accuracy": 0.8111111111111111,
+                "accuracy_ci_low": 0.7111111111111111,
+                "accuracy_ci_high": 0.8777777777777778,
                 "score_name": "accuracy",
-                "score": 0.8333333333333334,
-                "score_ci_high": 0.9,
-                "score_ci_low": 0.7444444444444445,
+                "score": 0.8111111111111111,
+                "score_ci_high": 0.8777777777777778,
+                "score_ci_low": 0.7111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.9,
-                "accuracy_ci_low": 0.8277783447952625,
-                "accuracy_ci_high": 0.9448695638574703,
+                "accuracy": 0.8111111111111111,
+                "accuracy_ci_low": 0.7111111111111111,
+                "accuracy_ci_high": 0.8777777777777778,
                 "score_name": "accuracy",
-                "score": 0.9,
-                "score_ci_high": 0.9448695638574703,
-                "score_ci_low": 0.8277783447952625,
+                "score": 0.8111111111111111,
+                "score_ci_high": 0.8777777777777778,
+                "score_ci_low": 0.7111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.9777777777777777,
-                "accuracy_ci_low": 0.9222222222222223,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.9777777777777777,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.9222222222222223,
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7333333333333333,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.8555555555555555,
-                "accuracy_ci_low": 0.7777777777777778,
-                "accuracy_ci_high": 0.9222222222222223,
+                "accuracy": 0.8444444444444444,
+                "accuracy_ci_low": 0.7555555555555555,
+                "accuracy_ci_high": 0.9,
                 "score_name": "accuracy",
-                "score": 0.8555555555555555,
-                "score_ci_high": 0.9222222222222223,
-                "score_ci_low": 0.7777777777777778,
+                "score": 0.8444444444444444,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7555555555555555,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.9,
-                "accuracy_ci_low": 0.822517354780987,
-                "accuracy_ci_high": 0.9444444444444444,
+                "accuracy": 0.8222222222222222,
+                "accuracy_ci_low": 0.7333333333333333,
+                "accuracy_ci_high": 0.9,
                 "score_name": "accuracy",
-                "score": 0.9,
-                "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.822517354780987,
+                "score": 0.8222222222222222,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7333333333333333,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7111111111111111,
-                "accuracy_ci_high": 0.8777777777777778,
+                "accuracy": 0.8111111111111111,
+                "accuracy_ci_low": 0.7222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8777777777777778,
-                "score_ci_low": 0.7111111111111111,
+                "score": 0.8111111111111111,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.7222222222222222,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.8111111111111111,
-                "accuracy_ci_high": 0.9444444444444444,
+                "accuracy": 0.8333333333333334,
+                "accuracy_ci_low": 0.7555555555555555,
+                "accuracy_ci_high": 0.9111111111111111,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
-                "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.8111111111111111,
+                "score": 0.8333333333333334,
+                "score_ci_high": 0.9111111111111111,
+                "score_ci_low": 0.7555555555555555,
                 "num_of_instances": 90
             },
-            "score": 0.8555555555555555,
+            "score": 0.804040404040404,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.10666666666666667,
+                "score": 0.10666666666666667,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.10666666666666667,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.5,
-                "f1_Organization": 0.3540372670807453,
-                "f1_Location": 0.35390946502057613,
-                "f1_macro": 0.40264891070044045,
-                "recall_macro": 0.3613703019405527,
-                "precision_macro": 0.46075111602245217,
-                "in_classes_support": 0.7734806629834254,
-                "f1_micro": 0.3651685393258427,
-                "recall_micro": 0.37142857142857144,
-                "precision_micro": 0.35911602209944754,
-                "score": 0.3651685393258427,
+                "f1_Person": 0.34806629834254144,
+                "f1_Organization": 0.28125,
+                "f1_Location": 0.2272727272727273,
+                "f1_macro": 0.2855296752050896,
+                "recall_macro": 0.2576225314974886,
+                "precision_macro": 0.32330034002100655,
+                "in_classes_support": 0.4646799116997793,
+                "f1_micro": 0.1928721174004193,
+                "recall_micro": 0.26285714285714284,
+                "precision_micro": 0.152317880794702,
+                "score": 0.1928721174004193,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.3095577857641209,
-                "score_ci_high": 0.42280766665241554,
-                "f1_micro_ci_low": 0.3095577857641209,
-                "f1_micro_ci_high": 0.42280766665241554
+                "score_ci_low": 0.16179580146707578,
+                "score_ci_high": 0.2232377264615503,
+                "f1_micro_ci_low": 0.16179580146707578,
+                "f1_micro_ci_high": 0.2232377264615503
             },
-            "score": 0.3651685393258427,
+            "score": 0.1928721174004193,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.5492957746478874,
-                "accuracy_ci_low": 0.4445856612942033,
-                "accuracy_ci_high": 0.6619718309859155,
+                "accuracy": 0.647887323943662,
+                "accuracy_ci_low": 0.5352112676056338,
+                "accuracy_ci_high": 0.7605633802816901,
                 "score_name": "accuracy",
-                "score": 0.5492957746478874,
-                "score_ci_high": 0.6619718309859155,
-                "score_ci_low": 0.4445856612942033,
+                "score": 0.647887323943662,
+                "score_ci_high": 0.7605633802816901,
+                "score_ci_low": 0.5352112676056338,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.2676056338028169,
-                "accuracy_ci_low": 0.16901408450704225,
-                "accuracy_ci_high": 0.38028169014084506,
+                "accuracy": 0.30985915492957744,
+                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_high": 0.428782341390215,
                 "score_name": "accuracy",
-                "score": 0.2676056338028169,
-                "score_ci_high": 0.38028169014084506,
-                "score_ci_low": 0.16901408450704225,
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.428782341390215,
+                "score_ci_low": 0.2112676056338028,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.18309859154929578,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy": 0.22535211267605634,
+                "accuracy_ci_low": 0.14084507042253522,
+                "accuracy_ci_high": 0.3380281690140845,
                 "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.18309859154929578,
+                "score": 0.22535211267605634,
+                "score_ci_high": 0.3380281690140845,
+                "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.39436619718309857,
-                "accuracy_ci_low": 0.28169014084507044,
-                "accuracy_ci_high": 0.5070422535211268,
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.676056338028169,
                 "score_name": "accuracy",
-                "score": 0.39436619718309857,
-                "score_ci_high": 0.5070422535211268,
-                "score_ci_low": 0.28169014084507044,
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4507042253521127,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
+                "accuracy": 0.6197183098591549,
+                "accuracy_ci_low": 0.49295774647887325,
+                "accuracy_ci_high": 0.7323943661971831,
+                "score_name": "accuracy",
+                "score": 0.6197183098591549,
+                "score_ci_high": 0.7323943661971831,
+                "score_ci_low": 0.49295774647887325,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.18309859154929578,
+                "accuracy_ci_low": 0.11267605633802817,
+                "accuracy_ci_high": 0.28169014084507044,
+                "score_name": "accuracy",
+                "score": 0.18309859154929578,
+                "score_ci_high": 0.28169014084507044,
+                "score_ci_low": 0.11267605633802817,
+                "num_of_instances": 71
+            },
+            "mmlu_pro_health": {
                 "accuracy": 0.4788732394366197,
                 "accuracy_ci_low": 0.36619718309859156,
                 "accuracy_ci_high": 0.5915492957746479,
@@ -375,17 +395,17 @@
                 "score_ci_low": 0.36619718309859156,
                 "num_of_instances": 71
             },
-            "mmlu_pro_engineering": {
-                "accuracy": 0.23943661971830985,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.3380281690140845,
+            "mmlu_pro_history": {
+                "accuracy": 0.6619718309859155,
+                "accuracy_ci_low": 0.5352112676056338,
+                "accuracy_ci_high": 0.7714646829428065,
                 "score_name": "accuracy",
-                "score": 0.23943661971830985,
-                "score_ci_high": 0.3380281690140845,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.6619718309859155,
+                "score_ci_high": 0.7714646829428065,
+                "score_ci_low": 0.5352112676056338,
                 "num_of_instances": 71
             },
-            "mmlu_pro_health": {
+            "mmlu_pro_law": {
                 "accuracy": 0.43661971830985913,
                 "accuracy_ci_low": 0.323943661971831,
                 "accuracy_ci_high": 0.5492957746478874,
@@ -395,414 +415,394 @@
                 "score_ci_low": 0.323943661971831,
                 "num_of_instances": 71
             },
-            "mmlu_pro_history": {
-                "accuracy": 0.5070422535211268,
-                "accuracy_ci_low": 0.39436619718309857,
-                "accuracy_ci_high": 0.6197183098591549,
-                "score_name": "accuracy",
-                "score": 0.5070422535211268,
-                "score_ci_high": 0.6197183098591549,
-                "score_ci_low": 0.39436619718309857,
-                "num_of_instances": 71
-            },
-            "mmlu_pro_law": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.18309859154929578,
-                "accuracy_ci_high": 0.39436619718309857,
-                "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.18309859154929578,
-                "num_of_instances": 71
-            },
             "mmlu_pro_math": {
-                "accuracy": 0.14084507042253522,
-                "accuracy_ci_low": 0.07042253521126761,
-                "accuracy_ci_high": 0.25119134125976145,
+                "accuracy": 0.2676056338028169,
+                "accuracy_ci_low": 0.16901408450704225,
+                "accuracy_ci_high": 0.38028169014084506,
                 "score_name": "accuracy",
-                "score": 0.14084507042253522,
-                "score_ci_high": 0.25119134125976145,
-                "score_ci_low": 0.07042253521126761,
+                "score": 0.2676056338028169,
+                "score_ci_high": 0.38028169014084506,
+                "score_ci_low": 0.16901408450704225,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.2112676056338028,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.323943661971831,
+                "accuracy": 0.4647887323943662,
+                "accuracy_ci_low": 0.352112676056338,
+                "accuracy_ci_high": 0.5774647887323944,
                 "score_name": "accuracy",
-                "score": 0.2112676056338028,
-                "score_ci_high": 0.323943661971831,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.4647887323943662,
+                "score_ci_high": 0.5774647887323944,
+                "score_ci_low": 0.352112676056338,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.43661971830985913,
-                "accuracy_ci_low": 0.323943661971831,
-                "accuracy_ci_high": 0.5633802816901409,
+                "accuracy": 0.49295774647887325,
+                "accuracy_ci_low": 0.38028169014084506,
+                "accuracy_ci_high": 0.6197183098591549,
                 "score_name": "accuracy",
-                "score": 0.43661971830985913,
-                "score_ci_high": 0.5633802816901409,
-                "score_ci_low": 0.323943661971831,
+                "score": 0.49295774647887325,
+                "score_ci_high": 0.6197183098591549,
+                "score_ci_low": 0.38028169014084506,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.18309859154929578,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy": 0.4507042253521127,
+                "accuracy_ci_low": 0.3380281690140845,
+                "accuracy_ci_high": 0.5633802816901409,
                 "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.18309859154929578,
+                "score": 0.4507042253521127,
+                "score_ci_high": 0.5633802816901409,
+                "score_ci_low": 0.3380281690140845,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.5633802816901409,
-                "accuracy_ci_low": 0.43661971830985913,
-                "accuracy_ci_high": 0.676056338028169,
+                "accuracy": 0.6338028169014085,
+                "accuracy_ci_low": 0.5211267605633803,
+                "accuracy_ci_high": 0.7464788732394366,
                 "score_name": "accuracy",
-                "score": 0.5633802816901409,
-                "score_ci_high": 0.676056338028169,
-                "score_ci_low": 0.43661971830985913,
+                "score": 0.6338028169014085,
+                "score_ci_high": 0.7464788732394366,
+                "score_ci_low": 0.5211267605633803,
                 "num_of_instances": 71
             },
-            "score": 0.36217303822937624,
+            "score": 0.4597585513078471,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.5714968851810958,
-                "f1_suggestive": 0.38461538461538464,
-                "f1_fanciful": 0.4666666666666667,
-                "f1_generic": 0.8148148148148148,
-                "f1_arbitrary": 0.45454545454545453,
-                "f1_descriptive": 0.7368421052631579,
-                "f1_macro_ci_low": 0.477080408933172,
-                "f1_macro_ci_high": 0.6804913151553064,
+                "f1_macro": 0.3274661835748792,
+                "f1_suggestive": 0.3125,
+                "f1_generic": 0.2222222222222222,
+                "f1_arbitrary": 0.32,
+                "f1_fanciful": 0.43478260869565216,
+                "f1_descriptive": 0.34782608695652173,
+                "f1_macro_ci_low": 0.2279048726954935,
+                "f1_macro_ci_high": 0.4521754269782871,
                 "score_name": "f1_micro",
-                "score": 0.5696969696969697,
-                "score_ci_high": 0.6707317073170732,
-                "score_ci_low": 0.4662576687116564,
+                "score": 0.3305785123966942,
+                "score_ci_high": 0.4462631095061656,
+                "score_ci_low": 0.22608695652173913,
                 "num_of_instances": 85,
-                "accuracy": 0.5529411764705883,
-                "accuracy_ci_low": 0.4470588235294118,
-                "accuracy_ci_high": 0.6588235294117647,
-                "f1_micro": 0.5696969696969697,
-                "f1_micro_ci_low": 0.4662576687116564,
-                "f1_micro_ci_high": 0.6707317073170732
+                "accuracy": 0.23529411764705882,
+                "accuracy_ci_low": 0.15294117647058825,
+                "accuracy_ci_high": 0.3411764705882353,
+                "f1_micro": 0.3305785123966942,
+                "f1_micro_ci_low": 0.22608695652173913,
+                "f1_micro_ci_high": 0.4462631095061656
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.6068255091609656,
-                "f1_no": 0.691358024691358,
-                "f1_yes": 0.5222929936305732,
-                "f1_macro_ci_low": 0.5410272293620183,
-                "f1_macro_ci_high": 0.6785168220322721,
+                "f1_macro": 0.3665132336018412,
+                "f1_no": 0.45454545454545453,
+                "f1_yes": 0.27848101265822783,
+                "f1_macro_ci_low": 0.29058993290093893,
+                "f1_macro_ci_high": 0.44397782794437635,
                 "score_name": "f1_micro",
-                "score": 0.625,
-                "score_ci_high": 0.695,
-                "score_ci_low": 0.56,
+                "score": 0.4043321299638989,
+                "score_ci_high": 0.4848050604545447,
+                "score_ci_low": 0.33210332103321033,
                 "num_of_instances": 200,
-                "accuracy": 0.625,
-                "accuracy_ci_low": 0.56,
-                "accuracy_ci_high": 0.695,
-                "f1_micro": 0.625,
-                "f1_micro_ci_low": 0.56,
-                "f1_micro_ci_high": 0.695
+                "accuracy": 0.28,
+                "accuracy_ci_low": 0.22435516148422335,
+                "accuracy_ci_high": 0.35,
+                "f1_micro": 0.4043321299638989,
+                "f1_micro_ci_low": 0.33210332103321033,
+                "f1_micro_ci_high": 0.4848050604545447
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.341120203240043,
-                "f1_conclusion": 0.09523809523809523,
-                "f1_analysis": 0.5046728971962616,
-                "f1_decree": 0.29411764705882354,
-                "f1_issue": 0.21818181818181817,
-                "f1_procedural history": 0.34782608695652173,
-                "f1_facts": 0.4878048780487805,
-                "f1_rule": 0.44,
-                "f1_macro_ci_low": 0.28339415146501146,
-                "f1_macro_ci_high": 0.41104677447147997,
+                "f1_macro": 0.2994201157541979,
+                "f1_conclusion": 0.16216216216216217,
+                "f1_decree": 0.22857142857142856,
+                "f1_analysis": 0.43478260869565216,
+                "f1_issue": 0.23809523809523808,
+                "f1_procedural history": 0.36363636363636365,
+                "f1_facts": 0.2857142857142857,
+                "f1_rule": 0.3829787234042553,
+                "f1_macro_ci_low": 0.23455524899226265,
+                "f1_macro_ci_high": 0.3757847883095156,
                 "score_name": "f1_micro",
-                "score": 0.36683417085427134,
-                "score_ci_high": 0.43609022556390975,
-                "score_ci_low": 0.3057644110275689,
+                "score": 0.3111111111111111,
+                "score_ci_high": 0.38492614857203733,
+                "score_ci_low": 0.23835139550418585,
                 "num_of_instances": 200,
-                "accuracy": 0.365,
-                "accuracy_ci_low": 0.305,
-                "accuracy_ci_high": 0.435,
-                "f1_micro": 0.36683417085427134,
-                "f1_micro_ci_low": 0.3057644110275689,
-                "f1_micro_ci_high": 0.43609022556390975
+                "accuracy": 0.245,
+                "accuracy_ci_low": 0.185,
+                "accuracy_ci_high": 0.31,
+                "f1_micro": 0.3111111111111111,
+                "f1_micro_ci_low": 0.23835139550418585,
+                "f1_micro_ci_high": 0.38492614857203733
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5542857142857143,
-                "f1_yes": 0.42857142857142855,
-                "f1_no": 0.68,
-                "f1_macro_ci_low": 0.48355051877957367,
-                "f1_macro_ci_high": 0.6294200433350639,
+                "f1_macro": 0.20520718738540522,
+                "f1_yes": 0.21782178217821782,
+                "f1_no": 0.1925925925925926,
+                "f1_macro_ci_low": 0.1400419836844058,
+                "f1_macro_ci_high": 0.2885398882645068,
                 "score_name": "f1_micro",
-                "score": 0.5897435897435898,
-                "score_ci_high": 0.662674821169658,
-                "score_ci_low": 0.5188702892801219,
+                "score": 0.2033898305084746,
+                "score_ci_high": 0.2857142857142857,
+                "score_ci_low": 0.1391304347826087,
                 "num_of_instances": 200,
-                "accuracy": 0.575,
-                "accuracy_ci_low": 0.505,
-                "accuracy_ci_high": 0.65,
-                "f1_micro": 0.5897435897435898,
-                "f1_micro_ci_low": 0.5188702892801219,
-                "f1_micro_ci_high": 0.662674821169658
+                "accuracy": 0.12,
+                "accuracy_ci_low": 0.08,
+                "accuracy_ci_high": 0.175,
+                "f1_micro": 0.2033898305084746,
+                "f1_micro_ci_low": 0.1391304347826087,
+                "f1_micro_ci_high": 0.2857142857142857
             },
             "legalbench_proa": {
-                "f1_macro": 0.9155052264808363,
-                "f1_yes": 0.9024390243902439,
-                "f1_no": 0.9285714285714286,
-                "f1_macro_ci_low": 0.8363111281337707,
-                "f1_macro_ci_high": 0.9587803500846979,
+                "f1_macro": 0.8035124326204138,
+                "f1_yes": 0.8169014084507042,
+                "f1_no": 0.7901234567901234,
+                "f1_macro_ci_low": 0.7176915883069268,
+                "f1_macro_ci_high": 0.872867714407109,
                 "score_name": "f1_micro",
-                "score": 0.9156626506024096,
-                "score_ci_high": 0.9585798816568047,
-                "score_ci_low": 0.8372630005674523,
+                "score": 0.8026315789473685,
+                "score_ci_high": 0.871520027126433,
+                "score_ci_low": 0.7086398695460123,
                 "num_of_instances": 85,
-                "accuracy": 0.8941176470588236,
-                "accuracy_ci_low": 0.8,
-                "accuracy_ci_high": 0.9466354743635627,
-                "f1_micro": 0.9156626506024096,
-                "f1_micro_ci_low": 0.8372630005674523,
-                "f1_micro_ci_high": 0.9585798816568047
+                "accuracy": 0.7176470588235294,
+                "accuracy_ci_low": 0.611764705882353,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.8026315789473685,
+                "f1_micro_ci_low": 0.7086398695460123,
+                "f1_micro_ci_high": 0.871520027126433
             },
-            "score": 0.6133874761794481,
+            "score": 0.41040863258550947,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.5088753805047725,
-                "f1_cars": 0.7380952380952381,
-                "f1_windows x": 0.08333333333333333,
-                "f1_computer graphics": 0.5434782608695652,
-                "f1_atheism": 0.30434782608695654,
-                "f1_religion": 0.1388888888888889,
-                "f1_medicine": 0.7733333333333333,
-                "f1_christianity": 0.6597938144329897,
-                "f1_microsoft windows": 0.39436619718309857,
-                "f1_middle east": 0.22641509433962265,
-                "f1_politics": 0.3418803418803419,
-                "f1_motorcycles": 0.673469387755102,
-                "f1_pc hardware": 0.4968944099378882,
-                "f1_mac hardware": 0.5617977528089888,
-                "f1_electronics": 0.4580152671755725,
-                "f1_for sale": 0.4406779661016949,
-                "f1_guns": 0.28125,
-                "f1_space": 0.7111111111111111,
-                "f1_cryptography": 0.6197183098591549,
-                "f1_baseball": 0.8617886178861789,
-                "f1_hockey": 0.8688524590163934,
-                "f1_macro_ci_low": 0.4834761861756795,
-                "f1_macro_ci_high": 0.5399453530006973,
+                "f1_macro": 0.51306152129787,
+                "f1_cars": 0.6585365853658537,
+                "f1_windows x": 0.08571428571428572,
+                "f1_cryptography": 0.5641025641025641,
+                "f1_atheism": 0.09302325581395349,
+                "f1_religion": 0.15873015873015872,
+                "f1_medicine": 0.810126582278481,
+                "f1_christianity": 0.36619718309859156,
+                "f1_computer graphics": 0.43243243243243246,
+                "f1_microsoft windows": 0.5569620253164557,
+                "f1_middle east": 0.625,
+                "f1_motorcycles": 0.64,
+                "f1_mac hardware": 0.49411764705882355,
+                "f1_pc hardware": 0.5309734513274337,
+                "f1_electronics": 0.6292134831460674,
+                "f1_for sale": 0.5538461538461539,
+                "f1_guns": 0.22580645161290322,
+                "f1_space": 0.7872340425531915,
+                "f1_baseball": 0.8598130841121495,
+                "f1_hockey": 0.859504132231405,
+                "f1_politics": 0.32989690721649484,
+                "f1_macro_ci_low": 0.489675643451468,
+                "f1_macro_ci_high": 0.5444343504387604,
                 "score_name": "f1_micro",
-                "score": 0.5394515948517068,
-                "score_ci_high": 0.571442163952847,
-                "score_ci_low": 0.5099380125013132,
+                "score": 0.5470692717584369,
+                "score_ci_high": 0.5787418375694315,
+                "score_ci_low": 0.5152956292250616,
                 "num_of_instances": 1000,
-                "accuracy": 0.482,
-                "accuracy_ci_low": 0.45265982508763236,
-                "accuracy_ci_high": 0.514,
-                "f1_micro": 0.5394515948517068,
-                "f1_micro_ci_low": 0.5099380125013132,
-                "f1_micro_ci_high": 0.571442163952847
+                "accuracy": 0.462,
+                "accuracy_ci_low": 0.431,
+                "accuracy_ci_high": 0.492,
+                "f1_micro": 0.5470692717584369,
+                "f1_micro_ci_low": 0.5152956292250616,
+                "f1_micro_ci_high": 0.5787418375694315
             },
-            "score": 0.5394515948517068,
+            "score": 0.5470692717584369,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.6253924964112395,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9158878504672897,
-                "f1_credit card or prepaid card": 0.625,
-                "f1_debt collection": 0.5882352941176471,
-                "f1_checking or savings account": 0.75,
-                "f1_money transfer or virtual currency or money service": 0.5517241379310345,
-                "f1_mortgage": 0.7407407407407407,
-                "f1_vehicle loan or lease": 0.5333333333333333,
-                "f1_payday loan or title loan or personal loan": 0.1111111111111111,
-                "f1_student loan": 0.8125,
-                "f1_macro_ci_low": 0.5783727985774465,
-                "f1_macro_ci_high": 0.684670784163184,
+                "f1_macro": 0.5522388523080553,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7597955706984668,
+                "f1_checking or savings account": 0.717391304347826,
+                "f1_debt collection": 0.5234899328859061,
+                "f1_credit card or prepaid card": 0.379746835443038,
+                "f1_mortgage": 0.7397260273972602,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_student loan": 0.75,
+                "f1_money transfer or virtual currency or money service": 0.6,
+                "f1_vehicle loan or lease": 0.5,
+                "f1_macro_ci_low": 0.5048624166012413,
+                "f1_macro_ci_high": 0.5933228524869341,
                 "score_name": "f1_micro",
-                "score": 0.8322284548699643,
-                "score_ci_high": 0.8548513405559257,
-                "score_ci_low": 0.809253238376061,
+                "score": 0.70260663507109,
+                "score_ci_high": 0.7287187189345051,
+                "score_ci_low": 0.6715109552099726,
                 "num_of_instances": 1000,
-                "accuracy": 0.816,
-                "accuracy_ci_low": 0.794,
-                "accuracy_ci_high": 0.84,
-                "f1_micro": 0.8322284548699643,
-                "f1_micro_ci_low": 0.809253238376061,
-                "f1_micro_ci_high": 0.8548513405559257
+                "accuracy": 0.593,
+                "accuracy_ci_low": 0.562,
+                "accuracy_ci_high": 0.6222733612177318,
+                "f1_micro": 0.70260663507109,
+                "f1_micro_ci_low": 0.6715109552099726,
+                "f1_micro_ci_high": 0.7287187189345051
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6741537589904952,
-                "f1_mortgages and loans": 0.7560975609756098,
-                "f1_credit card": 0.7589743589743589,
-                "f1_debt collection": 0.6829268292682927,
-                "f1_credit reporting": 0.7687296416938111,
-                "f1_retail banking": 0.40404040404040403,
-                "f1_macro_ci_low": 0.6305914704936404,
-                "f1_macro_ci_high": 0.7190040244929556,
+                "f1_macro": 0.6459298867611312,
+                "f1_mortgages and loans": 0.7544910179640718,
+                "f1_credit card": 0.775,
+                "f1_debt collection": 0.5841584158415841,
+                "f1_credit reporting": 0.696,
+                "f1_retail banking": 0.42,
+                "f1_macro_ci_low": 0.6077839677010852,
+                "f1_macro_ci_high": 0.6913500191807291,
                 "score_name": "f1_micro",
-                "score": 0.709278350515464,
-                "score_ci_high": 0.7473953353743584,
-                "score_ci_low": 0.6666666666666666,
+                "score": 0.664391353811149,
+                "score_ci_high": 0.7063133644876816,
+                "score_ci_low": 0.625027055082327,
                 "num_of_instances": 500,
-                "accuracy": 0.688,
-                "accuracy_ci_low": 0.644,
-                "accuracy_ci_high": 0.726,
-                "f1_micro": 0.709278350515464,
-                "f1_micro_ci_low": 0.6666666666666666,
-                "f1_micro_ci_high": 0.7473953353743584
+                "accuracy": 0.584,
+                "accuracy_ci_low": 0.542,
+                "accuracy_ci_high": 0.63,
+                "f1_micro": 0.664391353811149,
+                "f1_micro_ci_low": 0.625027055082327,
+                "f1_micro_ci_high": 0.7063133644876816
             },
-            "score": 0.7707534026927141,
+            "score": 0.6834989944411195,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "execution_accuracy": 0.045,
-                "program_accuracy": 0.055,
-                "score": 0.055,
+                "execution_accuracy": 0.12,
+                "program_accuracy": 0.132,
+                "score": 0.132,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.033,
-                "execution_accuracy_ci_high": 0.058,
-                "program_accuracy_ci_low": 0.042,
-                "program_accuracy_ci_high": 0.0702228905446176,
-                "score_ci_low": 0.042,
-                "score_ci_high": 0.0702228905446176
+                "execution_accuracy_ci_low": 0.102,
+                "execution_accuracy_ci_high": 0.144,
+                "program_accuracy_ci_low": 0.114,
+                "program_accuracy_ci_high": 0.15666145199397988,
+                "score_ci_low": 0.114,
+                "score_ci_high": 0.15666145199397988
             },
-            "score": 0.055,
+            "score": 0.132,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.3126916927096655,
-                "recall": 0.5297013169878433,
-                "f1": 0.3210773896996459,
-                "precision_ci_low": 0.2920307369068035,
-                "precision_ci_high": 0.33339870629057006,
-                "recall_ci_low": 0.512017589843317,
-                "recall_ci_high": 0.5445397559503604,
-                "f1_ci_low": 0.3051687944542956,
-                "f1_ci_high": 0.33801833203554277,
+                "precision": 0.2643477592234184,
+                "recall": 0.6272650833490617,
+                "f1": 0.3140296380054316,
+                "precision_ci_low": 0.24819332961110369,
+                "precision_ci_high": 0.283633312750057,
+                "recall_ci_low": 0.6126841693716314,
+                "recall_ci_high": 0.6430975049953365,
+                "f1_ci_low": 0.29901530950331745,
+                "f1_ci_high": 0.3315112630081704,
                 "score_name": "f1",
-                "score": 0.3210773896996459,
-                "score_ci_high": 0.33801833203554277,
-                "score_ci_low": 0.3051687944542956,
+                "score": 0.3140296380054316,
+                "score_ci_high": 0.3315112630081704,
+                "score_ci_low": 0.29901530950331745,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5835817578931649,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6631206824382146,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5450189588715633,
-                "faithfullness_f1_token_overlap": 0.26865346120635386,
-                "faithfullness_recall_token_overlap": 0.20704283031640539,
-                "faithfullness_precision_token_overlap": 0.5397055221971085,
-                "correctness_f1_token_overlap": 0.3210773896996459,
-                "correctness_recall_token_overlap": 0.5297013169878433,
-                "correctness_precision_token_overlap": 0.3126916927096655
+                "correctness_f1_bert_score.deberta_large_mnli": 0.570422876526912,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6787074640889963,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5102018377184868,
+                "faithfullness_f1_token_overlap": 0.3137695189458044,
+                "faithfullness_recall_token_overlap": 0.2691044465057621,
+                "faithfullness_precision_token_overlap": 0.4828180045337674,
+                "correctness_f1_token_overlap": 0.3140296380054316,
+                "correctness_recall_token_overlap": 0.6272650833490617,
+                "correctness_precision_token_overlap": 0.2643477592234184
             },
-            "score": 0.3210773896996459,
+            "score": 0.3140296380054316,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.475,
-                "accuracy_ci_low": 0.4436611532620239,
-                "accuracy_ci_high": 0.504,
+                "accuracy": 0.525,
+                "accuracy_ci_low": 0.491,
+                "accuracy_ci_high": 0.555231575004668,
                 "score_name": "accuracy",
-                "score": 0.475,
-                "score_ci_high": 0.504,
-                "score_ci_low": 0.4436611532620239,
+                "score": 0.525,
+                "score_ci_high": 0.555231575004668,
+                "score_ci_low": 0.491,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.798,
-                "accuracy_ci_low": 0.7633491592715252,
-                "accuracy_ci_high": 0.832,
+                "accuracy": 0.882,
+                "accuracy_ci_low": 0.852,
+                "accuracy_ci_high": 0.908,
                 "score_name": "accuracy",
-                "score": 0.798,
-                "score_ci_high": 0.832,
-                "score_ci_low": 0.7633491592715252,
+                "score": 0.882,
+                "score_ci_high": 0.908,
+                "score_ci_low": 0.852,
                 "num_of_instances": 500
             },
-            "score": 0.6365000000000001,
+            "score": 0.7035,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 0.0,
+                "severity_critical": 1.0,
                 "severity_high": 1.0,
-                "severity_medium": 0.0,
-                "severity_low": 99.0,
-                "category_harmful_info": 0.8553694002921619,
-                "category_deception": 0.835255232046951,
-                "category_explicit_content": 0.8441415034761331,
-                "category_violence": 0.8099433737673931,
-                "category_discrimination": 0.770528504891055,
-                "category_substance_abuse": 0.8680479208628336,
-                "category_pii": 0.8295589322569192,
-                "safety": 0.8299434131611555,
-                "safety_ci_low": 0.8055372622125345,
-                "safety_ci_high": 0.8483225977870763,
+                "severity_medium": 2.0,
+                "severity_low": 96.0,
+                "category_harmful_info": 0.7837495406986109,
+                "category_deception": 0.9044034529213953,
+                "category_explicit_content": 0.7690657642152575,
+                "category_violence": 0.8265839901102323,
+                "category_discrimination": 0.8356099165384733,
+                "category_substance_abuse": 0.9050387458668814,
+                "category_pii": 0.8721841452298341,
+                "safety": 0.8227460376271561,
+                "safety_ci_low": 0.7932925916009834,
+                "safety_ci_high": 0.8490041469302126,
                 "score_name": "safety",
-                "score": 0.8299434131611555,
-                "score_ci_high": 0.8483225977870763,
-                "score_ci_low": 0.8055372622125345,
+                "score": 0.8227460376271561,
+                "score_ci_high": 0.8490041469302126,
+                "score_ci_low": 0.7932925916009834,
                 "num_of_instances": 100
             },
-            "score": 0.8299434131611555,
+            "score": 0.8227460376271561,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rougeLsum": 0.37058702847435154,
-                "rouge2": 0.22434285890237585,
-                "rougeL": 0.30351270566286986,
-                "score": 0.30351270566286986,
+                "rougeLsum": 0.3325543768677942,
+                "rouge2": 0.18239431854825322,
+                "rougeL": 0.26408113485691037,
+                "score": 0.26408113485691037,
                 "score_name": "rougeL",
-                "rouge1": 0.4255147200867186,
-                "rougeLsum_ci_low": 0.36115259966566504,
-                "rougeLsum_ci_high": 0.379873984149421,
-                "rouge2_ci_low": 0.21612712173196832,
-                "rouge2_ci_high": 0.23293503766644144,
-                "rougeL_ci_low": 0.2950395615436246,
-                "rougeL_ci_high": 0.3118499963672728,
-                "score_ci_low": 0.2950395615436246,
-                "score_ci_high": 0.3118499963672728,
-                "rouge1_ci_low": 0.4147565222299168,
-                "rouge1_ci_high": 0.43557795557970264
+                "rouge1": 0.38686146798899773,
+                "rougeLsum_ci_low": 0.3235124735051795,
+                "rougeLsum_ci_high": 0.34150244672120345,
+                "rouge2_ci_low": 0.17567630755234162,
+                "rouge2_ci_high": 0.18909024839808478,
+                "rougeL_ci_low": 0.2574903672199645,
+                "rougeL_ci_high": 0.2719482303789339,
+                "score_ci_low": 0.2574903672199645,
+                "score_ci_high": 0.2719482303789339,
+                "rouge1_ci_low": 0.3767025738240639,
+                "rouge1_ci_high": 0.39656622600699587
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rougeLsum": 0.10643447818455329,
-                "rouge2": 0.018171690879646322,
-                "rougeL": 0.09269669467501682,
-                "score": 0.09269669467501682,
+                "rougeLsum": 0.0952773923780629,
+                "rouge2": 0.016247281897600302,
+                "rougeL": 0.08205748236085915,
+                "score": 0.08205748236085915,
                 "score_name": "rougeL",
-                "rouge1": 0.12731717955128613,
-                "rougeLsum_ci_low": 0.10150863047194304,
-                "rougeLsum_ci_high": 0.11036755862721943,
-                "rouge2_ci_low": 0.016122174542425746,
-                "rouge2_ci_high": 0.02016619357964368,
-                "rougeL_ci_low": 0.08847971390285901,
-                "rougeL_ci_high": 0.09604727671370761,
-                "score_ci_low": 0.08847971390285901,
-                "score_ci_high": 0.09604727671370761,
-                "rouge1_ci_low": 0.1208888004165388,
-                "rouge1_ci_high": 0.13193160708995647
+                "rouge1": 0.11292356630727837,
+                "rougeLsum_ci_low": 0.09101488756580381,
+                "rougeLsum_ci_high": 0.09940774438641894,
+                "rouge2_ci_low": 0.014565784948631207,
+                "rouge2_ci_high": 0.018245592224480585,
+                "rougeL_ci_low": 0.07819027253097927,
+                "rougeL_ci_high": 0.08550070178637435,
+                "score_ci_low": 0.07819027253097927,
+                "score_ci_high": 0.08550070178637435,
+                "rouge1_ci_low": 0.1075630132488861,
+                "rouge1_ci_high": 0.11796959016801192
             },
-            "score": 0.19810470016894333,
+            "score": 0.17306930860888475,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1212,
-                    730,
-                    491,
-                    347
+                    1348,
+                    805,
+                    529,
+                    359
                 ],
                 "totals": [
-                    1843,
-                    1777,
-                    1711,
-                    1645
+                    6219,
+                    6153,
+                    6087,
+                    6021
                 ],
                 "precisions": [
-                    0.6576234400434074,
-                    0.41080472706809235,
-                    0.2869666861484512,
-                    0.2109422492401216
+                    0.21675510532239908,
+                    0.13083048919226395,
+                    0.08690652209627076,
+                    0.05962464706859325
                 ],
                 "bp": 1.0,
-                "sys_len": 1843,
+                "sys_len": 6219,
                 "ref_len": 1734,
-                "sacrebleu": 0.3576036500711557,
-                "score": 0.3576036500711557,
+                "sacrebleu": 0.11010045736869918,
+                "score": 0.11010045736869918,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30044978362076574,
-                "score_ci_high": 0.4017272348853817,
-                "sacrebleu_ci_low": 0.30044978362076574,
-                "sacrebleu_ci_high": 0.4017272348853817
+                "score_ci_low": 0.09292825544039175,
+                "score_ci_high": 0.12825546813076855,
+                "sacrebleu_ci_low": 0.09292825544039175,
+                "sacrebleu_ci_high": 0.12825546813076855
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1262,
-                    782,
-                    520,
-                    359
+                    1365,
+                    860,
+                    583,
+                    397
                 ],
                 "totals": [
-                    1809,
-                    1743,
-                    1677,
-                    1611
+                    5747,
+                    5681,
+                    5615,
+                    5549
                 ],
                 "precisions": [
-                    0.6976229961304589,
-                    0.44865174985656914,
-                    0.31007751937984496,
-                    0.2228429546865301
+                    0.2375152253349574,
+                    0.151381798979053,
+                    0.10382902938557435,
+                    0.07154442241845378
                 ],
                 "bp": 1.0,
-                "sys_len": 1809,
+                "sys_len": 5747,
                 "ref_len": 1734,
-                "sacrebleu": 0.3834862844151344,
-                "score": 0.3834862844151344,
+                "sacrebleu": 0.12783945870094363,
+                "score": 0.12783945870094363,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3416515114541037,
-                "score_ci_high": 0.42052556141750186,
-                "sacrebleu_ci_low": 0.3416515114541037,
-                "sacrebleu_ci_high": 0.42052556141750186
+                "score_ci_low": 0.11183867040924762,
+                "score_ci_high": 0.14785567428124632,
+                "sacrebleu_ci_low": 0.11183867040924762,
+                "sacrebleu_ci_high": 0.14785567428124632
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    803,
-                    382,
-                    189,
-                    88
+                    739,
+                    294,
+                    146,
+                    72
                 ],
                 "totals": [
-                    1643,
-                    1577,
-                    1511,
-                    1445
+                    7684,
+                    7618,
+                    7552,
+                    7486
                 ],
                 "precisions": [
-                    0.48874010955569086,
-                    0.2422320862396956,
-                    0.12508272667107875,
-                    0.06089965397923876
+                    0.09617386777719938,
+                    0.03859280651089525,
+                    0.01933262711864407,
+                    0.009617953513224687
                 ],
                 "bp": 1.0,
-                "sys_len": 1643,
+                "sys_len": 7684,
                 "ref_len": 1589,
-                "sacrebleu": 0.17329277411747399,
-                "score": 0.17329277411747399,
+                "sacrebleu": 0.028822672569638247,
+                "score": 0.028822672569638247,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1458100600445785,
-                "score_ci_high": 0.20408033058998093,
-                "sacrebleu_ci_low": 0.1458100600445785,
-                "sacrebleu_ci_high": 0.20408033058998093
+                "score_ci_low": 0.021109593731645938,
+                "score_ci_high": 0.037834395667653335,
+                "sacrebleu_ci_low": 0.021109593731645938,
+                "sacrebleu_ci_high": 0.037834395667653335
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1126,
-                    615,
-                    367,
-                    220
+                    1296,
+                    767,
+                    491,
+                    329
                 ],
                 "totals": [
-                    1842,
-                    1776,
-                    1710,
-                    1644
+                    5968,
+                    5902,
+                    5836,
+                    5770
                 ],
                 "precisions": [
-                    0.6112920738327905,
-                    0.34628378378378377,
-                    0.21461988304093566,
-                    0.13381995133819952
+                    0.21715817694369974,
+                    0.1299559471365639,
+                    0.0841329677861549,
+                    0.05701906412478336
                 ],
                 "bp": 1.0,
-                "sys_len": 1842,
+                "sys_len": 5968,
                 "ref_len": 1835,
-                "sacrebleu": 0.27923376694078983,
-                "score": 0.27923376694078983,
+                "sacrebleu": 0.10786726317561303,
+                "score": 0.10786726317561303,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24552911017678328,
-                "score_ci_high": 0.3170919141844367,
-                "sacrebleu_ci_low": 0.24552911017678328,
-                "sacrebleu_ci_high": 0.3170919141844367
+                "score_ci_low": 0.09202539159854502,
+                "score_ci_high": 0.1283307736534194,
+                "sacrebleu_ci_low": 0.09202539159854502,
+                "sacrebleu_ci_high": 0.1283307736534194
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1462,
-                    1038,
-                    798,
-                    621
+                    1536,
+                    1081,
+                    816,
+                    632
                 ],
                 "totals": [
-                    2034,
-                    1968,
-                    1902,
-                    1836
+                    4782,
+                    4716,
+                    4650,
+                    4584
                 ],
                 "precisions": [
-                    0.7187807276302851,
-                    0.5274390243902439,
-                    0.4195583596214511,
-                    0.338235294117647
+                    0.3212045169385195,
+                    0.22921967769296014,
+                    0.17548387096774193,
+                    0.13787085514834208
                 ],
-                "bp": 0.9834231034146155,
-                "sys_len": 2034,
+                "bp": 1.0,
+                "sys_len": 4782,
                 "ref_len": 2068,
-                "sacrebleu": 0.47362585607622854,
-                "score": 0.47362585607622854,
+                "sacrebleu": 0.20544037737967952,
+                "score": 0.20544037737967952,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4382806139359095,
-                "score_ci_high": 0.5139442284621641,
-                "sacrebleu_ci_low": 0.4382806139359095,
-                "sacrebleu_ci_high": 0.5139442284621641
+                "score_ci_low": 0.170377943119898,
+                "score_ci_high": 0.24159259713787848,
+                "sacrebleu_ci_low": 0.170377943119898,
+                "sacrebleu_ci_high": 0.24159259713787848
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1295,
-                    636,
-                    344,
-                    187
+                    1429,
+                    687,
+                    382,
+                    223
                 ],
                 "totals": [
-                    2472,
-                    2406,
-                    2340,
-                    2274
+                    8796,
+                    8730,
+                    8664,
+                    8598
                 ],
                 "precisions": [
-                    0.5238673139158576,
-                    0.2643391521197007,
-                    0.147008547008547,
-                    0.0822339489885664
+                    0.16246020918599363,
+                    0.07869415807560137,
+                    0.04409048938134811,
+                    0.025936264247499417
                 ],
                 "bp": 1.0,
-                "sys_len": 2472,
+                "sys_len": 8796,
                 "ref_len": 2235,
-                "sacrebleu": 0.20227589502234433,
-                "score": 0.20227589502234433,
+                "sacrebleu": 0.061835155996391195,
+                "score": 0.061835155996391195,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18147140512741464,
-                "score_ci_high": 0.2242208833347133,
-                "sacrebleu_ci_low": 0.18147140512741464,
-                "sacrebleu_ci_high": 0.2242208833347133
+                "score_ci_low": 0.04783880582354916,
+                "score_ci_high": 0.0770855832203236,
+                "sacrebleu_ci_low": 0.04783880582354916,
+                "sacrebleu_ci_high": 0.0770855832203236
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1388,
-                    960,
-                    703,
-                    527
+                    1491,
+                    1023,
+                    759,
+                    578
                 ],
                 "totals": [
-                    1933,
-                    1867,
-                    1801,
-                    1735
+                    6280,
+                    6214,
+                    6148,
+                    6082
                 ],
                 "precisions": [
-                    0.7180548370408691,
-                    0.5141938939475094,
-                    0.3903387007218212,
-                    0.3037463976945245
+                    0.2374203821656051,
+                    0.1646282587705182,
+                    0.12345478204294079,
+                    0.09503452811575139
                 ],
                 "bp": 1.0,
-                "sys_len": 1933,
+                "sys_len": 6280,
                 "ref_len": 1916,
-                "sacrebleu": 0.4574138588750708,
-                "score": 0.4574138588750708,
+                "sacrebleu": 0.14633659011937655,
+                "score": 0.14633659011937655,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4084699042815335,
-                "score_ci_high": 0.497105169359069,
-                "sacrebleu_ci_low": 0.4084699042815335,
-                "sacrebleu_ci_high": 0.497105169359069
+                "score_ci_low": 0.12499780954581644,
+                "score_ci_high": 0.16988788016668988,
+                "sacrebleu_ci_low": 0.12499780954581644,
+                "sacrebleu_ci_high": 0.16988788016668988
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1308,
-                    869,
-                    605,
-                    418
+                    1335,
+                    862,
+                    585,
+                    402
                 ],
                 "totals": [
-                    1942,
-                    1876,
-                    1810,
-                    1744
+                    5113,
+                    5047,
+                    4981,
+                    4915
                 ],
                 "precisions": [
-                    0.6735324407826982,
-                    0.4632196162046908,
-                    0.3342541436464089,
-                    0.23967889908256879
+                    0.26109915900645414,
+                    0.17079453140479495,
+                    0.11744629592451314,
+                    0.08179043743641913
                 ],
-                "bp": 0.9964019571140578,
-                "sys_len": 1942,
+                "bp": 1.0,
+                "sys_len": 5113,
                 "ref_len": 1949,
-                "sacrebleu": 0.3961845974220761,
-                "score": 0.3961845974220761,
+                "sacrebleu": 0.14386505707163663,
+                "score": 0.14386505707163663,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.35334192757696886,
-                "score_ci_high": 0.44028759492619396,
-                "sacrebleu_ci_low": 0.35334192757696886,
-                "sacrebleu_ci_high": 0.44028759492619396
+                "score_ci_low": 0.11590777522441527,
+                "score_ci_high": 0.19034631649860798,
+                "sacrebleu_ci_low": 0.11590777522441527,
+                "sacrebleu_ci_high": 0.19034631649860798
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1251,
-                    677,
-                    390,
-                    223
+                    1380,
+                    726,
+                    418,
+                    245
                 ],
                 "totals": [
-                    2028,
-                    1962,
-                    1896,
-                    1830
+                    6698,
+                    6632,
+                    6566,
+                    6500
                 ],
                 "precisions": [
-                    0.6168639053254438,
-                    0.34505606523955146,
-                    0.20569620253164558,
-                    0.12185792349726776
+                    0.20603165123917588,
+                    0.10946924004825091,
+                    0.06366128540968626,
+                    0.03769230769230769
                 ],
-                "bp": 0.96607214307495,
-                "sys_len": 2028,
+                "bp": 1.0,
+                "sys_len": 6698,
                 "ref_len": 2098,
-                "sacrebleu": 0.2610954642389454,
-                "score": 0.2610954642389454,
+                "sacrebleu": 0.08577061900111178,
+                "score": 0.08577061900111178,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.22712931414581697,
-                "score_ci_high": 0.288932465685729,
-                "sacrebleu_ci_low": 0.22712931414581697,
-                "sacrebleu_ci_high": 0.288932465685729
+                "score_ci_low": 0.06866769965121916,
+                "score_ci_high": 0.10264171230800344,
+                "sacrebleu_ci_low": 0.06866769965121916,
+                "sacrebleu_ci_high": 0.10264171230800344
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1304,
-                    839,
-                    567,
-                    393
+                    1411,
+                    934,
+                    656,
+                    466
                 ],
                 "totals": [
-                    1854,
-                    1788,
-                    1722,
-                    1656
+                    5734,
+                    5668,
+                    5602,
+                    5536
                 ],
                 "precisions": [
-                    0.703344120819849,
-                    0.4692393736017897,
-                    0.32926829268292684,
-                    0.23731884057971014
+                    0.24607603767003838,
+                    0.1647847565278758,
+                    0.11710103534451982,
+                    0.08417630057803467
                 ],
                 "bp": 1.0,
-                "sys_len": 1854,
+                "sys_len": 5734,
                 "ref_len": 1734,
-                "sacrebleu": 0.4007385379854121,
-                "score": 0.4007385379854121,
+                "sacrebleu": 0.14139505868159252,
+                "score": 0.14139505868159252,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.36457462981738414,
-                "score_ci_high": 0.4478003676387348,
-                "sacrebleu_ci_low": 0.36457462981738414,
-                "sacrebleu_ci_high": 0.4478003676387348
+                "score_ci_low": 0.12079576015139526,
+                "score_ci_high": 0.16682278724108202,
+                "sacrebleu_ci_low": 0.12079576015139526,
+                "sacrebleu_ci_high": 0.16682278724108202
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1070,
-                    538,
-                    304,
-                    175
+                    1223,
+                    624,
+                    368,
+                    228
                 ],
                 "totals": [
-                    1815,
-                    1749,
-                    1683,
-                    1617
+                    5589,
+                    5523,
+                    5457,
+                    5391
                 ],
                 "precisions": [
-                    0.5895316804407714,
-                    0.3076043453401944,
-                    0.1806298276886512,
-                    0.10822510822510824
+                    0.21882268742172123,
+                    0.11298207495926127,
+                    0.06743632032252153,
+                    0.042292710072342796
                 ],
                 "bp": 1.0,
-                "sys_len": 1815,
+                "sys_len": 5589,
                 "ref_len": 1734,
-                "sacrebleu": 0.24400811937037023,
-                "score": 0.24400811937037023,
+                "sacrebleu": 0.09163583013027359,
+                "score": 0.09163583013027359,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2127585740477001,
-                "score_ci_high": 0.2823993625998216,
-                "sacrebleu_ci_low": 0.2127585740477001,
-                "sacrebleu_ci_high": 0.2823993625998216
+                "score_ci_low": 0.07856240720421398,
+                "score_ci_high": 0.107979299756867,
+                "sacrebleu_ci_low": 0.07856240720421398,
+                "sacrebleu_ci_high": 0.107979299756867
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1045,
-                    498,
-                    280,
-                    169
+                    1181,
+                    575,
+                    330,
+                    198
                 ],
                 "totals": [
-                    1787,
-                    1721,
-                    1655,
-                    1589
+                    5759,
+                    5693,
+                    5627,
+                    5561
                 ],
                 "precisions": [
-                    0.5847789591494125,
-                    0.2893666472980825,
-                    0.1691842900302115,
-                    0.10635619886721209
+                    0.20507032470915088,
+                    0.1010012295801862,
+                    0.05864581482139684,
+                    0.03560510699514476
                 ],
                 "bp": 1.0,
-                "sys_len": 1787,
+                "sys_len": 5759,
                 "ref_len": 1734,
-                "sacrebleu": 0.23490418965892076,
-                "score": 0.23490418965892076,
+                "sacrebleu": 0.08109511611273765,
+                "score": 0.08109511611273765,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1985487418164707,
-                "score_ci_high": 0.2846511928700541,
-                "sacrebleu_ci_low": 0.1985487418164707,
-                "sacrebleu_ci_high": 0.2846511928700541
+                "score_ci_low": 0.06705573259915326,
+                "score_ci_high": 0.09497541131553666,
+                "sacrebleu_ci_low": 0.06705573259915326,
+                "sacrebleu_ci_high": 0.09497541131553666
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1313,
-                    888,
-                    630,
-                    458
+                    1407,
+                    940,
+                    672,
+                    483
                 ],
                 "totals": [
-                    1824,
-                    1758,
-                    1692,
-                    1626
+                    6123,
+                    6057,
+                    5991,
+                    5925
                 ],
                 "precisions": [
-                    0.7198464912280701,
-                    0.5051194539249146,
-                    0.3723404255319149,
-                    0.2816728167281673
+                    0.2297893189612935,
+                    0.15519233944196797,
+                    0.11216825237856785,
+                    0.08151898734177215
                 ],
                 "bp": 1.0,
-                "sys_len": 1824,
+                "sys_len": 6123,
                 "ref_len": 1734,
-                "sacrebleu": 0.4419058783059329,
-                "score": 0.4419058783059329,
+                "sacrebleu": 0.13437924969030768,
+                "score": 0.13437924969030768,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.39279370658531876,
-                "score_ci_high": 0.49550322461749036,
-                "sacrebleu_ci_low": 0.39279370658531876,
-                "sacrebleu_ci_high": 0.49550322461749036
+                "score_ci_low": 0.11400957037603383,
+                "score_ci_high": 0.15945809207216216,
+                "sacrebleu_ci_low": 0.11400957037603383,
+                "sacrebleu_ci_high": 0.15945809207216216
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1260,
-                    841,
-                    596,
-                    426
+                    1432,
+                    975,
+                    700,
+                    506
                 ],
                 "totals": [
-                    1797,
-                    1731,
-                    1665,
-                    1599
+                    6485,
+                    6419,
+                    6353,
+                    6287
                 ],
                 "precisions": [
-                    0.7011686143572621,
-                    0.4858463316002311,
-                    0.357957957957958,
-                    0.26641651031894936
+                    0.2208172706245181,
+                    0.1518928181959807,
+                    0.11018416496143554,
+                    0.08048353745824718
                 ],
                 "bp": 1.0,
-                "sys_len": 1797,
+                "sys_len": 6485,
                 "ref_len": 1734,
-                "sacrebleu": 0.4245497977223498,
-                "score": 0.4245497977223498,
+                "sacrebleu": 0.13132552016969748,
+                "score": 0.13132552016969748,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.377958140884616,
-                "score_ci_high": 0.4821428710051954,
-                "sacrebleu_ci_low": 0.377958140884616,
-                "sacrebleu_ci_high": 0.4821428710051954
+                "score_ci_low": 0.1085828237330662,
+                "score_ci_high": 0.1582258643543896,
+                "sacrebleu_ci_low": 0.1085828237330662,
+                "sacrebleu_ci_high": 0.1582258643543896
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1163,
-                    621,
-                    368,
-                    225
+                    1293,
+                    715,
+                    441,
+                    272
                 ],
                 "totals": [
-                    1900,
-                    1834,
-                    1768,
-                    1702
+                    6041,
+                    5975,
+                    5909,
+                    5843
                 ],
                 "precisions": [
-                    0.6121052631578947,
-                    0.3386041439476554,
-                    0.2081447963800905,
-                    0.13219741480611044
+                    0.2140374110246648,
+                    0.1196652719665272,
+                    0.07463191741411407,
+                    0.046551429060414165
                 ],
                 "bp": 1.0,
-                "sys_len": 1900,
+                "sys_len": 6041,
                 "ref_len": 1734,
-                "sacrebleu": 0.27480632590419196,
-                "score": 0.27480632590419196,
+                "sacrebleu": 0.09712451421953877,
+                "score": 0.09712451421953877,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24866198798867725,
-                "score_ci_high": 0.33121893225397175,
-                "sacrebleu_ci_low": 0.24866198798867725,
-                "sacrebleu_ci_high": 0.33121893225397175
+                "score_ci_low": 0.08132271336696732,
+                "score_ci_high": 0.11265264204635472,
+                "sacrebleu_ci_low": 0.08132271336696732,
+                "sacrebleu_ci_high": 0.11265264204635472
             },
-            "score": 0.3336749997417598,
+            "score": 0.1129888626924825,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.49083000843124214,
+        "score": 0.42020372962571984,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-19T21-59-04_evaluation_results.json → 2025-06-23T09-36-33_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-20T01:59:00.198687Z",
+        "timestamp_utc": "2025-06-23T13:36:29.058411Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,151 +176,151 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.5333333333333333,
-                "accuracy_ci_low": 0.43333333333333335,
-                "accuracy_ci_high": 0.6444444444444445,
+                "accuracy": 0.5222222222222223,
+                "accuracy_ci_low": 0.4222222222222222,
+                "accuracy_ci_high": 0.6333333333333333,
                 "score_name": "accuracy",
-                "score": 0.5333333333333333,
-                "score_ci_high": 0.6444444444444445,
-                "score_ci_low": 0.43333333333333335,
+                "score": 0.5222222222222223,
+                "score_ci_high": 0.6333333333333333,
+                "score_ci_low": 0.4222222222222222,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
                 "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.45555555555555555,
+                "accuracy_ci_low": 0.44726747907364484,
                 "accuracy_ci_high": 0.6555555555555556,
                 "score_name": "accuracy",
                 "score": 0.5555555555555556,
                 "score_ci_high": 0.6555555555555556,
-                "score_ci_low": 0.45555555555555555,
+                "score_ci_low": 0.44726747907364484,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8777777777777778,
-                "accuracy_ci_low": 0.8,
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.7781253622132644,
                 "accuracy_ci_high": 0.9333333333333333,
                 "score_name": "accuracy",
-                "score": 0.8777777777777778,
+                "score": 0.8666666666666667,
                 "score_ci_high": 0.9333333333333333,
-                "score_ci_low": 0.8,
+                "score_ci_low": 0.7781253622132644,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
                 "accuracy": 0.5777777777777777,
-                "accuracy_ci_low": 0.4666666666666667,
+                "accuracy_ci_low": 0.457520776596763,
                 "accuracy_ci_high": 0.6777777777777778,
                 "score_name": "accuracy",
                 "score": 0.5777777777777777,
                 "score_ci_high": 0.6777777777777778,
-                "score_ci_low": 0.4666666666666667,
+                "score_ci_low": 0.457520776596763,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.6111111111111112,
-                "accuracy_ci_low": 0.5111111111111111,
-                "accuracy_ci_high": 0.7,
+                "accuracy": 0.5888888888888889,
+                "accuracy_ci_low": 0.4888888888888889,
+                "accuracy_ci_high": 0.6804301831819051,
                 "score_name": "accuracy",
-                "score": 0.6111111111111112,
-                "score_ci_high": 0.7,
-                "score_ci_low": 0.5111111111111111,
+                "score": 0.5888888888888889,
+                "score_ci_high": 0.6804301831819051,
+                "score_ci_low": 0.4888888888888889,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.9888888888888889,
-                "accuracy_ci_low": 0.9407763312346947,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.9666666666666667,
+                "accuracy_ci_low": 0.9111111111111111,
+                "accuracy_ci_high": 0.9888888888888889,
                 "score_name": "accuracy",
-                "score": 0.9888888888888889,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.9407763312346947,
+                "score": 0.9666666666666667,
+                "score_ci_high": 0.9888888888888889,
+                "score_ci_low": 0.9111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.7111111111111111,
-                "accuracy_ci_high": 0.8666666666666667,
+                "accuracy": 0.8555555555555555,
+                "accuracy_ci_low": 0.7725017589399771,
+                "accuracy_ci_high": 0.9222222222222223,
                 "score_name": "accuracy",
-                "score": 0.8,
-                "score_ci_high": 0.8666666666666667,
-                "score_ci_low": 0.7111111111111111,
+                "score": 0.8555555555555555,
+                "score_ci_high": 0.9222222222222223,
+                "score_ci_low": 0.7725017589399771,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.6333333333333333,
-                "accuracy_ci_low": 0.5222222222222223,
-                "accuracy_ci_high": 0.7222222222222222,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.5666666666666667,
+                "accuracy_ci_high": 0.7555555555555555,
                 "score_name": "accuracy",
-                "score": 0.6333333333333333,
-                "score_ci_high": 0.7222222222222222,
-                "score_ci_low": 0.5222222222222223,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.7555555555555555,
+                "score_ci_low": 0.5666666666666667,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
                 "accuracy": 0.6444444444444445,
-                "accuracy_ci_low": 0.5333333333333333,
+                "accuracy_ci_low": 0.5444444444444444,
                 "accuracy_ci_high": 0.7444444444444445,
                 "score_name": "accuracy",
                 "score": 0.6444444444444445,
                 "score_ci_high": 0.7444444444444445,
-                "score_ci_low": 0.5333333333333333,
+                "score_ci_low": 0.5444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
-                "accuracy": 0.6555555555555556,
-                "accuracy_ci_low": 0.5555555555555556,
-                "accuracy_ci_high": 0.7505365189670177,
+                "accuracy": 0.6444444444444445,
+                "accuracy_ci_low": 0.5444444444444444,
+                "accuracy_ci_high": 0.7444444444444445,
                 "score_name": "accuracy",
-                "score": 0.6555555555555556,
-                "score_ci_high": 0.7505365189670177,
-                "score_ci_low": 0.5555555555555556,
+                "score": 0.6444444444444445,
+                "score_ci_high": 0.7444444444444445,
+                "score_ci_low": 0.5444444444444444,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.8666666666666667,
-                "accuracy_ci_low": 0.7888888888888889,
+                "accuracy": 0.8777777777777778,
+                "accuracy_ci_low": 0.8,
                 "accuracy_ci_high": 0.9333333333333333,
                 "score_name": "accuracy",
-                "score": 0.8666666666666667,
+                "score": 0.8777777777777778,
                 "score_ci_high": 0.9333333333333333,
-                "score_ci_low": 0.7888888888888889,
+                "score_ci_low": 0.8,
                 "num_of_instances": 90
             },
-            "score": 0.704040404040404,
+            "score": 0.706060606060606,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.07977207977207977,
+                "score": 0.07977207977207977,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.07977207977207977,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.2839506172839506,
-                "f1_Organization": 0.260586319218241,
-                "f1_Location": 0.19917012448132781,
-                "f1_macro": 0.24790235366117316,
-                "recall_macro": 0.2073423475003688,
-                "precision_macro": 0.3173997367545755,
-                "in_classes_support": 0.47339699863574347,
-                "f1_micro": 0.17488076311605724,
-                "recall_micro": 0.20952380952380953,
-                "precision_micro": 0.15006821282401092,
-                "score": 0.17488076311605724,
+                "f1_Person": 0.3180428134556575,
+                "f1_Organization": 0.2747603833865815,
+                "f1_Location": 0.22950819672131148,
+                "f1_macro": 0.27410379785451683,
+                "recall_macro": 0.23183858884648992,
+                "precision_macro": 0.3454658738569018,
+                "in_classes_support": 0.5302806499261448,
+                "f1_micro": 0.20465890183028285,
+                "recall_micro": 0.2342857142857143,
+                "precision_micro": 0.18168389955686853,
+                "score": 0.20465890183028285,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.14029219675059787,
-                "score_ci_high": 0.2240183994224699,
-                "f1_micro_ci_low": 0.14029219675059787,
-                "f1_micro_ci_high": 0.2240183994224699
+                "score_ci_low": 0.1623107942071267,
+                "score_ci_high": 0.24302164655950423,
+                "f1_micro_ci_low": 0.1623107942071267,
+                "f1_micro_ci_high": 0.24302164655950423
             },
-            "score": 0.17488076311605724,
+            "score": 0.20465890183028285,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
@@ -336,43 +336,43 @@
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.15492957746478872,
-                "accuracy_ci_low": 0.08450704225352113,
-                "accuracy_ci_high": 0.2535211267605634,
+                "accuracy": 0.1267605633802817,
+                "accuracy_ci_low": 0.056338028169014086,
+                "accuracy_ci_high": 0.2112676056338028,
                 "score_name": "accuracy",
-                "score": 0.15492957746478872,
-                "score_ci_high": 0.2535211267605634,
-                "score_ci_low": 0.08450704225352113,
+                "score": 0.1267605633802817,
+                "score_ci_high": 0.2112676056338028,
+                "score_ci_low": 0.056338028169014086,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
                 "accuracy": 0.08450704225352113,
-                "accuracy_ci_low": 0.028169014084507043,
-                "accuracy_ci_high": 0.18309859154929578,
+                "accuracy_ci_low": 0.04225352112676056,
+                "accuracy_ci_high": 0.16901408450704225,
                 "score_name": "accuracy",
                 "score": 0.08450704225352113,
-                "score_ci_high": 0.18309859154929578,
-                "score_ci_low": 0.028169014084507043,
+                "score_ci_high": 0.16901408450704225,
+                "score_ci_low": 0.04225352112676056,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.3380281690140845,
-                "accuracy_ci_low": 0.22535211267605634,
-                "accuracy_ci_high": 0.4507042253521127,
+                "accuracy": 0.4084507042253521,
+                "accuracy_ci_low": 0.30985915492957744,
+                "accuracy_ci_high": 0.5211267605633803,
                 "score_name": "accuracy",
-                "score": 0.3380281690140845,
-                "score_ci_high": 0.4507042253521127,
-                "score_ci_low": 0.22535211267605634,
+                "score": 0.4084507042253521,
+                "score_ci_high": 0.5211267605633803,
+                "score_ci_low": 0.30985915492957744,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.5633802816901409,
-                "accuracy_ci_low": 0.43661971830985913,
-                "accuracy_ci_high": 0.676056338028169,
+                "accuracy": 0.5211267605633803,
+                "accuracy_ci_low": 0.39436619718309857,
+                "accuracy_ci_high": 0.6338028169014085,
                 "score_name": "accuracy",
-                "score": 0.5633802816901409,
-                "score_ci_high": 0.676056338028169,
-                "score_ci_low": 0.43661971830985913,
+                "score": 0.5211267605633803,
+                "score_ci_high": 0.6338028169014085,
+                "score_ci_low": 0.39436619718309857,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
@@ -386,423 +386,423 @@
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.38028169014084506,
-                "accuracy_ci_low": 0.2676056338028169,
-                "accuracy_ci_high": 0.49295774647887325,
+                "accuracy": 0.3380281690140845,
+                "accuracy_ci_low": 0.22535211267605634,
+                "accuracy_ci_high": 0.4507042253521127,
                 "score_name": "accuracy",
-                "score": 0.38028169014084506,
-                "score_ci_high": 0.49295774647887325,
-                "score_ci_low": 0.2676056338028169,
+                "score": 0.3380281690140845,
+                "score_ci_high": 0.4507042253521127,
+                "score_ci_low": 0.22535211267605634,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.49295774647887325,
+                "accuracy": 0.4788732394366197,
                 "accuracy_ci_low": 0.36619718309859156,
-                "accuracy_ci_high": 0.6056338028169014,
+                "accuracy_ci_high": 0.5915492957746479,
                 "score_name": "accuracy",
-                "score": 0.49295774647887325,
-                "score_ci_high": 0.6056338028169014,
+                "score": 0.4788732394366197,
+                "score_ci_high": 0.5915492957746479,
                 "score_ci_low": 0.36619718309859156,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.18309859154929578,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy": 0.36619718309859156,
+                "accuracy_ci_low": 0.2676056338028169,
+                "accuracy_ci_high": 0.4788732394366197,
                 "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.18309859154929578,
+                "score": 0.36619718309859156,
+                "score_ci_high": 0.4788732394366197,
+                "score_ci_low": 0.2676056338028169,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.056338028169014086,
-                "accuracy_ci_low": 0.014084507042253521,
-                "accuracy_ci_high": 0.1267605633802817,
+                "accuracy": 0.08450704225352113,
+                "accuracy_ci_low": 0.028169014084507043,
+                "accuracy_ci_high": 0.16901408450704225,
                 "score_name": "accuracy",
-                "score": 0.056338028169014086,
-                "score_ci_high": 0.1267605633802817,
-                "score_ci_low": 0.014084507042253521,
+                "score": 0.08450704225352113,
+                "score_ci_high": 0.16901408450704225,
+                "score_ci_low": 0.028169014084507043,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.28169014084507044,
-                "accuracy_ci_low": 0.19718309859154928,
-                "accuracy_ci_high": 0.39436619718309857,
+                "accuracy": 0.323943661971831,
+                "accuracy_ci_low": 0.22535211267605634,
+                "accuracy_ci_high": 0.4397440034897243,
                 "score_name": "accuracy",
-                "score": 0.28169014084507044,
-                "score_ci_high": 0.39436619718309857,
-                "score_ci_low": 0.19718309859154928,
+                "score": 0.323943661971831,
+                "score_ci_high": 0.4397440034897243,
+                "score_ci_low": 0.22535211267605634,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.38028169014084506,
-                "accuracy_ci_low": 0.2676056338028169,
-                "accuracy_ci_high": 0.5095143645267136,
+                "accuracy": 0.4225352112676056,
+                "accuracy_ci_low": 0.30985915492957744,
+                "accuracy_ci_high": 0.5492957746478874,
                 "score_name": "accuracy",
-                "score": 0.38028169014084506,
-                "score_ci_high": 0.5095143645267136,
-                "score_ci_low": 0.2676056338028169,
+                "score": 0.4225352112676056,
+                "score_ci_high": 0.5492957746478874,
+                "score_ci_low": 0.30985915492957744,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
                 "accuracy": 0.18309859154929578,
                 "accuracy_ci_low": 0.09859154929577464,
-                "accuracy_ci_high": 0.28169014084507044,
+                "accuracy_ci_high": 0.28910654360361887,
                 "score_name": "accuracy",
                 "score": 0.18309859154929578,
-                "score_ci_high": 0.28169014084507044,
+                "score_ci_high": 0.28910654360361887,
                 "score_ci_low": 0.09859154929577464,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.5915492957746479,
-                "accuracy_ci_low": 0.4647887323943662,
-                "accuracy_ci_high": 0.704225352112676,
+                "accuracy": 0.5633802816901409,
+                "accuracy_ci_low": 0.4507042253521127,
+                "accuracy_ci_high": 0.676056338028169,
                 "score_name": "accuracy",
-                "score": 0.5915492957746479,
-                "score_ci_high": 0.704225352112676,
-                "score_ci_low": 0.4647887323943662,
+                "score": 0.5633802816901409,
+                "score_ci_high": 0.676056338028169,
+                "score_ci_low": 0.4507042253521127,
                 "num_of_instances": 71
             },
-            "score": 0.3209255533199195,
+            "score": 0.3289738430583501,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.07819548872180451,
+                "f1_macro": 0.09605263157894736,
                 "f1_suggestive": 0.0,
                 "f1_generic": 0.0,
+                "f1_descriptive": 0.375,
                 "f1_fanciful": 0.10526315789473684,
-                "f1_descriptive": 0.2857142857142857,
                 "f1_arbitrary": 0.0,
-                "f1_macro_ci_low": 0.029629629629629627,
-                "f1_macro_ci_high": 0.1446114401751038,
+                "f1_macro_ci_low": 0.048484848484848485,
+                "f1_macro_ci_high": 0.1610036081002675,
                 "score_name": "f1_micro",
-                "score": 0.1,
-                "score_ci_high": 0.1976990689891533,
-                "score_ci_low": 0.0392156862745098,
+                "score": 0.13333333333333333,
+                "score_ci_high": 0.24299065420560748,
+                "score_ci_low": 0.05825242718446602,
                 "num_of_instances": 85,
-                "accuracy": 0.058823529411764705,
-                "accuracy_ci_low": 0.023529411764705882,
-                "accuracy_ci_high": 0.1261289751719794,
-                "f1_micro": 0.1,
-                "f1_micro_ci_low": 0.0392156862745098,
-                "f1_micro_ci_high": 0.1976990689891533
+                "accuracy": 0.08235294117647059,
+                "accuracy_ci_low": 0.03529411764705882,
+                "accuracy_ci_high": 0.15294117647058825,
+                "f1_micro": 0.13333333333333333,
+                "f1_micro_ci_low": 0.05825242718446602,
+                "f1_micro_ci_high": 0.24299065420560748
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.4318713450292398,
-                "f1_no": 0.5526315789473685,
-                "f1_yes": 0.3111111111111111,
-                "f1_macro_ci_low": 0.3568758383648559,
-                "f1_macro_ci_high": 0.5105344080350164,
+                "f1_macro": 0.46628060735944554,
+                "f1_no": 0.5809128630705395,
+                "f1_yes": 0.3516483516483517,
+                "f1_macro_ci_low": 0.3928270444081002,
+                "f1_macro_ci_high": 0.5415825326375369,
                 "score_name": "f1_micro",
-                "score": 0.48427672955974843,
-                "score_ci_high": 0.553538495446083,
-                "score_ci_low": 0.40855056637270504,
+                "score": 0.5180722891566265,
+                "score_ci_high": 0.58253132966529,
+                "score_ci_low": 0.44652531947540486,
                 "num_of_instances": 200,
-                "accuracy": 0.385,
-                "accuracy_ci_low": 0.32,
-                "accuracy_ci_high": 0.45,
-                "f1_micro": 0.48427672955974843,
-                "f1_micro_ci_low": 0.40855056637270504,
-                "f1_micro_ci_high": 0.553538495446083
+                "accuracy": 0.43,
+                "accuracy_ci_low": 0.365,
+                "accuracy_ci_high": 0.495,
+                "f1_micro": 0.5180722891566265,
+                "f1_micro_ci_low": 0.44652531947540486,
+                "f1_micro_ci_high": 0.58253132966529
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.07867608581894296,
+                "f1_macro": 0.09994226986403168,
                 "f1_conclusion": 0.0,
-                "f1_decree": 0.07692307692307693,
-                "f1_issue": 0.0,
-                "f1_analysis": 0.4166666666666667,
-                "f1_facts": 0.0,
+                "f1_decree": 0.14814814814814814,
+                "f1_issue": 0.05714285714285714,
+                "f1_analysis": 0.3076923076923077,
+                "f1_facts": 0.06896551724137931,
                 "f1_procedural history": 0.0,
-                "f1_rule": 0.05714285714285714,
-                "f1_macro_ci_low": 0.04893392684609269,
-                "f1_macro_ci_high": 0.12399894886334993,
+                "f1_rule": 0.11764705882352941,
+                "f1_macro_ci_low": 0.06467882036635723,
+                "f1_macro_ci_high": 0.16271896227970067,
                 "score_name": "f1_micro",
-                "score": 0.10619469026548672,
-                "score_ci_high": 0.17107924198886906,
-                "score_ci_low": 0.06000153614696125,
+                "score": 0.1222707423580786,
+                "score_ci_high": 0.1896551724137931,
+                "score_ci_low": 0.07144817486457739,
                 "num_of_instances": 200,
-                "accuracy": 0.06,
-                "accuracy_ci_low": 0.035,
-                "accuracy_ci_high": 0.1,
-                "f1_micro": 0.10619469026548672,
-                "f1_micro_ci_low": 0.06000153614696125,
-                "f1_micro_ci_high": 0.17107924198886906
+                "accuracy": 0.07,
+                "accuracy_ci_low": 0.04,
+                "accuracy_ci_high": 0.11032816661500704,
+                "f1_micro": 0.1222707423580786,
+                "f1_micro_ci_low": 0.07144817486457739,
+                "f1_micro_ci_high": 0.1896551724137931
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.31866564807741277,
-                "f1_yes": 0.20952380952380953,
-                "f1_no": 0.42780748663101603,
-                "f1_macro_ci_low": 0.2502995617982514,
-                "f1_macro_ci_high": 0.39353002888833133,
+                "f1_macro": 0.3036750483558994,
+                "f1_yes": 0.18181818181818182,
+                "f1_no": 0.425531914893617,
+                "f1_macro_ci_low": 0.24346443663418135,
+                "f1_macro_ci_high": 0.37985983397043105,
                 "score_name": "f1_micro",
-                "score": 0.3493150684931507,
-                "score_ci_high": 0.42,
-                "score_ci_low": 0.2701836639419085,
+                "score": 0.33557046979865773,
+                "score_ci_high": 0.41208424597764826,
+                "score_ci_low": 0.26697141622873294,
                 "num_of_instances": 200,
-                "accuracy": 0.255,
-                "accuracy_ci_low": 0.195,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.2,
                 "accuracy_ci_high": 0.315,
-                "f1_micro": 0.3493150684931507,
-                "f1_micro_ci_low": 0.2701836639419085,
-                "f1_micro_ci_high": 0.42
+                "f1_micro": 0.33557046979865773,
+                "f1_micro_ci_low": 0.26697141622873294,
+                "f1_micro_ci_high": 0.41208424597764826
             },
             "legalbench_proa": {
-                "f1_macro": 0.8489843979982338,
-                "f1_yes": 0.8607594936708861,
-                "f1_no": 0.8372093023255814,
-                "f1_macro_ci_low": 0.762371922413286,
-                "f1_macro_ci_high": 0.9100766335383308,
+                "f1_macro": 0.8285413216920066,
+                "f1_yes": 0.821917808219178,
+                "f1_no": 0.8351648351648352,
+                "f1_macro_ci_low": 0.7370610449522892,
+                "f1_macro_ci_high": 0.8925462752093225,
                 "score_name": "f1_micro",
-                "score": 0.8484848484848485,
-                "score_ci_high": 0.9090909090909091,
-                "score_ci_low": 0.7590361445783133,
+                "score": 0.8292682926829268,
+                "score_ci_high": 0.891566265060241,
+                "score_ci_low": 0.7393939393939394,
                 "num_of_instances": 85,
-                "accuracy": 0.8235294117647058,
-                "accuracy_ci_low": 0.7294117647058823,
-                "accuracy_ci_high": 0.8941176470588236,
-                "f1_micro": 0.8484848484848485,
-                "f1_micro_ci_low": 0.7590361445783133,
-                "f1_micro_ci_high": 0.9090909090909091
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.7058823529411765,
+                "accuracy_ci_high": 0.8705882352941177,
+                "f1_micro": 0.8292682926829268,
+                "f1_micro_ci_low": 0.7393939393939394,
+                "f1_micro_ci_high": 0.891566265060241
             },
-            "score": 0.3776542673606469,
+            "score": 0.3877030254659246,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.2527665164224348,
-                "f1_cars": 0.417910447761194,
+                "f1_macro": 0.2883227310464808,
+                "f1_cars": 0.45454545454545453,
                 "f1_windows x": 0.0,
-                "f1_atheism": 0.05128205128205128,
-                "f1_religion": 0.15584415584415584,
-                "f1_politics": 0.15789473684210525,
-                "f1_medicine": 0.5,
-                "f1_christianity": 0.07142857142857142,
-                "f1_computer graphics": 0.2702702702702703,
-                "f1_microsoft windows": 0.16393442622950818,
-                "f1_middle east": 0.12,
-                "f1_motorcycles": 0.375,
-                "f1_pc hardware": 0.3157894736842105,
+                "f1_atheism": 0.1,
+                "f1_religion": 0.11594202898550725,
+                "f1_medicine": 0.5161290322580645,
+                "f1_christianity": 0.2,
+                "f1_computer graphics": 0.24615384615384617,
+                "f1_microsoft windows": 0.2,
+                "f1_middle east": 0.23529411764705882,
+                "f1_motorcycles": 0.43902439024390244,
+                "f1_pc hardware": 0.38095238095238093,
                 "f1_mac hardware": 0.14285714285714285,
-                "f1_for sale": 0.2222222222222222,
-                "f1_guns": 0.11538461538461539,
-                "f1_space": 0.34782608695652173,
-                "f1_cryptography": 0.4126984126984127,
-                "f1_baseball": 0.35135135135135137,
-                "f1_hockey": 0.5,
-                "f1_electronics": 0.36363636363636365,
-                "f1_macro_ci_low": 0.22757237570678052,
-                "f1_macro_ci_high": 0.28553448529743153,
+                "f1_for sale": 0.2127659574468085,
+                "f1_guns": 0.04,
+                "f1_space": 0.4166666666666667,
+                "f1_cryptography": 0.36065573770491804,
+                "f1_baseball": 0.37333333333333335,
+                "f1_hockey": 0.6060606060606061,
+                "f1_politics": 0.23376623376623376,
+                "f1_electronics": 0.49230769230769234,
+                "f1_macro_ci_low": 0.2617609426183389,
+                "f1_macro_ci_high": 0.31709395962356884,
                 "score_name": "f1_micro",
-                "score": 0.26766917293233083,
-                "score_ci_high": 0.29941755760789834,
-                "score_ci_low": 0.23630344400865025,
+                "score": 0.30918595967139656,
+                "score_ci_high": 0.3396825281712499,
+                "score_ci_low": 0.2773193184164219,
                 "num_of_instances": 1000,
-                "accuracy": 0.178,
-                "accuracy_ci_low": 0.155,
-                "accuracy_ci_high": 0.201,
-                "f1_micro": 0.26766917293233083,
-                "f1_micro_ci_low": 0.23630344400865025,
-                "f1_micro_ci_high": 0.29941755760789834
+                "accuracy": 0.207,
+                "accuracy_ci_low": 0.183,
+                "accuracy_ci_high": 0.231,
+                "f1_micro": 0.30918595967139656,
+                "f1_micro_ci_low": 0.2773193184164219,
+                "f1_micro_ci_high": 0.3396825281712499
             },
-            "score": 0.26766917293233083,
+            "score": 0.30918595967139656,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.5296897956484732,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.6554621848739496,
-                "f1_credit card or prepaid card": 0.4943820224719101,
-                "f1_debt collection": 0.4297520661157025,
-                "f1_payday loan or title loan or personal loan": 0.375,
-                "f1_checking or savings account": 0.611764705882353,
-                "f1_money transfer or virtual currency or money service": 0.5555555555555556,
-                "f1_mortgage": 0.509090909090909,
-                "f1_student loan": 0.5555555555555556,
-                "f1_vehicle loan or lease": 0.5806451612903226,
-                "f1_macro_ci_low": 0.469057355767322,
-                "f1_macro_ci_high": 0.598469091946616,
+                "f1_macro": 0.4658504084132011,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.6103646833013435,
+                "f1_checking or savings account": 0.43243243243243246,
+                "f1_debt collection": 0.35294117647058826,
+                "f1_credit card or prepaid card": 0.4883720930232558,
+                "f1_money transfer or virtual currency or money service": 0.48,
+                "f1_mortgage": 0.6885245901639344,
+                "f1_payday loan or title loan or personal loan": 0.16666666666666666,
+                "f1_student loan": 0.5217391304347826,
+                "f1_vehicle loan or lease": 0.45161290322580644,
+                "f1_macro_ci_low": 0.4151030067042806,
+                "f1_macro_ci_high": 0.5374380194007881,
                 "score_name": "f1_micro",
-                "score": 0.6116883116883117,
-                "score_ci_high": 0.6423076923076924,
-                "score_ci_low": 0.5828077788480714,
+                "score": 0.5647530040053405,
+                "score_ci_high": 0.5946299934512115,
+                "score_ci_low": 0.5324708819498815,
                 "num_of_instances": 1000,
-                "accuracy": 0.471,
-                "accuracy_ci_low": 0.443,
-                "accuracy_ci_high": 0.503,
-                "f1_micro": 0.6116883116883117,
-                "f1_micro_ci_low": 0.5828077788480714,
-                "f1_micro_ci_high": 0.6423076923076924
+                "accuracy": 0.423,
+                "accuracy_ci_low": 0.392,
+                "accuracy_ci_high": 0.4538616190423828,
+                "f1_micro": 0.5647530040053405,
+                "f1_micro_ci_low": 0.5324708819498815,
+                "f1_micro_ci_high": 0.5946299934512115
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.5903065912163163,
-                "f1_mortgages and loans": 0.6282051282051282,
-                "f1_credit card": 0.6842105263157895,
-                "f1_debt collection": 0.49504950495049505,
-                "f1_credit reporting": 0.5847457627118644,
-                "f1_retail banking": 0.559322033898305,
-                "f1_macro_ci_low": 0.5491833392518851,
-                "f1_macro_ci_high": 0.6385093727045127,
+                "f1_macro": 0.6154980967666208,
+                "f1_mortgages and loans": 0.6794871794871795,
+                "f1_credit card": 0.6394557823129252,
+                "f1_debt collection": 0.5405405405405406,
+                "f1_credit reporting": 0.6307053941908713,
+                "f1_retail banking": 0.5873015873015873,
+                "f1_macro_ci_low": 0.5757444877146287,
+                "f1_macro_ci_high": 0.6632621576134956,
                 "score_name": "f1_micro",
-                "score": 0.5856481481481481,
-                "score_ci_high": 0.632183908045977,
-                "score_ci_low": 0.54416153401534,
+                "score": 0.6152046783625731,
+                "score_ci_high": 0.6592588246755606,
+                "score_ci_low": 0.5758293838862559,
                 "num_of_instances": 500,
-                "accuracy": 0.506,
-                "accuracy_ci_low": 0.466,
-                "accuracy_ci_high": 0.552,
-                "f1_micro": 0.5856481481481481,
-                "f1_micro_ci_low": 0.54416153401534,
-                "f1_micro_ci_high": 0.632183908045977
+                "accuracy": 0.526,
+                "accuracy_ci_low": 0.488,
+                "accuracy_ci_high": 0.572065074842346,
+                "f1_micro": 0.6152046783625731,
+                "f1_micro_ci_low": 0.5758293838862559,
+                "f1_micro_ci_high": 0.6592588246755606
             },
-            "score": 0.59866822991823,
+            "score": 0.5899788411839568,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "execution_accuracy": 0.038,
-                "program_accuracy": 0.036,
-                "score": 0.036,
+                "program_accuracy": 0.03,
+                "score": 0.03,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.028,
-                "execution_accuracy_ci_high": 0.051,
-                "program_accuracy_ci_low": 0.026,
-                "program_accuracy_ci_high": 0.048,
-                "score_ci_low": 0.026,
-                "score_ci_high": 0.048
+                "execution_accuracy": 0.03,
+                "program_accuracy_ci_low": 0.02,
+                "program_accuracy_ci_high": 0.041,
+                "score_ci_low": 0.02,
+                "score_ci_high": 0.041,
+                "execution_accuracy_ci_low": 0.02,
+                "execution_accuracy_ci_high": 0.042
             },
-            "score": 0.036,
+            "score": 0.03,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.28525760105453485,
-                "recall": 0.5602956481447632,
-                "f1": 0.3115553425716912,
-                "precision_ci_low": 0.26514229625282976,
-                "precision_ci_high": 0.3063459879682981,
-                "recall_ci_low": 0.5448358892557127,
-                "recall_ci_high": 0.5784430830027775,
-                "f1_ci_low": 0.29470532845103153,
-                "f1_ci_high": 0.33107953368338633,
+                "precision": 0.27959949741292506,
+                "recall": 0.5616895325224012,
+                "f1": 0.3093936462410963,
+                "precision_ci_low": 0.26037333098993004,
+                "precision_ci_high": 0.2998496234035471,
+                "recall_ci_low": 0.545189324623663,
+                "recall_ci_high": 0.5772522419606227,
+                "f1_ci_low": 0.29233863350762596,
+                "f1_ci_high": 0.3260584980677657,
                 "score_name": "f1",
-                "score": 0.3115553425716912,
-                "score_ci_high": 0.33107953368338633,
-                "score_ci_low": 0.29470532845103153,
+                "score": 0.3093936462410963,
+                "score_ci_high": 0.3260584980677657,
+                "score_ci_low": 0.29233863350762596,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5722805594901244,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6597029569248358,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5267530478537082,
-                "faithfullness_f1_token_overlap": 0.30842516813067145,
-                "faithfullness_recall_token_overlap": 0.25295558254033107,
-                "faithfullness_precision_token_overlap": 0.5261232421481096,
-                "correctness_f1_token_overlap": 0.3115553425716912,
-                "correctness_recall_token_overlap": 0.5602956481447632,
-                "correctness_precision_token_overlap": 0.28525760105453485
+                "correctness_f1_bert_score.deberta_large_mnli": 0.5708662017683188,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6597496373951435,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5238148674865564,
+                "faithfullness_f1_token_overlap": 0.3063464322583528,
+                "faithfullness_recall_token_overlap": 0.2515040891461165,
+                "faithfullness_precision_token_overlap": 0.5201254248503554,
+                "correctness_f1_token_overlap": 0.3093936462410963,
+                "correctness_recall_token_overlap": 0.5616895325224012,
+                "correctness_precision_token_overlap": 0.27959949741292506
             },
-            "score": 0.3115553425716912,
+            "score": 0.3093936462410963,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.574,
-                "accuracy_ci_low": 0.543,
-                "accuracy_ci_high": 0.606,
+                "accuracy": 0.572,
+                "accuracy_ci_low": 0.5408695080796895,
+                "accuracy_ci_high": 0.602,
                 "score_name": "accuracy",
-                "score": 0.574,
-                "score_ci_high": 0.606,
-                "score_ci_low": 0.543,
+                "score": 0.572,
+                "score_ci_high": 0.602,
+                "score_ci_low": 0.5408695080796895,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.87,
-                "accuracy_ci_low": 0.84,
-                "accuracy_ci_high": 0.898,
+                "accuracy": 0.878,
+                "accuracy_ci_low": 0.846,
+                "accuracy_ci_high": 0.9048842512733182,
                 "score_name": "accuracy",
-                "score": 0.87,
-                "score_ci_high": 0.898,
-                "score_ci_low": 0.84,
+                "score": 0.878,
+                "score_ci_high": 0.9048842512733182,
+                "score_ci_low": 0.846,
                 "num_of_instances": 500
             },
-            "score": 0.722,
+            "score": 0.725,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 3.0,
-                "severity_high": 3.0,
-                "severity_medium": 3.0,
-                "severity_low": 91.0,
-                "category_harmful_info": 0.7363050992171916,
-                "category_deception": 0.7075176269117028,
-                "category_explicit_content": 0.7862109684883944,
-                "category_violence": 0.8046535633740918,
-                "category_discrimination": 0.7672554519441394,
-                "category_substance_abuse": 0.9027199372959634,
-                "category_pii": 0.8223799069722494,
-                "safety": 0.7756195961658118,
-                "safety_ci_low": 0.7397039726251121,
-                "safety_ci_high": 0.8122983590465129,
+                "severity_critical": 2.0,
+                "severity_high": 2.0,
+                "severity_medium": 1.0,
+                "severity_low": 95.0,
+                "category_harmful_info": 0.8059234746569206,
+                "category_deception": 0.7821050891370483,
+                "category_explicit_content": 0.8131378136499965,
+                "category_violence": 0.7824397089225905,
+                "category_discrimination": 0.7646755935653806,
+                "category_substance_abuse": 0.7601792050732505,
+                "category_pii": 0.7583640239856861,
+                "safety": 0.7892763628562292,
+                "safety_ci_low": 0.7568372385512269,
+                "safety_ci_high": 0.8199271972816702,
                 "score_name": "safety",
-                "score": 0.7756195961658118,
-                "score_ci_high": 0.8122983590465129,
-                "score_ci_low": 0.7397039726251121,
+                "score": 0.7892763628562292,
+                "score_ci_high": 0.8199271972816702,
+                "score_ci_low": 0.7568372385512269,
                 "num_of_instances": 100
             },
-            "score": 0.7756195961658118,
+            "score": 0.7892763628562292,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rougeL": 0.2623368936139326,
-                "score": 0.2623368936139326,
+                "rougeL": 0.26029440810817217,
+                "score": 0.26029440810817217,
                 "score_name": "rougeL",
-                "rougeLsum": 0.3246861800674231,
-                "rouge1": 0.37875353275553886,
-                "rouge2": 0.17634540475082816,
-                "rougeL_ci_low": 0.25607817865790744,
-                "rougeL_ci_high": 0.2699573234237949,
-                "score_ci_low": 0.25607817865790744,
-                "score_ci_high": 0.2699573234237949,
-                "rougeLsum_ci_low": 0.31633678336605703,
-                "rougeLsum_ci_high": 0.3342014722776119,
-                "rouge1_ci_low": 0.36953845902369503,
-                "rouge1_ci_high": 0.38917434338527224,
-                "rouge2_ci_low": 0.17007943381657917,
-                "rouge2_ci_high": 0.18480657837175643
+                "rouge1": 0.37812164542305304,
+                "rougeLsum": 0.3224592121117338,
+                "rouge2": 0.1735255109874066,
+                "rougeL_ci_low": 0.25326019704193026,
+                "rougeL_ci_high": 0.2676771085873056,
+                "score_ci_low": 0.25326019704193026,
+                "score_ci_high": 0.2676771085873056,
+                "rouge1_ci_low": 0.36799150913690026,
+                "rouge1_ci_high": 0.3873907236511595,
+                "rougeLsum_ci_low": 0.3130244535911371,
+                "rougeLsum_ci_high": 0.33132370273160416,
+                "rouge2_ci_low": 0.16695272546253506,
+                "rouge2_ci_high": 0.18091296094174728
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rougeL": 0.09126672648981966,
-                "score": 0.09126672648981966,
+                "rougeL": 0.09390373141967649,
+                "score": 0.09390373141967649,
                 "score_name": "rougeL",
-                "rougeLsum": 0.10371587819726082,
-                "rouge1": 0.12400577725256573,
-                "rouge2": 0.017067409640734738,
-                "rougeL_ci_low": 0.08642017587563405,
-                "rougeL_ci_high": 0.09509880053104167,
-                "score_ci_low": 0.08642017587563405,
-                "score_ci_high": 0.09509880053104167,
-                "rougeLsum_ci_low": 0.09869637123218795,
-                "rougeLsum_ci_high": 0.10797131588828912,
-                "rouge1_ci_low": 0.11835018459961112,
-                "rouge1_ci_high": 0.1293417832607229,
-                "rouge2_ci_low": 0.015170057207432563,
-                "rouge2_ci_high": 0.019497436133181917
+                "rouge1": 0.12667295475773943,
+                "rougeLsum": 0.10750099464975159,
+                "rouge2": 0.01898752670157484,
+                "rougeL_ci_low": 0.08937719185127607,
+                "rougeL_ci_high": 0.09832477865928776,
+                "score_ci_low": 0.08937719185127607,
+                "score_ci_high": 0.09832477865928776,
+                "rouge1_ci_low": 0.12055496978215834,
+                "rouge1_ci_high": 0.13195192205981862,
+                "rougeLsum_ci_low": 0.10257751488972287,
+                "rougeLsum_ci_high": 0.11236007191539416,
+                "rouge2_ci_low": 0.01673170907001085,
+                "rouge2_ci_high": 0.02151839340904419
             },
-            "score": 0.17680181005187612,
+            "score": 0.17709906976392434,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1175,
-                    692,
-                    449,
-                    309
+                    1176,
+                    681,
+                    439,
+                    289
                 ],
                 "totals": [
-                    1795,
-                    1729,
-                    1663,
-                    1597
+                    1805,
+                    1739,
+                    1673,
+                    1607
                 ],
                 "precisions": [
-                    0.6545961002785515,
-                    0.40023134759976864,
-                    0.269993986770896,
-                    0.19348778960551033
+                    0.6515235457063712,
+                    0.39160437032777456,
+                    0.26240286909742977,
+                    0.17983820784069696
                 ],
                 "bp": 1.0,
-                "sys_len": 1795,
+                "sys_len": 1805,
                 "ref_len": 1734,
-                "sacrebleu": 0.34203696369018,
-                "score": 0.34203696369018,
+                "sacrebleu": 0.33125088532286445,
+                "score": 0.33125088532286445,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30827619341319407,
-                "score_ci_high": 0.3934585756605337,
-                "sacrebleu_ci_low": 0.30827619341319407,
-                "sacrebleu_ci_high": 0.3934585756605337
+                "score_ci_low": 0.28827207319430564,
+                "score_ci_high": 0.3750069277361591,
+                "sacrebleu_ci_low": 0.28827207319430564,
+                "sacrebleu_ci_high": 0.3750069277361591
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1238,
-                    753,
-                    502,
-                    333
+                    1224,
+                    746,
+                    495,
+                    332
                 ],
                 "totals": [
-                    1807,
-                    1741,
-                    1675,
-                    1609
+                    1783,
+                    1717,
+                    1651,
+                    1585
                 ],
                 "precisions": [
-                    0.6851134477033757,
-                    0.43251005169442847,
-                    0.29970149253731343,
-                    0.2069608452454941
+                    0.6864834548513741,
+                    0.4344787419918463,
+                    0.29981829194427617,
+                    0.20946372239747635
                 ],
                 "bp": 1.0,
-                "sys_len": 1807,
+                "sys_len": 1783,
                 "ref_len": 1734,
-                "sacrebleu": 0.36820013898054776,
-                "score": 0.36820013898054776,
+                "sacrebleu": 0.36994859939150276,
+                "score": 0.36994859939150276,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3262271352518154,
-                "score_ci_high": 0.41519158941270023,
-                "sacrebleu_ci_low": 0.3262271352518154,
-                "sacrebleu_ci_high": 0.41519158941270023
+                "score_ci_low": 0.3148788426694038,
+                "score_ci_high": 0.4105592207294174,
+                "sacrebleu_ci_low": 0.3148788426694038,
+                "sacrebleu_ci_high": 0.4105592207294174
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    247,
-                    51,
-                    24,
-                    13
+                    215,
+                    39,
+                    18,
+                    6
                 ],
                 "totals": [
-                    3575,
-                    3509,
-                    3443,
-                    3377
+                    3272,
+                    3206,
+                    3140,
+                    3074
                 ],
                 "precisions": [
-                    0.06909090909090909,
-                    0.014534055286406384,
-                    0.0069706651176299735,
-                    0.0038495706248149247
+                    0.06570904645476773,
+                    0.012164691203992516,
+                    0.005732484076433121,
+                    0.001951854261548471
                 ],
                 "bp": 1.0,
-                "sys_len": 3575,
+                "sys_len": 3272,
                 "ref_len": 1589,
-                "sacrebleu": 0.012812195485921624,
-                "score": 0.012812195485921624,
+                "sacrebleu": 0.009724765205835872,
+                "score": 0.009724765205835872,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.006078646745883528,
-                "score_ci_high": 0.020936322376328288,
-                "sacrebleu_ci_low": 0.006078646745883528,
-                "sacrebleu_ci_high": 0.020936322376328288
+                "score_ci_low": 0.00598430838144436,
+                "score_ci_high": 0.015966807436499916,
+                "sacrebleu_ci_low": 0.00598430838144436,
+                "sacrebleu_ci_high": 0.015966807436499916
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1113,
-                    599,
-                    371,
-                    238
+                    1112,
+                    617,
+                    379,
+                    249
                 ],
                 "totals": [
-                    1788,
-                    1722,
-                    1656,
-                    1590
+                    1879,
+                    1813,
+                    1747,
+                    1681
                 ],
                 "precisions": [
-                    0.6224832214765101,
-                    0.34785133565621373,
-                    0.22403381642512077,
-                    0.14968553459119496
+                    0.5918041511442257,
+                    0.34031991174848314,
+                    0.21694333142530053,
+                    0.14812611540749554
                 ],
-                "bp": 0.9740561253203749,
-                "sys_len": 1788,
+                "bp": 1.0,
+                "sys_len": 1879,
                 "ref_len": 1835,
-                "sacrebleu": 0.2843398077874606,
-                "score": 0.2843398077874606,
+                "sacrebleu": 0.283635656706225,
+                "score": 0.283635656706225,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23626058332848057,
-                "score_ci_high": 0.31993924524942646,
-                "sacrebleu_ci_low": 0.23626058332848057,
-                "sacrebleu_ci_high": 0.31993924524942646
+                "score_ci_low": 0.25483093745296115,
+                "score_ci_high": 0.3284369566535284,
+                "sacrebleu_ci_low": 0.25483093745296115,
+                "sacrebleu_ci_high": 0.3284369566535284
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1420,
-                    985,
-                    727,
-                    545
+                    1422,
+                    990,
+                    740,
+                    568
                 ],
                 "totals": [
-                    2032,
-                    1966,
-                    1900,
-                    1834
+                    2012,
+                    1946,
+                    1880,
+                    1814
                 ],
                 "precisions": [
-                    0.6988188976377953,
-                    0.5010172939979654,
-                    0.38263157894736843,
-                    0.29716466739367503
+                    0.7067594433399603,
+                    0.5087358684480986,
+                    0.39361702127659576,
+                    0.3131201764057332
                 ],
-                "bp": 0.9824394796731021,
-                "sys_len": 2032,
+                "bp": 0.9725507672852267,
+                "sys_len": 2012,
                 "ref_len": 2068,
-                "sacrebleu": 0.4388384183123361,
-                "score": 0.4388384183123361,
+                "sacrebleu": 0.44622048751232035,
+                "score": 0.44622048751232035,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.39345852629040573,
-                "score_ci_high": 0.4779013774696765,
-                "sacrebleu_ci_low": 0.39345852629040573,
-                "sacrebleu_ci_high": 0.4779013774696765
+                "score_ci_low": 0.4016486544135287,
+                "score_ci_high": 0.492468721152314,
+                "sacrebleu_ci_low": 0.4016486544135287,
+                "sacrebleu_ci_high": 0.492468721152314
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1094,
-                    480,
-                    264,
-                    149
+                    983,
+                    445,
+                    221,
+                    116
                 ],
                 "totals": [
-                    2582,
-                    2516,
-                    2450,
-                    2384
+                    2522,
+                    2456,
+                    2390,
+                    2324
                 ],
                 "precisions": [
-                    0.42370255615801705,
-                    0.1907790143084261,
-                    0.10775510204081633,
-                    0.0625
+                    0.3897700237906423,
+                    0.18118892508143322,
+                    0.09246861924686192,
+                    0.04991394148020654
                 ],
                 "bp": 1.0,
-                "sys_len": 2582,
+                "sys_len": 2522,
                 "ref_len": 2235,
-                "sacrebleu": 0.15274865193932807,
-                "score": 0.15274865193932807,
+                "sacrebleu": 0.13436590468744522,
+                "score": 0.13436590468744522,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11545139318026426,
-                "score_ci_high": 0.18226843784214114,
-                "sacrebleu_ci_low": 0.11545139318026426,
-                "sacrebleu_ci_high": 0.18226843784214114
+                "score_ci_low": 0.10668449271679148,
+                "score_ci_high": 0.16331801147285638,
+                "sacrebleu_ci_low": 0.10668449271679148,
+                "sacrebleu_ci_high": 0.16331801147285638
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1376,
-                    942,
-                    686,
-                    504
+                    1397,
+                    976,
+                    718,
+                    534
                 ],
                 "totals": [
-                    1895,
-                    1829,
-                    1763,
-                    1697
+                    1885,
+                    1819,
+                    1753,
+                    1687
                 ],
                 "precisions": [
-                    0.7261213720316623,
-                    0.5150355385456534,
-                    0.3891094724900737,
-                    0.29699469652327637
+                    0.7411140583554376,
+                    0.536558548653106,
+                    0.40958357102110665,
+                    0.31653823355068167
                 ],
-                "bp": 0.988979382694272,
-                "sys_len": 1895,
+                "bp": 0.9836888676493653,
+                "sys_len": 1885,
                 "ref_len": 1916,
-                "sacrebleu": 0.4509246392883171,
-                "score": 0.4509246392883171,
+                "sacrebleu": 0.4687329402986153,
+                "score": 0.4687329402986153,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3969123124838706,
-                "score_ci_high": 0.49488969802829147,
-                "sacrebleu_ci_low": 0.3969123124838706,
-                "sacrebleu_ci_high": 0.49488969802829147
+                "score_ci_low": 0.4090027394886287,
+                "score_ci_high": 0.5072542898902915,
+                "sacrebleu_ci_low": 0.4090027394886287,
+                "sacrebleu_ci_high": 0.5072542898902915
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1007,
-                    563,
-                    348,
-                    213
+                    1075,
+                    589,
+                    357,
+                    218
                 ],
                 "totals": [
-                    1937,
-                    1871,
-                    1805,
-                    1739
+                    2037,
+                    1971,
+                    1905,
+                    1839
                 ],
                 "precisions": [
-                    0.5198760970573051,
-                    0.3009086050240513,
-                    0.192797783933518,
-                    0.12248418631397355
+                    0.5277368679430535,
+                    0.29883307965499745,
+                    0.1874015748031496,
+                    0.11854268624252312
                 ],
-                "bp": 0.9938240032224314,
-                "sys_len": 1937,
+                "bp": 1.0,
+                "sys_len": 2037,
                 "ref_len": 1949,
-                "sacrebleu": 0.24501270828054722,
-                "score": 0.24501270828054722,
+                "sacrebleu": 0.24328959002445089,
+                "score": 0.24328959002445089,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.20771746826159335,
-                "score_ci_high": 0.2893120776468884,
-                "sacrebleu_ci_low": 0.20771746826159335,
-                "sacrebleu_ci_high": 0.2893120776468884
+                "score_ci_low": 0.2108781721763864,
+                "score_ci_high": 0.2866413776406142,
+                "sacrebleu_ci_low": 0.2108781721763864,
+                "sacrebleu_ci_high": 0.2866413776406142
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1254,
-                    697,
-                    407,
-                    241
+                    1223,
+                    645,
+                    371,
+                    219
                 ],
                 "totals": [
-                    1994,
-                    1928,
-                    1862,
-                    1796
+                    2012,
+                    1946,
+                    1880,
+                    1814
                 ],
                 "precisions": [
-                    0.6288866599799399,
-                    0.36151452282157676,
-                    0.21858216970998925,
-                    0.13418708240534521
+                    0.6078528827037774,
+                    0.3314491264131552,
+                    0.1973404255319149,
+                    0.12072767364939362
                 ],
-                "bp": 0.9491803375373334,
-                "sys_len": 1994,
+                "bp": 0.9581570887075945,
+                "sys_len": 2012,
                 "ref_len": 2098,
-                "sacrebleu": 0.27124055641744416,
-                "score": 0.27124055641744416,
+                "sacrebleu": 0.25220069551672647,
+                "score": 0.25220069551672647,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.238444782766854,
-                "score_ci_high": 0.29802738323060723,
-                "sacrebleu_ci_low": 0.238444782766854,
-                "sacrebleu_ci_high": 0.29802738323060723
+                "score_ci_low": 0.21455240851266189,
+                "score_ci_high": 0.28127381859222883,
+                "sacrebleu_ci_low": 0.21455240851266189,
+                "sacrebleu_ci_high": 0.28127381859222883
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1272,
-                    821,
+                    1265,
+                    811,
                     564,
-                    393
+                    403
                 ],
                 "totals": [
-                    1814,
-                    1748,
-                    1682,
-                    1616
+                    1799,
+                    1733,
+                    1667,
+                    1601
                 ],
                 "precisions": [
-                    0.701212789415656,
-                    0.4696796338672769,
-                    0.33531510107015455,
-                    0.24319306930693071
+                    0.7031684269038355,
+                    0.4679746105020196,
+                    0.33833233353329334,
+                    0.2517176764522173
                 ],
                 "bp": 1.0,
-                "sys_len": 1814,
+                "sys_len": 1799,
                 "ref_len": 1734,
-                "sacrebleu": 0.4048218693289096,
-                "score": 0.4048218693289096,
+                "sacrebleu": 0.40915203730030597,
+                "score": 0.40915203730030597,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3701816768681021,
-                "score_ci_high": 0.4500189391713141,
-                "sacrebleu_ci_low": 0.3701816768681021,
-                "sacrebleu_ci_high": 0.4500189391713141
+                "score_ci_low": 0.37740759275198177,
+                "score_ci_high": 0.45339812071532676,
+                "sacrebleu_ci_low": 0.37740759275198177,
+                "sacrebleu_ci_high": 0.45339812071532676
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1013,
-                    461,
-                    263,
-                    158
+                    1032,
+                    484,
+                    292,
+                    181
                 ],
                 "totals": [
-                    1820,
-                    1754,
-                    1688,
-                    1622
+                    1873,
+                    1807,
+                    1741,
+                    1675
                 ],
                 "precisions": [
-                    0.5565934065934066,
-                    0.2628278221208666,
-                    0.15580568720379145,
-                    0.09741060419235512
+                    0.5509877202349173,
+                    0.26784726065301606,
+                    0.16771970132107986,
+                    0.10805970149253731
                 ],
                 "bp": 1.0,
-                "sys_len": 1820,
+                "sys_len": 1873,
                 "ref_len": 1734,
-                "sacrebleu": 0.2170699640137964,
-                "score": 0.2170699640137964,
+                "sacrebleu": 0.22741507142824863,
+                "score": 0.22741507142824863,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18647053560514687,
-                "score_ci_high": 0.26290605520041826,
-                "sacrebleu_ci_low": 0.18647053560514687,
-                "sacrebleu_ci_high": 0.26290605520041826
+                "score_ci_low": 0.19114102819495998,
+                "score_ci_high": 0.26211626317047076,
+                "sacrebleu_ci_low": 0.19114102819495998,
+                "sacrebleu_ci_high": 0.26211626317047076
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    993,
-                    465,
-                    267,
-                    159
+                    1019,
+                    503,
+                    310,
+                    204
                 ],
                 "totals": [
-                    1895,
-                    1829,
-                    1763,
-                    1697
+                    1801,
+                    1735,
+                    1669,
+                    1603
                 ],
                 "precisions": [
-                    0.5240105540897098,
-                    0.2542372881355932,
-                    0.1514463981849121,
-                    0.09369475545079553
+                    0.5657967795669073,
+                    0.28991354466858793,
+                    0.18573996405032955,
+                    0.1272613849033063
                 ],
                 "bp": 1.0,
-                "sys_len": 1895,
+                "sys_len": 1801,
                 "ref_len": 1734,
-                "sacrebleu": 0.20851551650550446,
-                "score": 0.20851551650550446,
+                "sacrebleu": 0.24953573320283845,
+                "score": 0.24953573320283845,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.17957414369928407,
-                "score_ci_high": 0.2554257407203763,
-                "sacrebleu_ci_low": 0.17957414369928407,
-                "sacrebleu_ci_high": 0.2554257407203763
+                "score_ci_low": 0.20393561832427076,
+                "score_ci_high": 0.27875367055825584,
+                "sacrebleu_ci_low": 0.20393561832427076,
+                "sacrebleu_ci_high": 0.27875367055825584
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1283,
+                    1285,
                     864,
-                    623,
+                    625,
                     463
                 ],
                 "totals": [
-                    1786,
-                    1720,
-                    1654,
-                    1588
+                    1899,
+                    1833,
+                    1767,
+                    1701
                 ],
                 "precisions": [
-                    0.7183650615901456,
-                    0.5023255813953488,
-                    0.3766626360338573,
-                    0.2915617128463476
+                    0.6766719325961031,
+                    0.4713584288052373,
+                    0.35370684776457273,
+                    0.2721928277483833
                 ],
                 "bp": 1.0,
-                "sys_len": 1786,
+                "sys_len": 1899,
                 "ref_len": 1734,
-                "sacrebleu": 0.4461731000368479,
-                "score": 0.4461731000368479,
+                "sacrebleu": 0.4186126965889709,
+                "score": 0.4186126965889709,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3937462177900075,
-                "score_ci_high": 0.5074742125274534,
-                "sacrebleu_ci_low": 0.3937462177900075,
-                "sacrebleu_ci_high": 0.5074742125274534
+                "score_ci_low": 0.3580478494686714,
+                "score_ci_high": 0.47721854835474,
+                "sacrebleu_ci_low": 0.3580478494686714,
+                "sacrebleu_ci_high": 0.47721854835474
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1272,
-                    815,
-                    567,
-                    394
+                    1286,
+                    859,
+                    598,
+                    417
                 ],
                 "totals": [
-                    1883,
-                    1817,
-                    1751,
-                    1685
+                    1799,
+                    1733,
+                    1667,
+                    1601
                 ],
                 "precisions": [
-                    0.6755177907594265,
-                    0.4485415520088057,
-                    0.32381496287835526,
-                    0.23382789317507416
+                    0.7148415786548082,
+                    0.4956722446624351,
+                    0.3587282543491302,
+                    0.26046221111805123
                 ],
                 "bp": 1.0,
-                "sys_len": 1883,
+                "sys_len": 1799,
                 "ref_len": 1734,
-                "sacrebleu": 0.3891868659594801,
-                "score": 0.3891868659594801,
+                "sacrebleu": 0.42655857647405626,
+                "score": 0.42655857647405626,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3320885601116078,
-                "score_ci_high": 0.43155018915229526,
-                "sacrebleu_ci_low": 0.3320885601116078,
-                "sacrebleu_ci_high": 0.43155018915229526
+                "score_ci_low": 0.37843554301324095,
+                "score_ci_high": 0.4676738192216758,
+                "sacrebleu_ci_low": 0.37843554301324095,
+                "sacrebleu_ci_high": 0.4676738192216758
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1163,
-                    647,
-                    405,
-                    255
+                    1178,
+                    657,
+                    409,
+                    266
                 ],
                 "totals": [
-                    1850,
-                    1784,
-                    1718,
-                    1652
+                    1874,
+                    1808,
+                    1742,
+                    1676
                 ],
                 "precisions": [
-                    0.6286486486486487,
-                    0.36266816143497754,
-                    0.23573923166472643,
-                    0.15435835351089588
+                    0.6286019210245464,
+                    0.36338495575221236,
+                    0.23478760045924227,
+                    0.15871121718377088
                 ],
                 "bp": 1.0,
-                "sys_len": 1850,
+                "sys_len": 1874,
                 "ref_len": 1734,
-                "sacrebleu": 0.30180043021927255,
-                "score": 0.30180043021927255,
+                "sacrebleu": 0.3037430550349969,
+                "score": 0.3037430550349969,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2688254114664501,
-                "score_ci_high": 0.3447493011880723,
-                "sacrebleu_ci_low": 0.2688254114664501,
-                "sacrebleu_ci_high": 0.3447493011880723
+                "score_ci_low": 0.26128349825222114,
+                "score_ci_high": 0.3531214265467947,
+                "sacrebleu_ci_low": 0.26128349825222114,
+                "sacrebleu_ci_high": 0.3531214265467947
             },
-            "score": 0.30224812174972626,
+            "score": 0.3049591129796936,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.40523563547897645,
+        "score": 0.38015857299104155,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-21T09-36-54_evaluation_results.json → 2025-06-23T14-18-29_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-21T13:36:49.808623Z",
+        "timestamp_utc": "2025-06-23T18:18:25.502854Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,13 +176,13 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.6777777777777778,
+                "accuracy": 0.7888888888888889,
+                "accuracy_ci_low": 0.7,
                 "accuracy_ci_high": 0.8555555555555555,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.7888888888888889,
                 "score_ci_high": 0.8555555555555555,
-                "score_ci_low": 0.6777777777777778,
+                "score_ci_low": 0.7,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
@@ -216,13 +216,13 @@
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9555555555555556,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.9888888888888889,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.9555555555555556,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
@@ -276,51 +276,51 @@
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.8666666666666667,
-                "accuracy_ci_low": 0.7888888888888889,
-                "accuracy_ci_high": 0.9333333333333333,
+                "accuracy": 0.8555555555555555,
+                "accuracy_ci_low": 0.7777777777777778,
+                "accuracy_ci_high": 0.9222222222222223,
                 "score_name": "accuracy",
-                "score": 0.8666666666666667,
-                "score_ci_high": 0.9333333333333333,
-                "score_ci_low": 0.7888888888888889,
+                "score": 0.8555555555555555,
+                "score_ci_high": 0.9222222222222223,
+                "score_ci_low": 0.7777777777777778,
                 "num_of_instances": 90
             },
-            "score": 0.9636363636363636,
+            "score": 0.9626262626262626,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.14026236125126135,
+                "score": 0.14026236125126135,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.14026236125126135,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.5744680851063829,
-                "f1_Organization": 0.3625,
-                "f1_Location": 0.3764705882352941,
-                "f1_macro": 0.4378128911138923,
-                "recall_macro": 0.39500837147693896,
-                "precision_macro": 0.49432322991751926,
-                "in_classes_support": 0.6292466765140325,
-                "f1_micro": 0.3560732113144759,
-                "recall_micro": 0.4076190476190476,
-                "precision_micro": 0.31610044313146235,
-                "score": 0.3560732113144759,
+                "f1_Person": 0.5721784776902887,
+                "f1_Organization": 0.37837837837837834,
+                "f1_Location": 0.3692307692307692,
+                "f1_macro": 0.4399292084331454,
+                "recall_macro": 0.40673591832987227,
+                "precision_macro": 0.48338733915656995,
+                "in_classes_support": 0.6414285714285715,
+                "f1_micro": 0.35918367346938773,
+                "recall_micro": 0.41904761904761906,
+                "precision_micro": 0.3142857142857143,
+                "score": 0.35918367346938773,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.31585267626167035,
-                "score_ci_high": 0.4082105948099384,
-                "f1_micro_ci_low": 0.31585267626167035,
-                "f1_micro_ci_high": 0.4082105948099384
+                "score_ci_low": 0.311485746926114,
+                "score_ci_high": 0.40594536569316764,
+                "f1_micro_ci_low": 0.311485746926114,
+                "f1_micro_ci_high": 0.40594536569316764
             },
-            "score": 0.3560732113144759,
+            "score": 0.35918367346938773,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
@@ -336,23 +336,23 @@
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.19718309859154928,
+                "accuracy": 0.18309859154929578,
                 "accuracy_ci_low": 0.11267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy_ci_high": 0.28169014084507044,
                 "score_name": "accuracy",
-                "score": 0.19718309859154928,
-                "score_ci_high": 0.30985915492957744,
+                "score": 0.18309859154929578,
+                "score_ci_high": 0.28169014084507044,
                 "score_ci_low": 0.11267605633802817,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.1267605633802817,
-                "accuracy_ci_low": 0.056338028169014086,
-                "accuracy_ci_high": 0.22535211267605634,
+                "accuracy": 0.15492957746478872,
+                "accuracy_ci_low": 0.08450704225352113,
+                "accuracy_ci_high": 0.2535211267605634,
                 "score_name": "accuracy",
-                "score": 0.1267605633802817,
-                "score_ci_high": 0.22535211267605634,
-                "score_ci_low": 0.056338028169014086,
+                "score": 0.15492957746478872,
+                "score_ci_high": 0.2535211267605634,
+                "score_ci_low": 0.08450704225352113,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
@@ -377,52 +377,52 @@
             },
             "mmlu_pro_engineering": {
                 "accuracy": 0.323943661971831,
-                "accuracy_ci_low": 0.2112676056338028,
+                "accuracy_ci_low": 0.22535211267605634,
                 "accuracy_ci_high": 0.43661971830985913,
                 "score_name": "accuracy",
                 "score": 0.323943661971831,
                 "score_ci_high": 0.43661971830985913,
-                "score_ci_low": 0.2112676056338028,
+                "score_ci_low": 0.22535211267605634,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.5492957746478874,
-                "accuracy_ci_low": 0.43661971830985913,
-                "accuracy_ci_high": 0.6619718309859155,
+                "accuracy": 0.5352112676056338,
+                "accuracy_ci_low": 0.4225352112676056,
+                "accuracy_ci_high": 0.647887323943662,
                 "score_name": "accuracy",
-                "score": 0.5492957746478874,
-                "score_ci_high": 0.6619718309859155,
-                "score_ci_low": 0.43661971830985913,
+                "score": 0.5352112676056338,
+                "score_ci_high": 0.647887323943662,
+                "score_ci_low": 0.4225352112676056,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.6901408450704225,
-                "accuracy_ci_low": 0.5774647887323944,
-                "accuracy_ci_high": 0.7887323943661971,
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6310963819783834,
+                "accuracy_ci_high": 0.8309859154929577,
                 "score_name": "accuracy",
-                "score": 0.6901408450704225,
-                "score_ci_high": 0.7887323943661971,
-                "score_ci_low": 0.5774647887323944,
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6310963819783834,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.5774647887323944,
-                "accuracy_ci_low": 0.4647887323943662,
+                "accuracy": 0.5915492957746479,
+                "accuracy_ci_low": 0.4788732394366197,
                 "accuracy_ci_high": 0.6901408450704225,
                 "score_name": "accuracy",
-                "score": 0.5774647887323944,
+                "score": 0.5915492957746479,
                 "score_ci_high": 0.6901408450704225,
-                "score_ci_low": 0.4647887323943662,
+                "score_ci_low": 0.4788732394366197,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
                 "accuracy": 0.29577464788732394,
-                "accuracy_ci_low": 0.2112676056338028,
-                "accuracy_ci_high": 0.415768498221757,
+                "accuracy_ci_low": 0.19718309859154928,
+                "accuracy_ci_high": 0.41750158298380896,
                 "score_name": "accuracy",
                 "score": 0.29577464788732394,
-                "score_ci_high": 0.415768498221757,
-                "score_ci_low": 0.2112676056338028,
+                "score_ci_high": 0.41750158298380896,
+                "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
@@ -438,21 +438,21 @@
             "mmlu_pro_philosophy": {
                 "accuracy": 0.5915492957746479,
                 "accuracy_ci_low": 0.4647887323943662,
-                "accuracy_ci_high": 0.6901408450704225,
+                "accuracy_ci_high": 0.704225352112676,
                 "score_name": "accuracy",
                 "score": 0.5915492957746479,
-                "score_ci_high": 0.6901408450704225,
+                "score_ci_high": 0.704225352112676,
                 "score_ci_low": 0.4647887323943662,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.3380281690140845,
-                "accuracy_ci_low": 0.22535211267605634,
-                "accuracy_ci_high": 0.4507042253521127,
+                "accuracy": 0.30985915492957744,
+                "accuracy_ci_low": 0.19718309859154928,
+                "accuracy_ci_high": 0.4084507042253521,
                 "score_name": "accuracy",
-                "score": 0.3380281690140845,
-                "score_ci_high": 0.4507042253521127,
-                "score_ci_low": 0.22535211267605634,
+                "score": 0.30985915492957744,
+                "score_ci_high": 0.4084507042253521,
+                "score_ci_low": 0.19718309859154928,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
@@ -465,90 +465,90 @@
                 "score_ci_low": 0.5915492957746479,
                 "num_of_instances": 71
             },
-            "score": 0.48893360160965793,
+            "score": 0.4909456740442656,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.6576598179941833,
+                "f1_macro": 0.6597867569632275,
                 "f1_suggestive": 0.5882352941176471,
                 "f1_generic": 0.72,
                 "f1_descriptive": 0.6818181818181818,
-                "f1_fanciful": 0.6666666666666666,
-                "f1_arbitrary": 0.631578947368421,
-                "f1_macro_ci_low": 0.5539989867710009,
-                "f1_macro_ci_high": 0.7630121059181559,
+                "f1_fanciful": 0.7142857142857143,
+                "f1_arbitrary": 0.5945945945945946,
+                "f1_macro_ci_low": 0.548464495780585,
+                "f1_macro_ci_high": 0.7575557118629758,
                 "score_name": "f1_micro",
                 "score": 0.6547619047619048,
-                "score_ci_high": 0.75,
-                "score_ci_low": 0.5476190476190477,
+                "score_ci_high": 0.7425149700598802,
+                "score_ci_low": 0.5437048440428358,
                 "num_of_instances": 85,
                 "accuracy": 0.6470588235294118,
                 "accuracy_ci_low": 0.5411764705882353,
                 "accuracy_ci_high": 0.7411764705882353,
                 "f1_micro": 0.6547619047619048,
-                "f1_micro_ci_low": 0.5476190476190477,
-                "f1_micro_ci_high": 0.75
+                "f1_micro_ci_low": 0.5437048440428358,
+                "f1_micro_ci_high": 0.7425149700598802
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.5644255319148936,
-                "f1_no": 0.6808510638297872,
-                "f1_yes": 0.448,
-                "f1_macro_ci_low": 0.4955233440030641,
-                "f1_macro_ci_high": 0.637195719539535,
+                "f1_macro": 0.5899229232086782,
+                "f1_no": 0.6637168141592921,
+                "f1_yes": 0.5161290322580645,
+                "f1_macro_ci_low": 0.5156319338532668,
+                "f1_macro_ci_high": 0.6621360959437094,
                 "score_name": "f1_micro",
-                "score": 0.6,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.5292645910446814,
+                "score": 0.6114285714285714,
+                "score_ci_high": 0.6798739003144315,
+                "score_ci_low": 0.5364733968179762,
                 "num_of_instances": 200,
-                "accuracy": 0.54,
-                "accuracy_ci_low": 0.47,
+                "accuracy": 0.535,
+                "accuracy_ci_low": 0.465,
                 "accuracy_ci_high": 0.605,
-                "f1_micro": 0.6,
-                "f1_micro_ci_low": 0.5292645910446814,
-                "f1_micro_ci_high": 0.6666666666666666
+                "f1_micro": 0.6114285714285714,
+                "f1_micro_ci_low": 0.5364733968179762,
+                "f1_micro_ci_high": 0.6798739003144315
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.26417086507814297,
-                "f1_conclusion": 0.15384615384615385,
-                "f1_decree": 0.13333333333333333,
+                "f1_macro": 0.24082948201110896,
+                "f1_conclusion": 0.10810810810810811,
+                "f1_decree": 0.12903225806451613,
                 "f1_issue": 0.18181818181818182,
-                "f1_analysis": 0.5633802816901409,
-                "f1_facts": 0.13793103448275862,
-                "f1_procedural history": 0.3384615384615385,
-                "f1_rule": 0.3404255319148936,
-                "f1_macro_ci_low": 0.2096003414946908,
-                "f1_macro_ci_high": 0.3335047284451988,
+                "f1_analysis": 0.5833333333333334,
+                "f1_facts": 0.06666666666666667,
+                "f1_procedural history": 0.3125,
+                "f1_rule": 0.30434782608695654,
+                "f1_macro_ci_low": 0.19135126191537805,
+                "f1_macro_ci_high": 0.31037088994163425,
                 "score_name": "f1_micro",
-                "score": 0.3076923076923077,
-                "score_ci_high": 0.37785141927318056,
-                "score_ci_low": 0.24264738787974433,
+                "score": 0.29012345679012347,
+                "score_ci_high": 0.3634815160611135,
+                "score_ci_low": 0.22855500349415586,
                 "num_of_instances": 200,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.195,
-                "accuracy_ci_high": 0.315,
-                "f1_micro": 0.3076923076923077,
-                "f1_micro_ci_low": 0.24264738787974433,
-                "f1_micro_ci_high": 0.37785141927318056
+                "accuracy": 0.235,
+                "accuracy_ci_low": 0.185,
+                "accuracy_ci_high": 0.3,
+                "f1_micro": 0.29012345679012347,
+                "f1_micro_ci_low": 0.22855500349415586,
+                "f1_micro_ci_high": 0.3634815160611135
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.463898065125059,
-                "f1_yes": 0.4662576687116564,
-                "f1_no": 0.46153846153846156,
-                "f1_macro_ci_low": 0.39561838155337925,
-                "f1_macro_ci_high": 0.5401290279855109,
+                "f1_macro": 0.4444123800591588,
+                "f1_yes": 0.4550898203592814,
+                "f1_no": 0.43373493975903615,
+                "f1_macro_ci_low": 0.3748865543059881,
+                "f1_macro_ci_high": 0.5178711641402558,
                 "score_name": "f1_micro",
-                "score": 0.463855421686747,
-                "score_ci_high": 0.5388200914855517,
-                "score_ci_low": 0.3937587086040863,
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.5162242117942616,
+                "score_ci_low": 0.37379019448718637,
                 "num_of_instances": 200,
-                "accuracy": 0.385,
-                "accuracy_ci_low": 0.32,
-                "accuracy_ci_high": 0.455,
-                "f1_micro": 0.463855421686747,
-                "f1_micro_ci_low": 0.3937587086040863,
-                "f1_micro_ci_high": 0.5388200914855517
+                "accuracy": 0.37,
+                "accuracy_ci_low": 0.305,
+                "accuracy_ci_high": 0.435,
+                "f1_micro": 0.4444444444444444,
+                "f1_micro_ci_low": 0.37379019448718637,
+                "f1_micro_ci_high": 0.5162242117942616
             },
             "legalbench_proa": {
                 "f1_macro": 0.75,
@@ -568,172 +568,172 @@
                 "f1_micro_ci_low": 0.6573326079878734,
                 "f1_micro_ci_high": 0.8234882632928148
             },
-            "score": 0.5552619268281919,
+            "score": 0.5501516754850089,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.6134758195135697,
-                "f1_cars": 0.8297872340425532,
-                "f1_windows x": 0.14492753623188406,
-                "f1_computer graphics": 0.41379310344827586,
-                "f1_atheism": 0.48148148148148145,
-                "f1_religion": 0.24561403508771928,
-                "f1_medicine": 0.8372093023255814,
-                "f1_christianity": 0.7878787878787878,
-                "f1_microsoft windows": 0.611764705882353,
-                "f1_middle east": 0.5970149253731343,
-                "f1_motorcycles": 0.7128712871287128,
-                "f1_pc hardware": 0.5736434108527132,
-                "f1_mac hardware": 0.6382978723404256,
-                "f1_electronics": 0.6436781609195402,
-                "f1_for sale": 0.7027027027027027,
-                "f1_guns": 0.34375,
-                "f1_space": 0.8269230769230769,
-                "f1_cryptography": 0.6486486486486487,
-                "f1_baseball": 0.9122807017543859,
+                "f1_macro": 0.625283027988775,
+                "f1_cars": 0.8541666666666666,
+                "f1_windows x": 0.11940298507462686,
+                "f1_computer graphics": 0.4297520661157025,
+                "f1_atheism": 0.509090909090909,
+                "f1_religion": 0.2222222222222222,
+                "f1_medicine": 0.8705882352941177,
+                "f1_christianity": 0.7755102040816326,
+                "f1_microsoft windows": 0.6436781609195402,
+                "f1_middle east": 0.6666666666666666,
+                "f1_motorcycles": 0.7326732673267327,
+                "f1_pc hardware": 0.5846153846153846,
+                "f1_mac hardware": 0.6458333333333334,
+                "f1_electronics": 0.6666666666666666,
+                "f1_for sale": 0.6944444444444444,
+                "f1_guns": 0.36923076923076925,
+                "f1_space": 0.8235294117647058,
+                "f1_cryptography": 0.6575342465753424,
+                "f1_baseball": 0.9310344827586207,
                 "f1_politics": 0.3787878787878788,
-                "f1_hockey": 0.9384615384615385,
-                "f1_macro_ci_low": 0.5870232871618599,
-                "f1_macro_ci_high": 0.6399401092520309,
+                "f1_hockey": 0.9302325581395349,
+                "f1_macro_ci_low": 0.5984293289041998,
+                "f1_macro_ci_high": 0.65230217299566,
                 "score_name": "f1_micro",
-                "score": 0.6360655737704918,
-                "score_ci_high": 0.6652078896587189,
-                "score_ci_low": 0.606009312496833,
+                "score": 0.6474114441416894,
+                "score_ci_high": 0.6749803309845304,
+                "score_ci_low": 0.6158904109589041,
                 "num_of_instances": 1000,
-                "accuracy": 0.582,
-                "accuracy_ci_low": 0.549,
-                "accuracy_ci_high": 0.61,
-                "f1_micro": 0.6360655737704918,
-                "f1_micro_ci_low": 0.606009312496833,
-                "f1_micro_ci_high": 0.6652078896587189
+                "accuracy": 0.594,
+                "accuracy_ci_low": 0.561,
+                "accuracy_ci_high": 0.622,
+                "f1_micro": 0.6474114441416894,
+                "f1_micro_ci_low": 0.6158904109589041,
+                "f1_micro_ci_high": 0.6749803309845304
             },
-            "score": 0.6360655737704918,
+            "score": 0.6474114441416894,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.7310250832696968,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9301994301994302,
-                "f1_payday loan or title loan or personal loan": 0.2222222222222222,
-                "f1_mortgage": 0.8450704225352113,
-                "f1_credit card or prepaid card": 0.7596899224806202,
-                "f1_debt collection": 0.6887417218543046,
-                "f1_vehicle loan or lease": 0.7906976744186046,
-                "f1_checking or savings account": 0.8210526315789474,
-                "f1_money transfer or virtual currency or money service": 0.625,
-                "f1_student loan": 0.896551724137931,
-                "f1_macro_ci_low": 0.6859846742091775,
-                "f1_macro_ci_high": 0.7782039487360795,
+                "f1_macro": 0.7879467396802322,
+                "f1_student loan": 0.8888888888888888,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9461756373937678,
+                "f1_debt collection": 0.6428571428571429,
+                "f1_checking or savings account": 0.8222222222222222,
+                "f1_mortgage": 0.9705882352941176,
+                "f1_payday loan or title loan or personal loan": 0.5333333333333333,
+                "f1_credit card or prepaid card": 0.8666666666666667,
+                "f1_money transfer or virtual currency or money service": 0.7111111111111111,
+                "f1_vehicle loan or lease": 0.7096774193548387,
+                "f1_macro_ci_low": 0.7309747205142173,
+                "f1_macro_ci_high": 0.8394377629013812,
                 "score_name": "f1_micro",
-                "score": 0.8752515090543259,
-                "score_ci_high": 0.8950697745652848,
-                "score_ci_low": 0.8535042875641251,
+                "score": 0.8977732793522267,
+                "score_ci_high": 0.9148163850441952,
+                "score_ci_low": 0.8791739655658123,
                 "num_of_instances": 1000,
-                "accuracy": 0.87,
-                "accuracy_ci_low": 0.848,
-                "accuracy_ci_high": 0.8900650893159325,
-                "f1_micro": 0.8752515090543259,
-                "f1_micro_ci_low": 0.8535042875641251,
-                "f1_micro_ci_high": 0.8950697745652848
+                "accuracy": 0.887,
+                "accuracy_ci_low": 0.8679599560953464,
+                "accuracy_ci_high": 0.906,
+                "f1_micro": 0.8977732793522267,
+                "f1_micro_ci_low": 0.8791739655658123,
+                "f1_micro_ci_high": 0.9148163850441952
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.7446539429987555,
-                "f1_mortgages and loans": 0.8,
+                "f1_macro": 0.755452767113785,
+                "f1_mortgages and loans": 0.8135593220338984,
                 "f1_credit card": 0.7865168539325843,
-                "f1_debt collection": 0.6972477064220184,
-                "f1_credit reporting": 0.7835051546391752,
-                "f1_retail banking": 0.656,
-                "f1_macro_ci_low": 0.7076662872810797,
-                "f1_macro_ci_high": 0.7839689190392729,
+                "f1_debt collection": 0.7069767441860465,
+                "f1_credit reporting": 0.7876712328767124,
+                "f1_retail banking": 0.6825396825396826,
+                "f1_macro_ci_low": 0.7186153400597911,
+                "f1_macro_ci_high": 0.7954872943230671,
                 "score_name": "f1_micro",
-                "score": 0.75177304964539,
-                "score_ci_high": 0.7885153719683515,
-                "score_ci_low": 0.7144890867826504,
+                "score": 0.7611336032388664,
+                "score_ci_high": 0.797979797979798,
+                "score_ci_low": 0.7233852933885262,
                 "num_of_instances": 500,
-                "accuracy": 0.742,
-                "accuracy_ci_low": 0.7044918597766052,
-                "accuracy_ci_high": 0.78,
-                "f1_micro": 0.75177304964539,
-                "f1_micro_ci_low": 0.7144890867826504,
-                "f1_micro_ci_high": 0.7885153719683515
+                "accuracy": 0.752,
+                "accuracy_ci_low": 0.712,
+                "accuracy_ci_high": 0.79,
+                "f1_micro": 0.7611336032388664,
+                "f1_micro_ci_low": 0.7233852933885262,
+                "f1_micro_ci_high": 0.797979797979798
             },
-            "score": 0.8135122793498579,
+            "score": 0.8294534412955465,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "program_accuracy": 0.206,
-                "score": 0.206,
+                "program_accuracy": 0.21,
+                "score": 0.21,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.183,
-                "program_accuracy_ci_low": 0.18192581131748398,
-                "program_accuracy_ci_high": 0.233,
-                "score_ci_low": 0.18192581131748398,
-                "score_ci_high": 0.233,
-                "execution_accuracy_ci_low": 0.16,
-                "execution_accuracy_ci_high": 0.206
+                "execution_accuracy": 0.19,
+                "program_accuracy_ci_low": 0.1850718210152138,
+                "program_accuracy_ci_high": 0.236,
+                "score_ci_low": 0.1850718210152138,
+                "score_ci_high": 0.236,
+                "execution_accuracy_ci_low": 0.167,
+                "execution_accuracy_ci_high": 0.214
             },
-            "score": 0.206,
+            "score": 0.21,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.2829195176672266,
-                "recall": 0.5895069651787405,
-                "f1": 0.3247642449571121,
-                "precision_ci_low": 0.2632314020521836,
-                "precision_ci_high": 0.3018747287495355,
-                "recall_ci_low": 0.5757048544050242,
-                "recall_ci_high": 0.605461197768975,
-                "f1_ci_low": 0.3070149800069332,
-                "f1_ci_high": 0.3415261837814938,
+                "precision": 0.2793294516338928,
+                "recall": 0.5853975586330095,
+                "f1": 0.32244875528474853,
+                "precision_ci_low": 0.2608509633355691,
+                "precision_ci_high": 0.2992012655344498,
+                "recall_ci_low": 0.5709386747947787,
+                "recall_ci_high": 0.6004552716919522,
+                "f1_ci_low": 0.3057656195014431,
+                "f1_ci_high": 0.34039188672012827,
                 "score_name": "f1",
-                "score": 0.3247642449571121,
-                "score_ci_high": 0.3415261837814938,
-                "score_ci_low": 0.3070149800069332,
+                "score": 0.32244875528474853,
+                "score_ci_high": 0.34039188672012827,
+                "score_ci_low": 0.3057656195014431,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5840552796920141,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6764627607663473,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5322982901086409,
-                "faithfullness_f1_token_overlap": 0.3161988054051969,
-                "faithfullness_recall_token_overlap": 0.2527973404819663,
-                "faithfullness_precision_token_overlap": 0.5297932448406707,
-                "correctness_f1_token_overlap": 0.3247642449571121,
-                "correctness_recall_token_overlap": 0.5895069651787405,
-                "correctness_precision_token_overlap": 0.2829195176672266
+                "correctness_f1_bert_score.deberta_large_mnli": 0.583492674678564,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6757716050744057,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5312159793823957,
+                "faithfullness_f1_token_overlap": 0.31789392117250886,
+                "faithfullness_recall_token_overlap": 0.25491265786006334,
+                "faithfullness_precision_token_overlap": 0.5284813664748244,
+                "correctness_f1_token_overlap": 0.32244875528474853,
+                "correctness_recall_token_overlap": 0.5853975586330095,
+                "correctness_precision_token_overlap": 0.2793294516338928
             },
-            "score": 0.3247642449571121,
+            "score": 0.32244875528474853,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.568,
-                "accuracy_ci_low": 0.536,
-                "accuracy_ci_high": 0.597,
+                "accuracy": 0.565,
+                "accuracy_ci_low": 0.5335774990883203,
+                "accuracy_ci_high": 0.5932386235436204,
                 "score_name": "accuracy",
-                "score": 0.568,
-                "score_ci_high": 0.597,
-                "score_ci_low": 0.536,
+                "score": 0.565,
+                "score_ci_high": 0.5932386235436204,
+                "score_ci_low": 0.5335774990883203,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.922,
-                "accuracy_ci_low": 0.8937961961124387,
-                "accuracy_ci_high": 0.9425849640401368,
+                "accuracy": 0.92,
+                "accuracy_ci_low": 0.892,
+                "accuracy_ci_high": 0.94,
                 "score_name": "accuracy",
-                "score": 0.922,
-                "score_ci_high": 0.9425849640401368,
-                "score_ci_low": 0.8937961961124387,
+                "score": 0.92,
+                "score_ci_high": 0.94,
+                "score_ci_low": 0.892,
                 "num_of_instances": 500
             },
-            "score": 0.745,
+            "score": 0.7424999999999999,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
@@ -741,68 +741,68 @@
             "attaq_500": {
                 "severity_critical": 0.0,
                 "severity_high": 0.0,
-                "severity_medium": 1.0,
-                "severity_low": 99.0,
-                "category_harmful_info": 0.8334540272453089,
-                "category_deception": 0.8512123076602666,
-                "category_explicit_content": 0.8684844600431846,
-                "category_violence": 0.8207998569862591,
-                "category_discrimination": 0.7722197345324925,
-                "category_substance_abuse": 0.8600103222661548,
-                "category_pii": 0.8025329068855003,
-                "safety": 0.8309456067584041,
-                "safety_ci_low": 0.812541088262446,
-                "safety_ci_high": 0.8500828095785149,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.8275590381670063,
+                "category_deception": 0.870389111565821,
+                "category_explicit_content": 0.8619736799537534,
+                "category_violence": 0.839932223764204,
+                "category_discrimination": 0.7697427272796631,
+                "category_substance_abuse": 0.8608841189907658,
+                "category_pii": 0.7655655101493553,
+                "safety": 0.8361458782349815,
+                "safety_ci_low": 0.8192866587424859,
+                "safety_ci_high": 0.8545105900812325,
                 "score_name": "safety",
-                "score": 0.8309456067584041,
-                "score_ci_high": 0.8500828095785149,
-                "score_ci_low": 0.812541088262446,
+                "score": 0.8361458782349815,
+                "score_ci_high": 0.8545105900812325,
+                "score_ci_low": 0.8192866587424859,
                 "num_of_instances": 100
             },
-            "score": 0.8309456067584041,
+            "score": 0.8361458782349815,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge1": 0.4363286082688575,
-                "rouge2": 0.22072608165229177,
-                "rougeL": 0.3067201488969288,
-                "score": 0.3067201488969288,
+                "rougeLsum": 0.3756142003700213,
+                "rouge1": 0.43798911333105917,
+                "rouge2": 0.22301544369181786,
+                "rougeL": 0.30881611781607615,
+                "score": 0.30881611781607615,
                 "score_name": "rougeL",
-                "rougeLsum": 0.37348424474826925,
-                "rouge1_ci_low": 0.42622540172251744,
-                "rouge1_ci_high": 0.445479768051287,
-                "rouge2_ci_low": 0.2133469390728694,
-                "rouge2_ci_high": 0.22852486335907615,
-                "rougeL_ci_low": 0.2990472341490495,
-                "rougeL_ci_high": 0.31469167779345575,
-                "score_ci_low": 0.2990472341490495,
-                "score_ci_high": 0.31469167779345575,
-                "rougeLsum_ci_low": 0.36439724635298765,
-                "rougeLsum_ci_high": 0.3821672398567385
+                "rougeLsum_ci_low": 0.3665764288194776,
+                "rougeLsum_ci_high": 0.3845640981112231,
+                "rouge1_ci_low": 0.42803373903414665,
+                "rouge1_ci_high": 0.44745782528977346,
+                "rouge2_ci_low": 0.21565182499600158,
+                "rouge2_ci_high": 0.23166608743475037,
+                "rougeL_ci_low": 0.3011644909175205,
+                "rougeL_ci_high": 0.31722072238042875,
+                "score_ci_low": 0.3011644909175205,
+                "score_ci_high": 0.31722072238042875
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge1": 0.13281253260941045,
-                "rouge2": 0.020216836126571446,
-                "rougeL": 0.0953008337162679,
-                "score": 0.0953008337162679,
+                "rougeLsum": 0.1094469715392792,
+                "rouge1": 0.13301752816755213,
+                "rouge2": 0.020621633401068214,
+                "rougeL": 0.09635378924374519,
+                "score": 0.09635378924374519,
                 "score_name": "rougeL",
-                "rougeLsum": 0.10871871026938495,
-                "rouge1_ci_low": 0.1270654167753335,
-                "rouge1_ci_high": 0.13798234251944885,
-                "rouge2_ci_low": 0.01830191124480353,
-                "rouge2_ci_high": 0.022113898574922495,
-                "rougeL_ci_low": 0.09124510500356316,
-                "rougeL_ci_high": 0.098681444438937,
-                "score_ci_low": 0.09124510500356316,
-                "score_ci_high": 0.098681444438937,
-                "rougeLsum_ci_low": 0.10412203835097511,
-                "rougeLsum_ci_high": 0.11266943648159043
+                "rougeLsum_ci_low": 0.10482036952689613,
+                "rougeLsum_ci_high": 0.11359992785988014,
+                "rouge1_ci_low": 0.12738784135782572,
+                "rouge1_ci_high": 0.13845012766033873,
+                "rouge2_ci_low": 0.01856074580818113,
+                "rouge2_ci_high": 0.02259991124480518,
+                "rougeL_ci_low": 0.09230760346929477,
+                "rougeL_ci_high": 0.09984237535822288,
+                "score_ci_low": 0.09230760346929477,
+                "score_ci_high": 0.09984237535822288
             },
-            "score": 0.20101049130659834,
+            "score": 0.20258495352991068,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1297,
-                    853,
-                    602,
-                    432
+                    1310,
+                    862,
+                    608,
+                    433
                 ],
                 "totals": [
-                    1773,
-                    1707,
-                    1641,
-                    1575
+                    1791,
+                    1725,
+                    1659,
+                    1593
                 ],
                 "precisions": [
-                    0.7315284827975184,
-                    0.4997070884592853,
-                    0.3668494820231566,
-                    0.27428571428571424
+                    0.7314349525404802,
+                    0.49971014492753624,
+                    0.36648583484026526,
+                    0.27181418706842436
                 ],
                 "bp": 1.0,
-                "sys_len": 1773,
+                "sys_len": 1791,
                 "ref_len": 1734,
-                "sacrebleu": 0.4379348051104114,
-                "score": 0.4379348051104114,
+                "sacrebleu": 0.43682330208953546,
+                "score": 0.43682330208953546,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3839572779473718,
-                "score_ci_high": 0.47542268740294724,
-                "sacrebleu_ci_low": 0.3839572779473718,
-                "sacrebleu_ci_high": 0.47542268740294724
+                "score_ci_low": 0.3952567851744898,
+                "score_ci_high": 0.4779782047825724,
+                "sacrebleu_ci_low": 0.3952567851744898,
+                "sacrebleu_ci_high": 0.4779782047825724
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1336,
-                    889,
-                    624,
-                    446
+                    1330,
+                    885,
+                    621,
+                    444
                 ],
                 "totals": [
-                    1793,
-                    1727,
-                    1661,
-                    1595
+                    1803,
+                    1737,
+                    1671,
+                    1605
                 ],
                 "precisions": [
-                    0.7451199107640825,
-                    0.5147654892877823,
-                    0.37567730282962075,
-                    0.27962382445141065
+                    0.7376594564614531,
+                    0.5094991364421416,
+                    0.37163375224416517,
+                    0.2766355140186916
                 ],
                 "bp": 1.0,
-                "sys_len": 1793,
+                "sys_len": 1803,
                 "ref_len": 1734,
-                "sacrebleu": 0.4480290559468694,
-                "score": 0.4480290559468694,
+                "sacrebleu": 0.44335908539973706,
+                "score": 0.44335908539973706,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.40758551741171895,
-                "score_ci_high": 0.49424014474329564,
-                "sacrebleu_ci_low": 0.40758551741171895,
-                "sacrebleu_ci_high": 0.49424014474329564
+                "score_ci_low": 0.407676677665113,
+                "score_ci_high": 0.49444892387735484,
+                "sacrebleu_ci_low": 0.407676677665113,
+                "sacrebleu_ci_high": 0.49444892387735484
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    908,
-                    498,
-                    301,
-                    181
+                    904,
+                    502,
+                    300,
+                    175
                 ],
                 "totals": [
-                    1600,
-                    1534,
-                    1468,
-                    1402
+                    1585,
+                    1519,
+                    1453,
+                    1387
                 ],
                 "precisions": [
-                    0.5675,
-                    0.3246414602346806,
-                    0.20504087193460488,
-                    0.1291012838801712
+                    0.5703470031545741,
+                    0.3304805793285056,
+                    0.20646937370956642,
+                    0.12617159336697908
                 ],
-                "bp": 1.0,
-                "sys_len": 1600,
+                "bp": 0.9974795224450381,
+                "sys_len": 1585,
                 "ref_len": 1589,
-                "sacrebleu": 0.26426230123240096,
-                "score": 0.26426230123240096,
+                "sacrebleu": 0.26404598765634385,
+                "score": 0.26404598765634385,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2206088677339497,
-                "score_ci_high": 0.3043911445742578,
-                "sacrebleu_ci_low": 0.2206088677339497,
-                "sacrebleu_ci_high": 0.3043911445742578
+                "score_ci_low": 0.23497301086217232,
+                "score_ci_high": 0.29259086123320976,
+                "sacrebleu_ci_low": 0.23497301086217232,
+                "sacrebleu_ci_high": 0.29259086123320976
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1252,
-                    762,
-                    513,
-                    371
+                    1246,
+                    765,
+                    517,
+                    376
                 ],
                 "totals": [
-                    1850,
-                    1784,
-                    1718,
-                    1652
+                    1853,
+                    1787,
+                    1721,
+                    1655
                 ],
                 "precisions": [
-                    0.6767567567567567,
-                    0.42713004484304934,
-                    0.29860302677532014,
-                    0.2245762711864407
+                    0.6724230976794388,
+                    0.42809177392277564,
+                    0.3004067402672865,
+                    0.22719033232628397
                 ],
                 "bp": 1.0,
-                "sys_len": 1850,
+                "sys_len": 1853,
                 "ref_len": 1835,
-                "sacrebleu": 0.37313217401540816,
-                "score": 0.37313217401540816,
+                "sacrebleu": 0.3743861346447394,
+                "score": 0.3743861346447394,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3292059035458714,
-                "score_ci_high": 0.4247888842695035,
-                "sacrebleu_ci_low": 0.3292059035458714,
-                "sacrebleu_ci_high": 0.4247888842695035
+                "score_ci_low": 0.3331443738925502,
+                "score_ci_high": 0.4167892583826109,
+                "sacrebleu_ci_low": 0.3331443738925502,
+                "sacrebleu_ci_high": 0.4167892583826109
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1553,
-                    1160,
-                    918,
-                    734
+                    1562,
+                    1176,
+                    936,
+                    755
                 ],
                 "totals": [
-                    2035,
-                    1969,
-                    1903,
-                    1837
+                    2040,
+                    1974,
+                    1908,
+                    1842
                 ],
                 "precisions": [
-                    0.763144963144963,
-                    0.5891315388522093,
-                    0.4823962165002627,
-                    0.3995645073489385
+                    0.7656862745098039,
+                    0.5957446808510638,
+                    0.49056603773584906,
+                    0.40988056460369166
                 ],
-                "bp": 0.9839145587719164,
-                "sys_len": 2035,
+                "bp": 0.9863682748637871,
+                "sys_len": 2040,
                 "ref_len": 2068,
-                "sacrebleu": 0.5338385284906572,
-                "score": 0.5338385284906572,
+                "sacrebleu": 0.5428196432331734,
+                "score": 0.5428196432331734,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4960508343377122,
-                "score_ci_high": 0.5778992367091751,
-                "sacrebleu_ci_low": 0.4960508343377122,
-                "sacrebleu_ci_high": 0.5778992367091751
+                "score_ci_low": 0.509892185181122,
+                "score_ci_high": 0.5871694828358561,
+                "sacrebleu_ci_low": 0.509892185181122,
+                "sacrebleu_ci_high": 0.5871694828358561
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1322,
-                    700,
-                    421,
-                    259
+                    1356,
+                    727,
+                    432,
+                    274
                 ],
                 "totals": [
-                    2388,
-                    2322,
-                    2256,
-                    2190
+                    2382,
+                    2316,
+                    2250,
+                    2184
                 ],
                 "precisions": [
-                    0.5536013400335008,
-                    0.301464254952627,
-                    0.18661347517730498,
-                    0.1182648401826484
+                    0.5692695214105793,
+                    0.3139032815198618,
+                    0.192,
+                    0.12545787545787546
                 ],
                 "bp": 1.0,
-                "sys_len": 2388,
+                "sys_len": 2382,
                 "ref_len": 2235,
-                "sacrebleu": 0.2463530263425343,
-                "score": 0.2463530263425343,
+                "sacrebleu": 0.25614049024804236,
+                "score": 0.25614049024804236,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21902218576525906,
-                "score_ci_high": 0.28268262467368716,
-                "sacrebleu_ci_low": 0.21902218576525906,
-                "sacrebleu_ci_high": 0.28268262467368716
+                "score_ci_low": 0.2240052628669271,
+                "score_ci_high": 0.2935558382274424,
+                "sacrebleu_ci_low": 0.2240052628669271,
+                "sacrebleu_ci_high": 0.2935558382274424
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1463,
-                    1077,
-                    833,
-                    645
+                    1455,
+                    1063,
+                    821,
+                    637
                 ],
                 "totals": [
-                    1890,
-                    1824,
-                    1758,
-                    1692
+                    1900,
+                    1834,
+                    1768,
+                    1702
                 ],
                 "precisions": [
-                    0.774074074074074,
-                    0.5904605263157895,
-                    0.47383390216154725,
-                    0.3812056737588652
+                    0.7657894736842106,
+                    0.579607415485278,
+                    0.4643665158371041,
+                    0.37426556991774385
                 ],
-                "bp": 0.9863375760488048,
-                "sys_len": 1890,
+                "bp": 0.9916143051127146,
+                "sys_len": 1900,
                 "ref_len": 1916,
-                "sacrebleu": 0.5287072892340219,
-                "score": 0.5287072892340219,
+                "sacrebleu": 0.5225932644775685,
+                "score": 0.5225932644775685,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.47933696238952217,
-                "score_ci_high": 0.5714963999565859,
-                "sacrebleu_ci_low": 0.47933696238952217,
-                "sacrebleu_ci_high": 0.5714963999565859
+                "score_ci_low": 0.4737364435189365,
+                "score_ci_high": 0.5631623567289689,
+                "sacrebleu_ci_low": 0.4737364435189365,
+                "sacrebleu_ci_high": 0.5631623567289689
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1402,
-                    981,
-                    716,
-                    523
+                    1404,
+                    989,
+                    719,
+                    525
                 ],
                 "totals": [
-                    1969,
-                    1903,
-                    1837,
-                    1771
+                    1962,
+                    1896,
+                    1830,
+                    1764
                 ],
                 "precisions": [
-                    0.71203656678517,
-                    0.5155018392012611,
-                    0.3897659227000544,
-                    0.295313382269904
+                    0.7155963302752294,
+                    0.5216244725738397,
+                    0.39289617486338796,
+                    0.2976190476190476
                 ],
                 "bp": 1.0,
-                "sys_len": 1969,
+                "sys_len": 1962,
                 "ref_len": 1949,
-                "sacrebleu": 0.4533721907576224,
-                "score": 0.4533721907576224,
+                "sacrebleu": 0.45707887169863065,
+                "score": 0.45707887169863065,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.41581094731566015,
-                "score_ci_high": 0.498033544687375,
-                "sacrebleu_ci_low": 0.41581094731566015,
-                "sacrebleu_ci_high": 0.498033544687375
+                "score_ci_low": 0.4211360031313033,
+                "score_ci_high": 0.5197096344136953,
+                "sacrebleu_ci_low": 0.4211360031313033,
+                "sacrebleu_ci_high": 0.5197096344136953
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1285,
-                    735,
-                    453,
-                    283
+                    1297,
+                    753,
+                    472,
+                    301
                 ],
                 "totals": [
-                    1994,
-                    1928,
-                    1862,
-                    1796
+                    2014,
+                    1948,
+                    1882,
+                    1816
                 ],
                 "precisions": [
-                    0.6444332998996991,
-                    0.38122406639004147,
-                    0.24328678839957035,
-                    0.15757238307349666
+                    0.6439920556107249,
+                    0.38655030800821355,
+                    0.2507970244420829,
+                    0.1657488986784141
                 ],
-                "bp": 0.9491803375373334,
-                "sys_len": 1994,
+                "bp": 0.9591497695217011,
+                "sys_len": 2014,
                 "ref_len": 2098,
-                "sacrebleu": 0.2956909013134059,
-                "score": 0.2956909013134059,
+                "sacrebleu": 0.3059153842651481,
+                "score": 0.3059153842651481,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2691627285024834,
-                "score_ci_high": 0.32834717984418194,
-                "sacrebleu_ci_low": 0.2691627285024834,
-                "sacrebleu_ci_high": 0.32834717984418194
+                "score_ci_low": 0.2738471419962368,
+                "score_ci_high": 0.33744567062204633,
+                "sacrebleu_ci_low": 0.2738471419962368,
+                "sacrebleu_ci_high": 0.33744567062204633
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1366,
+                    1371,
                     964,
-                    696,
-                    499
+                    693,
+                    491
                 ],
                 "totals": [
-                    1844,
-                    1778,
-                    1712,
-                    1646
+                    1839,
+                    1773,
+                    1707,
+                    1641
                 ],
                 "precisions": [
-                    0.7407809110629067,
-                    0.5421822272215973,
-                    0.40654205607476634,
-                    0.30315917375455653
+                    0.7455138662316476,
+                    0.5437112239142696,
+                    0.40597539543058,
+                    0.2992078001218769
                 ],
                 "bp": 1.0,
-                "sys_len": 1844,
+                "sys_len": 1839,
                 "ref_len": 1734,
-                "sacrebleu": 0.47168581235777585,
-                "score": 0.47168581235777585,
+                "sacrebleu": 0.4710577594991048,
+                "score": 0.4710577594991048,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.41832464742975967,
-                "score_ci_high": 0.5113720236775223,
-                "sacrebleu_ci_low": 0.41832464742975967,
-                "sacrebleu_ci_high": 0.5113720236775223
+                "score_ci_low": 0.4268855553367512,
+                "score_ci_high": 0.5049624748525423,
+                "sacrebleu_ci_low": 0.4268855553367512,
+                "sacrebleu_ci_high": 0.5049624748525423
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1121,
-                    627,
-                    398,
-                    260
+                    1130,
+                    638,
+                    409,
+                    268
                 ],
                 "totals": [
-                    1831,
-                    1765,
-                    1699,
-                    1633
+                    1794,
+                    1728,
+                    1662,
+                    1596
                 ],
                 "precisions": [
-                    0.6122337520480612,
-                    0.35524079320113316,
-                    0.23425544437904647,
-                    0.15921616656460502
+                    0.6298773690078038,
+                    0.36921296296296297,
+                    0.24608904933814682,
+                    0.16791979949874686
                 ],
                 "bp": 1.0,
-                "sys_len": 1831,
+                "sys_len": 1794,
                 "ref_len": 1734,
-                "sacrebleu": 0.30010915149877077,
-                "score": 0.30010915149877077,
+                "sacrebleu": 0.31309907547937593,
+                "score": 0.31309907547937593,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25220636468972635,
-                "score_ci_high": 0.34253906716484656,
-                "sacrebleu_ci_low": 0.25220636468972635,
-                "sacrebleu_ci_high": 0.34253906716484656
+                "score_ci_low": 0.26572456707025455,
+                "score_ci_high": 0.3514896703047057,
+                "sacrebleu_ci_low": 0.26572456707025455,
+                "sacrebleu_ci_high": 0.3514896703047057
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1108,
-                    607,
-                    365,
-                    233
+                    1112,
+                    613,
+                    383,
+                    250
                 ],
                 "totals": [
-                    1751,
-                    1685,
-                    1619,
-                    1553
+                    1725,
+                    1659,
+                    1593,
+                    1527
                 ],
                 "precisions": [
-                    0.6327812678469446,
-                    0.3602373887240356,
-                    0.22544780728844965,
-                    0.150032195750161
+                    0.6446376811594203,
+                    0.3694996986136227,
+                    0.24042686754551162,
+                    0.16371971185330714
                 ],
-                "bp": 1.0,
-                "sys_len": 1751,
+                "bp": 0.9947961956419216,
+                "sys_len": 1725,
                 "ref_len": 1734,
-                "sacrebleu": 0.2963249503420515,
-                "score": 0.2963249503420515,
+                "sacrebleu": 0.3095548068732939,
+                "score": 0.3095548068732939,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25949260863076823,
-                "score_ci_high": 0.3478140976036152,
-                "sacrebleu_ci_low": 0.25949260863076823,
-                "sacrebleu_ci_high": 0.3478140976036152
+                "score_ci_low": 0.27129838546648216,
+                "score_ci_high": 0.3652127158245783,
+                "sacrebleu_ci_low": 0.27129838546648216,
+                "sacrebleu_ci_high": 0.3652127158245783
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1380,
-                    1014,
-                    776,
-                    594
+                    1382,
+                    1013,
+                    771,
+                    586
                 ],
                 "totals": [
-                    1815,
-                    1749,
-                    1683,
-                    1617
+                    1816,
+                    1750,
+                    1684,
+                    1618
                 ],
                 "precisions": [
-                    0.7603305785123967,
-                    0.5797598627787307,
-                    0.46108140225787286,
-                    0.3673469387755102
+                    0.7610132158590308,
+                    0.5788571428571428,
+                    0.4578384798099763,
+                    0.3621755253399258
                 ],
                 "bp": 1.0,
-                "sys_len": 1815,
+                "sys_len": 1816,
                 "ref_len": 1734,
-                "sacrebleu": 0.5227284747817351,
-                "score": 0.5227284747817351,
+                "sacrebleu": 0.5198747760500348,
+                "score": 0.5198747760500348,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.46746182790145246,
-                "score_ci_high": 0.5686400938537622,
-                "sacrebleu_ci_low": 0.46746182790145246,
-                "sacrebleu_ci_high": 0.5686400938537622
+                "score_ci_low": 0.470424249351864,
+                "score_ci_high": 0.5665291609523828,
+                "sacrebleu_ci_low": 0.470424249351864,
+                "sacrebleu_ci_high": 0.5665291609523828
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1375,
-                    972,
-                    711,
-                    529
+                    1376,
+                    980,
+                    722,
+                    541
                 ],
                 "totals": [
-                    1800,
-                    1734,
-                    1668,
-                    1602
+                    1801,
+                    1735,
+                    1669,
+                    1603
                 ],
                 "precisions": [
-                    0.7638888888888888,
-                    0.560553633217993,
-                    0.4262589928057554,
-                    0.33021223470661676
+                    0.7640199888950583,
+                    0.5648414985590778,
+                    0.43259436788496103,
+                    0.3374922021210231
                 ],
                 "bp": 1.0,
-                "sys_len": 1800,
+                "sys_len": 1801,
                 "ref_len": 1734,
-                "sacrebleu": 0.49548272782749336,
-                "score": 0.49548272782749336,
+                "sacrebleu": 0.5010072151606186,
+                "score": 0.5010072151606186,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.45134581590558903,
-                "score_ci_high": 0.5288737168538972,
-                "sacrebleu_ci_low": 0.45134581590558903,
-                "sacrebleu_ci_high": 0.5288737168538972
+                "score_ci_low": 0.4614513028457302,
+                "score_ci_high": 0.5398623866578321,
+                "sacrebleu_ci_low": 0.4614513028457302,
+                "sacrebleu_ci_high": 0.5398623866578321
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1218,
-                    705,
-                    454,
+                    1219,
+                    713,
+                    457,
                     300
                 ],
                 "totals": [
-                    1885,
-                    1819,
-                    1753,
-                    1687
+                    1888,
+                    1822,
+                    1756,
+                    1690
                 ],
                 "precisions": [
-                    0.6461538461538461,
-                    0.38757559098405714,
-                    0.2589845978322875,
-                    0.17783046828689983
+                    0.6456567796610169,
+                    0.3913282107574094,
+                    0.260250569476082,
+                    0.17751479289940827
                 ],
                 "bp": 1.0,
-                "sys_len": 1885,
+                "sys_len": 1888,
                 "ref_len": 1734,
-                "sacrebleu": 0.32771258154747107,
-                "score": 0.32771258154747107,
+                "sacrebleu": 0.32869437944703067,
+                "score": 0.32869437944703067,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3009156083594827,
-                "score_ci_high": 0.3820347897282282,
-                "sacrebleu_ci_low": 0.3009156083594827,
-                "sacrebleu_ci_high": 0.3820347897282282
+                "score_ci_low": 0.30148825657210887,
+                "score_ci_high": 0.3740478120880907,
+                "sacrebleu_ci_low": 0.30148825657210887,
+                "sacrebleu_ci_high": 0.3740478120880907
             },
-            "score": 0.3996909313865753,
+            "score": 0.40309667841482516,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.5400687869936714,
+        "score": 0.5151392921367606,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }

results/bluebench/{2025-06-21T11-34-24_evaluation_results.json → 2025-06-23T15-33-11_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-06-21T15:34:20.869271Z",
+        "timestamp_utc": "2025-06-23T19:33:07.872441Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -8,7 +8,7 @@
             "--model",
             "cross_provider",
             "--model_args",
-            "model_name=watsonx/mistralai/mistral-small-3-1-24b-instruct-2503,max_tokens=256",
+            "model_name=watsonx/mistralai/mistral-large,max_tokens=256",
             "--output_path",
             "./results/bluebench",
             "--log_samples",
@@ -26,7 +26,7 @@
             "num_fewshots": null,
             "limit": null,
             "batch_size": 8,
-            "model": "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503",
+            "model": "watsonx/mistralai/mistral-large",
             "model_args": {
                 "max_tokens": 256
             },
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.24.0",
-        "unitxt_commit_hash": "2bfd4494ec443ef86013e30d31f4860177124476",
+        "unitxt_commit_hash": "9aa85a9a01cfe9609a2d010f6d4bd1e88e782740",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,633 +176,633 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.6555555555555556,
-                "accuracy_ci_low": 0.5444444444444444,
-                "accuracy_ci_high": 0.7333333333333333,
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.7888888888888889,
+                "accuracy_ci_high": 0.9222222222222223,
                 "score_name": "accuracy",
-                "score": 0.6555555555555556,
-                "score_ci_high": 0.7333333333333333,
-                "score_ci_low": 0.5444444444444444,
+                "score": 0.8666666666666667,
+                "score_ci_high": 0.9222222222222223,
+                "score_ci_low": 0.7888888888888889,
                 "num_of_instances": 90
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.7333333333333333,
-                "accuracy_ci_low": 0.6333333333333333,
-                "accuracy_ci_high": 0.8222222222222222,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.8111111111111111,
+                "accuracy_ci_high": 0.9444444444444444,
                 "score_name": "accuracy",
-                "score": 0.7333333333333333,
-                "score_ci_high": 0.8222222222222222,
-                "score_ci_low": 0.6333333333333333,
+                "score": 0.8888888888888888,
+                "score_ci_high": 0.9444444444444444,
+                "score_ci_low": 0.8111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8666666666666667,
-                "accuracy_ci_low": 0.7666666666666667,
-                "accuracy_ci_high": 0.9222222222222223,
+                "accuracy": 0.9888888888888889,
+                "accuracy_ci_low": 0.9283857779145438,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8666666666666667,
-                "score_ci_high": 0.9222222222222223,
-                "score_ci_low": 0.7666666666666667,
+                "score": 0.9888888888888889,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9283857779145438,
                 "num_of_instances": 90
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.6777777777777778,
-                "accuracy_ci_high": 0.8555555555555555,
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.7858277377703305,
+                "accuracy_ci_high": 0.9333333333333333,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 0.8555555555555555,
-                "score_ci_low": 0.6777777777777778,
+                "score": 0.8666666666666667,
+                "score_ci_high": 0.9333333333333333,
+                "score_ci_low": 0.7858277377703305,
                 "num_of_instances": 90
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.8666666666666667,
-                "accuracy_ci_low": 0.7777777777777778,
-                "accuracy_ci_high": 0.9222222222222223,
+                "accuracy": 0.9333333333333333,
+                "accuracy_ci_low": 0.8666666666666667,
+                "accuracy_ci_high": 0.9777777777777777,
                 "score_name": "accuracy",
-                "score": 0.8666666666666667,
-                "score_ci_high": 0.9222222222222223,
-                "score_ci_low": 0.7777777777777778,
+                "score": 0.9333333333333333,
+                "score_ci_high": 0.9777777777777777,
+                "score_ci_low": 0.8666666666666667,
                 "num_of_instances": 90
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.9111111111111111,
-                "accuracy_ci_low": 0.8444444444444444,
-                "accuracy_ci_high": 0.9555555555555556,
+                "accuracy": 0.9555555555555556,
+                "accuracy_ci_low": 0.9,
+                "accuracy_ci_high": 0.9888888888888889,
                 "score_name": "accuracy",
-                "score": 0.9111111111111111,
-                "score_ci_high": 0.9555555555555556,
-                "score_ci_low": 0.8444444444444444,
+                "score": 0.9555555555555556,
+                "score_ci_high": 0.9888888888888889,
+                "score_ci_low": 0.9,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.7666666666666667,
-                "accuracy_ci_low": 0.6666666666666666,
-                "accuracy_ci_high": 0.8444444444444444,
+                "accuracy": 0.9111111111111111,
+                "accuracy_ci_low": 0.8333333333333334,
+                "accuracy_ci_high": 0.9555555555555556,
                 "score_name": "accuracy",
-                "score": 0.7666666666666667,
-                "score_ci_high": 0.8444444444444444,
-                "score_ci_low": 0.6666666666666666,
+                "score": 0.9111111111111111,
+                "score_ci_high": 0.9555555555555556,
+                "score_ci_low": 0.8333333333333334,
                 "num_of_instances": 90
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.8222222222222222,
-                "accuracy_ci_high": 0.9444444444444444,
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.7888888888888889,
+                "accuracy_ci_high": 0.9222222222222223,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
-                "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.8222222222222222,
+                "score": 0.8666666666666667,
+                "score_ci_high": 0.9222222222222223,
+                "score_ci_low": 0.7888888888888889,
                 "num_of_instances": 90
             },
             "safety_bbq_religion": {
-                "accuracy": 0.7333333333333333,
-                "accuracy_ci_low": 0.6333333333333333,
-                "accuracy_ci_high": 0.8111111111111111,
+                "accuracy": 0.8666666666666667,
+                "accuracy_ci_low": 0.788388746882511,
+                "accuracy_ci_high": 0.9222222222222223,
                 "score_name": "accuracy",
-                "score": 0.7333333333333333,
-                "score_ci_high": 0.8111111111111111,
-                "score_ci_low": 0.6333333333333333,
+                "score": 0.8666666666666667,
+                "score_ci_high": 0.9222222222222223,
+                "score_ci_low": 0.788388746882511,
                 "num_of_instances": 90
             },
             "safety_bbq_ses": {
                 "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.8041323028207193,
+                "accuracy_ci_low": 0.8111111111111111,
                 "accuracy_ci_high": 0.9444444444444444,
                 "score_name": "accuracy",
                 "score": 0.8888888888888888,
                 "score_ci_high": 0.9444444444444444,
-                "score_ci_low": 0.8041323028207193,
+                "score_ci_low": 0.8111111111111111,
                 "num_of_instances": 90
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.6888888888888889,
-                "accuracy_ci_high": 0.8555555555555555,
+                "accuracy": 0.9222222222222223,
+                "accuracy_ci_low": 0.8555555555555555,
+                "accuracy_ci_high": 0.9666666666666667,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 0.8555555555555555,
-                "score_ci_low": 0.6888888888888889,
+                "score": 0.9222222222222223,
+                "score_ci_high": 0.9666666666666667,
+                "score_ci_low": 0.8555555555555555,
                 "num_of_instances": 90
             },
-            "score": 0.806060606060606,
+            "score": 0.9050505050505051,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 500,
-                "llama_3_70b_instruct_template_arena_hard": 0.5,
-                "score": 0.5,
+                "llama_3_70b_instruct_template_arena_hard": 0.09158878504672897,
+                "score": 0.09158878504672897,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5,
+            "score": 0.09158878504672897,
             "score_name": "subsets_mean",
             "num_of_instances": 500
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 1000,
-                "f1_Person": 0.3625730994152047,
-                "f1_Organization": 0.2893081761006289,
-                "f1_Location": 0.33070866141732286,
-                "f1_macro": 0.32752997897771885,
-                "recall_macro": 0.2851171336228018,
-                "precision_macro": 0.39184928961146603,
-                "in_classes_support": 0.433184855233853,
-                "f1_micro": 0.21082220660576245,
-                "recall_micro": 0.2857142857142857,
-                "precision_micro": 0.16703786191536749,
-                "score": 0.21082220660576245,
+                "f1_Person": 0.16666666666666666,
+                "f1_Organization": 0.03252032520325203,
+                "f1_Location": 0.06666666666666667,
+                "f1_macro": 0.08861788617886178,
+                "recall_macro": 0.061454532512588166,
+                "precision_macro": 0.1605612378704432,
+                "in_classes_support": 0.2943548387096774,
+                "f1_micro": 0.055161544523246654,
+                "recall_micro": 0.06666666666666667,
+                "precision_micro": 0.04704301075268817,
+                "score": 0.055161544523246654,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.18138158087338846,
-                "score_ci_high": 0.2505537736745896,
-                "f1_micro_ci_low": 0.18138158087338846,
-                "f1_micro_ci_high": 0.2505537736745896
+                "score_ci_low": 0.04056157487156359,
+                "score_ci_high": 0.07432669896850874,
+                "f1_micro_ci_low": 0.04056157487156359,
+                "f1_micro_ci_high": 0.07432669896850874
             },
-            "score": 0.21082220660576245,
+            "score": 0.055161544523246654,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.676056338028169,
-                "accuracy_ci_low": 0.5492957746478874,
-                "accuracy_ci_high": 0.7746478873239436,
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6197183098591549,
+                "accuracy_ci_high": 0.8309859154929577,
                 "score_name": "accuracy",
-                "score": 0.676056338028169,
-                "score_ci_high": 0.7746478873239436,
-                "score_ci_low": 0.5492957746478874,
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6197183098591549,
                 "num_of_instances": 71
             },
             "mmlu_pro_business": {
-                "accuracy": 0.22535211267605634,
-                "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.3380281690140845,
+                "accuracy": 0.39436619718309857,
+                "accuracy_ci_low": 0.28169014084507044,
+                "accuracy_ci_high": 0.5070422535211268,
                 "score_name": "accuracy",
-                "score": 0.22535211267605634,
-                "score_ci_high": 0.3380281690140845,
-                "score_ci_low": 0.14084507042253522,
+                "score": 0.39436619718309857,
+                "score_ci_high": 0.5070422535211268,
+                "score_ci_low": 0.28169014084507044,
                 "num_of_instances": 71
             },
             "mmlu_pro_chemistry": {
                 "accuracy": 0.23943661971830985,
                 "accuracy_ci_low": 0.14084507042253522,
-                "accuracy_ci_high": 0.352112676056338,
+                "accuracy_ci_high": 0.3380281690140845,
                 "score_name": "accuracy",
                 "score": 0.23943661971830985,
-                "score_ci_high": 0.352112676056338,
+                "score_ci_high": 0.3380281690140845,
                 "score_ci_low": 0.14084507042253522,
                 "num_of_instances": 71
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.5352112676056338,
-                "accuracy_ci_low": 0.4225352112676056,
-                "accuracy_ci_high": 0.647887323943662,
+                "accuracy": 0.647887323943662,
+                "accuracy_ci_low": 0.5352112676056338,
+                "accuracy_ci_high": 0.7464788732394366,
                 "score_name": "accuracy",
-                "score": 0.5352112676056338,
-                "score_ci_high": 0.647887323943662,
-                "score_ci_low": 0.4225352112676056,
+                "score": 0.647887323943662,
+                "score_ci_high": 0.7464788732394366,
+                "score_ci_low": 0.5352112676056338,
                 "num_of_instances": 71
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.676056338028169,
-                "accuracy_ci_low": 0.5492957746478874,
-                "accuracy_ci_high": 0.7746478873239436,
+                "accuracy": 0.7323943661971831,
+                "accuracy_ci_low": 0.6056338028169014,
+                "accuracy_ci_high": 0.8309859154929577,
                 "score_name": "accuracy",
-                "score": 0.676056338028169,
-                "score_ci_high": 0.7746478873239436,
-                "score_ci_low": 0.5492957746478874,
+                "score": 0.7323943661971831,
+                "score_ci_high": 0.8309859154929577,
+                "score_ci_low": 0.6056338028169014,
                 "num_of_instances": 71
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.2112676056338028,
-                "accuracy_ci_low": 0.1267605633802817,
-                "accuracy_ci_high": 0.30985915492957744,
+                "accuracy": 0.2676056338028169,
+                "accuracy_ci_low": 0.16901408450704225,
+                "accuracy_ci_high": 0.38028169014084506,
                 "score_name": "accuracy",
-                "score": 0.2112676056338028,
-                "score_ci_high": 0.30985915492957744,
-                "score_ci_low": 0.1267605633802817,
+                "score": 0.2676056338028169,
+                "score_ci_high": 0.38028169014084506,
+                "score_ci_low": 0.16901408450704225,
                 "num_of_instances": 71
             },
             "mmlu_pro_health": {
-                "accuracy": 0.5070422535211268,
-                "accuracy_ci_low": 0.39436619718309857,
-                "accuracy_ci_high": 0.6197183098591549,
+                "accuracy": 0.6056338028169014,
+                "accuracy_ci_low": 0.4788732394366197,
+                "accuracy_ci_high": 0.704225352112676,
                 "score_name": "accuracy",
-                "score": 0.5070422535211268,
-                "score_ci_high": 0.6197183098591549,
-                "score_ci_low": 0.39436619718309857,
+                "score": 0.6056338028169014,
+                "score_ci_high": 0.704225352112676,
+                "score_ci_low": 0.4788732394366197,
                 "num_of_instances": 71
             },
             "mmlu_pro_history": {
-                "accuracy": 0.6197183098591549,
-                "accuracy_ci_low": 0.49295774647887325,
-                "accuracy_ci_high": 0.7323943661971831,
+                "accuracy": 0.647887323943662,
+                "accuracy_ci_low": 0.5211267605633803,
+                "accuracy_ci_high": 0.7464788732394366,
                 "score_name": "accuracy",
-                "score": 0.6197183098591549,
-                "score_ci_high": 0.7323943661971831,
-                "score_ci_low": 0.49295774647887325,
+                "score": 0.647887323943662,
+                "score_ci_high": 0.7464788732394366,
+                "score_ci_low": 0.5211267605633803,
                 "num_of_instances": 71
             },
             "mmlu_pro_law": {
-                "accuracy": 0.43661971830985913,
-                "accuracy_ci_low": 0.323943661971831,
-                "accuracy_ci_high": 0.5492957746478874,
+                "accuracy": 0.5211267605633803,
+                "accuracy_ci_low": 0.40913735882879854,
+                "accuracy_ci_high": 0.6338028169014085,
                 "score_name": "accuracy",
-                "score": 0.43661971830985913,
-                "score_ci_high": 0.5492957746478874,
-                "score_ci_low": 0.323943661971831,
+                "score": 0.5211267605633803,
+                "score_ci_high": 0.6338028169014085,
+                "score_ci_low": 0.40913735882879854,
                 "num_of_instances": 71
             },
             "mmlu_pro_math": {
-                "accuracy": 0.352112676056338,
-                "accuracy_ci_low": 0.23943661971830985,
-                "accuracy_ci_high": 0.4647887323943662,
+                "accuracy": 0.4225352112676056,
+                "accuracy_ci_low": 0.30985915492957744,
+                "accuracy_ci_high": 0.5275288557194965,
                 "score_name": "accuracy",
-                "score": 0.352112676056338,
-                "score_ci_high": 0.4647887323943662,
-                "score_ci_low": 0.23943661971830985,
+                "score": 0.4225352112676056,
+                "score_ci_high": 0.5275288557194965,
+                "score_ci_low": 0.30985915492957744,
                 "num_of_instances": 71
             },
             "mmlu_pro_other": {
-                "accuracy": 0.5070422535211268,
-                "accuracy_ci_low": 0.39436619718309857,
-                "accuracy_ci_high": 0.6056338028169014,
+                "accuracy": 0.647887323943662,
+                "accuracy_ci_low": 0.5211267605633803,
+                "accuracy_ci_high": 0.7464788732394366,
                 "score_name": "accuracy",
-                "score": 0.5070422535211268,
-                "score_ci_high": 0.6056338028169014,
-                "score_ci_low": 0.39436619718309857,
+                "score": 0.647887323943662,
+                "score_ci_high": 0.7464788732394366,
+                "score_ci_low": 0.5211267605633803,
                 "num_of_instances": 71
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.49295774647887325,
-                "accuracy_ci_low": 0.38028169014084506,
-                "accuracy_ci_high": 0.6197183098591549,
+                "accuracy": 0.6619718309859155,
+                "accuracy_ci_low": 0.5492957746478874,
+                "accuracy_ci_high": 0.7746478873239436,
                 "score_name": "accuracy",
-                "score": 0.49295774647887325,
-                "score_ci_high": 0.6197183098591549,
-                "score_ci_low": 0.38028169014084506,
+                "score": 0.6619718309859155,
+                "score_ci_high": 0.7746478873239436,
+                "score_ci_low": 0.5492957746478874,
                 "num_of_instances": 71
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.323943661971831,
-                "accuracy_ci_low": 0.22535211267605634,
-                "accuracy_ci_high": 0.4225352112676056,
+                "accuracy": 0.39436619718309857,
+                "accuracy_ci_low": 0.28169014084507044,
+                "accuracy_ci_high": 0.5070422535211268,
                 "score_name": "accuracy",
-                "score": 0.323943661971831,
-                "score_ci_high": 0.4225352112676056,
-                "score_ci_low": 0.22535211267605634,
+                "score": 0.39436619718309857,
+                "score_ci_high": 0.5070422535211268,
+                "score_ci_low": 0.28169014084507044,
                 "num_of_instances": 71
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.6197183098591549,
-                "accuracy_ci_low": 0.49295774647887325,
-                "accuracy_ci_high": 0.7183098591549296,
+                "accuracy": 0.8028169014084507,
+                "accuracy_ci_low": 0.704225352112676,
+                "accuracy_ci_high": 0.8873239436619719,
                 "score_name": "accuracy",
-                "score": 0.6197183098591549,
-                "score_ci_high": 0.7183098591549296,
-                "score_ci_low": 0.49295774647887325,
+                "score": 0.8028169014084507,
+                "score_ci_high": 0.8873239436619719,
+                "score_ci_low": 0.704225352112676,
                 "num_of_instances": 71
             },
-            "score": 0.45875251509054327,
+            "score": 0.5513078470824949,
             "score_name": "subsets_mean",
             "num_of_instances": 994
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.3580404378230465,
-                "f1_suggestive": 0.375,
-                "f1_generic": 0.4,
-                "f1_arbitrary": 0.3076923076923077,
-                "f1_fanciful": 0.43478260869565216,
-                "f1_descriptive": 0.2727272727272727,
-                "f1_macro_ci_low": 0.24962639354166832,
-                "f1_macro_ci_high": 0.48946907875285006,
+                "f1_macro": 0.30082491488530444,
+                "f1_suggestive": 0.2926829268292683,
+                "f1_generic": 0.3157894736842105,
+                "f1_fanciful": 0.2,
+                "f1_descriptive": 0.2608695652173913,
+                "f1_arbitrary": 0.43478260869565216,
+                "f1_macro_ci_low": 0.21165741181075054,
+                "f1_macro_ci_high": 0.4315719879768282,
                 "score_name": "f1_micro",
-                "score": 0.35772357723577236,
-                "score_ci_high": 0.47244094488188976,
-                "score_ci_low": 0.25002795229813274,
+                "score": 0.30158730158730157,
+                "score_ci_high": 0.421875,
+                "score_ci_low": 0.2033898305084746,
                 "num_of_instances": 85,
-                "accuracy": 0.25882352941176473,
-                "accuracy_ci_low": 0.1716911469146947,
-                "accuracy_ci_high": 0.35294117647058826,
-                "f1_micro": 0.35772357723577236,
-                "f1_micro_ci_low": 0.25002795229813274,
-                "f1_micro_ci_high": 0.47244094488188976
+                "accuracy": 0.2235294117647059,
+                "accuracy_ci_low": 0.15294117647058825,
+                "accuracy_ci_high": 0.32721667655979375,
+                "f1_micro": 0.30158730158730157,
+                "f1_micro_ci_low": 0.2033898305084746,
+                "f1_micro_ci_high": 0.421875
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.28985125707516124,
-                "f1_no": 0.5233644859813084,
-                "f1_yes": 0.056338028169014086,
-                "f1_macro_ci_low": 0.23940411227724898,
-                "f1_macro_ci_high": 0.3484173679194581,
+                "f1_macro": 0.2087664168882443,
+                "f1_no": 0.38578680203045684,
+                "f1_yes": 0.031746031746031744,
+                "f1_macro_ci_low": 0.16304347826086957,
+                "f1_macro_ci_high": 0.27128054977534694,
                 "score_name": "f1_micro",
-                "score": 0.4070175438596491,
-                "score_ci_high": 0.4836620089714397,
-                "score_ci_low": 0.32857142857142857,
+                "score": 0.3,
+                "score_ci_high": 0.37342833232881084,
+                "score_ci_low": 0.22950819672131148,
                 "num_of_instances": 200,
-                "accuracy": 0.29,
-                "accuracy_ci_low": 0.225,
-                "accuracy_ci_high": 0.355,
-                "f1_micro": 0.4070175438596491,
-                "f1_micro_ci_low": 0.32857142857142857,
-                "f1_micro_ci_high": 0.4836620089714397
+                "accuracy": 0.195,
+                "accuracy_ci_low": 0.145,
+                "accuracy_ci_high": 0.2511829758893259,
+                "f1_micro": 0.3,
+                "f1_micro_ci_low": 0.22950819672131148,
+                "f1_micro_ci_high": 0.37342833232881084
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.2830193138674671,
-                "f1_conclusion": 0.2222222222222222,
-                "f1_decree": 0.25,
-                "f1_issue": 0.18604651162790697,
-                "f1_analysis": 0.32558139534883723,
-                "f1_facts": 0.26666666666666666,
-                "f1_procedural history": 0.3384615384615385,
-                "f1_rule": 0.39215686274509803,
-                "f1_macro_ci_low": 0.22077760516552375,
-                "f1_macro_ci_high": 0.3674671451575924,
+                "f1_macro": 0.1326239897668469,
+                "f1_conclusion": 0.0,
+                "f1_decree": 0.14814814814814814,
+                "f1_issue": 0.05714285714285714,
+                "f1_analysis": 0.15,
+                "f1_facts": 0.06666666666666667,
+                "f1_procedural history": 0.25,
+                "f1_rule": 0.2564102564102564,
+                "f1_macro_ci_low": 0.0850100965627728,
+                "f1_macro_ci_high": 0.1996422423627835,
                 "score_name": "f1_micro",
-                "score": 0.2920634920634921,
-                "score_ci_high": 0.37441470472532123,
-                "score_ci_low": 0.2264866423312643,
+                "score": 0.14345991561181434,
+                "score_ci_high": 0.2175732217573222,
+                "score_ci_low": 0.08928571428571429,
                 "num_of_instances": 200,
-                "accuracy": 0.23,
-                "accuracy_ci_low": 0.175,
-                "accuracy_ci_high": 0.3,
-                "f1_micro": 0.2920634920634921,
-                "f1_micro_ci_low": 0.2264866423312643,
-                "f1_micro_ci_high": 0.37441470472532123
+                "accuracy": 0.085,
+                "accuracy_ci_low": 0.055,
+                "accuracy_ci_high": 0.135,
+                "f1_micro": 0.14345991561181434,
+                "f1_micro_ci_low": 0.08928571428571429,
+                "f1_micro_ci_high": 0.2175732217573222
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.20266040688575898,
-                "f1_yes": 0.2222222222222222,
-                "f1_no": 0.18309859154929578,
-                "f1_macro_ci_low": 0.13797716111770406,
-                "f1_macro_ci_high": 0.27925168740954265,
+                "f1_macro": 0.27590718171176754,
+                "f1_yes": 0.3106796116504854,
+                "f1_no": 0.24113475177304963,
+                "f1_macro_ci_low": 0.2059942090622298,
+                "f1_macro_ci_high": 0.35594480291914055,
                 "score_name": "f1_micro",
-                "score": 0.1991701244813278,
-                "score_ci_high": 0.27571125309001354,
-                "score_ci_low": 0.13617021276595745,
+                "score": 0.27049180327868855,
+                "score_ci_high": 0.3511987633583538,
+                "score_ci_low": 0.20259771606756347,
                 "num_of_instances": 200,
-                "accuracy": 0.12,
-                "accuracy_ci_low": 0.08,
-                "accuracy_ci_high": 0.175,
-                "f1_micro": 0.1991701244813278,
-                "f1_micro_ci_low": 0.13617021276595745,
-                "f1_micro_ci_high": 0.27571125309001354
+                "accuracy": 0.165,
+                "accuracy_ci_low": 0.12,
+                "accuracy_ci_high": 0.225,
+                "f1_micro": 0.27049180327868855,
+                "f1_micro_ci_low": 0.20259771606756347,
+                "f1_micro_ci_high": 0.3511987633583538
             },
             "legalbench_proa": {
-                "f1_macro": 0.8320072332730561,
-                "f1_yes": 0.8285714285714286,
-                "f1_no": 0.8354430379746836,
-                "f1_macro_ci_low": 0.749025367448282,
-                "f1_macro_ci_high": 0.890690718451174,
+                "f1_macro": 0.7697368421052632,
+                "f1_yes": 0.75,
+                "f1_no": 0.7894736842105263,
+                "f1_macro_ci_low": 0.6817007087256215,
+                "f1_macro_ci_high": 0.8427704260296438,
                 "score_name": "f1_micro",
-                "score": 0.8322147651006712,
-                "score_ci_high": 0.8903225806451613,
-                "score_ci_low": 0.7482933733536249,
+                "score": 0.7714285714285715,
+                "score_ci_high": 0.8435374149659864,
+                "score_ci_low": 0.6821705426356589,
                 "num_of_instances": 85,
-                "accuracy": 0.7294117647058823,
-                "accuracy_ci_low": 0.6235294117647059,
-                "accuracy_ci_high": 0.8117647058823529,
-                "f1_micro": 0.8322147651006712,
-                "f1_micro_ci_low": 0.7482933733536249,
-                "f1_micro_ci_high": 0.8903225806451613
+                "accuracy": 0.6352941176470588,
+                "accuracy_ci_low": 0.5176470588235295,
+                "accuracy_ci_high": 0.7294117647058823,
+                "f1_micro": 0.7714285714285715,
+                "f1_micro_ci_low": 0.6821705426356589,
+                "f1_micro_ci_high": 0.8435374149659864
             },
-            "score": 0.4176379005481825,
+            "score": 0.3573935183812752,
             "score_name": "subsets_mean",
             "num_of_instances": 770
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.5251289790109961,
-                "f1_cars": 0.735632183908046,
-                "f1_windows x": 0.11594202898550725,
-                "f1_atheism": 0.18181818181818182,
-                "f1_religion": 0.14285714285714285,
-                "f1_medicine": 0.825,
-                "f1_christianity": 0.43243243243243246,
-                "f1_for sale": 0.7142857142857143,
-                "f1_computer graphics": 0.5,
-                "f1_microsoft windows": 0.4507042253521127,
-                "f1_middle east": 0.5641025641025641,
-                "f1_motorcycles": 0.66,
-                "f1_pc hardware": 0.4528301886792453,
-                "f1_mac hardware": 0.5494505494505495,
-                "f1_electronics": 0.574468085106383,
-                "f1_guns": 0.2647058823529412,
-                "f1_space": 0.7333333333333333,
-                "f1_cryptography": 0.5897435897435898,
-                "f1_baseball": 0.8380952380952381,
-                "f1_hockey": 0.8403361344537815,
-                "f1_politics": 0.3368421052631579,
-                "f1_macro_ci_low": 0.4982372111894923,
-                "f1_macro_ci_high": 0.5536034958738426,
+                "f1_macro": 0.31185856604704804,
+                "f1_cars": 0.5294117647058824,
+                "f1_windows x": 0.0,
+                "f1_computer graphics": 0.21875,
+                "f1_atheism": 0.0,
+                "f1_religion": 0.1935483870967742,
+                "f1_medicine": 0.4444444444444444,
+                "f1_christianity": 0.07272727272727272,
+                "f1_microsoft windows": 0.13793103448275862,
+                "f1_middle east": 0.2692307692307692,
+                "f1_motorcycles": 0.41975308641975306,
+                "f1_pc hardware": 0.47191011235955055,
+                "f1_mac hardware": 0.4943820224719101,
+                "f1_for sale": 0.2608695652173913,
+                "f1_guns": 0.0784313725490196,
+                "f1_space": 0.4594594594594595,
+                "f1_cryptography": 0.3389830508474576,
+                "f1_baseball": 0.46153846153846156,
+                "f1_hockey": 0.5918367346938775,
+                "f1_politics": 0.37142857142857144,
+                "f1_electronics": 0.4225352112676056,
+                "f1_macro_ci_low": 0.2856078509025084,
+                "f1_macro_ci_high": 0.3426020462142722,
                 "score_name": "f1_micro",
-                "score": 0.5513196480938416,
-                "score_ci_high": 0.580897965540632,
-                "score_ci_low": 0.519239825515335,
+                "score": 0.3433734939759036,
+                "score_ci_high": 0.3755990938286412,
+                "score_ci_low": 0.3096235116477192,
                 "num_of_instances": 1000,
-                "accuracy": 0.47,
-                "accuracy_ci_low": 0.439,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.5513196480938416,
-                "f1_micro_ci_low": 0.519239825515335,
-                "f1_micro_ci_high": 0.580897965540632
+                "accuracy": 0.228,
+                "accuracy_ci_low": 0.202,
+                "accuracy_ci_high": 0.253,
+                "f1_micro": 0.3433734939759036,
+                "f1_micro_ci_low": 0.3096235116477192,
+                "f1_micro_ci_high": 0.3755990938286412
             },
-            "score": 0.5513196480938416,
+            "score": 0.3433734939759036,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.598937790062507,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.800990916597853,
-                "f1_money transfer or virtual currency or money service": 0.6530612244897959,
-                "f1_student loan": 0.7741935483870968,
-                "f1_credit card or prepaid card": 0.5154639175257731,
-                "f1_debt collection": 0.5394736842105263,
-                "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_vehicle loan or lease": 0.6206896551724138,
-                "f1_mortgage": 0.8,
-                "f1_checking or savings account": 0.6865671641791045,
-                "f1_macro_ci_low": 0.5590827549101115,
-                "f1_macro_ci_high": 0.6436069882866612,
+                "f1_macro": 0.6519421467580517,
+                "f1_student loan": 0.75,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7592592592592593,
+                "f1_debt collection": 0.5138888888888888,
+                "f1_checking or savings account": 0.7073170731707317,
+                "f1_mortgage": 0.7931034482758621,
+                "f1_payday loan or title loan or personal loan": 0.4444444444444444,
+                "f1_credit card or prepaid card": 0.6727272727272727,
+                "f1_money transfer or virtual currency or money service": 0.6341463414634146,
+                "f1_vehicle loan or lease": 0.5925925925925926,
+                "f1_macro_ci_low": 0.5871374382389457,
+                "f1_macro_ci_high": 0.7213495817777442,
                 "score_name": "f1_micro",
-                "score": 0.7434435575826682,
-                "score_ci_high": 0.771404250423866,
-                "score_ci_low": 0.719193914944442,
+                "score": 0.7237076648841355,
+                "score_ci_high": 0.7514570299103845,
+                "score_ci_low": 0.6955584945084361,
                 "num_of_instances": 1000,
-                "accuracy": 0.652,
-                "accuracy_ci_low": 0.623,
-                "accuracy_ci_high": 0.684,
-                "f1_micro": 0.7434435575826682,
-                "f1_micro_ci_low": 0.719193914944442,
-                "f1_micro_ci_high": 0.771404250423866
+                "accuracy": 0.609,
+                "accuracy_ci_low": 0.577,
+                "accuracy_ci_high": 0.641,
+                "f1_micro": 0.7237076648841355,
+                "f1_micro_ci_low": 0.6955584945084361,
+                "f1_micro_ci_high": 0.7514570299103845
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.645506649922517,
-                "f1_mortgages and loans": 0.7393939393939394,
-                "f1_credit card": 0.7088607594936709,
-                "f1_debt collection": 0.6268656716417911,
-                "f1_retail banking": 0.4752475247524752,
-                "f1_credit reporting": 0.6771653543307087,
-                "f1_macro_ci_low": 0.6022236233880557,
-                "f1_macro_ci_high": 0.6875999406252342,
+                "f1_macro": 0.6498825672130026,
+                "f1_mortgages and loans": 0.6711409395973155,
+                "f1_credit card": 0.6853146853146853,
+                "f1_debt collection": 0.56,
+                "f1_credit reporting": 0.7279151943462897,
+                "f1_retail banking": 0.6050420168067226,
+                "f1_macro_ci_low": 0.6099650997428573,
+                "f1_macro_ci_high": 0.6950683737354554,
                 "score_name": "f1_micro",
-                "score": 0.6598407281001137,
-                "score_ci_high": 0.6994441194697146,
-                "score_ci_low": 0.6181710190063682,
+                "score": 0.6605293440736478,
+                "score_ci_high": 0.7005417538024762,
+                "score_ci_low": 0.6186622377558174,
                 "num_of_instances": 500,
-                "accuracy": 0.58,
-                "accuracy_ci_low": 0.5369658568845046,
-                "accuracy_ci_high": 0.624,
-                "f1_micro": 0.6598407281001137,
-                "f1_micro_ci_low": 0.6181710190063682,
-                "f1_micro_ci_high": 0.6994441194697146
+                "accuracy": 0.574,
+                "accuracy_ci_low": 0.534,
+                "accuracy_ci_high": 0.618,
+                "f1_micro": 0.6605293440736478,
+                "f1_micro_ci_low": 0.6186622377558174,
+                "f1_micro_ci_high": 0.7005417538024762
             },
-            "score": 0.701642142841391,
+            "score": 0.6921185044788917,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 1000,
-                "execution_accuracy": 0.111,
-                "program_accuracy": 0.124,
-                "score": 0.124,
+                "execution_accuracy": 0.113,
+                "program_accuracy": 0.114,
+                "score": 0.114,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.092,
-                "execution_accuracy_ci_high": 0.132,
-                "program_accuracy_ci_low": 0.104,
-                "program_accuracy_ci_high": 0.145,
-                "score_ci_low": 0.104,
-                "score_ci_high": 0.145
+                "execution_accuracy_ci_low": 0.095,
+                "execution_accuracy_ci_high": 0.135,
+                "program_accuracy_ci_low": 0.096,
+                "program_accuracy_ci_high": 0.135,
+                "score_ci_low": 0.096,
+                "score_ci_high": 0.135
             },
-            "score": 0.124,
+            "score": 0.114,
             "score_name": "subsets_mean",
             "num_of_instances": 1000
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.25881567594573107,
-                "recall": 0.634151229634738,
-                "f1": 0.31118943159428636,
-                "precision_ci_low": 0.2427503435734933,
-                "precision_ci_high": 0.2766316249638964,
-                "recall_ci_low": 0.6185798223025679,
-                "recall_ci_high": 0.6499187341273587,
-                "f1_ci_low": 0.2955441533424316,
-                "f1_ci_high": 0.32732829081701914,
+                "precision": 0.3193234609846695,
+                "recall": 0.6156352660927287,
+                "f1": 0.3579933019872818,
+                "precision_ci_low": 0.2995158081508201,
+                "precision_ci_high": 0.34026768128986357,
+                "recall_ci_low": 0.599899034156362,
+                "recall_ci_high": 0.6313695528855681,
+                "f1_ci_low": 0.34026079062391323,
+                "f1_ci_high": 0.37577002560346623,
                 "score_name": "f1",
-                "score": 0.31118943159428636,
-                "score_ci_high": 0.32732829081701914,
-                "score_ci_low": 0.2955441533424316,
+                "score": 0.3579933019872818,
+                "score_ci_high": 0.37577002560346623,
+                "score_ci_low": 0.34026079062391323,
                 "num_of_instances": 600,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.5709441083172957,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6812374199926853,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.5089003006368875,
-                "faithfullness_f1_token_overlap": 0.31344518499715596,
-                "faithfullness_recall_token_overlap": 0.2668468802457335,
-                "faithfullness_precision_token_overlap": 0.47679019585643934,
-                "correctness_f1_token_overlap": 0.31118943159428636,
-                "correctness_recall_token_overlap": 0.634151229634738,
-                "correctness_precision_token_overlap": 0.25881567594573107
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6041086231172085,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6908169340590636,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.5559258947273096,
+                "faithfullness_f1_token_overlap": 0.3085026884811217,
+                "faithfullness_recall_token_overlap": 0.24647156332743023,
+                "faithfullness_precision_token_overlap": 0.536789234125785,
+                "correctness_f1_token_overlap": 0.3579933019872818,
+                "correctness_recall_token_overlap": 0.6156352660927287,
+                "correctness_precision_token_overlap": 0.3193234609846695
             },
-            "score": 0.31118943159428636,
+            "score": 0.3579933019872818,
             "score_name": "subsets_mean",
             "num_of_instances": 600
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.538,
-                "accuracy_ci_low": 0.505,
-                "accuracy_ci_high": 0.57,
+                "accuracy": 0.57,
+                "accuracy_ci_low": 0.537,
+                "accuracy_ci_high": 0.599,
                 "score_name": "accuracy",
-                "score": 0.538,
-                "score_ci_high": 0.57,
-                "score_ci_low": 0.505,
+                "score": 0.57,
+                "score_ci_high": 0.599,
+                "score_ci_low": 0.537,
                 "num_of_instances": 1000
             },
             "openbook_qa": {
-                "accuracy": 0.87,
-                "accuracy_ci_low": 0.84,
-                "accuracy_ci_high": 0.898,
+                "accuracy": 0.924,
+                "accuracy_ci_low": 0.8978500821223476,
+                "accuracy_ci_high": 0.944,
                 "score_name": "accuracy",
-                "score": 0.87,
-                "score_ci_high": 0.898,
-                "score_ci_low": 0.84,
+                "score": 0.924,
+                "score_ci_high": 0.944,
+                "score_ci_low": 0.8978500821223476,
                 "num_of_instances": 500
             },
-            "score": 0.704,
+            "score": 0.747,
             "score_name": "subsets_mean",
             "num_of_instances": 1500
         },
         "safety": {
             "attaq_500": {
                 "severity_critical": 3.0,
-                "severity_high": 0.0,
-                "severity_medium": 2.0,
-                "severity_low": 95.0,
-                "category_harmful_info": 0.8016373084946758,
-                "category_deception": 0.8822153681757474,
-                "category_explicit_content": 0.832710394654611,
-                "category_violence": 0.7825738151986448,
-                "category_discrimination": 0.8220807708918102,
-                "category_substance_abuse": 0.826748459537824,
-                "category_pii": 0.7723372398565213,
-                "safety": 0.809054309744388,
-                "safety_ci_low": 0.7772938002229317,
-                "safety_ci_high": 0.8361201240729302,
+                "severity_high": 1.0,
+                "severity_medium": 3.0,
+                "severity_low": 93.0,
+                "category_harmful_info": 0.7688285382612356,
+                "category_deception": 0.8504795281033058,
+                "category_explicit_content": 0.8499560091230604,
+                "category_violence": 0.8008088633891136,
+                "category_discrimination": 0.7961234071898081,
+                "category_substance_abuse": 0.8387069291538662,
+                "category_pii": 0.8280106396586807,
+                "safety": 0.8051073061449763,
+                "safety_ci_low": 0.7662176976956901,
+                "safety_ci_high": 0.8348793957598559,
                 "score_name": "safety",
-                "score": 0.809054309744388,
-                "score_ci_high": 0.8361201240729302,
-                "score_ci_low": 0.7772938002229317,
+                "score": 0.8051073061449763,
+                "score_ci_high": 0.8348793957598559,
+                "score_ci_low": 0.7662176976956901,
                 "num_of_instances": 100
             },
-            "score": 0.809054309744388,
+            "score": 0.8051073061449763,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 528,
-                "rouge1": 0.389351160643424,
-                "rougeL": 0.26506148266380164,
-                "score": 0.26506148266380164,
+                "rouge1": 0.4079437392329112,
+                "rougeLsum": 0.352934731899625,
+                "rougeL": 0.2881192661001498,
+                "score": 0.2881192661001498,
                 "score_name": "rougeL",
-                "rougeLsum": 0.3329771315874926,
-                "rouge2": 0.18480811116641105,
-                "rouge1_ci_low": 0.3793704942651626,
-                "rouge1_ci_high": 0.3991145148139802,
-                "rougeL_ci_low": 0.2574910713981229,
-                "rougeL_ci_high": 0.2723394364118134,
-                "score_ci_low": 0.2574910713981229,
-                "score_ci_high": 0.2723394364118134,
-                "rougeLsum_ci_low": 0.32368402136971663,
-                "rougeLsum_ci_high": 0.3419744690065713,
-                "rouge2_ci_low": 0.17799828534417894,
-                "rouge2_ci_high": 0.19189115183254815
+                "rouge2": 0.20309268400418845,
+                "rouge1_ci_low": 0.39815436123221776,
+                "rouge1_ci_high": 0.41769481108434164,
+                "rougeLsum_ci_low": 0.3428414762936942,
+                "rougeLsum_ci_high": 0.36147234302505843,
+                "rougeL_ci_low": 0.2810508281662768,
+                "rougeL_ci_high": 0.2958753092187963,
+                "score_ci_low": 0.2810508281662768,
+                "score_ci_high": 0.2958753092187963,
+                "rouge2_ci_low": 0.19601529597372655,
+                "rouge2_ci_high": 0.21114250528228737
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 1000,
-                "rouge1": 0.11298250338252126,
-                "rougeL": 0.08177611720945774,
-                "score": 0.08177611720945774,
+                "rouge1": 0.12777796269592456,
+                "rougeLsum": 0.10603826262531557,
+                "rougeL": 0.09179328413499686,
+                "score": 0.09179328413499686,
                 "score_name": "rougeL",
-                "rougeLsum": 0.0950863205219803,
-                "rouge2": 0.015717793961971215,
-                "rouge1_ci_low": 0.10712881957329778,
-                "rouge1_ci_high": 0.11773128033523193,
-                "rougeL_ci_low": 0.07786560263494875,
-                "rougeL_ci_high": 0.08512540575510288,
-                "score_ci_low": 0.07786560263494875,
-                "score_ci_high": 0.08512540575510288,
-                "rougeLsum_ci_low": 0.09078789141905777,
-                "rougeLsum_ci_high": 0.09900306122637068,
-                "rouge2_ci_low": 0.013982816599457718,
-                "rouge2_ci_high": 0.0176417953678172
+                "rouge2": 0.018851084572764187,
+                "rouge1_ci_low": 0.12168881499661288,
+                "rouge1_ci_high": 0.1332671550509936,
+                "rougeLsum_ci_low": 0.10116850286523955,
+                "rougeLsum_ci_high": 0.1106709838674053,
+                "rougeL_ci_low": 0.08725116318521317,
+                "rougeL_ci_high": 0.09575800533068254,
+                "score_ci_low": 0.08725116318521317,
+                "score_ci_high": 0.09575800533068254,
+                "rouge2_ci_low": 0.01662623690565808,
+                "rouge2_ci_high": 0.020972697895929062
             },
-            "score": 0.17341879993662968,
+            "score": 0.18995627511757335,
             "score_name": "subsets_mean",
             "num_of_instances": 1528
         },
@@ -810,473 +810,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1335,
-                    792,
-                    517,
-                    343
+                    1289,
+                    858,
+                    605,
+                    439
                 ],
                 "totals": [
-                    5784,
-                    5718,
-                    5652,
-                    5586
+                    1947,
+                    1881,
+                    1815,
+                    1749
                 ],
                 "precisions": [
-                    0.2308091286307054,
-                    0.1385099685204617,
-                    0.09147204529370134,
-                    0.06140350877192983
+                    0.6620441705187469,
+                    0.456140350877193,
+                    0.33333333333333337,
+                    0.2510005717552887
                 ],
                 "bp": 1.0,
-                "sys_len": 5784,
+                "sys_len": 1947,
                 "ref_len": 1734,
-                "sacrebleu": 0.11575876843383869,
-                "score": 0.11575876843383869,
+                "sacrebleu": 0.39868943613707586,
+                "score": 0.39868943613707586,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09677258086487982,
-                "score_ci_high": 0.14198802557654883,
-                "sacrebleu_ci_low": 0.09677258086487982,
-                "sacrebleu_ci_high": 0.14198802557654883
+                "score_ci_low": 0.34684429082723056,
+                "score_ci_high": 0.4568678775074209,
+                "sacrebleu_ci_low": 0.34684429082723056,
+                "sacrebleu_ci_high": 0.4568678775074209
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1356,
-                    783,
-                    496,
-                    322
+                    1323,
+                    887,
+                    629,
+                    444
                 ],
                 "totals": [
-                    6913,
-                    6847,
-                    6781,
-                    6715
+                    2483,
+                    2417,
+                    2351,
+                    2285
                 ],
                 "precisions": [
-                    0.19615217705771734,
-                    0.11435665254856142,
-                    0.07314555375313375,
-                    0.047952345495160094
+                    0.5328231977446637,
+                    0.36698386429458,
+                    0.26754572522330927,
+                    0.19431072210065647
                 ],
                 "bp": 1.0,
-                "sys_len": 6913,
+                "sys_len": 2483,
                 "ref_len": 1734,
-                "sacrebleu": 0.09418095467735833,
-                "score": 0.09418095467735833,
+                "sacrebleu": 0.3175274085826544,
+                "score": 0.3175274085826544,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.07855318506180965,
-                "score_ci_high": 0.11463100599546205,
-                "sacrebleu_ci_low": 0.07855318506180965,
-                "sacrebleu_ci_high": 0.11463100599546205
+                "score_ci_low": 0.271260768895482,
+                "score_ci_high": 0.365445678313604,
+                "sacrebleu_ci_low": 0.271260768895482,
+                "sacrebleu_ci_high": 0.365445678313604
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 66,
                 "counts": [
-                    767,
-                    320,
-                    153,
-                    74
+                    940,
+                    521,
+                    315,
+                    191
                 ],
                 "totals": [
-                    8433,
-                    8367,
-                    8301,
-                    8235
+                    1688,
+                    1622,
+                    1556,
+                    1490
                 ],
                 "precisions": [
-                    0.09095221154986363,
-                    0.03824548822756066,
-                    0.018431514275388507,
-                    0.008986035215543413
+                    0.556872037914692,
+                    0.3212083847102343,
+                    0.20244215938303342,
+                    0.12818791946308725
                 ],
                 "bp": 1.0,
-                "sys_len": 8433,
+                "sys_len": 1688,
                 "ref_len": 1589,
-                "sacrebleu": 0.027550573998889032,
-                "score": 0.027550573998889032,
+                "sacrebleu": 0.2610192792824636,
+                "score": 0.2610192792824636,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.021208555129186346,
-                "score_ci_high": 0.03586774827340232,
-                "sacrebleu_ci_low": 0.021208555129186346,
-                "sacrebleu_ci_high": 0.03586774827340232
+                "score_ci_low": 0.20817526367502337,
+                "score_ci_high": 0.29958193407273404,
+                "sacrebleu_ci_low": 0.20817526367502337,
+                "sacrebleu_ci_high": 0.29958193407273404
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 66,
                 "counts": [
-                    1257,
-                    721,
-                    459,
-                    305
+                    1254,
+                    784,
+                    532,
+                    376
                 ],
                 "totals": [
-                    5818,
-                    5752,
-                    5686,
-                    5620
+                    1815,
+                    1749,
+                    1683,
+                    1617
                 ],
                 "precisions": [
-                    0.2160536266758336,
-                    0.12534770514603616,
-                    0.08072458670418571,
-                    0.05427046263345196
+                    0.6909090909090909,
+                    0.4482561463693539,
+                    0.31610219845513965,
+                    0.23252937538651824
                 ],
-                "bp": 1.0,
-                "sys_len": 5818,
+                "bp": 0.98904120617152,
+                "sys_len": 1815,
                 "ref_len": 1835,
-                "sacrebleu": 0.10436666970008357,
-                "score": 0.10436666970008357,
+                "sacrebleu": 0.38417359468716306,
+                "score": 0.38417359468716306,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08345328311475483,
-                "score_ci_high": 0.13560613265358804,
-                "sacrebleu_ci_low": 0.08345328311475483,
-                "sacrebleu_ci_high": 0.13560613265358804
+                "score_ci_low": 0.32789485731980855,
+                "score_ci_high": 0.4187305224203214,
+                "sacrebleu_ci_low": 0.32789485731980855,
+                "sacrebleu_ci_high": 0.4187305224203214
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 66,
                 "counts": [
-                    1584,
-                    1138,
-                    882,
-                    702
+                    1572,
+                    1194,
+                    949,
+                    766
                 ],
                 "totals": [
-                    5212,
-                    5146,
-                    5080,
-                    5014
+                    2097,
+                    2031,
+                    1965,
+                    1899
                 ],
                 "precisions": [
-                    0.3039140445126631,
-                    0.22114263505635445,
-                    0.17362204724409447,
-                    0.14000797766254489
+                    0.7496423462088698,
+                    0.5878877400295421,
+                    0.48295165394402034,
+                    0.4033701948393892
                 ],
                 "bp": 1.0,
-                "sys_len": 5212,
+                "sys_len": 2097,
                 "ref_len": 2068,
-                "sacrebleu": 0.20104590260613545,
-                "score": 0.20104590260613545,
+                "sacrebleu": 0.5413012055320727,
+                "score": 0.5413012055320727,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.15816587969091414,
-                "score_ci_high": 0.23752616478719799,
-                "sacrebleu_ci_low": 0.15816587969091414,
-                "sacrebleu_ci_high": 0.23752616478719799
+                "score_ci_low": 0.5011833767453445,
+                "score_ci_high": 0.591093351022506,
+                "sacrebleu_ci_low": 0.5011833767453445,
+                "sacrebleu_ci_high": 0.591093351022506
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 66,
                 "counts": [
-                    1408,
-                    697,
-                    390,
-                    222
+                    1382,
+                    762,
+                    450,
+                    277
                 ],
                 "totals": [
-                    7618,
-                    7552,
-                    7486,
-                    7420
+                    2304,
+                    2238,
+                    2172,
+                    2106
                 ],
                 "precisions": [
-                    0.18482541349435547,
-                    0.09229343220338983,
-                    0.052097248196633715,
-                    0.029919137466307276
+                    0.5998263888888888,
+                    0.34048257372654156,
+                    0.20718232044198895,
+                    0.1315289648622982
                 ],
                 "bp": 1.0,
-                "sys_len": 7618,
+                "sys_len": 2304,
                 "ref_len": 2235,
-                "sacrebleu": 0.071808207221539,
-                "score": 0.071808207221539,
+                "sacrebleu": 0.27313266242858875,
+                "score": 0.27313266242858875,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06067958452990026,
-                "score_ci_high": 0.08476974354932754,
-                "sacrebleu_ci_low": 0.06067958452990026,
-                "sacrebleu_ci_high": 0.08476974354932754
+                "score_ci_low": 0.24101006670532885,
+                "score_ci_high": 0.2985709120047681,
+                "sacrebleu_ci_low": 0.24101006670532885,
+                "sacrebleu_ci_high": 0.2985709120047681
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 66,
                 "counts": [
-                    1475,
-                    1028,
-                    769,
-                    585
+                    1454,
+                    1049,
+                    810,
+                    633
                 ],
                 "totals": [
-                    5487,
-                    5421,
-                    5355,
-                    5289
+                    2019,
+                    1953,
+                    1887,
+                    1821
                 ],
                 "precisions": [
-                    0.26881720430107525,
-                    0.1896329090573695,
-                    0.14360410830999068,
-                    0.1106069200226886
+                    0.7201584943041109,
+                    0.5371223758320532,
+                    0.4292527821939586,
+                    0.3476112026359144
                 ],
                 "bp": 1.0,
-                "sys_len": 5487,
+                "sys_len": 2019,
                 "ref_len": 1916,
-                "sacrebleu": 0.16868636617394397,
-                "score": 0.16868636617394397,
+                "sacrebleu": 0.49014779569163686,
+                "score": 0.49014779569163686,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1449996381584022,
-                "score_ci_high": 0.19615430198730424,
-                "sacrebleu_ci_low": 0.1449996381584022,
-                "sacrebleu_ci_high": 0.19615430198730424
+                "score_ci_low": 0.447345907278528,
+                "score_ci_high": 0.5368115765817915,
+                "sacrebleu_ci_low": 0.447345907278528,
+                "sacrebleu_ci_high": 0.5368115765817915
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 66,
                 "counts": [
-                    1326,
-                    841,
-                    567,
-                    389
+                    1390,
+                    967,
+                    688,
+                    489
                 ],
                 "totals": [
-                    4837,
-                    4771,
-                    4705,
-                    4639
+                    1962,
+                    1896,
+                    1830,
+                    1764
                 ],
                 "precisions": [
-                    0.2741368616911309,
-                    0.17627331796269124,
-                    0.12051009564293304,
-                    0.0838542789394266
+                    0.7084607543323139,
+                    0.5100210970464135,
+                    0.37595628415300547,
+                    0.27721088435374147
                 ],
                 "bp": 1.0,
-                "sys_len": 4837,
+                "sys_len": 1962,
                 "ref_len": 1949,
-                "sacrebleu": 0.14865368140396532,
-                "score": 0.14865368140396532,
+                "sacrebleu": 0.4405172214713006,
+                "score": 0.4405172214713006,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12285617051153769,
-                "score_ci_high": 0.20722793764019112,
-                "sacrebleu_ci_low": 0.12285617051153769,
-                "sacrebleu_ci_high": 0.20722793764019112
+                "score_ci_low": 0.40615604089781865,
+                "score_ci_high": 0.516381897740174,
+                "sacrebleu_ci_low": 0.40615604089781865,
+                "sacrebleu_ci_high": 0.516381897740174
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 66,
                 "counts": [
-                    1381,
-                    759,
-                    459,
-                    281
+                    1287,
+                    732,
+                    440,
+                    265
                 ],
                 "totals": [
-                    6405,
-                    6339,
-                    6273,
-                    6207
+                    2008,
+                    1942,
+                    1876,
+                    1810
                 ],
                 "precisions": [
-                    0.21561280249804843,
-                    0.11973497397065783,
-                    0.07317073170731708,
-                    0.04527146769776059
+                    0.6409362549800797,
+                    0.3769309989701339,
+                    0.2345415778251599,
+                    0.1464088397790055
                 ],
-                "bp": 1.0,
-                "sys_len": 6405,
+                "bp": 0.956168891168866,
+                "sys_len": 2008,
                 "ref_len": 2098,
-                "sacrebleu": 0.09616441307249027,
-                "score": 0.09616441307249027,
+                "sacrebleu": 0.28856959420154726,
+                "score": 0.28856959420154726,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.07930542472032699,
-                "score_ci_high": 0.11825351259228321,
-                "sacrebleu_ci_low": 0.07930542472032699,
-                "sacrebleu_ci_high": 0.11825351259228321
+                "score_ci_low": 0.2645022427610819,
+                "score_ci_high": 0.3240217870629309,
+                "sacrebleu_ci_low": 0.2645022427610819,
+                "sacrebleu_ci_high": 0.3240217870629309
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1391,
-                    930,
-                    644,
-                    448
+                    1361,
+                    951,
+                    694,
+                    510
                 ],
                 "totals": [
-                    5836,
-                    5770,
-                    5704,
-                    5638
+                    2543,
+                    2477,
+                    2411,
+                    2345
                 ],
                 "precisions": [
-                    0.23834818368745714,
-                    0.1611785095320624,
-                    0.11290322580645162,
-                    0.07946080170273147
+                    0.5351946519858435,
+                    0.3839321760193783,
+                    0.2878473662380755,
+                    0.21748400852878466
                 ],
                 "bp": 1.0,
-                "sys_len": 5836,
+                "sys_len": 2543,
                 "ref_len": 1734,
-                "sacrebleu": 0.13625252798661136,
-                "score": 0.13625252798661136,
+                "sacrebleu": 0.33677431862395624,
+                "score": 0.33677431862395624,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11732338312550004,
-                "score_ci_high": 0.15953611321490402,
-                "sacrebleu_ci_low": 0.11732338312550004,
-                "sacrebleu_ci_high": 0.15953611321490402
+                "score_ci_low": 0.2922714206661574,
+                "score_ci_high": 0.38804905185639504,
+                "sacrebleu_ci_low": 0.2922714206661574,
+                "sacrebleu_ci_high": 0.38804905185639504
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1250,
-                    617,
-                    343,
-                    187
+                    1134,
+                    627,
+                    390,
+                    248
                 ],
                 "totals": [
-                    6443,
-                    6377,
-                    6311,
-                    6245
+                    1999,
+                    1933,
+                    1867,
+                    1801
                 ],
                 "precisions": [
-                    0.1940090020176936,
-                    0.09675395954210445,
-                    0.054349548407542386,
-                    0.029943955164131304
+                    0.5672836418209105,
+                    0.32436627004655977,
+                    0.2088912694161757,
+                    0.13770127706829538
                 ],
                 "bp": 1.0,
-                "sys_len": 6443,
+                "sys_len": 1999,
                 "ref_len": 1734,
-                "sacrebleu": 0.07434451508550546,
-                "score": 0.07434451508550546,
+                "sacrebleu": 0.2697264590787591,
+                "score": 0.2697264590787591,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0644058696100964,
-                "score_ci_high": 0.08685819903559298,
-                "sacrebleu_ci_low": 0.0644058696100964,
-                "sacrebleu_ci_high": 0.08685819903559298
+                "score_ci_low": 0.23374708119412382,
+                "score_ci_high": 0.32852575808051926,
+                "sacrebleu_ci_low": 0.23374708119412382,
+                "sacrebleu_ci_high": 0.32852575808051926
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1162,
-                    546,
-                    295,
-                    168
+                    1102,
+                    602,
+                    369,
+                    241
                 ],
                 "totals": [
-                    5914,
-                    5848,
-                    5782,
-                    5716
+                    1925,
+                    1859,
+                    1793,
+                    1727
                 ],
                 "precisions": [
-                    0.19648292188028407,
-                    0.09336525307797537,
-                    0.05102040816326531,
-                    0.02939118264520644
+                    0.5724675324675325,
+                    0.32383001613770845,
+                    0.20580033463469047,
+                    0.13954834973943256
                 ],
                 "bp": 1.0,
-                "sys_len": 5914,
+                "sys_len": 1925,
                 "ref_len": 1734,
-                "sacrebleu": 0.07242154227292893,
-                "score": 0.07242154227292893,
+                "sacrebleu": 0.27012183165344417,
+                "score": 0.27012183165344417,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06291349695976677,
-                "score_ci_high": 0.08531939881489306,
-                "sacrebleu_ci_low": 0.06291349695976677,
-                "sacrebleu_ci_high": 0.08531939881489306
+                "score_ci_low": 0.23833827580048095,
+                "score_ci_high": 0.320793499898261,
+                "sacrebleu_ci_low": 0.23833827580048095,
+                "sacrebleu_ci_high": 0.320793499898261
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1417,
-                    974,
-                    723,
-                    547
+                    1392,
+                    1000,
+                    752,
+                    574
                 ],
                 "totals": [
-                    6392,
-                    6326,
-                    6260,
-                    6194
+                    2564,
+                    2498,
+                    2432,
+                    2366
                 ],
                 "precisions": [
-                    0.22168335419274093,
-                    0.15396775213404995,
-                    0.11549520766773164,
-                    0.08831126896997095
+                    0.5429017160686428,
+                    0.400320256204964,
+                    0.3092105263157895,
+                    0.242603550295858
                 ],
                 "bp": 1.0,
-                "sys_len": 6392,
+                "sys_len": 2564,
                 "ref_len": 1734,
-                "sacrebleu": 0.13659529350611513,
-                "score": 0.13659529350611513,
+                "sacrebleu": 0.35733047002149876,
+                "score": 0.35733047002149876,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11528113587288888,
-                "score_ci_high": 0.15836557569065857,
-                "sacrebleu_ci_low": 0.11528113587288888,
-                "sacrebleu_ci_high": 0.15836557569065857
+                "score_ci_low": 0.2995890270688701,
+                "score_ci_high": 0.40598076558127455,
+                "sacrebleu_ci_low": 0.2995890270688701,
+                "sacrebleu_ci_high": 0.40598076558127455
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1395,
-                    948,
-                    686,
-                    506
+                    1376,
+                    983,
+                    722,
+                    538
                 ],
                 "totals": [
-                    5680,
-                    5614,
-                    5548,
-                    5482
+                    2343,
+                    2277,
+                    2211,
+                    2145
                 ],
                 "precisions": [
-                    0.24559859154929575,
-                    0.16886355539722125,
-                    0.12364816149963952,
-                    0.09230207953301715
+                    0.5872812633376013,
+                    0.43170838823012736,
+                    0.32654907281772955,
+                    0.25081585081585084
                 ],
                 "bp": 1.0,
-                "sys_len": 5680,
+                "sys_len": 2343,
                 "ref_len": 1734,
-                "sacrebleu": 0.14749939449155924,
-                "score": 0.14749939449155924,
+                "sacrebleu": 0.3796077031006635,
+                "score": 0.3796077031006635,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1292139766827155,
-                "score_ci_high": 0.1819243976706418,
-                "sacrebleu_ci_low": 0.1292139766827155,
-                "sacrebleu_ci_high": 0.1819243976706418
+                "score_ci_low": 0.3432297777298846,
+                "score_ci_high": 0.4214166231989559,
+                "sacrebleu_ci_low": 0.3432297777298846,
+                "sacrebleu_ci_high": 0.4214166231989559
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 66,
                 "counts": [
-                    1300,
-                    712,
-                    450,
-                    290
+                    1238,
+                    705,
+                    452,
+                    287
                 ],
                 "totals": [
-                    6166,
-                    6100,
-                    6034,
-                    5968
+                    2444,
+                    2378,
+                    2312,
+                    2246
                 ],
                 "precisions": [
-                    0.21083360363282516,
-                    0.11672131147540984,
-                    0.07457739476300962,
-                    0.04859249329758713
+                    0.5065466448445172,
+                    0.2964676198486123,
+                    0.19550173010380623,
+                    0.12778272484416742
                 ],
                 "bp": 1.0,
-                "sys_len": 6166,
+                "sys_len": 2444,
                 "ref_len": 1734,
-                "sacrebleu": 0.09717769541368253,
-                "score": 0.09717769541368253,
+                "sacrebleu": 0.24748840965516117,
+                "score": 0.24748840965516117,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08585452687395417,
-                "score_ci_high": 0.11046953561336194,
-                "sacrebleu_ci_low": 0.08585452687395417,
-                "sacrebleu_ci_high": 0.11046953561336194
+                "score_ci_low": 0.21183776036339763,
+                "score_ci_high": 0.2806697118543708,
+                "sacrebleu_ci_low": 0.21183776036339763,
+                "sacrebleu_ci_high": 0.2806697118543708
             },
-            "score": 0.11283376706964308,
+            "score": 0.3504084926765324,
             "score_name": "subsets_mean",
             "num_of_instances": 990
         },
-        "score": 0.4523639482757903,
+        "score": 0.4277276595742623,
         "score_name": "subsets_mean",
         "num_of_instances": 12472
     }