AudioBench-Leaderboard-Extend

Running

App Files Files Community

binwang commited on Mar 20

Commit

aa729a4

verified ·

1 Parent(s): 61dd7eb

Upload organize_model_results.json with huggingface_hub

Browse files

Files changed (1) hide show

organize_model_results.json +112 -108

organize_model_results.json CHANGED Viewed

@@ -16,6 +16,118 @@
             "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894
         }
     },
     "slue_p2_sqa5_test": {
         "llama3_70b_judge": {
             "gpt-4o-audio": 89.41176470588235,
@@ -1003,114 +1115,6 @@
             "cascade_whisper_large_v3_llama_3_8b_instruct": 5.987143868370054
         }
     },
-    "mmau_mini": {
-        "string_match": {
-            "Qwen-Audio-Chat": 38.5,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.5,
-            "Qwen2-Audio-7B-Instruct": 44.4,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 46.7,
-            "gemini-1.5-flash": 31.4,
-            "phi_4_multimodal_instruct": 54.50000000000001,
-            "seallms_audio_7b": 51.5,
-            "SALMONN_7B": 40.5,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 51.0
-        },
-        "llama3_70b_judge": {
-            "Qwen-Audio-Chat": 53.6,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.60000000000001,
-            "Qwen2-Audio-7B-Instruct": 58.9,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 52.7,
-            "gemini-1.5-flash": 58.199999999999996,
-            "phi_4_multimodal_instruct": 59.4,
-            "seallms_audio_7b": 60.199999999999996,
-            "SALMONN_7B": 48.4,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001
-        },
-        "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9
-        }
-    },
-    "mmau_mini_music": {
-        "string_match": {
-            "Qwen-Audio-Chat": 0.4311377245508982,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6077844311377245,
-            "Qwen2-Audio-7B-Instruct": 0.45808383233532934,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.39520958083832336,
-            "gemini-1.5-flash": 0.2904191616766467,
-            "phi_4_multimodal_instruct": 0.6377245508982036,
-            "seallms_audio_7b": 0.6047904191616766,
-            "SALMONN_7B": 0.4820359281437126,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5
-        },
-        "llama3_70b_judge": {
-            "Qwen-Audio-Chat": 0.5958083832335329,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6437125748502994,
-            "Qwen2-Audio-7B-Instruct": 0.6017964071856288,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.44011976047904194,
-            "gemini-1.5-flash": 0.5868263473053892,
-            "phi_4_multimodal_instruct": 0.688622754491018,
-            "seallms_audio_7b": 0.6646706586826348,
-            "SALMONN_7B": 0.5598802395209581,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748
-        },
-        "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557
-        }
-    },
-    "mmau_mini_sound": {
-        "string_match": {
-            "Qwen-Audio-Chat": 0.43543543543543545,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6606606606606606,
-            "Qwen2-Audio-7B-Instruct": 0.4744744744744745,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.45045045045045046,
-            "gemini-1.5-flash": 0.3483483483483483,
-            "phi_4_multimodal_instruct": 0.5975975975975976,
-            "seallms_audio_7b": 0.5165165165165165,
-            "SALMONN_7B": 0.4594594594594595,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.46546546546546547
-        },
-        "llama3_70b_judge": {
-            "Qwen-Audio-Chat": 0.5945945945945946,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.7027027027027027,
-            "Qwen2-Audio-7B-Instruct": 0.6306306306306306,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5345345345345346,
-            "gemini-1.5-flash": 0.5885885885885885,
-            "phi_4_multimodal_instruct": 0.6456456456456456,
-            "seallms_audio_7b": 0.6486486486486487,
-            "SALMONN_7B": 0.5105105105105106,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106
-        },
-        "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997
-        }
-    },
-    "mmau_mini_speech": {
-        "string_match": {
-            "Qwen-Audio-Chat": 0.2882882882882883,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5465465465465466,
-            "Qwen2-Audio-7B-Instruct": 0.3993993993993994,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5555555555555556,
-            "gemini-1.5-flash": 0.3033033033033033,
-            "phi_4_multimodal_instruct": 0.3993993993993994,
-            "seallms_audio_7b": 0.42342342342342343,
-            "SALMONN_7B": 0.2732732732732733,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5645645645645646
-        },
-        "llama3_70b_judge": {
-            "Qwen-Audio-Chat": 0.4174174174174174,
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5915915915915916,
-            "Qwen2-Audio-7B-Instruct": 0.5345345345345346,
-            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.6066066066066066,
-            "gemini-1.5-flash": 0.5705705705705706,
-            "phi_4_multimodal_instruct": 0.44744744744744747,
-            "seallms_audio_7b": 0.4924924924924925,
-            "SALMONN_7B": 0.3813813813813814,
-            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216
-        },
-        "gpt4o_judge": {
-            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826
-        }
-    },
     "imda_part5_30s_sqa_test": {
         "llama3_70b_judge": {
             "Qwen-Audio-Chat": 61.260000000000005,

             "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894
         }
     },
+    "mmau_mini": {
+        "string_match": {
+            "gpt-4o-audio": 0.0,
+            "Qwen-Audio-Chat": 38.5,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 60.5,
+            "Qwen2-Audio-7B-Instruct": 44.4,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 46.7,
+            "gemini-1.5-flash": 31.4,
+            "phi_4_multimodal_instruct": 54.50000000000001,
+            "seallms_audio_7b": 51.5,
+            "SALMONN_7B": 40.5,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 51.0
+        },
+        "llama3_70b_judge": {
+            "Qwen-Audio-Chat": 53.6,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 64.60000000000001,
+            "Qwen2-Audio-7B-Instruct": 58.9,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 52.7,
+            "gemini-1.5-flash": 58.199999999999996,
+            "phi_4_multimodal_instruct": 59.4,
+            "seallms_audio_7b": 60.199999999999996,
+            "SALMONN_7B": 48.4,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001
+        },
+        "gpt4o_judge": {
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9
+        }
+    },
+    "mmau_mini_music": {
+        "string_match": {
+            "gpt-4o-audio": 0.0,
+            "Qwen-Audio-Chat": 0.4311377245508982,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6077844311377245,
+            "Qwen2-Audio-7B-Instruct": 0.45808383233532934,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.39520958083832336,
+            "gemini-1.5-flash": 0.2904191616766467,
+            "phi_4_multimodal_instruct": 0.6377245508982036,
+            "seallms_audio_7b": 0.6047904191616766,
+            "SALMONN_7B": 0.4820359281437126,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5
+        },
+        "llama3_70b_judge": {
+            "Qwen-Audio-Chat": 0.5958083832335329,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6437125748502994,
+            "Qwen2-Audio-7B-Instruct": 0.6017964071856288,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.44011976047904194,
+            "gemini-1.5-flash": 0.5868263473053892,
+            "phi_4_multimodal_instruct": 0.688622754491018,
+            "seallms_audio_7b": 0.6646706586826348,
+            "SALMONN_7B": 0.5598802395209581,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748
+        },
+        "gpt4o_judge": {
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557
+        }
+    },
+    "mmau_mini_sound": {
+        "string_match": {
+            "gpt-4o-audio": 0.0,
+            "Qwen-Audio-Chat": 0.43543543543543545,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6606606606606606,
+            "Qwen2-Audio-7B-Instruct": 0.4744744744744745,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.45045045045045046,
+            "gemini-1.5-flash": 0.3483483483483483,
+            "phi_4_multimodal_instruct": 0.5975975975975976,
+            "seallms_audio_7b": 0.5165165165165165,
+            "SALMONN_7B": 0.4594594594594595,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.46546546546546547
+        },
+        "llama3_70b_judge": {
+            "Qwen-Audio-Chat": 0.5945945945945946,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.7027027027027027,
+            "Qwen2-Audio-7B-Instruct": 0.6306306306306306,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5345345345345346,
+            "gemini-1.5-flash": 0.5885885885885885,
+            "phi_4_multimodal_instruct": 0.6456456456456456,
+            "seallms_audio_7b": 0.6486486486486487,
+            "SALMONN_7B": 0.5105105105105106,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106
+        },
+        "gpt4o_judge": {
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997
+        }
+    },
+    "mmau_mini_speech": {
+        "string_match": {
+            "gpt-4o-audio": 0.0,
+            "Qwen-Audio-Chat": 0.2882882882882883,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5465465465465466,
+            "Qwen2-Audio-7B-Instruct": 0.3993993993993994,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.5555555555555556,
+            "gemini-1.5-flash": 0.3033033033033033,
+            "phi_4_multimodal_instruct": 0.3993993993993994,
+            "seallms_audio_7b": 0.42342342342342343,
+            "SALMONN_7B": 0.2732732732732733,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5645645645645646
+        },
+        "llama3_70b_judge": {
+            "Qwen-Audio-Chat": 0.4174174174174174,
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5915915915915916,
+            "Qwen2-Audio-7B-Instruct": 0.5345345345345346,
+            "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 0.6066066066066066,
+            "gemini-1.5-flash": 0.5705705705705706,
+            "phi_4_multimodal_instruct": 0.44744744744744747,
+            "seallms_audio_7b": 0.4924924924924925,
+            "SALMONN_7B": 0.3813813813813814,
+            "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216
+        },
+        "gpt4o_judge": {
+            "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826
+        }
+    },
     "slue_p2_sqa5_test": {
         "llama3_70b_judge": {
             "gpt-4o-audio": 89.41176470588235,
             "cascade_whisper_large_v3_llama_3_8b_instruct": 5.987143868370054
         }
     },
     "imda_part5_30s_sqa_test": {
         "llama3_70b_judge": {
             "Qwen-Audio-Chat": 61.260000000000005,