binwang commited on
Commit
1c954ce
·
verified ·
1 Parent(s): 36560f7

Upload organize_model_results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. organize_model_results.json +49 -34
organize_model_results.json CHANGED
@@ -1,4 +1,39 @@
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "voxceleb_accent_test": {
3
  "llama3_70b_judge": {
4
  "Qwen-Audio-Chat": 48.05088223225277,
@@ -606,22 +641,6 @@
606
  "cascade_whisper_large_v3_llama_3_8b_instruct": 52.252747252747255
607
  }
608
  },
609
- "cn_college_listen_mcq_test": {
610
- "llama3_70b_judge": {
611
- "Qwen-Audio-Chat": 63.232056362835756,
612
- "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.50726552179657,
613
- "Qwen2-Audio-7B-Instruct": 74.7247908410392,
614
- "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 91.85380889476001,
615
- "gemini-1.5-flash": 89.25583443416997,
616
- "phi_4_multimodal_instruct": 73.18361955085865,
617
- "WavLLM_fairseq": 66.31439894319684,
618
- "SALMONN_7B": 50.99075297225891,
619
- "cascade_whisper_large_v3_llama_3_8b_instruct": 85.2928225451343
620
- },
621
- "gpt4o_judge": {
622
- "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894
623
- }
624
- },
625
  "dream_tts_mcq_test": {
626
  "llama3_70b_judge": {
627
  "Qwen-Audio-Chat": 59.749085206481965,
@@ -1006,6 +1025,9 @@
1006
  "seallms_audio_7b": 60.199999999999996,
1007
  "SALMONN_7B": 48.4,
1008
  "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001
 
 
 
1009
  }
1010
  },
1011
  "mmau_mini_music": {
@@ -1030,6 +1052,9 @@
1030
  "seallms_audio_7b": 0.6646706586826348,
1031
  "SALMONN_7B": 0.5598802395209581,
1032
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748
 
 
 
1033
  }
1034
  },
1035
  "mmau_mini_sound": {
@@ -1054,6 +1079,9 @@
1054
  "seallms_audio_7b": 0.6486486486486487,
1055
  "SALMONN_7B": 0.5105105105105106,
1056
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106
 
 
 
1057
  }
1058
  },
1059
  "mmau_mini_speech": {
@@ -1078,6 +1106,9 @@
1078
  "seallms_audio_7b": 0.4924924924924925,
1079
  "SALMONN_7B": 0.3813813813813814,
1080
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216
 
 
 
1081
  }
1082
  },
1083
  "imda_part5_30s_sqa_test": {
@@ -1108,23 +1139,6 @@
1108
  "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.9
1109
  }
1110
  },
1111
- "slue_p2_sqa5_test": {
1112
- "llama3_70b_judge": {
1113
- "Qwen-Audio-Chat": 79.36274509803921,
1114
- "MERaLiON-AudioLLM-Whisper-SEA-LION": 86.76470588235293,
1115
- "Qwen2-Audio-7B-Instruct": 80.04901960784315,
1116
- "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 88.57843137254902,
1117
- "phi_4_multimodal_instruct": 88.33333333333334,
1118
- "seallms_audio_7b": 83.52941176470588,
1119
- "WavLLM_fairseq": 83.92156862745098,
1120
- "SALMONN_7B": 83.48039215686273,
1121
- "cascade_whisper_large_v3_llama_3_8b_instruct": 82.99019607843137
1122
- },
1123
- "gpt4o_judge": {
1124
- "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.23529411764707,
1125
- "cascade_whisper_large_v3_llama_3_8b_instruct": 87.79411764705883
1126
- }
1127
- },
1128
  "tedlium3_long_form_test": {
1129
  "wer": {
1130
  "Qwen-Audio-Chat": 0.2911540507002305,
@@ -1286,7 +1300,8 @@
1286
  },
1287
  "gigaspeech2_thai": {
1288
  "wer": {
1289
- "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.8901628256099774
 
1290
  }
1291
  },
1292
  "gigaspeech2_indo": {
 
1
  {
2
+ "cn_college_listen_mcq_test": {
3
+ "llama3_70b_judge": {
4
+ "gpt-4o-audio": 90.88507265521797,
5
+ "Qwen-Audio-Chat": 63.232056362835756,
6
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.50726552179657,
7
+ "Qwen2-Audio-7B-Instruct": 74.7247908410392,
8
+ "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 91.85380889476001,
9
+ "gemini-1.5-flash": 89.25583443416997,
10
+ "phi_4_multimodal_instruct": 73.18361955085865,
11
+ "WavLLM_fairseq": 66.31439894319684,
12
+ "SALMONN_7B": 50.99075297225891,
13
+ "cascade_whisper_large_v3_llama_3_8b_instruct": 85.2928225451343
14
+ },
15
+ "gpt4o_judge": {
16
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.77146631439894
17
+ }
18
+ },
19
+ "slue_p2_sqa5_test": {
20
+ "llama3_70b_judge": {
21
+ "gpt-4o-audio": 89.41176470588235,
22
+ "Qwen-Audio-Chat": 79.36274509803921,
23
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 86.76470588235293,
24
+ "Qwen2-Audio-7B-Instruct": 80.04901960784315,
25
+ "cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct": 88.57843137254902,
26
+ "phi_4_multimodal_instruct": 88.33333333333334,
27
+ "seallms_audio_7b": 83.52941176470588,
28
+ "WavLLM_fairseq": 83.92156862745098,
29
+ "SALMONN_7B": 83.48039215686273,
30
+ "cascade_whisper_large_v3_llama_3_8b_instruct": 82.99019607843137
31
+ },
32
+ "gpt4o_judge": {
33
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 88.23529411764707,
34
+ "cascade_whisper_large_v3_llama_3_8b_instruct": 87.79411764705883
35
+ }
36
+ },
37
  "voxceleb_accent_test": {
38
  "llama3_70b_judge": {
39
  "Qwen-Audio-Chat": 48.05088223225277,
 
641
  "cascade_whisper_large_v3_llama_3_8b_instruct": 52.252747252747255
642
  }
643
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
644
  "dream_tts_mcq_test": {
645
  "llama3_70b_judge": {
646
  "Qwen-Audio-Chat": 59.749085206481965,
 
1025
  "seallms_audio_7b": 60.199999999999996,
1026
  "SALMONN_7B": 48.4,
1027
  "cascade_whisper_large_v3_llama_3_8b_instruct": 55.60000000000001
1028
+ },
1029
+ "gpt4o_judge": {
1030
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 63.9
1031
  }
1032
  },
1033
  "mmau_mini_music": {
 
1052
  "seallms_audio_7b": 0.6646706586826348,
1053
  "SALMONN_7B": 0.5598802395209581,
1054
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5359281437125748
1055
+ },
1056
+ "gpt4o_judge": {
1057
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6347305389221557
1058
  }
1059
  },
1060
  "mmau_mini_sound": {
 
1079
  "seallms_audio_7b": 0.6486486486486487,
1080
  "SALMONN_7B": 0.5105105105105106,
1081
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.5105105105105106
1082
+ },
1083
+ "gpt4o_judge": {
1084
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.6996996996996997
1085
  }
1086
  },
1087
  "mmau_mini_speech": {
 
1106
  "seallms_audio_7b": 0.4924924924924925,
1107
  "SALMONN_7B": 0.3813813813813814,
1108
  "cascade_whisper_large_v3_llama_3_8b_instruct": 0.6216216216216216
1109
+ },
1110
+ "gpt4o_judge": {
1111
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.5825825825825826
1112
  }
1113
  },
1114
  "imda_part5_30s_sqa_test": {
 
1139
  "MERaLiON-AudioLLM-Whisper-SEA-LION": 66.9
1140
  }
1141
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1142
  "tedlium3_long_form_test": {
1143
  "wer": {
1144
  "Qwen-Audio-Chat": 0.2911540507002305,
 
1300
  },
1301
  "gigaspeech2_thai": {
1302
  "wer": {
1303
+ "MERaLiON-AudioLLM-Whisper-SEA-LION": 0.8901628256099774,
1304
+ "seallms_audio_7b": 0.3332398502070376
1305
  }
1306
  },
1307
  "gigaspeech2_indo": {