diff --git "a/scores/colpali-v1.3.json" "b/scores/colpali-v1.3.json" --- "a/scores/colpali-v1.3.json" +++ "b/scores/colpali-v1.3.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "colpali-v1.3", - "report_generated_date": "2025-06-09T07:08:13.841120", - "model_size": 2.92, - "url": "https://huggingface.co/vidore/colpali-v1.3", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.701533" }, "metrics": { "image": { @@ -1017,765 +1014,287 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.76, - "hit@5": 0.872, - "hit@10": 0.916, - "ndcg_linear@1": 0.76, - "ndcg_linear@5": 0.8173769173240496, - "ndcg_linear@10": 0.8316391463184296, - "ndcg_exponential@1": 0.76, - "ndcg_exponential@5": 0.8173769173240496, - "ndcg_exponential@10": 0.8316391463184296, - "precision@1": 0.76, - "precision@5": 0.17439999999999997, - "precision@10": 0.0916, - "recall@1": 0.76, - "recall@5": 0.872, - "recall@10": 0.916, - "f1@1": 0.76, - "f1@5": 0.29066666666666674, - "f1@10": 0.16654545454545455, - "map@1": 0.76, - "map@5": 0.7994, - "map@10": 0.8053031746031745, - "mrr@1": 0.76, - "mrr@5": 0.7994, - "mrr@10": 0.8053031746031745, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.4878048780487805, - "hit@5": 0.6385809312638581, - "hit@10": 0.6895787139689579, - "ndcg_linear@1": 0.4878048780487805, - "ndcg_linear@5": 0.5663817406801894, - "ndcg_linear@10": 0.5828545584108937, - "ndcg_exponential@1": 0.4878048780487805, - "ndcg_exponential@5": 0.5663817406801894, - "ndcg_exponential@10": 0.5828545584108937, - "precision@1": 0.4878048780487805, - "precision@5": 0.12860310421286028, - "precision@10": 0.06962305986696231, - "recall@1": 0.4835550628233555, - "recall@5": 0.635439763488544, - "recall@10": 0.685328898743533, - "f1@1": 0.4844505088407527, - "f1@5": 0.21333233958771985, - "f1@10": 0.1261170462944299, - "map@1": 0.4878048780487805, - "map@5": 0.542461197339246, - "map@10": 0.5494527153063739, - "mrr@1": 0.4878048780487805, - "mrr@5": 0.545269770879527, - "mrr@10": 0.5523598352866645, - "num_pred": 451, - "num_data": 451 - }, - "ViDoRe_infovqa": { - "hit@1": 0.791497975708502, - "hit@5": 0.8967611336032388, - "hit@10": 0.9311740890688259, - "ndcg_linear@1": 0.791497975708502, - "ndcg_linear@5": 0.8493870405671953, - "ndcg_linear@10": 0.8606301954292797, - "ndcg_exponential@1": 0.791497975708502, - "ndcg_exponential@5": 0.8493870405671953, - "ndcg_exponential@10": 0.8606301954292797, - "precision@1": 0.791497975708502, - "precision@5": 0.17975708502024293, - "precision@10": 0.09331983805668018, - "recall@1": 0.791497975708502, - "recall@5": 0.8967611336032388, - "recall@10": 0.930161943319838, - "f1@1": 0.791497975708502, - "f1@5": 0.299402352033931, - "f1@10": 0.16958041958041958, - "map@1": 0.791497975708502, - "map@5": 0.8334008097165991, - "map@10": 0.838223764539554, - "mrr@1": 0.791497975708502, - "mrr@5": 0.8333333333333333, - "mrr@10": 0.8383008804061435, - "num_pred": 494, - "num_data": 494 - }, - "ViDoRe_tabfquad": { - "hit@1": 0.8178571428571428, - "hit@5": 0.9107142857142857, - "hit@10": 0.9535714285714286, - "ndcg_linear@1": 0.8178571428571428, - "ndcg_linear@5": 0.8693088177463265, - "ndcg_linear@10": 0.8831829118752064, - "ndcg_exponential@1": 0.8178571428571428, - "ndcg_exponential@5": 0.8693088177463265, - "ndcg_exponential@10": 0.8831829118752064, - "precision@1": 0.8178571428571428, - "precision@5": 0.18214285714285713, - "precision@10": 0.09535714285714285, - "recall@1": 0.8178571428571428, - "recall@5": 0.9107142857142857, - "recall@10": 0.9535714285714286, - "f1@1": 0.8178571428571428, - "f1@5": 0.30357142857142866, - "f1@10": 0.17337662337662343, - "map@1": 0.8178571428571428, - "map@5": 0.8552976190476191, - "map@10": 0.8610359977324263, - "mrr@1": 0.8178571428571428, - "mrr@5": 0.8552976190476191, - "mrr@10": 0.8610359977324263, - "num_pred": 280, - "num_data": 280 - }, - "ViDoRe_tatdqa": { - "hit@1": 0.5771567436208992, - "hit@5": 0.8207776427703524, - "hit@10": 0.8845686512758202, - "ndcg_linear@1": 0.5771567436208992, - "ndcg_linear@5": 0.7086500725869258, - "ndcg_linear@10": 0.729683893384968, - "ndcg_exponential@1": 0.5771567436208992, - "ndcg_exponential@5": 0.7086500725869258, - "ndcg_exponential@10": 0.729683893384968, - "precision@1": 0.5771567436208992, - "precision@5": 0.1645200486026731, - "precision@10": 0.08876063183475091, - "recall@1": 0.5762454434993924, - "recall@5": 0.8198663426488457, - "recall@10": 0.8842648845686513, - "f1@1": 0.5765492102065614, - "f1@5": 0.2739397095411677, - "f1@10": 0.16128171140321815, - "map@1": 0.5771567436208992, - "map@5": 0.6712991089509924, - "map@10": 0.6801273891492604, - "mrr@1": 0.5771567436208992, - "mrr@5": 0.6716281895504252, - "mrr@10": 0.6803197747304673, - "num_pred": 1646, - "num_data": 1646 - }, - "ViDoRe_shiftproject": { - "hit@1": 0.61, - "hit@5": 0.88, - "hit@10": 0.95, - "ndcg_linear@1": 0.61, - "ndcg_linear@5": 0.7511606311644851, - "ndcg_linear@10": 0.774534113357672, - "ndcg_exponential@1": 0.61, - "ndcg_exponential@5": 0.7511606311644851, - "ndcg_exponential@10": 0.774534113357672, - "precision@1": 0.61, - "precision@5": 0.176, - "precision@10": 0.09499999999999999, - "recall@1": 0.61, - "recall@5": 0.88, - "recall@10": 0.95, - "f1@1": 0.61, - "f1@5": 0.2933333333333334, - "f1@10": 0.1727272727272728, - "map@1": 0.61, - "map@5": 0.7083333333333333, - "map@10": 0.7184126984126985, - "mrr@1": 0.61, - "mrr@5": 0.7083333333333333, - "mrr@10": 0.7184126984126985, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.92, - "hit@5": 0.98, - "hit@10": 0.99, - "ndcg_linear@1": 0.92, - "ndcg_linear@5": 0.9565464876785729, - "ndcg_linear@10": 0.9601085595496531, - "ndcg_exponential@1": 0.92, - "ndcg_exponential@5": 0.9565464876785729, - "ndcg_exponential@10": 0.9601085595496531, - "precision@1": 0.92, - "precision@5": 0.19599999999999995, - "precision@10": 0.09899999999999999, - "recall@1": 0.92, - "recall@5": 0.98, - "recall@10": 0.99, - "f1@1": 0.92, - "f1@5": 0.32666666666666677, - "f1@10": 0.18000000000000008, - "map@1": 0.92, - "map@5": 0.9483333333333333, - "map@10": 0.95, - "mrr@1": 0.92, - "mrr@5": 0.9483333333333333, - "mrr@10": 0.95, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.92, - "hit@5": 0.97, - "hit@10": 0.97, - "ndcg_linear@1": 0.92, - "ndcg_linear@5": 0.9466649487518345, - "ndcg_linear@10": 0.9466649487518345, - "ndcg_exponential@1": 0.92, - "ndcg_exponential@5": 0.9466649487518345, - "ndcg_exponential@10": 0.9466649487518345, - "precision@1": 0.92, - "precision@5": 0.19399999999999995, - "precision@10": 0.09699999999999998, - "recall@1": 0.92, - "recall@5": 0.97, - "recall@10": 0.97, - "f1@1": 0.92, - "f1@5": 0.3233333333333334, - "f1@10": 0.17636363636363647, - "map@1": 0.92, - "map@5": 0.9390000000000001, - "map@10": 0.9390000000000001, - "mrr@1": 0.92, - "mrr@5": 0.9390000000000001, - "mrr@10": 0.9390000000000001, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.86, - "hit@5": 0.99, - "hit@10": 1.0, - "ndcg_linear@1": 0.86, - "ndcg_linear@5": 0.9354743802857166, - "ndcg_linear@10": 0.9388077136190499, - "ndcg_exponential@1": 0.86, - "ndcg_exponential@5": 0.9354743802857166, - "ndcg_exponential@10": 0.9388077136190499, - "precision@1": 0.86, - "precision@5": 0.19799999999999998, - "precision@10": 0.09999999999999998, - "recall@1": 0.86, - "recall@5": 0.99, - "recall@10": 1.0, - "f1@1": 0.86, - "f1@5": 0.3300000000000001, - "f1@10": 0.1818181818181819, - "map@1": 0.86, - "map@5": 0.9166666666666665, - "map@10": 0.918095238095238, - "mrr@1": 0.86, - "mrr@5": 0.9166666666666665, - "mrr@10": 0.918095238095238, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.91, - "hit@5": 0.99, - "hit@10": 1.0, - "ndcg_linear@1": 0.91, - "ndcg_linear@5": 0.9591650827500021, - "ndcg_linear@10": 0.9621753827066419, - "ndcg_exponential@1": 0.91, - "ndcg_exponential@5": 0.9591650827500021, - "ndcg_exponential@10": 0.9621753827066419, - "precision@1": 0.91, - "precision@5": 0.19799999999999998, - "precision@10": 0.09999999999999998, - "recall@1": 0.91, - "recall@5": 0.99, - "recall@10": 1.0, - "f1@1": 0.91, - "f1@5": 0.3300000000000001, - "f1@10": 0.1818181818181819, - "map@1": 0.91, - "map@5": 0.9483333333333335, - "map@10": 0.9494444444444445, - "mrr@1": 0.91, - "mrr@5": 0.9483333333333335, - "mrr@10": 0.9494444444444445, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.4230769230769231, - "hit@5": 0.75, - "hit@10": 0.8653846153846154, - "ndcg_linear@1": 0.4102564102564103, - "ndcg_linear@5": 0.5129471948247265, - "ndcg_linear@10": 0.562683866445629, - "ndcg_exponential@1": 0.4065934065934066, - "ndcg_exponential@5": 0.5103409095742849, - "ndcg_exponential@10": 0.5610704127395229, - "precision@1": 0.4230769230769231, - "precision@5": 0.20769230769230768, - "precision@10": 0.1423076923076923, - "recall@1": 0.3169871794871795, - "recall@5": 0.5976190476190476, - "recall@10": 0.7128746253746254, - "f1@1": 0.3403846153846154, - "f1@5": 0.2807916756446169, - "f1@10": 0.21769245083724723, - "map@1": 0.4230769230769231, - "map@5": 0.45147435897435895, - "map@10": 0.4801182844932845, - "mrr@1": 0.4230769230769231, - "mrr@5": 0.5532051282051282, - "mrr@10": 0.5671474358974359, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.54375, - "hit@5": 0.84375, - "hit@10": 0.9, - "ndcg_linear@1": 0.54375, - "ndcg_linear@5": 0.5813917241540179, - "ndcg_linear@10": 0.6164790283371475, - "ndcg_exponential@1": 0.54375, - "ndcg_exponential@5": 0.5813917241540177, - "ndcg_exponential@10": 0.6164790283371475, - "precision@1": 0.54375, - "precision@5": 0.2625, - "precision@10": 0.169375, - "recall@1": 0.34021847943722944, - "recall@5": 0.6122186147186147, - "recall@10": 0.7155228625541126, - "f1@1": 0.3868535804473304, - "f1@5": 0.32523640248640245, - "f1@10": 0.24737585025268852, - "map@1": 0.54375, - "map@5": 0.5063940972222222, - "map@10": 0.525980220734127, - "mrr@1": 0.54375, - "mrr@5": 0.6589583333333333, - "mrr@10": 0.6664409722222222, - "num_pred": 160, - "num_data": 160 - }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.5171875, - "hit@5": 0.7984375, - "hit@10": 0.878125, - "ndcg_linear@1": 0.5171875, - "ndcg_linear@5": 0.5472043616227252, - "ndcg_linear@10": 0.5826984113580178, - "ndcg_exponential@1": 0.5171875, - "ndcg_exponential@5": 0.5472043616227253, - "ndcg_exponential@10": 0.5826984113580178, - "precision@1": 0.5171875, - "precision@5": 0.24874999999999997, - "precision@10": 0.16234375, - "recall@1": 0.3132504734848485, - "recall@5": 0.5801439155207537, - "recall@10": 0.6875872763878279, - "f1@1": 0.35874357413419916, - "f1@5": 0.3077863438644689, - "f1@10": 0.23607504359066858, - "map@1": 0.5171875, - "map@5": 0.4755620659722222, - "map@10": 0.49229102595899477, - "mrr@1": 0.5171875, - "mrr@5": 0.6226822916666667, - "mrr@10": 0.6335261656746032, - "num_pred": 640, - "num_data": 640 - }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.5689655172413793, - "hit@5": 0.8620689655172413, - "hit@10": 0.9482758620689655, - "ndcg_linear@1": 0.5689655172413793, - "ndcg_linear@5": 0.5268152906479108, - "ndcg_linear@10": 0.5186082985031036, - "ndcg_exponential@1": 0.5689655172413793, - "ndcg_exponential@5": 0.5268152906479108, - "ndcg_exponential@10": 0.5186082985031036, - "precision@1": 0.5689655172413793, - "precision@5": 0.4827586206896552, - "precision@10": 0.39655172413793105, - "recall@1": 0.05668839113649585, - "recall@5": 0.2779816308965305, - "recall@10": 0.4264394514841731, - "f1@1": 0.09919901824675957, - "f1@5": 0.2897727049291107, - "f1@10": 0.333622932433892, - "map@1": 0.5689655172413793, - "map@5": 0.4241666666666667, - "map@10": 0.3765042712487294, - "mrr@1": 0.5689655172413793, - "mrr@5": 0.6887931034482759, - "mrr@10": 0.7002326217843459, - "num_pred": 58, - "num_data": 58 - }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.5086206896551724, - "hit@5": 0.8577586206896551, - "hit@10": 0.9353448275862069, - "ndcg_linear@1": 0.5086206896551724, - "ndcg_linear@5": 0.48968646101987606, - "ndcg_linear@10": 0.4855016494653571, - "ndcg_exponential@1": 0.5086206896551724, - "ndcg_exponential@5": 0.48968646101987606, - "ndcg_exponential@10": 0.4855016494653571, - "precision@1": 0.5086206896551724, - "precision@5": 0.4456896551724138, - "precision@10": 0.3732758620689655, - "recall@1": 0.049608559736213115, - "recall@5": 0.2650305856260614, - "recall@10": 0.4040109375386366, - "f1@1": 0.0867748494784438, - "f1@5": 0.2676800330504101, - "f1@10": 0.31406186428354066, - "map@1": 0.5086206896551724, - "map@5": 0.38345545977011497, - "map@10": 0.3455301240866717, - "mrr@1": 0.5086206896551724, - "mrr@5": 0.6462643678160919, - "mrr@10": 0.6564638067870827, - "num_pred": 232, - "num_data": 232 - }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.45614035087719296, - "hit@5": 0.8245614035087719, - "hit@10": 0.8947368421052632, - "ndcg_linear@1": 0.45614035087719296, - "ndcg_linear@5": 0.5175125572897402, - "ndcg_linear@10": 0.5646719292113425, - "ndcg_exponential@1": 0.45614035087719296, - "ndcg_exponential@5": 0.5175125572897402, - "ndcg_exponential@10": 0.5646719292113425, - "precision@1": 0.45614035087719296, - "precision@5": 0.2631578947368421, - "precision@10": 0.17894736842105266, - "recall@1": 0.25818713450292397, - "recall@5": 0.5447159565580618, - "recall@10": 0.7023600668337511, - "f1@1": 0.29706122337701285, - "f1@5": 0.30604060285388357, - "f1@10": 0.2493310710436342, - "map@1": 0.45614035087719296, - "map@5": 0.4327923976608188, - "map@10": 0.45467383737120576, - "mrr@1": 0.45614035087719296, - "mrr@5": 0.5976608187134502, - "mrr@10": 0.6073517126148705, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.49122807017543857, - "hit@5": 0.8070175438596491, - "hit@10": 0.8728070175438597, - "ndcg_linear@1": 0.49122807017543857, - "ndcg_linear@5": 0.5294466383615495, - "ndcg_linear@10": 0.5608887645448074, - "ndcg_exponential@1": 0.49122807017543857, - "ndcg_exponential@5": 0.5294466383615495, - "ndcg_exponential@10": 0.5608887645448074, - "precision@1": 0.49122807017543857, - "precision@5": 0.26754385964912286, - "precision@10": 0.17631578947368426, - "recall@1": 0.25065267335004177, - "recall@5": 0.5506109022556391, - "recall@10": 0.6676691729323309, - "f1@1": 0.29780297238554954, - "f1@5": 0.3099469090154475, - "f1@10": 0.2428282645000911, - "map@1": 0.49122807017543857, - "map@5": 0.4436610623781677, - "map@10": 0.4549566044741483, - "mrr@1": 0.49122807017543857, - "mrr@5": 0.6209795321637426, - "mrr@10": 0.6295652325257587, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.7561274509803921, - "hit@5": 0.8553921568627451, - "hit@10": 0.8848039215686274, - "ndcg_linear@1": 0.7561274509803921, - "ndcg_linear@5": 0.8087241849861875, - "ndcg_linear@10": 0.8184673504656934, - "ndcg_exponential@1": 0.7561274509803921, - "ndcg_exponential@5": 0.8087241849861875, - "ndcg_exponential@10": 0.8184673504656934, - "precision@1": 0.7561274509803921, - "precision@5": 0.17107843137254902, - "precision@10": 0.08848039215686274, - "recall@1": 0.7561274509803921, - "recall@5": 0.8553921568627451, - "recall@10": 0.8848039215686274, - "f1@1": 0.7561274509803921, - "f1@5": 0.28513071895424846, - "f1@10": 0.16087344028520503, - "map@1": 0.7561274509803921, - "map@5": 0.7931985294117647, - "map@10": 0.7973593604108311, - "mrr@1": 0.7561274509803921, - "mrr@5": 0.7931985294117647, - "mrr@10": 0.7973593604108311, - "num_pred": 816, - "num_data": 816 + "video": { + "K700": { + "hit@1": 0.234, + "hit@5": 0.411, + "hit@10": 0.47, + "ndcg_linear@1": 0.234, + "ndcg_linear@5": 0.3273147185066129, + "ndcg_linear@10": 0.3462654041192875, + "ndcg_exponential@1": 0.234, + "ndcg_exponential@5": 0.3273147185066129, + "ndcg_exponential@10": 0.3462654041192875, + "precision@1": 0.234, + "precision@5": 0.0822, + "precision@10": 0.047, + "recall@1": 0.234, + "recall@5": 0.411, + "recall@10": 0.47, + "f1@1": 0.234, + "f1@5": 0.13700000000000004, + "f1@10": 0.08545454545454546, + "map@1": 0.234, + "map@5": 0.2995, + "map@10": 0.30724523809523807, + "mrr@1": 0.234, + "mrr@5": 0.2995, + "mrr@10": 0.30724523809523807, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_ChartQA": { - "hit@1": 0.6825396825396826, - "hit@5": 0.8571428571428571, - "hit@10": 0.9047619047619048, - "ndcg_linear@1": 0.6825396825396826, - "ndcg_linear@5": 0.781492660957272, - "ndcg_linear@10": 0.7972160024827536, - "ndcg_exponential@1": 0.6825396825396826, - "ndcg_exponential@5": 0.781492660957272, - "ndcg_exponential@10": 0.7972160024827536, - "precision@1": 0.6825396825396826, - "precision@5": 0.17142857142857135, - "precision@10": 0.09047619047619043, - "recall@1": 0.6825396825396826, - "recall@5": 0.8571428571428571, - "recall@10": 0.9047619047619048, - "f1@1": 0.6825396825396826, - "f1@5": 0.28571428571428564, - "f1@10": 0.16450216450216454, - "map@1": 0.6825396825396826, - "map@5": 0.7558201058201058, - "map@10": 0.7624968505920886, - "mrr@1": 0.6825396825396826, - "mrr@5": 0.7558201058201058, - "mrr@10": 0.7624968505920886, - "num_pred": 63, - "num_data": 63 + "SmthSmthV2": { + "hit@1": 0.251, + "hit@5": 0.485, + "hit@10": 0.592, + "ndcg_linear@1": 0.251, + "ndcg_linear@5": 0.37516969884345464, + "ndcg_linear@10": 0.4100591996704718, + "ndcg_exponential@1": 0.251, + "ndcg_exponential@5": 0.37516969884345464, + "ndcg_exponential@10": 0.4100591996704718, + "precision@1": 0.251, + "precision@5": 0.097, + "precision@10": 0.059199999999999996, + "recall@1": 0.251, + "recall@5": 0.485, + "recall@10": 0.592, + "f1@1": 0.251, + "f1@5": 0.1616666666666667, + "f1@10": 0.10763636363636366, + "map@1": 0.251, + "map@5": 0.33863333333333334, + "map@10": 0.35319563492063494, + "mrr@1": 0.251, + "mrr@5": 0.33863333333333334, + "mrr@10": 0.35319563492063494, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.7918781725888325, - "hit@5": 0.9289340101522843, - "hit@10": 0.9509306260575296, - "ndcg_linear@1": 0.7918781725888325, - "ndcg_linear@5": 0.8676285427363506, - "ndcg_linear@10": 0.8748306491817542, - "ndcg_exponential@1": 0.7918781725888325, - "ndcg_exponential@5": 0.8676285427363506, - "ndcg_exponential@10": 0.8748306491817542, - "precision@1": 0.7918781725888325, - "precision@5": 0.18578680203045683, - "precision@10": 0.09509306260575294, - "recall@1": 0.7918781725888325, - "recall@5": 0.9289340101522843, - "recall@10": 0.9509306260575296, - "f1@1": 0.7918781725888325, - "f1@5": 0.3096446700507615, - "f1@10": 0.17289647746500544, - "map@1": 0.7918781725888325, - "map@5": 0.8469543147208122, - "map@10": 0.8499758278946096, - "mrr@1": 0.7918781725888325, - "mrr@5": 0.8469543147208122, - "mrr@10": 0.8499758278946096, - "num_pred": 591, - "num_data": 591 + "HMDB51": { + "hit@1": 0.248, + "hit@5": 0.447, + "hit@10": 0.578, + "ndcg_linear@1": 0.248, + "ndcg_linear@5": 0.35078743678844715, + "ndcg_linear@10": 0.39292321098555394, + "ndcg_exponential@1": 0.248, + "ndcg_exponential@5": 0.35078743678844715, + "ndcg_exponential@10": 0.39292321098555394, + "precision@1": 0.248, + "precision@5": 0.0894, + "precision@10": 0.0578, + "recall@1": 0.248, + "recall@5": 0.447, + "recall@10": 0.578, + "f1@1": 0.248, + "f1@5": 0.14900000000000005, + "f1@10": 0.10509090909090911, + "map@1": 0.248, + "map@5": 0.31906666666666667, + "map@10": 0.33631944444444445, + "mrr@1": 0.248, + "mrr@5": 0.31906666666666667, + "mrr@10": 0.33631944444444445, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_SlideVQA": { - "hit@1": 0.9388489208633094, - "hit@5": 0.9892086330935251, - "hit@10": 0.9892086330935251, - "ndcg_linear@1": 0.9388489208633094, - "ndcg_linear@5": 0.9502736353073175, - "ndcg_linear@10": 0.9527678727475414, - "ndcg_exponential@1": 0.9388489208633094, - "ndcg_exponential@5": 0.9502736353073175, - "ndcg_exponential@10": 0.9527678727475414, - "precision@1": 0.9388489208633094, - "precision@5": 0.24388489208633093, - "precision@10": 0.12320143884892086, - "recall@1": 0.8120503597122302, - "recall@5": 0.9739208633093526, - "recall@10": 0.9802158273381295, - "f1@1": 0.85431654676259, - "f1@5": 0.38309352517985623, - "f1@10": 0.2163723566601265, - "map@1": 0.9388489208633094, - "map@5": 0.9327038369304556, - "map@10": 0.9343789254310837, - "mrr@1": 0.9388489208633094, - "mrr@5": 0.9612410071942447, - "mrr@10": 0.9612410071942447, - "num_pred": 556, - "num_data": 556 + "UCF101": { + "hit@1": 0.494, + "hit@5": 0.675, + "hit@10": 0.752, + "ndcg_linear@1": 0.494, + "ndcg_linear@5": 0.5961521274121583, + "ndcg_linear@10": 0.6208342304490151, + "ndcg_exponential@1": 0.494, + "ndcg_exponential@5": 0.5961521274121583, + "ndcg_exponential@10": 0.6208342304490151, + "precision@1": 0.494, + "precision@5": 0.135, + "precision@10": 0.0752, + "recall@1": 0.494, + "recall@5": 0.675, + "recall@10": 0.752, + "f1@1": 0.494, + "f1@5": 0.22500000000000006, + "f1@10": 0.13672727272727275, + "map@1": 0.494, + "map@5": 0.56945, + "map@10": 0.5795154761904762, + "mrr@1": 0.494, + "mrr@5": 0.56945, + "mrr@10": 0.5795154761904762, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_InfoVQA": { - "hit@1": 0.7729805013927576, - "hit@5": 0.924791086350975, - "hit@10": 0.9512534818941504, - "ndcg_linear@1": 0.7729805013927576, - "ndcg_linear@5": 0.8569356912112719, - "ndcg_linear@10": 0.8656693780762971, - "ndcg_exponential@1": 0.7729805013927576, - "ndcg_exponential@5": 0.8569356912112719, - "ndcg_exponential@10": 0.8656693780762971, - "precision@1": 0.7729805013927576, - "precision@5": 0.18495821727019496, - "precision@10": 0.09512534818941502, - "recall@1": 0.7729805013927576, - "recall@5": 0.924791086350975, - "recall@10": 0.9512534818941504, - "f1@1": 0.7729805013927576, - "f1@5": 0.308263695450325, - "f1@10": 0.1729551785262092, - "map@1": 0.7729805013927576, - "map@5": 0.8340529247910862, - "map@10": 0.8377569969491975, - "mrr@1": 0.7729805013927576, - "mrr@5": 0.8340529247910862, - "mrr@10": 0.8377569969491975, - "num_pred": 718, - "num_data": 718 + "Breakfast": { + "hit@1": 0.10854503464203233, + "hit@5": 0.5519630484988453, + "hit@10": 1.0, + "ndcg_linear@1": 0.10854503464203233, + "ndcg_linear@5": 0.3325565915387777, + "ndcg_linear@10": 0.47501706939610605, + "ndcg_exponential@1": 0.10854503464203233, + "ndcg_exponential@5": 0.3325565915387777, + "ndcg_exponential@10": 0.47501706939610605, + "precision@1": 0.10854503464203233, + "precision@5": 0.11039260969976905, + "precision@10": 0.1, + "recall@1": 0.10854503464203233, + "recall@5": 0.5519630484988453, + "recall@10": 1.0, + "f1@1": 0.10854503464203233, + "f1@5": 0.18398768283294847, + "f1@10": 0.1818181818181818, + "map@1": 0.10854503464203233, + "map@5": 0.2607005388760585, + "map@10": 0.3181110011364053, + "mrr@1": 0.10854503464203233, + "mrr@5": 0.2607005388760585, + "mrr@10": 0.3181110011364053, + "num_pred": 433, + "num_data": 433 }, - "VisRAG_PlotQA": { - "hit@1": 0.46349942062572425, - "hit@5": 0.7265353418308227, - "hit@10": 0.831981460023175, - "ndcg_linear@1": 0.46349942062572425, - "ndcg_linear@5": 0.6029827126488958, - "ndcg_linear@10": 0.6373604951260508, - "ndcg_exponential@1": 0.46349942062572425, - "ndcg_exponential@5": 0.6029827126488958, - "ndcg_exponential@10": 0.6373604951260508, - "precision@1": 0.46349942062572425, - "precision@5": 0.14530706836616453, - "precision@10": 0.08319814600231748, - "recall@1": 0.46349942062572425, - "recall@5": 0.7265353418308227, - "recall@10": 0.831981460023175, - "f1@1": 0.46349942062572425, - "f1@5": 0.24217844727694096, - "f1@10": 0.15126935636785002, - "map@1": 0.46349942062572425, - "map@5": 0.5619351100811124, - "map@10": 0.5762865787489194, - "mrr@1": 0.46349942062572425, - "mrr@5": 0.5619351100811124, - "mrr@10": 0.5762865787489194, - "num_pred": 863, - "num_data": 863 + "MVBench": { + "hit@1": 0.337, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.337, + "ndcg_linear@5": 0.6944068117467572, + "ndcg_linear@10": 0.6944068117467572, + "ndcg_exponential@1": 0.337, + "ndcg_exponential@5": 0.6944068117467572, + "ndcg_exponential@10": 0.6944068117467572, + "precision@1": 0.337, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.337, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.337, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.337, + "map@5": 0.5917374999999999, + "map@10": 0.5917374999999999, + "mrr@1": 0.337, + "mrr@5": 0.5917374999999999, + "mrr@10": 0.5917374999999999, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoSeek-page": { - "hit@1": 0.023642732049036778, - "hit@5": 0.4141856392294221, - "hit@10": 0.6348511383537653, - "ndcg_linear@1": 0.023642732049036778, - "ndcg_linear@5": 0.2215998227373851, - "ndcg_linear@10": 0.29328787953138075, - "ndcg_exponential@1": 0.023642732049036778, - "ndcg_exponential@5": 0.2215998227373851, - "ndcg_exponential@10": 0.29328787953138075, - "precision@1": 0.023642732049036778, - "precision@5": 0.08283712784588441, - "precision@10": 0.06348511383537653, - "recall@1": 0.023642732049036778, - "recall@5": 0.4141856392294221, - "recall@10": 0.6348511383537653, - "f1@1": 0.023642732049036778, - "f1@5": 0.13806187974314071, - "f1@10": 0.11542747970068462, - "map@1": 0.023642732049036778, - "map@5": 0.15847927612375948, - "map@10": 0.1882595974758847, - "mrr@1": 0.023642732049036778, - "mrr@5": 0.15847927612375948, - "mrr@10": 0.1882595974758847, - "num_pred": 1142, - "num_data": 1142 + "Video-MME": { + "hit@1": 0.30592592592592593, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.30592592592592593, + "ndcg_linear@5": 0.6721188097820419, + "ndcg_linear@10": 0.6721188097820419, + "ndcg_exponential@1": 0.30592592592592593, + "ndcg_exponential@5": 0.6721188097820419, + "ndcg_exponential@10": 0.6721188097820419, + "precision@1": 0.30592592592592593, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.30592592592592593, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.30592592592592593, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.30592592592592593, + "map@5": 0.5626543209876542, + "map@10": 0.5626543209876542, + "mrr@1": 0.30592592592592593, + "mrr@5": 0.5626543209876542, + "mrr@10": 0.5626543209876542, + "num_pred": 2700, + "num_data": 2700 }, - "ViDoSeek-doc": { - "hit@1": 0.9973730297723292, - "hit@5": 0.999124343257443, - "hit@10": 0.999124343257443, - "ndcg_linear@1": 0.680385288966725, - "ndcg_linear@5": 0.8366080322665971, - "ndcg_linear@10": 0.8228477305778173, - "ndcg_exponential@1": 0.4539654741055791, - "ndcg_exponential@5": 0.7156185424033248, - "ndcg_exponential@10": 0.7459741664587741, - "precision@1": 0.9973730297723292, - "precision@5": 0.9252189141856392, - "precision@10": 0.8394045534150613, - "recall@1": 0.05880569019029663, - "recall@5": 0.272128058121516, - "recall@10": 0.48450076291066557, - "f1@1": 0.11031252785994464, - "f1@5": 0.4120671799650714, - "f1@10": 0.6005622170825162, - "map@1": 0.9973730297723292, - "map@5": 0.9193753648569761, - "map@10": 0.8319699422384248, - "mrr@1": 0.9973730297723292, - "mrr@5": 0.998029772329247, - "mrr@10": 0.998029772329247, - "num_pred": 1142, - "num_data": 1142 + "NExTQA": { + "hit@1": 0.3520551144325082, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3520551144325082, + "ndcg_linear@5": 0.677249628478771, + "ndcg_linear@10": 0.677249628478771, + "ndcg_exponential@1": 0.3520551144325082, + "ndcg_exponential@5": 0.677249628478771, + "ndcg_exponential@10": 0.677249628478771, + "precision@1": 0.3520551144325082, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.3520551144325082, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3520551144325082, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.3520551144325082, + "map@5": 0.5717772069126577, + "map@10": 0.5717772069126577, + "mrr@1": 0.3520551144325082, + "mrr@5": 0.5716020551144325, + "mrr@10": 0.5716020551144325, + "num_pred": 8564, + "num_data": 8564 }, - "MMLongBench-page": { - "hit@1": 0.06563245823389022, - "hit@5": 0.2863961813842482, - "hit@10": 0.3914081145584726, - "ndcg_linear@1": 0.06563245823389022, - "ndcg_linear@5": 0.14173090844476535, - "ndcg_linear@10": 0.17778697624105239, - "ndcg_exponential@1": 0.06563245823389022, - "ndcg_exponential@5": 0.14173090844476535, - "ndcg_exponential@10": 0.17778697624105239, - "precision@1": 0.06563245823389022, - "precision@5": 0.06897374701670644, - "precision@10": 0.0513126491646778, - "recall@1": 0.037907716785998406, - "recall@5": 0.20580889873849303, - "recall@10": 0.3072328294881994, - "f1@1": 0.045729568760595, - "f1@5": 0.09743296504141655, - "f1@10": 0.08383696942332947, - "map@1": 0.06563245823389022, - "map@5": 0.10388988332007426, - "map@10": 0.1198034589067661, - "mrr@1": 0.06563245823389022, - "mrr@5": 0.1441129673826571, - "mrr@10": 0.15842093798537715, - "num_pred": 838, - "num_data": 838 + "EgoSchema": { + "hit@1": 0.384, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.384, + "ndcg_linear@5": 0.6958150458511005, + "ndcg_linear@10": 0.6958150458511005, + "ndcg_exponential@1": 0.384, + "ndcg_exponential@5": 0.6958150458511005, + "ndcg_exponential@10": 0.6958150458511005, + "precision@1": 0.384, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.384, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.384, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.384, + "map@5": 0.5960666666666666, + "map@10": 0.5960666666666666, + "mrr@1": 0.384, + "mrr@5": 0.5960666666666666, + "mrr@10": 0.5960666666666666, + "num_pred": 500, + "num_data": 500 + }, + "ActivityNetQA": { + "hit@1": 0.513, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.513, + "ndcg_linear@5": 0.8202627899893, + "ndcg_linear@10": 0.8202627899893, + "ndcg_exponential@1": 0.513, + "ndcg_exponential@5": 0.8202627899893, + "ndcg_exponential@10": 0.8202627899893, + "precision@1": 0.513, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.513, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.513, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.513, + "map@5": 0.7565, + "map@10": 0.7565, + "mrr@1": 0.513, + "mrr@5": 0.7565, + "mrr@10": 0.7565, + "num_pred": 1000, + "num_data": 1000 }, - "MMLongBench-doc": { - "hit@1": 0.7326968973747017, - "hit@5": 0.8186157517899761, - "hit@10": 0.847255369928401, - "ndcg_linear@1": 0.5103420843277645, - "ndcg_linear@5": 0.5251195991901183, - "ndcg_linear@10": 0.4977133933501543, - "ndcg_exponential@1": 0.35151721786566653, - "ndcg_exponential@5": 0.43679570479418406, - "ndcg_exponential@10": 0.44114798529060906, - "precision@1": 0.7326968973747017, - "precision@5": 0.5794749403341289, - "precision@10": 0.5013126491646778, - "recall@1": 0.024172213173396184, - "recall@5": 0.09148792471264464, - "recall@10": 0.15302173843210523, - "f1@1": 0.04631909415766511, - "f1@5": 0.15159508610283381, - "f1@10": 0.21973685679091384, - "map@1": 0.7326968973747017, - "map@5": 0.5482895783611774, - "map@10": 0.4582356896617039, - "mrr@1": 0.7326968973747017, - "mrr@5": 0.7665871121718377, - "mrr@10": 0.7705226919725726, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { "DiDeMo": { "hit@1": 0.22808764940239043, "hit@5": 0.4302788844621514, @@ -1804,481 +1323,875 @@ "num_pred": 1004, "num_data": 1004 }, - "MSR-VTT": { - "hit@1": 0.176, - "hit@5": 0.337, - "hit@10": 0.441, - "ndcg_linear@1": 0.176, - "ndcg_linear@5": 0.26090414668509254, - "ndcg_linear@10": 0.29375988725520075, - "ndcg_exponential@1": 0.176, - "ndcg_exponential@5": 0.26090414668509254, - "ndcg_exponential@10": 0.29375988725520075, - "precision@1": 0.176, - "precision@5": 0.0674, - "precision@10": 0.04409999999999999, - "recall@1": 0.176, - "recall@5": 0.337, - "recall@10": 0.441, - "f1@1": 0.176, - "f1@5": 0.11233333333333335, - "f1@10": 0.08018181818181819, - "map@1": 0.176, - "map@5": 0.23563333333333336, - "map@10": 0.24873690476190477, - "mrr@1": 0.176, - "mrr@5": 0.23563333333333336, - "mrr@10": 0.24873690476190477, - "num_pred": 1000, - "num_data": 1000 + "MSR-VTT": { + "hit@1": 0.176, + "hit@5": 0.337, + "hit@10": 0.441, + "ndcg_linear@1": 0.176, + "ndcg_linear@5": 0.26090414668509254, + "ndcg_linear@10": 0.29375988725520075, + "ndcg_exponential@1": 0.176, + "ndcg_exponential@5": 0.26090414668509254, + "ndcg_exponential@10": 0.29375988725520075, + "precision@1": 0.176, + "precision@5": 0.0674, + "precision@10": 0.04409999999999999, + "recall@1": 0.176, + "recall@5": 0.337, + "recall@10": 0.441, + "f1@1": 0.176, + "f1@5": 0.11233333333333335, + "f1@10": 0.08018181818181819, + "map@1": 0.176, + "map@5": 0.23563333333333336, + "map@10": 0.24873690476190477, + "mrr@1": 0.176, + "mrr@5": 0.23563333333333336, + "mrr@10": 0.24873690476190477, + "num_pred": 1000, + "num_data": 1000 + }, + "MSVD": { + "hit@1": 0.4537313432835821, + "hit@5": 0.7164179104477612, + "hit@10": 0.7865671641791044, + "ndcg_linear@1": 0.4537313432835821, + "ndcg_linear@5": 0.5961390786820799, + "ndcg_linear@10": 0.6187825557676115, + "ndcg_exponential@1": 0.4537313432835821, + "ndcg_exponential@5": 0.5961390786820799, + "ndcg_exponential@10": 0.6187825557676115, + "precision@1": 0.4537313432835821, + "precision@5": 0.14328358208955225, + "precision@10": 0.07865671641791044, + "recall@1": 0.4537313432835821, + "recall@5": 0.7164179104477612, + "recall@10": 0.7865671641791044, + "f1@1": 0.4537313432835821, + "f1@5": 0.2388059701492538, + "f1@10": 0.14301221166892814, + "map@1": 0.4537313432835821, + "map@5": 0.555870646766169, + "map@10": 0.5651930822080076, + "mrr@1": 0.4537313432835821, + "mrr@5": 0.555870646766169, + "mrr@10": 0.5651930822080076, + "num_pred": 670, + "num_data": 670 + }, + "VATEX": { + "hit@1": 0.16681554265297008, + "hit@5": 0.32402858418937025, + "hit@10": 0.40576150066994193, + "ndcg_linear@1": 0.16681554265297008, + "ndcg_linear@5": 0.24876051238618194, + "ndcg_linear@10": 0.2751625761091204, + "ndcg_exponential@1": 0.16681554265297008, + "ndcg_exponential@5": 0.24876051238618194, + "ndcg_exponential@10": 0.2751625761091204, + "precision@1": 0.16681554265297008, + "precision@5": 0.06480571683787406, + "precision@10": 0.0405761500669942, + "recall@1": 0.16681554265297008, + "recall@5": 0.32402858418937025, + "recall@10": 0.40576150066994193, + "f1@1": 0.16681554265297008, + "f1@5": 0.10800952806312343, + "f1@10": 0.07377481830362581, + "map@1": 0.16681554265297008, + "map@5": 0.22386109870477894, + "map@10": 0.23473578057097488, + "mrr@1": 0.16681554265297008, + "mrr@5": 0.22386109870477894, + "mrr@10": 0.23473578057097488, + "num_pred": 4478, + "num_data": 4478 + }, + "YouCook2": { + "hit@1": 0.05316137150047184, + "hit@5": 0.13337527524378737, + "hit@10": 0.18433469644542308, + "ndcg_linear@1": 0.05316137150047184, + "ndcg_linear@5": 0.09392223767158699, + "ndcg_linear@10": 0.11028455847469672, + "ndcg_exponential@1": 0.05316137150047184, + "ndcg_exponential@5": 0.09392223767158699, + "ndcg_exponential@10": 0.11028455847469672, + "precision@1": 0.05316137150047184, + "precision@5": 0.026675055048757473, + "precision@10": 0.01843346964454231, + "recall@1": 0.05316137150047184, + "recall@5": 0.13337527524378737, + "recall@10": 0.18433469644542308, + "f1@1": 0.05316137150047184, + "f1@5": 0.04445842508126246, + "f1@10": 0.03351539935371329, + "map@1": 0.05316137150047184, + "map@5": 0.08096885813148788, + "map@10": 0.08765359976432639, + "mrr@1": 0.05316137150047184, + "mrr@5": 0.08096885813148788, + "mrr@10": 0.08765359976432639, + "num_pred": 3179, + "num_data": 3179 + }, + "QVHighlight": { + "hit@1": 0.1994459833795014, + "hit@5": 0.5115420129270545, + "hit@10": 1.0, + "ndcg_linear@1": 0.1994459833795014, + "ndcg_linear@5": 0.35187697350343916, + "ndcg_linear@10": 0.5052964341688586, + "ndcg_exponential@1": 0.1994459833795014, + "ndcg_exponential@5": 0.35187697350343916, + "ndcg_exponential@10": 0.5052964341688586, + "precision@1": 0.1994459833795014, + "precision@5": 0.10230840258541089, + "precision@10": 0.09999999999999998, + "recall@1": 0.1994459833795014, + "recall@5": 0.5115420129270545, + "recall@10": 1.0, + "f1@1": 0.1994459833795014, + "f1@5": 0.17051400430901817, + "f1@10": 0.18181818181818185, + "map@1": 0.1994459833795014, + "map@5": 0.3000461680517082, + "map@10": 0.3607542247431444, + "mrr@1": 0.1994459833795014, + "mrr@5": 0.3000461680517082, + "mrr@10": 0.3607542247431444, + "num_pred": 1083, + "num_data": 1083 + }, + "Charades-STA": { + "hit@1": 0.2902338376891334, + "hit@5": 0.6217331499312242, + "hit@10": 1.0, + "ndcg_linear@1": 0.2902338376891334, + "ndcg_linear@5": 0.4587900118805714, + "ndcg_linear@10": 0.5779627989573906, + "ndcg_exponential@1": 0.2902338376891334, + "ndcg_exponential@5": 0.4587900118805714, + "ndcg_exponential@10": 0.5779627989573906, + "precision@1": 0.2902338376891334, + "precision@5": 0.12434662998624484, + "precision@10": 0.09999999999999998, + "recall@1": 0.2902338376891334, + "recall@5": 0.6217331499312242, + "recall@10": 1.0, + "f1@1": 0.2902338376891334, + "f1@5": 0.2072443833104081, + "f1@10": 0.18181818181818185, + "map@1": 0.2902338376891334, + "map@5": 0.40529573590096285, + "map@10": 0.4526838933647737, + "mrr@1": 0.2902338376891334, + "mrr@5": 0.40529573590096285, + "mrr@10": 0.4526838933647737, + "num_pred": 727, + "num_data": 727 + }, + "MomentSeeker": { + "hit@1": 0.2761111111111111, + "hit@5": 0.7705555555555555, + "hit@10": 1.0, + "ndcg_linear@1": 0.2761111111111111, + "ndcg_linear@5": 0.5263229165531387, + "ndcg_linear@10": 0.6067454069462798, + "ndcg_exponential@1": 0.2761111111111111, + "ndcg_exponential@5": 0.5263229165531387, + "ndcg_exponential@10": 0.6067454069462798, + "precision@1": 0.2761111111111111, + "precision@5": 0.1698888888888889, + "precision@10": 0.11261111111111113, + "recall@1": 0.2555092592592592, + "recall@5": 0.7533333333333333, + "recall@10": 0.997962962962963, + "f1@1": 0.2614717813051146, + "f1@5": 0.2724406303572971, + "f1@10": 0.19986989861989862, + "map@1": 0.2761111111111111, + "map@5": 0.4455154320987654, + "map@10": 0.4805357785861258, + "mrr@1": 0.2761111111111111, + "mrr@5": 0.45693518518518517, + "mrr@10": 0.48703725749559085, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.76, + "hit@5": 0.872, + "hit@10": 0.916, + "ndcg_linear@1": 0.76, + "ndcg_linear@5": 0.8173769173240496, + "ndcg_linear@10": 0.8316391463184296, + "ndcg_exponential@1": 0.76, + "ndcg_exponential@5": 0.8173769173240496, + "ndcg_exponential@10": 0.8316391463184296, + "precision@1": 0.76, + "precision@5": 0.17439999999999997, + "precision@10": 0.0916, + "recall@1": 0.76, + "recall@5": 0.872, + "recall@10": 0.916, + "f1@1": 0.76, + "f1@5": 0.29066666666666674, + "f1@10": 0.16654545454545455, + "map@1": 0.76, + "map@5": 0.7994, + "map@10": 0.8053031746031745, + "mrr@1": 0.76, + "mrr@5": 0.7994, + "mrr@10": 0.8053031746031745, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.4878048780487805, + "hit@5": 0.6385809312638581, + "hit@10": 0.6895787139689579, + "ndcg_linear@1": 0.4878048780487805, + "ndcg_linear@5": 0.5663817406801894, + "ndcg_linear@10": 0.5828545584108937, + "ndcg_exponential@1": 0.4878048780487805, + "ndcg_exponential@5": 0.5663817406801894, + "ndcg_exponential@10": 0.5828545584108937, + "precision@1": 0.4878048780487805, + "precision@5": 0.12860310421286028, + "precision@10": 0.06962305986696231, + "recall@1": 0.4835550628233555, + "recall@5": 0.635439763488544, + "recall@10": 0.685328898743533, + "f1@1": 0.4844505088407527, + "f1@5": 0.21333233958771985, + "f1@10": 0.1261170462944299, + "map@1": 0.4878048780487805, + "map@5": 0.542461197339246, + "map@10": 0.5494527153063739, + "mrr@1": 0.4878048780487805, + "mrr@5": 0.545269770879527, + "mrr@10": 0.5523598352866645, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.791497975708502, + "hit@5": 0.8967611336032388, + "hit@10": 0.9311740890688259, + "ndcg_linear@1": 0.791497975708502, + "ndcg_linear@5": 0.8493870405671953, + "ndcg_linear@10": 0.8606301954292797, + "ndcg_exponential@1": 0.791497975708502, + "ndcg_exponential@5": 0.8493870405671953, + "ndcg_exponential@10": 0.8606301954292797, + "precision@1": 0.791497975708502, + "precision@5": 0.17975708502024293, + "precision@10": 0.09331983805668018, + "recall@1": 0.791497975708502, + "recall@5": 0.8967611336032388, + "recall@10": 0.930161943319838, + "f1@1": 0.791497975708502, + "f1@5": 0.299402352033931, + "f1@10": 0.16958041958041958, + "map@1": 0.791497975708502, + "map@5": 0.8334008097165991, + "map@10": 0.838223764539554, + "mrr@1": 0.791497975708502, + "mrr@5": 0.8333333333333333, + "mrr@10": 0.8383008804061435, + "num_pred": 494, + "num_data": 494 }, - "MSVD": { - "hit@1": 0.4537313432835821, - "hit@5": 0.7164179104477612, - "hit@10": 0.7865671641791044, - "ndcg_linear@1": 0.4537313432835821, - "ndcg_linear@5": 0.5961390786820799, - "ndcg_linear@10": 0.6187825557676115, - "ndcg_exponential@1": 0.4537313432835821, - "ndcg_exponential@5": 0.5961390786820799, - "ndcg_exponential@10": 0.6187825557676115, - "precision@1": 0.4537313432835821, - "precision@5": 0.14328358208955225, - "precision@10": 0.07865671641791044, - "recall@1": 0.4537313432835821, - "recall@5": 0.7164179104477612, - "recall@10": 0.7865671641791044, - "f1@1": 0.4537313432835821, - "f1@5": 0.2388059701492538, - "f1@10": 0.14301221166892814, - "map@1": 0.4537313432835821, - "map@5": 0.555870646766169, - "map@10": 0.5651930822080076, - "mrr@1": 0.4537313432835821, - "mrr@5": 0.555870646766169, - "mrr@10": 0.5651930822080076, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tabfquad": { + "hit@1": 0.8178571428571428, + "hit@5": 0.9107142857142857, + "hit@10": 0.9535714285714286, + "ndcg_linear@1": 0.8178571428571428, + "ndcg_linear@5": 0.8693088177463265, + "ndcg_linear@10": 0.8831829118752064, + "ndcg_exponential@1": 0.8178571428571428, + "ndcg_exponential@5": 0.8693088177463265, + "ndcg_exponential@10": 0.8831829118752064, + "precision@1": 0.8178571428571428, + "precision@5": 0.18214285714285713, + "precision@10": 0.09535714285714285, + "recall@1": 0.8178571428571428, + "recall@5": 0.9107142857142857, + "recall@10": 0.9535714285714286, + "f1@1": 0.8178571428571428, + "f1@5": 0.30357142857142866, + "f1@10": 0.17337662337662343, + "map@1": 0.8178571428571428, + "map@5": 0.8552976190476191, + "map@10": 0.8610359977324263, + "mrr@1": 0.8178571428571428, + "mrr@5": 0.8552976190476191, + "mrr@10": 0.8610359977324263, + "num_pred": 280, + "num_data": 280 }, - "VATEX": { - "hit@1": 0.16681554265297008, - "hit@5": 0.32402858418937025, - "hit@10": 0.40576150066994193, - "ndcg_linear@1": 0.16681554265297008, - "ndcg_linear@5": 0.24876051238618194, - "ndcg_linear@10": 0.2751625761091204, - "ndcg_exponential@1": 0.16681554265297008, - "ndcg_exponential@5": 0.24876051238618194, - "ndcg_exponential@10": 0.2751625761091204, - "precision@1": 0.16681554265297008, - "precision@5": 0.06480571683787406, - "precision@10": 0.0405761500669942, - "recall@1": 0.16681554265297008, - "recall@5": 0.32402858418937025, - "recall@10": 0.40576150066994193, - "f1@1": 0.16681554265297008, - "f1@5": 0.10800952806312343, - "f1@10": 0.07377481830362581, - "map@1": 0.16681554265297008, - "map@5": 0.22386109870477894, - "map@10": 0.23473578057097488, - "mrr@1": 0.16681554265297008, - "mrr@5": 0.22386109870477894, - "mrr@10": 0.23473578057097488, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_tatdqa": { + "hit@1": 0.5771567436208992, + "hit@5": 0.8207776427703524, + "hit@10": 0.8845686512758202, + "ndcg_linear@1": 0.5771567436208992, + "ndcg_linear@5": 0.7086500725869258, + "ndcg_linear@10": 0.729683893384968, + "ndcg_exponential@1": 0.5771567436208992, + "ndcg_exponential@5": 0.7086500725869258, + "ndcg_exponential@10": 0.729683893384968, + "precision@1": 0.5771567436208992, + "precision@5": 0.1645200486026731, + "precision@10": 0.08876063183475091, + "recall@1": 0.5762454434993924, + "recall@5": 0.8198663426488457, + "recall@10": 0.8842648845686513, + "f1@1": 0.5765492102065614, + "f1@5": 0.2739397095411677, + "f1@10": 0.16128171140321815, + "map@1": 0.5771567436208992, + "map@5": 0.6712991089509924, + "map@10": 0.6801273891492604, + "mrr@1": 0.5771567436208992, + "mrr@5": 0.6716281895504252, + "mrr@10": 0.6803197747304673, + "num_pred": 1646, + "num_data": 1646 }, - "YouCook2": { - "hit@1": 0.05316137150047184, - "hit@5": 0.13337527524378737, - "hit@10": 0.18433469644542308, - "ndcg_linear@1": 0.05316137150047184, - "ndcg_linear@5": 0.09392223767158699, - "ndcg_linear@10": 0.11028455847469672, - "ndcg_exponential@1": 0.05316137150047184, - "ndcg_exponential@5": 0.09392223767158699, - "ndcg_exponential@10": 0.11028455847469672, - "precision@1": 0.05316137150047184, - "precision@5": 0.026675055048757473, - "precision@10": 0.01843346964454231, - "recall@1": 0.05316137150047184, - "recall@5": 0.13337527524378737, - "recall@10": 0.18433469644542308, - "f1@1": 0.05316137150047184, - "f1@5": 0.04445842508126246, - "f1@10": 0.03351539935371329, - "map@1": 0.05316137150047184, - "map@5": 0.08096885813148788, - "map@10": 0.08765359976432639, - "mrr@1": 0.05316137150047184, - "mrr@5": 0.08096885813148788, - "mrr@10": 0.08765359976432639, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_shiftproject": { + "hit@1": 0.61, + "hit@5": 0.88, + "hit@10": 0.95, + "ndcg_linear@1": 0.61, + "ndcg_linear@5": 0.7511606311644851, + "ndcg_linear@10": 0.774534113357672, + "ndcg_exponential@1": 0.61, + "ndcg_exponential@5": 0.7511606311644851, + "ndcg_exponential@10": 0.774534113357672, + "precision@1": 0.61, + "precision@5": 0.176, + "precision@10": 0.09499999999999999, + "recall@1": 0.61, + "recall@5": 0.88, + "recall@10": 0.95, + "f1@1": 0.61, + "f1@5": 0.2933333333333334, + "f1@10": 0.1727272727272728, + "map@1": 0.61, + "map@5": 0.7083333333333333, + "map@10": 0.7184126984126985, + "mrr@1": 0.61, + "mrr@5": 0.7083333333333333, + "mrr@10": 0.7184126984126985, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.92, + "hit@5": 0.98, + "hit@10": 0.99, + "ndcg_linear@1": 0.92, + "ndcg_linear@5": 0.9565464876785729, + "ndcg_linear@10": 0.9601085595496531, + "ndcg_exponential@1": 0.92, + "ndcg_exponential@5": 0.9565464876785729, + "ndcg_exponential@10": 0.9601085595496531, + "precision@1": 0.92, + "precision@5": 0.19599999999999995, + "precision@10": 0.09899999999999999, + "recall@1": 0.92, + "recall@5": 0.98, + "recall@10": 0.99, + "f1@1": 0.92, + "f1@5": 0.32666666666666677, + "f1@10": 0.18000000000000008, + "map@1": 0.92, + "map@5": 0.9483333333333333, + "map@10": 0.95, + "mrr@1": 0.92, + "mrr@5": 0.9483333333333333, + "mrr@10": 0.95, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.92, + "hit@5": 0.97, + "hit@10": 0.97, + "ndcg_linear@1": 0.92, + "ndcg_linear@5": 0.9466649487518345, + "ndcg_linear@10": 0.9466649487518345, + "ndcg_exponential@1": 0.92, + "ndcg_exponential@5": 0.9466649487518345, + "ndcg_exponential@10": 0.9466649487518345, + "precision@1": 0.92, + "precision@5": 0.19399999999999995, + "precision@10": 0.09699999999999998, + "recall@1": 0.92, + "recall@5": 0.97, + "recall@10": 0.97, + "f1@1": 0.92, + "f1@5": 0.3233333333333334, + "f1@10": 0.17636363636363647, + "map@1": 0.92, + "map@5": 0.9390000000000001, + "map@10": 0.9390000000000001, + "mrr@1": 0.92, + "mrr@5": 0.9390000000000001, + "mrr@10": 0.9390000000000001, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.1994459833795014, - "hit@5": 0.5115420129270545, + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.86, + "hit@5": 0.99, "hit@10": 1.0, - "ndcg_linear@1": 0.1994459833795014, - "ndcg_linear@5": 0.35187697350343916, - "ndcg_linear@10": 0.5052964341688586, - "ndcg_exponential@1": 0.1994459833795014, - "ndcg_exponential@5": 0.35187697350343916, - "ndcg_exponential@10": 0.5052964341688586, - "precision@1": 0.1994459833795014, - "precision@5": 0.10230840258541089, + "ndcg_linear@1": 0.86, + "ndcg_linear@5": 0.9354743802857166, + "ndcg_linear@10": 0.9388077136190499, + "ndcg_exponential@1": 0.86, + "ndcg_exponential@5": 0.9354743802857166, + "ndcg_exponential@10": 0.9388077136190499, + "precision@1": 0.86, + "precision@5": 0.19799999999999998, "precision@10": 0.09999999999999998, - "recall@1": 0.1994459833795014, - "recall@5": 0.5115420129270545, + "recall@1": 0.86, + "recall@5": 0.99, "recall@10": 1.0, - "f1@1": 0.1994459833795014, - "f1@5": 0.17051400430901817, - "f1@10": 0.18181818181818185, - "map@1": 0.1994459833795014, - "map@5": 0.3000461680517082, - "map@10": 0.3607542247431444, - "mrr@1": 0.1994459833795014, - "mrr@5": 0.3000461680517082, - "mrr@10": 0.3607542247431444, - "num_pred": 1083, - "num_data": 1083 + "f1@1": 0.86, + "f1@5": 0.3300000000000001, + "f1@10": 0.1818181818181819, + "map@1": 0.86, + "map@5": 0.9166666666666665, + "map@10": 0.918095238095238, + "mrr@1": 0.86, + "mrr@5": 0.9166666666666665, + "mrr@10": 0.918095238095238, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.2902338376891334, - "hit@5": 0.6217331499312242, + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.91, + "hit@5": 0.99, "hit@10": 1.0, - "ndcg_linear@1": 0.2902338376891334, - "ndcg_linear@5": 0.4587900118805714, - "ndcg_linear@10": 0.5779627989573906, - "ndcg_exponential@1": 0.2902338376891334, - "ndcg_exponential@5": 0.4587900118805714, - "ndcg_exponential@10": 0.5779627989573906, - "precision@1": 0.2902338376891334, - "precision@5": 0.12434662998624484, + "ndcg_linear@1": 0.91, + "ndcg_linear@5": 0.9591650827500021, + "ndcg_linear@10": 0.9621753827066419, + "ndcg_exponential@1": 0.91, + "ndcg_exponential@5": 0.9591650827500021, + "ndcg_exponential@10": 0.9621753827066419, + "precision@1": 0.91, + "precision@5": 0.19799999999999998, "precision@10": 0.09999999999999998, - "recall@1": 0.2902338376891334, - "recall@5": 0.6217331499312242, + "recall@1": 0.91, + "recall@5": 0.99, "recall@10": 1.0, - "f1@1": 0.2902338376891334, - "f1@5": 0.2072443833104081, - "f1@10": 0.18181818181818185, - "map@1": 0.2902338376891334, - "map@5": 0.40529573590096285, - "map@10": 0.4526838933647737, - "mrr@1": 0.2902338376891334, - "mrr@5": 0.40529573590096285, - "mrr@10": 0.4526838933647737, - "num_pred": 727, - "num_data": 727 + "f1@1": 0.91, + "f1@5": 0.3300000000000001, + "f1@10": 0.1818181818181819, + "map@1": 0.91, + "map@5": 0.9483333333333335, + "map@10": 0.9494444444444445, + "mrr@1": 0.91, + "mrr@5": 0.9483333333333335, + "mrr@10": 0.9494444444444445, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.296, - "hit@5": 0.775, - "hit@10": 1.0, - "ndcg_linear@1": 0.296, - "ndcg_linear@5": 0.5401914138460506, - "ndcg_linear@10": 0.6153961412718059, - "ndcg_exponential@1": 0.296, - "ndcg_exponential@5": 0.5401914138460506, - "ndcg_exponential@10": 0.6153961412718059, - "precision@1": 0.296, - "precision@5": 0.16240000000000002, - "precision@10": 0.10610000000000003, - "recall@1": 0.285625, - "recall@5": 0.7684583333333332, - "recall@10": 0.9989583333333334, - "f1@1": 0.2886888888888889, - "f1@5": 0.26520823620823625, - "f1@10": 0.18997745310245306, - "map@1": 0.296, - "map@5": 0.46244166666666664, - "map@10": 0.494421378968254, - "mrr@1": 0.296, - "mrr@5": 0.46720000000000006, - "mrr@10": 0.4972261904761905, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.4230769230769231, + "hit@5": 0.75, + "hit@10": 0.8653846153846154, + "ndcg_linear@1": 0.4102564102564103, + "ndcg_linear@5": 0.5129471948247265, + "ndcg_linear@10": 0.562683866445629, + "ndcg_exponential@1": 0.4065934065934066, + "ndcg_exponential@5": 0.5103409095742849, + "ndcg_exponential@10": 0.5610704127395229, + "precision@1": 0.4230769230769231, + "precision@5": 0.20769230769230768, + "precision@10": 0.1423076923076923, + "recall@1": 0.3169871794871795, + "recall@5": 0.5976190476190476, + "recall@10": 0.7128746253746254, + "f1@1": 0.3403846153846154, + "f1@5": 0.2807916756446169, + "f1@10": 0.21769245083724723, + "map@1": 0.4230769230769231, + "map@5": 0.45147435897435895, + "map@10": 0.4801182844932845, + "mrr@1": 0.4230769230769231, + "mrr@5": 0.5532051282051282, + "mrr@10": 0.5671474358974359, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.234, - "hit@5": 0.411, - "hit@10": 0.47, - "ndcg_linear@1": 0.234, - "ndcg_linear@5": 0.3273147185066129, - "ndcg_linear@10": 0.3462654041192875, - "ndcg_exponential@1": 0.234, - "ndcg_exponential@5": 0.3273147185066129, - "ndcg_exponential@10": 0.3462654041192875, - "precision@1": 0.234, - "precision@5": 0.0822, - "precision@10": 0.047, - "recall@1": 0.234, - "recall@5": 0.411, - "recall@10": 0.47, - "f1@1": 0.234, - "f1@5": 0.13700000000000004, - "f1@10": 0.08545454545454546, - "map@1": 0.234, - "map@5": 0.2995, - "map@10": 0.30724523809523807, - "mrr@1": 0.234, - "mrr@5": 0.2995, - "mrr@10": 0.30724523809523807, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.5171875, + "hit@5": 0.7984375, + "hit@10": 0.878125, + "ndcg_linear@1": 0.5171875, + "ndcg_linear@5": 0.5472043616227252, + "ndcg_linear@10": 0.5826984113580178, + "ndcg_exponential@1": 0.5171875, + "ndcg_exponential@5": 0.5472043616227253, + "ndcg_exponential@10": 0.5826984113580178, + "precision@1": 0.5171875, + "precision@5": 0.24874999999999997, + "precision@10": 0.16234375, + "recall@1": 0.3132504734848485, + "recall@5": 0.5801439155207537, + "recall@10": 0.6875872763878279, + "f1@1": 0.35874357413419916, + "f1@5": 0.3077863438644689, + "f1@10": 0.23607504359066858, + "map@1": 0.5171875, + "map@5": 0.4755620659722222, + "map@10": 0.49229102595899477, + "mrr@1": 0.5171875, + "mrr@5": 0.6226822916666667, + "mrr@10": 0.6335261656746032, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.251, - "hit@5": 0.485, - "hit@10": 0.592, - "ndcg_linear@1": 0.251, - "ndcg_linear@5": 0.37516969884345464, - "ndcg_linear@10": 0.4100591996704718, - "ndcg_exponential@1": 0.251, - "ndcg_exponential@5": 0.37516969884345464, - "ndcg_exponential@10": 0.4100591996704718, - "precision@1": 0.251, - "precision@5": 0.097, - "precision@10": 0.059199999999999996, - "recall@1": 0.251, - "recall@5": 0.485, - "recall@10": 0.592, - "f1@1": 0.251, - "f1@5": 0.1616666666666667, - "f1@10": 0.10763636363636366, - "map@1": 0.251, - "map@5": 0.33863333333333334, - "map@10": 0.35319563492063494, - "mrr@1": 0.251, - "mrr@5": 0.33863333333333334, - "mrr@10": 0.35319563492063494, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.5086206896551724, + "hit@5": 0.8577586206896551, + "hit@10": 0.9353448275862069, + "ndcg_linear@1": 0.5086206896551724, + "ndcg_linear@5": 0.48968646101987606, + "ndcg_linear@10": 0.4855016494653571, + "ndcg_exponential@1": 0.5086206896551724, + "ndcg_exponential@5": 0.48968646101987606, + "ndcg_exponential@10": 0.4855016494653571, + "precision@1": 0.5086206896551724, + "precision@5": 0.4456896551724138, + "precision@10": 0.3732758620689655, + "recall@1": 0.049608559736213115, + "recall@5": 0.2650305856260614, + "recall@10": 0.4040109375386366, + "f1@1": 0.0867748494784438, + "f1@5": 0.2676800330504101, + "f1@10": 0.31406186428354066, + "map@1": 0.5086206896551724, + "map@5": 0.38345545977011497, + "map@10": 0.3455301240866717, + "mrr@1": 0.5086206896551724, + "mrr@5": 0.6462643678160919, + "mrr@10": 0.6564638067870827, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.49122807017543857, + "hit@5": 0.8070175438596491, + "hit@10": 0.8728070175438597, + "ndcg_linear@1": 0.49122807017543857, + "ndcg_linear@5": 0.5294466383615495, + "ndcg_linear@10": 0.5608887645448074, + "ndcg_exponential@1": 0.49122807017543857, + "ndcg_exponential@5": 0.5294466383615495, + "ndcg_exponential@10": 0.5608887645448074, + "precision@1": 0.49122807017543857, + "precision@5": 0.26754385964912286, + "precision@10": 0.17631578947368426, + "recall@1": 0.25065267335004177, + "recall@5": 0.5506109022556391, + "recall@10": 0.6676691729323309, + "f1@1": 0.29780297238554954, + "f1@5": 0.3099469090154475, + "f1@10": 0.2428282645000911, + "map@1": 0.49122807017543857, + "map@5": 0.4436610623781677, + "map@10": 0.4549566044741483, + "mrr@1": 0.49122807017543857, + "mrr@5": 0.6209795321637426, + "mrr@10": 0.6295652325257587, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.248, - "hit@5": 0.447, - "hit@10": 0.578, - "ndcg_linear@1": 0.248, - "ndcg_linear@5": 0.35078743678844715, - "ndcg_linear@10": 0.39292321098555394, - "ndcg_exponential@1": 0.248, - "ndcg_exponential@5": 0.35078743678844715, - "ndcg_exponential@10": 0.39292321098555394, - "precision@1": 0.248, - "precision@5": 0.0894, - "precision@10": 0.0578, - "recall@1": 0.248, - "recall@5": 0.447, - "recall@10": 0.578, - "f1@1": 0.248, - "f1@5": 0.14900000000000005, - "f1@10": 0.10509090909090911, - "map@1": 0.248, - "map@5": 0.31906666666666667, - "map@10": 0.33631944444444445, - "mrr@1": 0.248, - "mrr@5": 0.31906666666666667, - "mrr@10": 0.33631944444444445, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.7561274509803921, + "hit@5": 0.8553921568627451, + "hit@10": 0.8848039215686274, + "ndcg_linear@1": 0.7561274509803921, + "ndcg_linear@5": 0.8087241849861875, + "ndcg_linear@10": 0.8184673504656934, + "ndcg_exponential@1": 0.7561274509803921, + "ndcg_exponential@5": 0.8087241849861875, + "ndcg_exponential@10": 0.8184673504656934, + "precision@1": 0.7561274509803921, + "precision@5": 0.17107843137254902, + "precision@10": 0.08848039215686274, + "recall@1": 0.7561274509803921, + "recall@5": 0.8553921568627451, + "recall@10": 0.8848039215686274, + "f1@1": 0.7561274509803921, + "f1@5": 0.28513071895424846, + "f1@10": 0.16087344028520503, + "map@1": 0.7561274509803921, + "map@5": 0.7931985294117647, + "map@10": 0.7973593604108311, + "mrr@1": 0.7561274509803921, + "mrr@5": 0.7931985294117647, + "mrr@10": 0.7973593604108311, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.494, - "hit@5": 0.675, - "hit@10": 0.752, - "ndcg_linear@1": 0.494, - "ndcg_linear@5": 0.5961521274121583, - "ndcg_linear@10": 0.6208342304490151, - "ndcg_exponential@1": 0.494, - "ndcg_exponential@5": 0.5961521274121583, - "ndcg_exponential@10": 0.6208342304490151, - "precision@1": 0.494, - "precision@5": 0.135, - "precision@10": 0.0752, - "recall@1": 0.494, - "recall@5": 0.675, - "recall@10": 0.752, - "f1@1": 0.494, - "f1@5": 0.22500000000000006, - "f1@10": 0.13672727272727275, - "map@1": 0.494, - "map@5": 0.56945, - "map@10": 0.5795154761904762, - "mrr@1": 0.494, - "mrr@5": 0.56945, - "mrr@10": 0.5795154761904762, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.6825396825396826, + "hit@5": 0.8571428571428571, + "hit@10": 0.9047619047619048, + "ndcg_linear@1": 0.6825396825396826, + "ndcg_linear@5": 0.781492660957272, + "ndcg_linear@10": 0.7972160024827536, + "ndcg_exponential@1": 0.6825396825396826, + "ndcg_exponential@5": 0.781492660957272, + "ndcg_exponential@10": 0.7972160024827536, + "precision@1": 0.6825396825396826, + "precision@5": 0.17142857142857135, + "precision@10": 0.09047619047619043, + "recall@1": 0.6825396825396826, + "recall@5": 0.8571428571428571, + "recall@10": 0.9047619047619048, + "f1@1": 0.6825396825396826, + "f1@5": 0.28571428571428564, + "f1@10": 0.16450216450216454, + "map@1": 0.6825396825396826, + "map@5": 0.7558201058201058, + "map@10": 0.7624968505920886, + "mrr@1": 0.6825396825396826, + "mrr@5": 0.7558201058201058, + "mrr@10": 0.7624968505920886, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.10854503464203233, - "hit@5": 0.5519630484988453, - "hit@10": 1.0, - "ndcg_linear@1": 0.10854503464203233, - "ndcg_linear@5": 0.3325565915387777, - "ndcg_linear@10": 0.47501706939610605, - "ndcg_exponential@1": 0.10854503464203233, - "ndcg_exponential@5": 0.3325565915387777, - "ndcg_exponential@10": 0.47501706939610605, - "precision@1": 0.10854503464203233, - "precision@5": 0.11039260969976905, - "precision@10": 0.1, - "recall@1": 0.10854503464203233, - "recall@5": 0.5519630484988453, - "recall@10": 1.0, - "f1@1": 0.10854503464203233, - "f1@5": 0.18398768283294847, - "f1@10": 0.1818181818181818, - "map@1": 0.10854503464203233, - "map@5": 0.2607005388760585, - "map@10": 0.3181110011364053, - "mrr@1": 0.10854503464203233, - "mrr@5": 0.2607005388760585, - "mrr@10": 0.3181110011364053, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.7918781725888325, + "hit@5": 0.9289340101522843, + "hit@10": 0.9509306260575296, + "ndcg_linear@1": 0.7918781725888325, + "ndcg_linear@5": 0.8676285427363506, + "ndcg_linear@10": 0.8748306491817542, + "ndcg_exponential@1": 0.7918781725888325, + "ndcg_exponential@5": 0.8676285427363506, + "ndcg_exponential@10": 0.8748306491817542, + "precision@1": 0.7918781725888325, + "precision@5": 0.18578680203045683, + "precision@10": 0.09509306260575294, + "recall@1": 0.7918781725888325, + "recall@5": 0.9289340101522843, + "recall@10": 0.9509306260575296, + "f1@1": 0.7918781725888325, + "f1@5": 0.3096446700507615, + "f1@10": 0.17289647746500544, + "map@1": 0.7918781725888325, + "map@5": 0.8469543147208122, + "map@10": 0.8499758278946096, + "mrr@1": 0.7918781725888325, + "mrr@5": 0.8469543147208122, + "mrr@10": 0.8499758278946096, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.337, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.337, - "ndcg_linear@5": 0.6944068117467572, - "ndcg_linear@10": 0.6944068117467572, - "ndcg_exponential@1": 0.337, - "ndcg_exponential@5": 0.6944068117467572, - "ndcg_exponential@10": 0.6944068117467572, - "precision@1": 0.337, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.337, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.337, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.337, - "map@5": 0.5917374999999999, - "map@10": 0.5917374999999999, - "mrr@1": 0.337, - "mrr@5": 0.5917374999999999, - "mrr@10": 0.5917374999999999, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.9388489208633094, + "hit@5": 0.9892086330935251, + "hit@10": 0.9892086330935251, + "ndcg_linear@1": 0.9388489208633094, + "ndcg_linear@5": 0.9502736353073175, + "ndcg_linear@10": 0.9527678727475414, + "ndcg_exponential@1": 0.9388489208633094, + "ndcg_exponential@5": 0.9502736353073175, + "ndcg_exponential@10": 0.9527678727475414, + "precision@1": 0.9388489208633094, + "precision@5": 0.24388489208633093, + "precision@10": 0.12320143884892086, + "recall@1": 0.8120503597122302, + "recall@5": 0.9739208633093526, + "recall@10": 0.9802158273381295, + "f1@1": 0.85431654676259, + "f1@5": 0.38309352517985623, + "f1@10": 0.2163723566601265, + "map@1": 0.9388489208633094, + "map@5": 0.9327038369304556, + "map@10": 0.9343789254310837, + "mrr@1": 0.9388489208633094, + "mrr@5": 0.9612410071942447, + "mrr@10": 0.9612410071942447, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.7729805013927576, + "hit@5": 0.924791086350975, + "hit@10": 0.9512534818941504, + "ndcg_linear@1": 0.7729805013927576, + "ndcg_linear@5": 0.8569356912112719, + "ndcg_linear@10": 0.8656693780762971, + "ndcg_exponential@1": 0.7729805013927576, + "ndcg_exponential@5": 0.8569356912112719, + "ndcg_exponential@10": 0.8656693780762971, + "precision@1": 0.7729805013927576, + "precision@5": 0.18495821727019496, + "precision@10": 0.09512534818941502, + "recall@1": 0.7729805013927576, + "recall@5": 0.924791086350975, + "recall@10": 0.9512534818941504, + "f1@1": 0.7729805013927576, + "f1@5": 0.308263695450325, + "f1@10": 0.1729551785262092, + "map@1": 0.7729805013927576, + "map@5": 0.8340529247910862, + "map@10": 0.8377569969491975, + "mrr@1": 0.7729805013927576, + "mrr@5": 0.8340529247910862, + "mrr@10": 0.8377569969491975, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.30592592592592593, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.30592592592592593, - "ndcg_linear@5": 0.6721188097820419, - "ndcg_linear@10": 0.6721188097820419, - "ndcg_exponential@1": 0.30592592592592593, - "ndcg_exponential@5": 0.6721188097820419, - "ndcg_exponential@10": 0.6721188097820419, - "precision@1": 0.30592592592592593, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.30592592592592593, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.30592592592592593, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.30592592592592593, - "map@5": 0.5626543209876542, - "map@10": 0.5626543209876542, - "mrr@1": 0.30592592592592593, - "mrr@5": 0.5626543209876542, - "mrr@10": 0.5626543209876542, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.46349942062572425, + "hit@5": 0.7265353418308227, + "hit@10": 0.831981460023175, + "ndcg_linear@1": 0.46349942062572425, + "ndcg_linear@5": 0.6029827126488958, + "ndcg_linear@10": 0.6373604951260508, + "ndcg_exponential@1": 0.46349942062572425, + "ndcg_exponential@5": 0.6029827126488958, + "ndcg_exponential@10": 0.6373604951260508, + "precision@1": 0.46349942062572425, + "precision@5": 0.14530706836616453, + "precision@10": 0.08319814600231748, + "recall@1": 0.46349942062572425, + "recall@5": 0.7265353418308227, + "recall@10": 0.831981460023175, + "f1@1": 0.46349942062572425, + "f1@5": 0.24217844727694096, + "f1@10": 0.15126935636785002, + "map@1": 0.46349942062572425, + "map@5": 0.5619351100811124, + "map@10": 0.5762865787489194, + "mrr@1": 0.46349942062572425, + "mrr@5": 0.5619351100811124, + "mrr@10": 0.5762865787489194, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.3520551144325082, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3520551144325082, - "ndcg_linear@5": 0.677249628478771, - "ndcg_linear@10": 0.677249628478771, - "ndcg_exponential@1": 0.3520551144325082, - "ndcg_exponential@5": 0.677249628478771, - "ndcg_exponential@10": 0.677249628478771, - "precision@1": 0.3520551144325082, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.3520551144325082, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3520551144325082, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.3520551144325082, - "map@5": 0.5717772069126577, - "map@10": 0.5717772069126577, - "mrr@1": 0.3520551144325082, - "mrr@5": 0.5716020551144325, - "mrr@10": 0.5716020551144325, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.023642732049036778, + "hit@5": 0.4141856392294221, + "hit@10": 0.6348511383537653, + "ndcg_linear@1": 0.023642732049036778, + "ndcg_linear@5": 0.2215998227373851, + "ndcg_linear@10": 0.29328787953138075, + "ndcg_exponential@1": 0.023642732049036778, + "ndcg_exponential@5": 0.2215998227373851, + "ndcg_exponential@10": 0.29328787953138075, + "precision@1": 0.023642732049036778, + "precision@5": 0.08283712784588441, + "precision@10": 0.06348511383537653, + "recall@1": 0.023642732049036778, + "recall@5": 0.4141856392294221, + "recall@10": 0.6348511383537653, + "f1@1": 0.023642732049036778, + "f1@5": 0.13806187974314071, + "f1@10": 0.11542747970068462, + "map@1": 0.023642732049036778, + "map@5": 0.15847927612375948, + "map@10": 0.1882595974758847, + "mrr@1": 0.023642732049036778, + "mrr@5": 0.15847927612375948, + "mrr@10": 0.1882595974758847, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.384, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.384, - "ndcg_linear@5": 0.6958150458511005, - "ndcg_linear@10": 0.6958150458511005, - "ndcg_exponential@1": 0.384, - "ndcg_exponential@5": 0.6958150458511005, - "ndcg_exponential@10": 0.6958150458511005, - "precision@1": 0.384, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.384, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.384, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.384, - "map@5": 0.5960666666666666, - "map@10": 0.5960666666666666, - "mrr@1": 0.384, - "mrr@5": 0.5960666666666666, - "mrr@10": 0.5960666666666666, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.9973730297723292, + "hit@5": 0.999124343257443, + "hit@10": 0.999124343257443, + "ndcg_linear@1": 0.680385288966725, + "ndcg_linear@5": 0.8366080322665971, + "ndcg_linear@10": 0.8228477305778173, + "ndcg_exponential@1": 0.4539654741055791, + "ndcg_exponential@5": 0.7156185424033248, + "ndcg_exponential@10": 0.7459741664587741, + "precision@1": 0.9973730297723292, + "precision@5": 0.9252189141856392, + "precision@10": 0.8394045534150613, + "recall@1": 0.05880569019029663, + "recall@5": 0.272128058121516, + "recall@10": 0.48450076291066557, + "f1@1": 0.11031252785994464, + "f1@5": 0.4120671799650714, + "f1@10": 0.6005622170825162, + "map@1": 0.9973730297723292, + "map@5": 0.9193753648569761, + "map@10": 0.8319699422384248, + "mrr@1": 0.9973730297723292, + "mrr@5": 0.998029772329247, + "mrr@10": 0.998029772329247, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.513, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.513, - "ndcg_linear@5": 0.8202627899893, - "ndcg_linear@10": 0.8202627899893, - "ndcg_exponential@1": 0.513, - "ndcg_exponential@5": 0.8202627899893, - "ndcg_exponential@10": 0.8202627899893, - "precision@1": 0.513, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.513, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.513, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.513, - "map@5": 0.7565, - "map@10": 0.7565, - "mrr@1": 0.513, - "mrr@5": 0.7565, - "mrr@10": 0.7565, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.06563245823389022, + "hit@5": 0.2863961813842482, + "hit@10": 0.3914081145584726, + "ndcg_linear@1": 0.06563245823389022, + "ndcg_linear@5": 0.14173090844476535, + "ndcg_linear@10": 0.17778697624105239, + "ndcg_exponential@1": 0.06563245823389022, + "ndcg_exponential@5": 0.14173090844476535, + "ndcg_exponential@10": 0.17778697624105239, + "precision@1": 0.06563245823389022, + "precision@5": 0.06897374701670644, + "precision@10": 0.0513126491646778, + "recall@1": 0.037907716785998406, + "recall@5": 0.20580889873849303, + "recall@10": 0.3072328294881994, + "f1@1": 0.045729568760595, + "f1@5": 0.09743296504141655, + "f1@10": 0.08383696942332947, + "map@1": 0.06563245823389022, + "map@5": 0.10388988332007426, + "map@10": 0.1198034589067661, + "mrr@1": 0.06563245823389022, + "mrr@5": 0.1441129673826571, + "mrr@10": 0.15842093798537715, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.7326968973747017, + "hit@5": 0.8186157517899761, + "hit@10": 0.847255369928401, + "ndcg_linear@1": 0.5103420843277645, + "ndcg_linear@5": 0.5251195991901183, + "ndcg_linear@10": 0.4977133933501543, + "ndcg_exponential@1": 0.35151721786566653, + "ndcg_exponential@5": 0.43679570479418406, + "ndcg_exponential@10": 0.44114798529060906, + "precision@1": 0.7326968973747017, + "precision@5": 0.5794749403341289, + "precision@10": 0.5013126491646778, + "recall@1": 0.024172213173396184, + "recall@5": 0.09148792471264464, + "recall@10": 0.15302173843210523, + "f1@1": 0.04631909415766511, + "f1@5": 0.15159508610283381, + "f1@10": 0.21973685679091384, + "map@1": 0.7326968973747017, + "map@5": 0.5482895783611774, + "map@10": 0.4582356896617039, + "mrr@1": 0.7326968973747017, + "mrr@5": 0.7665871121718377, + "mrr@10": 0.7705226919725726, + "num_pred": 838, + "num_data": 838 } } }