diff --git "a/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" "b/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" --- "a/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" +++ "b/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "VLM2Vec-V2.0-Qwen2VL-2B", - "report_generated_date": "2025-06-09T07:05:59.773788", - "model_size": 2.21, - "url": "https://huggingface.co/VLM2Vec/VLM2Vec-V2.0", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:17.882723" }, "metrics": { "image": { @@ -1017,765 +1014,287 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.734, - "hit@5": 0.866, - "hit@10": 0.906, - "ndcg_linear@1": 0.734, - "ndcg_linear@5": 0.8057573987022552, - "ndcg_linear@10": 0.8188064742981096, - "ndcg_exponential@1": 0.734, - "ndcg_exponential@5": 0.8057573987022552, - "ndcg_exponential@10": 0.8188064742981096, - "precision@1": 0.734, - "precision@5": 0.1732, - "precision@10": 0.0906, - "recall@1": 0.734, - "recall@5": 0.866, - "recall@10": 0.906, - "f1@1": 0.734, - "f1@5": 0.28866666666666674, - "f1@10": 0.16472727272727275, - "map@1": 0.734, - "map@5": 0.7856000000000001, - "map@10": 0.7910603174603175, - "mrr@1": 0.734, - "mrr@5": 0.7856000000000001, - "mrr@10": 0.7910603174603175, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.35033259423503327, - "hit@5": 0.541019955654102, - "hit@10": 0.6208425720620843, - "ndcg_linear@1": 0.35033259423503327, - "ndcg_linear@5": 0.4484886950768515, - "ndcg_linear@10": 0.47338485947788617, - "ndcg_exponential@1": 0.35033259423503327, - "ndcg_exponential@5": 0.4484886950768515, - "ndcg_exponential@10": 0.47338485947788617, - "precision@1": 0.35033259423503327, - "precision@5": 0.10953436807095342, - "precision@10": 0.06341463414634146, - "recall@1": 0.34320029563932003, - "recall@5": 0.5335920177383592, - "recall@10": 0.6106060606060606, - "f1@1": 0.34476093012678377, - "f1@5": 0.18059028489444562, - "f1@10": 0.11417931661834103, - "map@1": 0.35033259423503327, - "map@5": 0.41779995072678, - "map@10": 0.42790588814979064, - "mrr@1": 0.35033259423503327, - "mrr@5": 0.4246119733924612, - "mrr@10": 0.4349144757681343, - "num_pred": 451, - "num_data": 451 - }, - "ViDoRe_infovqa": { - "hit@1": 0.7732793522267206, - "hit@5": 0.8947368421052632, - "hit@10": 0.9412955465587044, - "ndcg_linear@1": 0.7732793522267206, - "ndcg_linear@5": 0.836933803347616, - "ndcg_linear@10": 0.852452939253696, - "ndcg_exponential@1": 0.7732793522267206, - "ndcg_exponential@5": 0.836933803347616, - "ndcg_exponential@10": 0.852452939253696, - "precision@1": 0.7732793522267206, - "precision@5": 0.17894736842105263, - "precision@10": 0.09433198380566803, - "recall@1": 0.7732793522267206, - "recall@5": 0.8937246963562753, - "recall@10": 0.9412955465587044, - "f1@1": 0.7732793522267206, - "f1@5": 0.29814921920185083, - "f1@10": 0.17145135566188197, - "map@1": 0.7732793522267206, - "map@5": 0.817914979757085, - "map@10": 0.8244569757727652, - "mrr@1": 0.7732793522267206, - "mrr@5": 0.8182523616734144, - "mrr@10": 0.8245051731893837, - "num_pred": 494, - "num_data": 494 - }, - "ViDoRe_tabfquad": { - "hit@1": 0.8107142857142857, - "hit@5": 0.9571428571428572, - "hit@10": 0.975, - "ndcg_linear@1": 0.8107142857142857, - "ndcg_linear@5": 0.8920767921191974, - "ndcg_linear@10": 0.8982104341600656, - "ndcg_exponential@1": 0.8107142857142857, - "ndcg_exponential@5": 0.8920767921191974, - "ndcg_exponential@10": 0.8982104341600656, - "precision@1": 0.8107142857142857, - "precision@5": 0.19142857142857142, - "precision@10": 0.09749999999999999, - "recall@1": 0.8107142857142857, - "recall@5": 0.9571428571428572, - "recall@10": 0.975, - "f1@1": 0.8107142857142857, - "f1@5": 0.31904761904761914, - "f1@10": 0.1772727272727273, - "map@1": 0.8107142857142857, - "map@5": 0.8701190476190476, - "map@10": 0.8728613945578231, - "mrr@1": 0.8107142857142857, - "mrr@5": 0.8701190476190476, - "mrr@10": 0.8728613945578231, - "num_pred": 280, - "num_data": 280 - }, - "ViDoRe_tatdqa": { - "hit@1": 0.30741190765492105, - "hit@5": 0.5577156743620899, - "hit@10": 0.6433778857837181, - "ndcg_linear@1": 0.30741190765492105, - "ndcg_linear@5": 0.4382195288905396, - "ndcg_linear@10": 0.4658056006683759, - "ndcg_exponential@1": 0.30741190765492105, - "ndcg_exponential@5": 0.4382195288905396, - "ndcg_exponential@10": 0.4658056006683759, - "precision@1": 0.30741190765492105, - "precision@5": 0.1117861482381531, - "precision@10": 0.06452004860267314, - "recall@1": 0.30650060753341435, - "recall@5": 0.5565613608748481, - "recall@10": 0.6422235722964762, - "f1@1": 0.3068043742405832, - "f1@5": 0.18601515940519592, - "f1@10": 0.11718583158437353, - "map@1": 0.30741190765492105, - "map@5": 0.39878898339408664, - "map@10": 0.41010757198788794, - "mrr@1": 0.30741190765492105, - "mrr@5": 0.39935196435803966, - "mrr@10": 0.41067257806322205, - "num_pred": 1646, - "num_data": 1646 - }, - "ViDoRe_shiftproject": { - "hit@1": 0.45, - "hit@5": 0.74, - "hit@10": 0.87, - "ndcg_linear@1": 0.45, - "ndcg_linear@5": 0.608217112664813, - "ndcg_linear@10": 0.650000125092289, - "ndcg_exponential@1": 0.45, - "ndcg_exponential@5": 0.608217112664813, - "ndcg_exponential@10": 0.650000125092289, - "precision@1": 0.45, - "precision@5": 0.14799999999999996, - "precision@10": 0.08699999999999998, - "recall@1": 0.45, - "recall@5": 0.74, - "recall@10": 0.87, - "f1@1": 0.45, - "f1@5": 0.24666666666666667, - "f1@10": 0.1581818181818182, - "map@1": 0.45, - "map@5": 0.564, - "map@10": 0.5810912698412699, - "mrr@1": 0.45, - "mrr@5": 0.564, - "mrr@10": 0.5810912698412699, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.76, - "hit@5": 0.97, - "hit@10": 0.98, - "ndcg_linear@1": 0.76, - "ndcg_linear@5": 0.8852555261521673, - "ndcg_linear@10": 0.8882658261088071, - "ndcg_exponential@1": 0.76, - "ndcg_exponential@5": 0.8852555261521673, - "ndcg_exponential@10": 0.8882658261088071, - "precision@1": 0.76, - "precision@5": 0.19399999999999995, - "precision@10": 0.09799999999999998, - "recall@1": 0.76, - "recall@5": 0.97, - "recall@10": 0.98, - "f1@1": 0.76, - "f1@5": 0.3233333333333334, - "f1@10": 0.17818181818181827, - "map@1": 0.76, - "map@5": 0.8558333333333333, - "map@10": 0.8569444444444444, - "mrr@1": 0.76, - "mrr@5": 0.8558333333333333, - "mrr@10": 0.8569444444444444, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.79, - "hit@5": 0.93, - "hit@10": 0.97, - "ndcg_linear@1": 0.79, - "ndcg_linear@5": 0.8650828475372231, - "ndcg_linear@10": 0.8780588117396043, - "ndcg_exponential@1": 0.79, - "ndcg_exponential@5": 0.8650828475372231, - "ndcg_exponential@10": 0.8780588117396043, - "precision@1": 0.79, - "precision@5": 0.18599999999999994, - "precision@10": 0.09699999999999999, - "recall@1": 0.79, - "recall@5": 0.93, - "recall@10": 0.97, - "f1@1": 0.79, - "f1@5": 0.31000000000000005, - "f1@10": 0.17636363636363647, - "map@1": 0.79, - "map@5": 0.8433333333333333, - "map@10": 0.8486904761904762, - "mrr@1": 0.79, - "mrr@5": 0.8433333333333333, - "mrr@10": 0.8486904761904762, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.74, - "hit@5": 0.94, - "hit@10": 0.98, - "ndcg_linear@1": 0.74, - "ndcg_linear@5": 0.8501367970825706, - "ndcg_linear@10": 0.8630184664460052, - "ndcg_exponential@1": 0.74, - "ndcg_exponential@5": 0.8501367970825706, - "ndcg_exponential@10": 0.8630184664460052, - "precision@1": 0.74, - "precision@5": 0.18799999999999997, - "precision@10": 0.09799999999999999, - "recall@1": 0.74, - "recall@5": 0.94, - "recall@10": 0.98, - "f1@1": 0.74, - "f1@5": 0.3133333333333334, - "f1@10": 0.17818181818181825, - "map@1": 0.74, - "map@5": 0.8198333333333332, - "map@10": 0.825111111111111, - "mrr@1": 0.74, - "mrr@5": 0.8198333333333332, - "mrr@10": 0.825111111111111, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.84, - "hit@5": 0.98, - "hit@10": 0.99, - "ndcg_linear@1": 0.84, - "ndcg_linear@5": 0.9217065065186136, - "ndcg_linear@10": 0.9252685783896938, - "ndcg_exponential@1": 0.84, - "ndcg_exponential@5": 0.9217065065186136, - "ndcg_exponential@10": 0.9252685783896938, - "precision@1": 0.84, - "precision@5": 0.19599999999999998, - "precision@10": 0.09899999999999999, - "recall@1": 0.84, - "recall@5": 0.98, - "recall@10": 0.99, - "f1@1": 0.84, - "f1@5": 0.32666666666666677, - "f1@10": 0.18000000000000008, - "map@1": 0.84, - "map@5": 0.9016666666666667, - "map@10": 0.9033333333333334, - "mrr@1": 0.84, - "mrr@5": 0.9016666666666667, - "mrr@10": 0.9033333333333334, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.38461538461538464, - "hit@5": 0.6730769230769231, - "hit@10": 0.8461538461538461, - "ndcg_linear@1": 0.375, - "ndcg_linear@5": 0.4555552886561195, - "ndcg_linear@10": 0.511493540486886, - "ndcg_exponential@1": 0.37179487179487186, - "ndcg_exponential@5": 0.4541859231025376, - "ndcg_exponential@10": 0.5099104815026472, - "precision@1": 0.38461538461538464, - "precision@5": 0.19230769230769226, - "precision@10": 0.13076923076923078, - "recall@1": 0.23621794871794868, - "recall@5": 0.5487179487179487, - "recall@10": 0.7033882783882782, - "f1@1": 0.2715483234714004, - "f1@5": 0.25723492194080433, - "f1@10": 0.2026450736292365, - "map@1": 0.38461538461538464, - "map@5": 0.3914850427350427, - "map@10": 0.415758547008547, - "mrr@1": 0.38461538461538464, - "mrr@5": 0.491025641025641, - "mrr@10": 0.5144993894993896, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.43125, - "hit@5": 0.78125, - "hit@10": 0.8625, - "ndcg_linear@1": 0.43125, - "ndcg_linear@5": 0.5037189683095866, - "ndcg_linear@10": 0.544890617781321, - "ndcg_exponential@1": 0.43125, - "ndcg_exponential@5": 0.5037189683095866, - "ndcg_exponential@10": 0.544890617781321, - "precision@1": 0.43125, - "precision@5": 0.2375, - "precision@10": 0.15750000000000003, - "recall@1": 0.27246279761904757, - "recall@5": 0.5615540648077413, - "recall@10": 0.6789739304812834, - "f1@1": 0.30771126443001445, - "f1@5": 0.29637853119103125, - "f1@10": 0.23083279540448656, - "map@1": 0.43125, - "map@5": 0.4270538194444445, - "map@10": 0.44736998653628113, - "mrr@1": 0.43125, - "mrr@5": 0.5565625, - "mrr@10": 0.5675992063492064, - "num_pred": 160, - "num_data": 160 - }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.36875, - "hit@5": 0.7171875, - "hit@10": 0.8203125, - "ndcg_linear@1": 0.36875, - "ndcg_linear@5": 0.4426897295015239, - "ndcg_linear@10": 0.4850950127175607, - "ndcg_exponential@1": 0.36875, - "ndcg_exponential@5": 0.4426897295015239, - "ndcg_exponential@10": 0.4850950127175607, - "precision@1": 0.36875, - "precision@5": 0.203125, - "precision@10": 0.13968750000000002, - "recall@1": 0.23270833333333335, - "recall@5": 0.49864999124649856, - "recall@10": 0.6150380777310923, - "f1@1": 0.2621073457792208, - "f1@5": 0.25601551313270066, - "f1@10": 0.20543311717898116, - "map@1": 0.36875, - "map@5": 0.3719822048611111, - "map@10": 0.3923365021789966, - "mrr@1": 0.36875, - "mrr@5": 0.497265625, - "mrr@10": 0.5111972966269841, - "num_pred": 640, - "num_data": 640 - }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.5689655172413793, - "hit@5": 0.9482758620689655, - "hit@10": 0.9482758620689655, - "ndcg_linear@1": 0.5689655172413793, - "ndcg_linear@5": 0.558322322048227, - "ndcg_linear@10": 0.5471300748455853, - "ndcg_exponential@1": 0.5689655172413793, - "ndcg_exponential@5": 0.5583223220482268, - "ndcg_exponential@10": 0.5471300748455852, - "precision@1": 0.5689655172413793, - "precision@5": 0.48275862068965525, - "precision@10": 0.39655172413793105, - "recall@1": 0.0780199437798001, - "recall@5": 0.3088989701472038, - "recall@10": 0.4354563545380438, - "f1@1": 0.1197688665774972, - "f1@5": 0.2988989952104816, - "f1@10": 0.33318335531888293, - "map@1": 0.5689655172413793, - "map@5": 0.444382183908046, - "map@10": 0.40631524975890737, - "mrr@1": 0.5689655172413793, - "mrr@5": 0.7333333333333333, - "mrr@10": 0.7333333333333333, - "num_pred": 58, - "num_data": 58 - }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.4525862068965517, - "hit@5": 0.8103448275862069, - "hit@10": 0.9267241379310345, - "ndcg_linear@1": 0.4525862068965517, - "ndcg_linear@5": 0.42995014841633666, - "ndcg_linear@10": 0.43467280891274934, - "ndcg_exponential@1": 0.4525862068965517, - "ndcg_exponential@5": 0.4299501484163365, - "ndcg_exponential@10": 0.43467280891274934, - "precision@1": 0.4525862068965517, - "precision@5": 0.38448275862068965, - "precision@10": 0.33103448275862074, - "recall@1": 0.049016392011628045, - "recall@5": 0.22829352926193736, - "recall@10": 0.36167397868493684, - "f1@1": 0.08115811949451532, - "f1@5": 0.22694280671921516, - "f1@10": 0.27660068249123476, - "map@1": 0.4525862068965517, - "map@5": 0.3282327586206897, - "map@10": 0.2973695391710324, - "mrr@1": 0.4525862068965517, - "mrr@5": 0.5951867816091955, - "mrr@10": 0.6120929118773947, - "num_pred": 232, - "num_data": 232 - }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.5614035087719298, - "hit@5": 0.7894736842105263, - "hit@10": 0.8596491228070176, - "ndcg_linear@1": 0.5614035087719298, - "ndcg_linear@5": 0.5129181248745351, - "ndcg_linear@10": 0.5494397287591082, - "ndcg_exponential@1": 0.5614035087719298, - "ndcg_exponential@5": 0.5129181248745351, - "ndcg_exponential@10": 0.5494397287591082, - "precision@1": 0.5614035087719298, - "precision@5": 0.22807017543859648, - "precision@10": 0.15964912280701754, - "recall@1": 0.28659147869674184, - "recall@5": 0.5196741854636592, - "recall@10": 0.6266290726817042, - "f1@1": 0.34330038449820666, - "f1@5": 0.2772425230843081, - "f1@10": 0.22506302456196542, - "map@1": 0.5614035087719298, - "map@5": 0.43352826510721243, - "map@10": 0.4492455096073517, - "mrr@1": 0.5614035087719298, - "mrr@5": 0.6467836257309941, - "mrr@10": 0.6572263993316625, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.4692982456140351, - "hit@5": 0.7719298245614035, - "hit@10": 0.8640350877192983, - "ndcg_linear@1": 0.4692982456140351, - "ndcg_linear@5": 0.466179429204456, - "ndcg_linear@10": 0.5017617250757785, - "ndcg_exponential@1": 0.4692982456140351, - "ndcg_exponential@5": 0.466179429204456, - "ndcg_exponential@10": 0.5017617250757785, - "precision@1": 0.4692982456140351, - "precision@5": 0.21929824561403508, - "precision@10": 0.15131578947368426, - "recall@1": 0.2380325814536341, - "recall@5": 0.4968671679197995, - "recall@10": 0.6020624477861319, - "f1@1": 0.28556649037592774, - "f1@5": 0.26698957296533893, - "f1@10": 0.21278941819922756, - "map@1": 0.4692982456140351, - "map@5": 0.38164961013645227, - "map@10": 0.39600202763854087, - "mrr@1": 0.4692982456140351, - "mrr@5": 0.5818713450292398, - "mrr@10": 0.594714216095795, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.7022058823529411, - "hit@5": 0.821078431372549, - "hit@10": 0.8615196078431373, - "ndcg_linear@1": 0.7022058823529411, - "ndcg_linear@5": 0.7687028213602233, - "ndcg_linear@10": 0.7817454060088397, - "ndcg_exponential@1": 0.7022058823529411, - "ndcg_exponential@5": 0.7687028213602233, - "ndcg_exponential@10": 0.7817454060088397, - "precision@1": 0.7022058823529411, - "precision@5": 0.1642156862745098, - "precision@10": 0.08615196078431371, - "recall@1": 0.7022058823529411, - "recall@5": 0.821078431372549, - "recall@10": 0.8615196078431373, - "f1@1": 0.7022058823529411, - "f1@5": 0.2736928104575164, - "f1@10": 0.15663992869875223, - "map@1": 0.7022058823529411, - "map@5": 0.7510212418300654, - "map@10": 0.7563827614379085, - "mrr@1": 0.7022058823529411, - "mrr@5": 0.7510212418300654, - "mrr@10": 0.7563827614379085, - "num_pred": 816, - "num_data": 816 - }, - "VisRAG_ChartQA": { - "hit@1": 0.7301587301587301, - "hit@5": 0.9206349206349206, - "hit@10": 0.9365079365079365, - "ndcg_linear@1": 0.7301587301587301, - "ndcg_linear@5": 0.8441010759070336, - "ndcg_linear@10": 0.849392081198039, - "ndcg_exponential@1": 0.7301587301587301, - "ndcg_exponential@5": 0.8441010759070336, - "ndcg_exponential@10": 0.849392081198039, - "precision@1": 0.7301587301587301, - "precision@5": 0.18412698412698403, - "precision@10": 0.0936507936507936, - "recall@1": 0.7301587301587301, - "recall@5": 0.9206349206349206, - "recall@10": 0.9365079365079365, - "f1@1": 0.7301587301587301, - "f1@5": 0.3068783068783068, - "f1@10": 0.1702741702741703, - "map@1": 0.7301587301587301, - "map@5": 0.8174603174603174, - "map@10": 0.8197278911564626, - "mrr@1": 0.7301587301587301, - "mrr@5": 0.8174603174603174, - "mrr@10": 0.8197278911564626, - "num_pred": 63, - "num_data": 63 + "video": { + "K700": { + "hit@1": 0.38, + "hit@5": 0.623, + "hit@10": 0.697, + "ndcg_linear@1": 0.38, + "ndcg_linear@5": 0.5087323881529896, + "ndcg_linear@10": 0.5328292310353233, + "ndcg_exponential@1": 0.38, + "ndcg_exponential@5": 0.5087323881529896, + "ndcg_exponential@10": 0.5328292310353233, + "precision@1": 0.38, + "precision@5": 0.12459999999999999, + "precision@10": 0.06969999999999998, + "recall@1": 0.38, + "recall@5": 0.623, + "recall@10": 0.697, + "f1@1": 0.38, + "f1@5": 0.20766666666666675, + "f1@10": 0.12672727272727274, + "map@1": 0.38, + "map@5": 0.4706833333333333, + "map@10": 0.4807186507936508, + "mrr@1": 0.38, + "mrr@5": 0.4706833333333333, + "mrr@10": 0.4807186507936508, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.5922165820642978, - "hit@5": 0.8206429780033841, - "hit@10": 0.8663282571912013, - "ndcg_linear@1": 0.5922165820642978, - "ndcg_linear@5": 0.717924714040525, - "ndcg_linear@10": 0.7327402059542735, - "ndcg_exponential@1": 0.5922165820642978, - "ndcg_exponential@5": 0.717924714040525, - "ndcg_exponential@10": 0.7327402059542735, - "precision@1": 0.5922165820642978, - "precision@5": 0.16412859560067683, - "precision@10": 0.08663282571912012, - "recall@1": 0.5922165820642978, - "recall@5": 0.8206429780033841, - "recall@10": 0.8663282571912013, - "f1@1": 0.5922165820642978, - "f1@5": 0.2735476593344614, - "f1@10": 0.15751422858021846, - "map@1": 0.5922165820642978, - "map@5": 0.6833051325437113, - "map@10": 0.6894455187602395, - "mrr@1": 0.5922165820642978, - "mrr@5": 0.6833051325437113, - "mrr@10": 0.6894455187602395, - "num_pred": 591, - "num_data": 591 + "SmthSmthV2": { + "hit@1": 0.428, + "hit@5": 0.692, + "hit@10": 0.786, + "ndcg_linear@1": 0.428, + "ndcg_linear@5": 0.5672834391090181, + "ndcg_linear@10": 0.5981163502873708, + "ndcg_exponential@1": 0.428, + "ndcg_exponential@5": 0.5672834391090181, + "ndcg_exponential@10": 0.5981163502873708, + "precision@1": 0.428, + "precision@5": 0.13839999999999997, + "precision@10": 0.07859999999999999, + "recall@1": 0.428, + "recall@5": 0.692, + "recall@10": 0.786, + "f1@1": 0.428, + "f1@5": 0.23066666666666674, + "f1@10": 0.14290909090909093, + "map@1": 0.428, + "map@5": 0.5259333333333333, + "map@10": 0.5389079365079364, + "mrr@1": 0.428, + "mrr@5": 0.5259333333333333, + "mrr@10": 0.5389079365079364, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_SlideVQA": { - "hit@1": 0.8794964028776978, - "hit@5": 0.9748201438848921, - "hit@10": 0.9802158273381295, - "ndcg_linear@1": 0.8794964028776978, - "ndcg_linear@5": 0.9150078016078801, - "ndcg_linear@10": 0.9207838812109942, - "ndcg_exponential@1": 0.8794964028776978, - "ndcg_exponential@5": 0.9150078016078801, - "ndcg_exponential@10": 0.9207838812109942, - "precision@1": 0.8794964028776978, - "precision@5": 0.2384892086330935, - "precision@10": 0.12176258992805757, - "recall@1": 0.7598920863309353, - "recall@5": 0.954136690647482, - "recall@10": 0.9694244604316546, - "f1@1": 0.7997601918465228, - "f1@5": 0.37478588557725256, - "f1@10": 0.2138652714192283, - "map@1": 0.8794964028776978, - "map@5": 0.8905575539568344, - "map@10": 0.894137547105173, - "mrr@1": 0.8794964028776978, - "mrr@5": 0.9226618705035972, - "mrr@10": 0.9233213429256595, - "num_pred": 556, - "num_data": 556 + "HMDB51": { + "hit@1": 0.409, + "hit@5": 0.638, + "hit@10": 0.713, + "ndcg_linear@1": 0.409, + "ndcg_linear@5": 0.5335348498920724, + "ndcg_linear@10": 0.5576824580140078, + "ndcg_exponential@1": 0.409, + "ndcg_exponential@5": 0.5335348498920724, + "ndcg_exponential@10": 0.5576824580140078, + "precision@1": 0.409, + "precision@5": 0.12760000000000002, + "precision@10": 0.0713, + "recall@1": 0.409, + "recall@5": 0.638, + "recall@10": 0.713, + "f1@1": 0.409, + "f1@5": 0.21266666666666673, + "f1@10": 0.12963636363636366, + "map@1": 0.409, + "map@5": 0.49854999999999994, + "map@10": 0.5084662698412699, + "mrr@1": 0.409, + "mrr@5": 0.49854999999999994, + "mrr@10": 0.5084662698412699, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_InfoVQA": { - "hit@1": 0.7465181058495822, - "hit@5": 0.9415041782729805, - "hit@10": 0.9665738161559888, - "ndcg_linear@1": 0.7465181058495822, - "ndcg_linear@5": 0.8565495992280632, - "ndcg_linear@10": 0.8647865140019099, - "ndcg_exponential@1": 0.7465181058495822, - "ndcg_exponential@5": 0.8565495992280632, - "ndcg_exponential@10": 0.8647865140019099, - "precision@1": 0.7465181058495822, - "precision@5": 0.18830083565459607, - "precision@10": 0.09665738161559885, - "recall@1": 0.7465181058495822, - "recall@5": 0.9415041782729805, - "recall@10": 0.9665738161559888, - "f1@1": 0.7465181058495822, - "f1@5": 0.31383472609099355, - "f1@10": 0.1757406938465435, - "map@1": 0.7465181058495822, - "map@5": 0.827669452181987, - "map@10": 0.8311491356059602, - "mrr@1": 0.7465181058495822, - "mrr@5": 0.827669452181987, - "mrr@10": 0.8311491356059602, - "num_pred": 718, - "num_data": 718 + "UCF101": { + "hit@1": 0.6, + "hit@5": 0.831, + "hit@10": 0.888, + "ndcg_linear@1": 0.6, + "ndcg_linear@5": 0.728875610371988, + "ndcg_linear@10": 0.7473187422682429, + "ndcg_exponential@1": 0.6, + "ndcg_exponential@5": 0.728875610371988, + "ndcg_exponential@10": 0.7473187422682429, + "precision@1": 0.6, + "precision@5": 0.16620000000000001, + "precision@10": 0.08880000000000002, + "recall@1": 0.6, + "recall@5": 0.831, + "recall@10": 0.888, + "f1@1": 0.6, + "f1@5": 0.277, + "f1@10": 0.16145454545454543, + "map@1": 0.6, + "map@5": 0.6944166666666668, + "map@10": 0.7020325396825396, + "mrr@1": 0.6, + "mrr@5": 0.6944166666666668, + "mrr@10": 0.7020325396825396, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_PlotQA": { - "hit@1": 0.5052143684820394, - "hit@5": 0.7937427578215527, - "hit@10": 0.8887601390498262, - "ndcg_linear@1": 0.5052143684820394, - "ndcg_linear@5": 0.6604877574932249, - "ndcg_linear@10": 0.6916336549342924, - "ndcg_exponential@1": 0.5052143684820394, - "ndcg_exponential@5": 0.6604877574932249, - "ndcg_exponential@10": 0.6916336549342924, - "precision@1": 0.5052143684820394, - "precision@5": 0.15874855156431056, - "precision@10": 0.08887601390498262, - "recall@1": 0.5052143684820394, - "recall@5": 0.7937427578215527, - "recall@10": 0.8887601390498262, - "f1@1": 0.5052143684820394, - "f1@5": 0.26458091927385097, - "f1@10": 0.1615927525545139, - "map@1": 0.5052143684820394, - "map@5": 0.6159907300115876, - "map@10": 0.629089646673656, - "mrr@1": 0.5052143684820394, - "mrr@5": 0.6159907300115876, - "mrr@10": 0.629089646673656, - "num_pred": 863, - "num_data": 863 + "Breakfast": { + "hit@1": 0.14780600461893764, + "hit@5": 0.6327944572748267, + "hit@10": 1.0, + "ndcg_linear@1": 0.14780600461893764, + "ndcg_linear@5": 0.3858298763211173, + "ndcg_linear@10": 0.5069598731135728, + "ndcg_exponential@1": 0.14780600461893764, + "ndcg_exponential@5": 0.3858298763211173, + "ndcg_exponential@10": 0.5069598731135728, + "precision@1": 0.14780600461893764, + "precision@5": 0.12655889145496535, + "precision@10": 0.1, + "recall@1": 0.14780600461893764, + "recall@5": 0.6327944572748267, + "recall@10": 1.0, + "f1@1": 0.14780600461893764, + "f1@5": 0.21093148575827564, + "f1@10": 0.1818181818181818, + "map@1": 0.14780600461893764, + "map@5": 0.30546574287913775, + "map@10": 0.3568477583489131, + "mrr@1": 0.14780600461893764, + "mrr@5": 0.30546574287913775, + "mrr@10": 0.3568477583489131, + "num_pred": 433, + "num_data": 433 }, - "ViDoSeek-page": { - "hit@1": 0.03502626970227671, - "hit@5": 0.3975481611208406, - "hit@10": 0.5674255691768827, - "ndcg_linear@1": 0.03502626970227671, - "ndcg_linear@5": 0.21944808482459283, - "ndcg_linear@10": 0.2752564930475152, - "ndcg_exponential@1": 0.03502626970227671, - "ndcg_exponential@5": 0.21944808482459285, - "ndcg_exponential@10": 0.2752564930475153, - "precision@1": 0.03502626970227671, - "precision@5": 0.07950963222416814, - "precision@10": 0.05674255691768828, - "recall@1": 0.03502626970227671, - "recall@5": 0.3975481611208406, - "recall@10": 0.5674255691768827, - "f1@1": 0.03502626970227671, - "f1@5": 0.13251605370694688, - "f1@10": 0.10316828530488775, - "map@1": 0.03502626970227671, - "map@5": 0.16103327495621714, - "map@10": 0.184575236983293, - "mrr@1": 0.03502626970227671, - "mrr@5": 0.16103327495621714, - "mrr@10": 0.184575236983293, - "num_pred": 1142, - "num_data": 1142 + "MVBench": { + "hit@1": 0.337, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.337, + "ndcg_linear@5": 0.7011998180757394, + "ndcg_linear@10": 0.7011998180757394, + "ndcg_exponential@1": 0.337, + "ndcg_exponential@5": 0.7011998180757394, + "ndcg_exponential@10": 0.7011998180757394, + "precision@1": 0.337, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.337, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.337, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.337, + "map@5": 0.6002625, + "map@10": 0.6002625, + "mrr@1": 0.337, + "mrr@5": 0.6002625, + "mrr@10": 0.6002625, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoSeek-doc": { - "hit@1": 0.9903677758318739, - "hit@5": 0.9973730297723292, - "hit@10": 0.9982486865148862, - "ndcg_linear@1": 0.6762988908347927, - "ndcg_linear@5": 0.8018406693375459, - "ndcg_linear@10": 0.767343136404217, - "ndcg_exponential@1": 0.4519639729797348, - "ndcg_exponential@5": 0.6874729261482034, - "ndcg_exponential@10": 0.6978420362198057, - "precision@1": 0.9903677758318739, - "precision@5": 0.8740805604203151, - "precision@10": 0.7643607705779335, - "recall@1": 0.05845834634908238, - "recall@5": 0.2559977615058973, - "recall@10": 0.4381450224554127, - "f1@1": 0.1096528022965413, - "f1@5": 0.388099043607817, - "f1@10": 0.5446572421004792, - "map@1": 0.9903677758318739, - "map@5": 0.8628546409807355, - "map@10": 0.7436259056064535, - "mrr@1": 0.9903677758318739, - "mrr@5": 0.9936077057793345, - "mrr@10": 0.9937327995996997, - "num_pred": 1142, - "num_data": 1142 + "Video-MME": { + "hit@1": 0.30703703703703705, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.30703703703703705, + "ndcg_linear@5": 0.6745941548531916, + "ndcg_linear@10": 0.6745941548531916, + "ndcg_exponential@1": 0.30703703703703705, + "ndcg_exponential@5": 0.6745941548531916, + "ndcg_exponential@10": 0.6745941548531916, + "precision@1": 0.30703703703703705, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.30703703703703705, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.30703703703703705, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.30703703703703705, + "map@5": 0.5658024691358025, + "map@10": 0.5658024691358025, + "mrr@1": 0.30703703703703705, + "mrr@5": 0.5658024691358025, + "mrr@10": 0.5658024691358025, + "num_pred": 2700, + "num_data": 2700 }, - "MMLongBench-page": { - "hit@1": 0.05250596658711217, - "hit@5": 0.24224343675417662, - "hit@10": 0.33890214797136037, - "ndcg_linear@1": 0.05250596658711217, - "ndcg_linear@5": 0.11893466223008214, - "ndcg_linear@10": 0.15008296554980238, - "ndcg_exponential@1": 0.05250596658711217, - "ndcg_exponential@5": 0.11893466223008214, - "ndcg_exponential@10": 0.15008296554980238, - "precision@1": 0.05250596658711217, - "precision@5": 0.056324582338902154, - "precision@10": 0.04307875894988067, - "recall@1": 0.028838504375497214, - "recall@5": 0.1757174110694397, - "recall@10": 0.2610182975338106, - "f1@1": 0.03586377822654672, - "f1@5": 0.08071268195513599, - "f1@10": 0.07038494595369645, - "map@1": 0.05250596658711217, - "map@5": 0.08618237867939538, - "map@10": 0.10006671765099569, - "mrr@1": 0.05250596658711217, - "mrr@5": 0.12171837708830548, - "mrr@10": 0.13489174906239346, - "num_pred": 838, - "num_data": 838 + "NExTQA": { + "hit@1": 0.20924801494628678, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.20924801494628678, + "ndcg_linear@5": 0.6098815656327321, + "ndcg_linear@10": 0.6098815656327321, + "ndcg_exponential@1": 0.20924801494628678, + "ndcg_exponential@5": 0.6098815656327321, + "ndcg_exponential@10": 0.6098815656327321, + "precision@1": 0.20924801494628678, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.20924801494628678, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.20924801494628678, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.20924801494628678, + "map@5": 0.4819554725206289, + "map@10": 0.4819554725206289, + "mrr@1": 0.20924801494628678, + "mrr@5": 0.4817803207224038, + "mrr@10": 0.4817803207224038, + "num_pred": 8564, + "num_data": 8564 + }, + "EgoSchema": { + "hit@1": 0.34, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.34, + "ndcg_linear@5": 0.6804792280158065, + "ndcg_linear@10": 0.6804792280158065, + "ndcg_exponential@1": 0.34, + "ndcg_exponential@5": 0.6804792280158065, + "ndcg_exponential@10": 0.6804792280158065, + "precision@1": 0.34, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.34, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.34, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.34, + "map@5": 0.5750333333333333, + "map@10": 0.5750333333333333, + "mrr@1": 0.34, + "mrr@5": 0.5750333333333333, + "mrr@10": 0.5750333333333333, + "num_pred": 500, + "num_data": 500 + }, + "ActivityNetQA": { + "hit@1": 0.523, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.523, + "ndcg_linear@5": 0.8239534924535853, + "ndcg_linear@10": 0.8239534924535853, + "ndcg_exponential@1": 0.523, + "ndcg_exponential@5": 0.8239534924535853, + "ndcg_exponential@10": 0.8239534924535853, + "precision@1": 0.523, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.523, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.523, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.523, + "map@5": 0.7615, + "map@10": 0.7615, + "mrr@1": 0.523, + "mrr@5": 0.7615, + "mrr@10": 0.7615, + "num_pred": 1000, + "num_data": 1000 }, - "MMLongBench-doc": { - "hit@1": 0.6157517899761337, - "hit@5": 0.7482100238663485, - "hit@10": 0.7947494033412887, - "ndcg_linear@1": 0.4287987271280827, - "ndcg_linear@5": 0.43709371712705786, - "ndcg_linear@10": 0.4147091530579679, - "ndcg_exponential@1": 0.2952608250937606, - "ndcg_exponential@5": 0.3633856276662273, - "ndcg_exponential@10": 0.36808988125041325, - "precision@1": 0.6157517899761337, - "precision@5": 0.4797136038186158, - "precision@10": 0.41563245823389017, - "recall@1": 0.01970187328257191, - "recall@5": 0.07488813025872536, - "recall@10": 0.12545835862481708, - "f1@1": 0.03781026144552173, - "f1@5": 0.12459954660237356, - "f1@10": 0.18139093061449052, - "map@1": 0.6157517899761337, - "map@5": 0.44125696101829753, - "map@10": 0.36061324308738785, - "mrr@1": 0.6157517899761337, - "mrr@5": 0.6687947494033414, - "mrr@10": 0.6751861006932606, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { "DiDeMo": { "hit@1": 0.30378486055776893, "hit@5": 0.5388446215139442, @@ -1804,481 +1323,875 @@ "num_pred": 1004, "num_data": 1004 }, - "MSR-VTT": { - "hit@1": 0.283, - "hit@5": 0.504, - "hit@10": 0.594, - "ndcg_linear@1": 0.283, - "ndcg_linear@5": 0.3981240408165469, - "ndcg_linear@10": 0.4272757830933265, - "ndcg_exponential@1": 0.283, - "ndcg_exponential@5": 0.3981240408165469, - "ndcg_exponential@10": 0.4272757830933265, - "precision@1": 0.283, - "precision@5": 0.1008, - "precision@10": 0.059399999999999994, - "recall@1": 0.283, - "recall@5": 0.504, - "recall@10": 0.594, - "f1@1": 0.283, - "f1@5": 0.16800000000000007, - "f1@10": 0.10800000000000003, - "map@1": 0.283, - "map@5": 0.36315000000000003, - "map@10": 0.3752111111111111, - "mrr@1": 0.283, - "mrr@5": 0.36315000000000003, - "mrr@10": 0.3752111111111111, - "num_pred": 1000, - "num_data": 1000 + "MSR-VTT": { + "hit@1": 0.283, + "hit@5": 0.504, + "hit@10": 0.594, + "ndcg_linear@1": 0.283, + "ndcg_linear@5": 0.3981240408165469, + "ndcg_linear@10": 0.4272757830933265, + "ndcg_exponential@1": 0.283, + "ndcg_exponential@5": 0.3981240408165469, + "ndcg_exponential@10": 0.4272757830933265, + "precision@1": 0.283, + "precision@5": 0.1008, + "precision@10": 0.059399999999999994, + "recall@1": 0.283, + "recall@5": 0.504, + "recall@10": 0.594, + "f1@1": 0.283, + "f1@5": 0.16800000000000007, + "f1@10": 0.10800000000000003, + "map@1": 0.283, + "map@5": 0.36315000000000003, + "map@10": 0.3752111111111111, + "mrr@1": 0.283, + "mrr@5": 0.36315000000000003, + "mrr@10": 0.3752111111111111, + "num_pred": 1000, + "num_data": 1000 + }, + "MSVD": { + "hit@1": 0.48059701492537316, + "hit@5": 0.7597014925373134, + "hit@10": 0.844776119402985, + "ndcg_linear@1": 0.48059701492537316, + "ndcg_linear@5": 0.63096871065978, + "ndcg_linear@10": 0.6588553080260654, + "ndcg_exponential@1": 0.48059701492537316, + "ndcg_exponential@5": 0.63096871065978, + "ndcg_exponential@10": 0.6588553080260654, + "precision@1": 0.48059701492537316, + "precision@5": 0.15194029850746266, + "precision@10": 0.0844776119402985, + "recall@1": 0.48059701492537316, + "recall@5": 0.7597014925373134, + "recall@10": 0.844776119402985, + "f1@1": 0.48059701492537316, + "f1@5": 0.2532338308457712, + "f1@10": 0.15359565807327005, + "map@1": 0.48059701492537316, + "map@5": 0.5879104477611939, + "map@10": 0.5996434494195687, + "mrr@1": 0.48059701492537316, + "mrr@5": 0.5879104477611939, + "mrr@10": 0.5996434494195687, + "num_pred": 670, + "num_data": 670 + }, + "VATEX": { + "hit@1": 0.26462706565430993, + "hit@5": 0.5015631978561857, + "hit@10": 0.6045109423849933, + "ndcg_linear@1": 0.26462706565430993, + "ndcg_linear@5": 0.3881950217969347, + "ndcg_linear@10": 0.4215820887400626, + "ndcg_exponential@1": 0.26462706565430993, + "ndcg_exponential@5": 0.3881950217969347, + "ndcg_exponential@10": 0.4215820887400626, + "precision@1": 0.26462706565430993, + "precision@5": 0.10031263957123716, + "precision@10": 0.06045109423849933, + "recall@1": 0.26462706565430993, + "recall@5": 0.5015631978561857, + "recall@10": 0.6045109423849933, + "f1@1": 0.26462706565430993, + "f1@5": 0.16718773261872863, + "f1@10": 0.10991108043363516, + "map@1": 0.26462706565430993, + "map@5": 0.35071460473425636, + "map@10": 0.364550678094671, + "mrr@1": 0.26462706565430993, + "mrr@5": 0.35071460473425636, + "mrr@10": 0.364550678094671, + "num_pred": 4478, + "num_data": 4478 + }, + "YouCook2": { + "hit@1": 0.10632274300094369, + "hit@5": 0.23057565272098143, + "hit@10": 0.30009436929852157, + "ndcg_linear@1": 0.10632274300094369, + "ndcg_linear@5": 0.17033555328162955, + "ndcg_linear@10": 0.19266164698283084, + "ndcg_exponential@1": 0.10632274300094369, + "ndcg_exponential@5": 0.17033555328162955, + "ndcg_exponential@10": 0.19266164698283084, + "precision@1": 0.10632274300094369, + "precision@5": 0.04611513054419629, + "precision@10": 0.030009436929852158, + "recall@1": 0.10632274300094369, + "recall@5": 0.23057565272098143, + "recall@10": 0.30009436929852157, + "f1@1": 0.10632274300094369, + "f1@5": 0.07685855090699381, + "f1@10": 0.054562612599731185, + "map@1": 0.10632274300094369, + "map@5": 0.150487574709028, + "map@10": 0.15961231194795208, + "mrr@1": 0.10632274300094369, + "mrr@5": 0.150487574709028, + "mrr@10": 0.15961231194795208, + "num_pred": 3179, + "num_data": 3179 + }, + "QVHighlight": { + "hit@1": 0.4939981532779317, + "hit@5": 0.8679593721144968, + "hit@10": 1.0, + "ndcg_linear@1": 0.4939981532779317, + "ndcg_linear@5": 0.6946135135669718, + "ndcg_linear@10": 0.7373262762832604, + "ndcg_exponential@1": 0.4939981532779317, + "ndcg_exponential@5": 0.6946135135669718, + "ndcg_exponential@10": 0.7373262762832604, + "precision@1": 0.4939981532779317, + "precision@5": 0.17359187442289936, + "precision@10": 0.09999999999999998, + "recall@1": 0.4939981532779317, + "recall@5": 0.8679593721144968, + "recall@10": 1.0, + "f1@1": 0.4939981532779317, + "f1@5": 0.2893197907048323, + "f1@10": 0.18181818181818185, + "map@1": 0.4939981532779317, + "map@5": 0.636734379809172, + "map@10": 0.6543771709976696, + "mrr@1": 0.4939981532779317, + "mrr@5": 0.636734379809172, + "mrr@10": 0.6543771709976696, + "num_pred": 1083, + "num_data": 1083 + }, + "Charades-STA": { + "hit@1": 0.20220082530949107, + "hit@5": 0.6011004126547456, + "hit@10": 1.0, + "ndcg_linear@1": 0.20220082530949107, + "ndcg_linear@5": 0.3987840994710363, + "ndcg_linear@10": 0.524544158533619, + "ndcg_exponential@1": 0.20220082530949107, + "ndcg_exponential@5": 0.3987840994710363, + "ndcg_exponential@10": 0.524544158533619, + "precision@1": 0.20220082530949107, + "precision@5": 0.1202200825309491, + "precision@10": 0.09999999999999998, + "recall@1": 0.20220082530949107, + "recall@5": 0.6011004126547456, + "recall@10": 1.0, + "f1@1": 0.20220082530949107, + "f1@5": 0.20036680421824854, + "f1@10": 0.18181818181818185, + "map@1": 0.20220082530949107, + "map@5": 0.3330353049060064, + "map@10": 0.38307897207484554, + "mrr@1": 0.20220082530949107, + "mrr@5": 0.3330353049060064, + "mrr@10": 0.38307897207484554, + "num_pred": 727, + "num_data": 727 + }, + "MomentSeeker": { + "hit@1": 0.4083333333333333, + "hit@5": 0.8861111111111111, + "hit@10": 1.0, + "ndcg_linear@1": 0.4083333333333333, + "ndcg_linear@5": 0.6586571278564146, + "ndcg_linear@10": 0.7020363910751584, + "ndcg_exponential@1": 0.4083333333333333, + "ndcg_exponential@5": 0.6586571278564146, + "ndcg_exponential@10": 0.7020363910751584, + "precision@1": 0.4083333333333333, + "precision@5": 0.1951111111111111, + "precision@10": 0.11277777777777781, + "recall@1": 0.3842824074074074, + "recall@5": 0.8708564814814814, + "recall@10": 0.9982175925925925, + "f1@1": 0.39165520282186944, + "f1@5": 0.3137015453682121, + "f1@10": 0.20007009194509193, + "map@1": 0.4083333333333333, + "map@5": 0.5822586419753087, + "map@10": 0.6026371527777779, + "mrr@1": 0.4083333333333333, + "mrr@5": 0.5929629629629629, + "mrr@10": 0.6089462081128747, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.734, + "hit@5": 0.866, + "hit@10": 0.906, + "ndcg_linear@1": 0.734, + "ndcg_linear@5": 0.8057573987022552, + "ndcg_linear@10": 0.8188064742981096, + "ndcg_exponential@1": 0.734, + "ndcg_exponential@5": 0.8057573987022552, + "ndcg_exponential@10": 0.8188064742981096, + "precision@1": 0.734, + "precision@5": 0.1732, + "precision@10": 0.0906, + "recall@1": 0.734, + "recall@5": 0.866, + "recall@10": 0.906, + "f1@1": 0.734, + "f1@5": 0.28866666666666674, + "f1@10": 0.16472727272727275, + "map@1": 0.734, + "map@5": 0.7856000000000001, + "map@10": 0.7910603174603175, + "mrr@1": 0.734, + "mrr@5": 0.7856000000000001, + "mrr@10": 0.7910603174603175, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.35033259423503327, + "hit@5": 0.541019955654102, + "hit@10": 0.6208425720620843, + "ndcg_linear@1": 0.35033259423503327, + "ndcg_linear@5": 0.4484886950768515, + "ndcg_linear@10": 0.47338485947788617, + "ndcg_exponential@1": 0.35033259423503327, + "ndcg_exponential@5": 0.4484886950768515, + "ndcg_exponential@10": 0.47338485947788617, + "precision@1": 0.35033259423503327, + "precision@5": 0.10953436807095342, + "precision@10": 0.06341463414634146, + "recall@1": 0.34320029563932003, + "recall@5": 0.5335920177383592, + "recall@10": 0.6106060606060606, + "f1@1": 0.34476093012678377, + "f1@5": 0.18059028489444562, + "f1@10": 0.11417931661834103, + "map@1": 0.35033259423503327, + "map@5": 0.41779995072678, + "map@10": 0.42790588814979064, + "mrr@1": 0.35033259423503327, + "mrr@5": 0.4246119733924612, + "mrr@10": 0.4349144757681343, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.7732793522267206, + "hit@5": 0.8947368421052632, + "hit@10": 0.9412955465587044, + "ndcg_linear@1": 0.7732793522267206, + "ndcg_linear@5": 0.836933803347616, + "ndcg_linear@10": 0.852452939253696, + "ndcg_exponential@1": 0.7732793522267206, + "ndcg_exponential@5": 0.836933803347616, + "ndcg_exponential@10": 0.852452939253696, + "precision@1": 0.7732793522267206, + "precision@5": 0.17894736842105263, + "precision@10": 0.09433198380566803, + "recall@1": 0.7732793522267206, + "recall@5": 0.8937246963562753, + "recall@10": 0.9412955465587044, + "f1@1": 0.7732793522267206, + "f1@5": 0.29814921920185083, + "f1@10": 0.17145135566188197, + "map@1": 0.7732793522267206, + "map@5": 0.817914979757085, + "map@10": 0.8244569757727652, + "mrr@1": 0.7732793522267206, + "mrr@5": 0.8182523616734144, + "mrr@10": 0.8245051731893837, + "num_pred": 494, + "num_data": 494 + }, + "ViDoRe_tabfquad": { + "hit@1": 0.8107142857142857, + "hit@5": 0.9571428571428572, + "hit@10": 0.975, + "ndcg_linear@1": 0.8107142857142857, + "ndcg_linear@5": 0.8920767921191974, + "ndcg_linear@10": 0.8982104341600656, + "ndcg_exponential@1": 0.8107142857142857, + "ndcg_exponential@5": 0.8920767921191974, + "ndcg_exponential@10": 0.8982104341600656, + "precision@1": 0.8107142857142857, + "precision@5": 0.19142857142857142, + "precision@10": 0.09749999999999999, + "recall@1": 0.8107142857142857, + "recall@5": 0.9571428571428572, + "recall@10": 0.975, + "f1@1": 0.8107142857142857, + "f1@5": 0.31904761904761914, + "f1@10": 0.1772727272727273, + "map@1": 0.8107142857142857, + "map@5": 0.8701190476190476, + "map@10": 0.8728613945578231, + "mrr@1": 0.8107142857142857, + "mrr@5": 0.8701190476190476, + "mrr@10": 0.8728613945578231, + "num_pred": 280, + "num_data": 280 }, - "MSVD": { - "hit@1": 0.48059701492537316, - "hit@5": 0.7597014925373134, - "hit@10": 0.844776119402985, - "ndcg_linear@1": 0.48059701492537316, - "ndcg_linear@5": 0.63096871065978, - "ndcg_linear@10": 0.6588553080260654, - "ndcg_exponential@1": 0.48059701492537316, - "ndcg_exponential@5": 0.63096871065978, - "ndcg_exponential@10": 0.6588553080260654, - "precision@1": 0.48059701492537316, - "precision@5": 0.15194029850746266, - "precision@10": 0.0844776119402985, - "recall@1": 0.48059701492537316, - "recall@5": 0.7597014925373134, - "recall@10": 0.844776119402985, - "f1@1": 0.48059701492537316, - "f1@5": 0.2532338308457712, - "f1@10": 0.15359565807327005, - "map@1": 0.48059701492537316, - "map@5": 0.5879104477611939, - "map@10": 0.5996434494195687, - "mrr@1": 0.48059701492537316, - "mrr@5": 0.5879104477611939, - "mrr@10": 0.5996434494195687, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tatdqa": { + "hit@1": 0.30741190765492105, + "hit@5": 0.5577156743620899, + "hit@10": 0.6433778857837181, + "ndcg_linear@1": 0.30741190765492105, + "ndcg_linear@5": 0.4382195288905396, + "ndcg_linear@10": 0.4658056006683759, + "ndcg_exponential@1": 0.30741190765492105, + "ndcg_exponential@5": 0.4382195288905396, + "ndcg_exponential@10": 0.4658056006683759, + "precision@1": 0.30741190765492105, + "precision@5": 0.1117861482381531, + "precision@10": 0.06452004860267314, + "recall@1": 0.30650060753341435, + "recall@5": 0.5565613608748481, + "recall@10": 0.6422235722964762, + "f1@1": 0.3068043742405832, + "f1@5": 0.18601515940519592, + "f1@10": 0.11718583158437353, + "map@1": 0.30741190765492105, + "map@5": 0.39878898339408664, + "map@10": 0.41010757198788794, + "mrr@1": 0.30741190765492105, + "mrr@5": 0.39935196435803966, + "mrr@10": 0.41067257806322205, + "num_pred": 1646, + "num_data": 1646 }, - "VATEX": { - "hit@1": 0.26462706565430993, - "hit@5": 0.5015631978561857, - "hit@10": 0.6045109423849933, - "ndcg_linear@1": 0.26462706565430993, - "ndcg_linear@5": 0.3881950217969347, - "ndcg_linear@10": 0.4215820887400626, - "ndcg_exponential@1": 0.26462706565430993, - "ndcg_exponential@5": 0.3881950217969347, - "ndcg_exponential@10": 0.4215820887400626, - "precision@1": 0.26462706565430993, - "precision@5": 0.10031263957123716, - "precision@10": 0.06045109423849933, - "recall@1": 0.26462706565430993, - "recall@5": 0.5015631978561857, - "recall@10": 0.6045109423849933, - "f1@1": 0.26462706565430993, - "f1@5": 0.16718773261872863, - "f1@10": 0.10991108043363516, - "map@1": 0.26462706565430993, - "map@5": 0.35071460473425636, - "map@10": 0.364550678094671, - "mrr@1": 0.26462706565430993, - "mrr@5": 0.35071460473425636, - "mrr@10": 0.364550678094671, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_shiftproject": { + "hit@1": 0.45, + "hit@5": 0.74, + "hit@10": 0.87, + "ndcg_linear@1": 0.45, + "ndcg_linear@5": 0.608217112664813, + "ndcg_linear@10": 0.650000125092289, + "ndcg_exponential@1": 0.45, + "ndcg_exponential@5": 0.608217112664813, + "ndcg_exponential@10": 0.650000125092289, + "precision@1": 0.45, + "precision@5": 0.14799999999999996, + "precision@10": 0.08699999999999998, + "recall@1": 0.45, + "recall@5": 0.74, + "recall@10": 0.87, + "f1@1": 0.45, + "f1@5": 0.24666666666666667, + "f1@10": 0.1581818181818182, + "map@1": 0.45, + "map@5": 0.564, + "map@10": 0.5810912698412699, + "mrr@1": 0.45, + "mrr@5": 0.564, + "mrr@10": 0.5810912698412699, + "num_pred": 100, + "num_data": 100 }, - "YouCook2": { - "hit@1": 0.10632274300094369, - "hit@5": 0.23057565272098143, - "hit@10": 0.30009436929852157, - "ndcg_linear@1": 0.10632274300094369, - "ndcg_linear@5": 0.17033555328162955, - "ndcg_linear@10": 0.19266164698283084, - "ndcg_exponential@1": 0.10632274300094369, - "ndcg_exponential@5": 0.17033555328162955, - "ndcg_exponential@10": 0.19266164698283084, - "precision@1": 0.10632274300094369, - "precision@5": 0.04611513054419629, - "precision@10": 0.030009436929852158, - "recall@1": 0.10632274300094369, - "recall@5": 0.23057565272098143, - "recall@10": 0.30009436929852157, - "f1@1": 0.10632274300094369, - "f1@5": 0.07685855090699381, - "f1@10": 0.054562612599731185, - "map@1": 0.10632274300094369, - "map@5": 0.150487574709028, - "map@10": 0.15961231194795208, - "mrr@1": 0.10632274300094369, - "mrr@5": 0.150487574709028, - "mrr@10": 0.15961231194795208, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.76, + "hit@5": 0.97, + "hit@10": 0.98, + "ndcg_linear@1": 0.76, + "ndcg_linear@5": 0.8852555261521673, + "ndcg_linear@10": 0.8882658261088071, + "ndcg_exponential@1": 0.76, + "ndcg_exponential@5": 0.8852555261521673, + "ndcg_exponential@10": 0.8882658261088071, + "precision@1": 0.76, + "precision@5": 0.19399999999999995, + "precision@10": 0.09799999999999998, + "recall@1": 0.76, + "recall@5": 0.97, + "recall@10": 0.98, + "f1@1": 0.76, + "f1@5": 0.3233333333333334, + "f1@10": 0.17818181818181827, + "map@1": 0.76, + "map@5": 0.8558333333333333, + "map@10": 0.8569444444444444, + "mrr@1": 0.76, + "mrr@5": 0.8558333333333333, + "mrr@10": 0.8569444444444444, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.79, + "hit@5": 0.93, + "hit@10": 0.97, + "ndcg_linear@1": 0.79, + "ndcg_linear@5": 0.8650828475372231, + "ndcg_linear@10": 0.8780588117396043, + "ndcg_exponential@1": 0.79, + "ndcg_exponential@5": 0.8650828475372231, + "ndcg_exponential@10": 0.8780588117396043, + "precision@1": 0.79, + "precision@5": 0.18599999999999994, + "precision@10": 0.09699999999999999, + "recall@1": 0.79, + "recall@5": 0.93, + "recall@10": 0.97, + "f1@1": 0.79, + "f1@5": 0.31000000000000005, + "f1@10": 0.17636363636363647, + "map@1": 0.79, + "map@5": 0.8433333333333333, + "map@10": 0.8486904761904762, + "mrr@1": 0.79, + "mrr@5": 0.8433333333333333, + "mrr@10": 0.8486904761904762, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.4939981532779317, - "hit@5": 0.8679593721144968, - "hit@10": 1.0, - "ndcg_linear@1": 0.4939981532779317, - "ndcg_linear@5": 0.6946135135669718, - "ndcg_linear@10": 0.7373262762832604, - "ndcg_exponential@1": 0.4939981532779317, - "ndcg_exponential@5": 0.6946135135669718, - "ndcg_exponential@10": 0.7373262762832604, - "precision@1": 0.4939981532779317, - "precision@5": 0.17359187442289936, - "precision@10": 0.09999999999999998, - "recall@1": 0.4939981532779317, - "recall@5": 0.8679593721144968, - "recall@10": 1.0, - "f1@1": 0.4939981532779317, - "f1@5": 0.2893197907048323, - "f1@10": 0.18181818181818185, - "map@1": 0.4939981532779317, - "map@5": 0.636734379809172, - "map@10": 0.6543771709976696, - "mrr@1": 0.4939981532779317, - "mrr@5": 0.636734379809172, - "mrr@10": 0.6543771709976696, - "num_pred": 1083, - "num_data": 1083 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.74, + "hit@5": 0.94, + "hit@10": 0.98, + "ndcg_linear@1": 0.74, + "ndcg_linear@5": 0.8501367970825706, + "ndcg_linear@10": 0.8630184664460052, + "ndcg_exponential@1": 0.74, + "ndcg_exponential@5": 0.8501367970825706, + "ndcg_exponential@10": 0.8630184664460052, + "precision@1": 0.74, + "precision@5": 0.18799999999999997, + "precision@10": 0.09799999999999999, + "recall@1": 0.74, + "recall@5": 0.94, + "recall@10": 0.98, + "f1@1": 0.74, + "f1@5": 0.3133333333333334, + "f1@10": 0.17818181818181825, + "map@1": 0.74, + "map@5": 0.8198333333333332, + "map@10": 0.825111111111111, + "mrr@1": 0.74, + "mrr@5": 0.8198333333333332, + "mrr@10": 0.825111111111111, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.20220082530949107, - "hit@5": 0.6011004126547456, - "hit@10": 1.0, - "ndcg_linear@1": 0.20220082530949107, - "ndcg_linear@5": 0.3987840994710363, - "ndcg_linear@10": 0.524544158533619, - "ndcg_exponential@1": 0.20220082530949107, - "ndcg_exponential@5": 0.3987840994710363, - "ndcg_exponential@10": 0.524544158533619, - "precision@1": 0.20220082530949107, - "precision@5": 0.1202200825309491, - "precision@10": 0.09999999999999998, - "recall@1": 0.20220082530949107, - "recall@5": 0.6011004126547456, - "recall@10": 1.0, - "f1@1": 0.20220082530949107, - "f1@5": 0.20036680421824854, - "f1@10": 0.18181818181818185, - "map@1": 0.20220082530949107, - "map@5": 0.3330353049060064, - "map@10": 0.38307897207484554, - "mrr@1": 0.20220082530949107, - "mrr@5": 0.3330353049060064, - "mrr@10": 0.38307897207484554, - "num_pred": 727, - "num_data": 727 + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.84, + "hit@5": 0.98, + "hit@10": 0.99, + "ndcg_linear@1": 0.84, + "ndcg_linear@5": 0.9217065065186136, + "ndcg_linear@10": 0.9252685783896938, + "ndcg_exponential@1": 0.84, + "ndcg_exponential@5": 0.9217065065186136, + "ndcg_exponential@10": 0.9252685783896938, + "precision@1": 0.84, + "precision@5": 0.19599999999999998, + "precision@10": 0.09899999999999999, + "recall@1": 0.84, + "recall@5": 0.98, + "recall@10": 0.99, + "f1@1": 0.84, + "f1@5": 0.32666666666666677, + "f1@10": 0.18000000000000008, + "map@1": 0.84, + "map@5": 0.9016666666666667, + "map@10": 0.9033333333333334, + "mrr@1": 0.84, + "mrr@5": 0.9016666666666667, + "mrr@10": 0.9033333333333334, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.458, - "hit@5": 0.883, - "hit@10": 1.0, - "ndcg_linear@1": 0.458, - "ndcg_linear@5": 0.6805748597853842, - "ndcg_linear@10": 0.7209509472387817, - "ndcg_exponential@1": 0.458, - "ndcg_exponential@5": 0.6805748597853842, - "ndcg_exponential@10": 0.7209509472387817, - "precision@1": 0.458, - "precision@5": 0.18520000000000006, - "precision@10": 0.10640000000000004, - "recall@1": 0.4467083333333334, - "recall@5": 0.8777083333333333, - "recall@10": 0.999625, - "f1@1": 0.44986031746031746, - "f1@5": 0.3026584526584527, - "f1@10": 0.19034848484848482, - "map@1": 0.458, - "map@5": 0.6129316666666668, - "map@10": 0.6307495866402116, - "mrr@1": 0.458, - "mrr@5": 0.6167166666666667, - "mrr@10": 0.6328710317460317, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.38461538461538464, + "hit@5": 0.6730769230769231, + "hit@10": 0.8461538461538461, + "ndcg_linear@1": 0.375, + "ndcg_linear@5": 0.4555552886561195, + "ndcg_linear@10": 0.511493540486886, + "ndcg_exponential@1": 0.37179487179487186, + "ndcg_exponential@5": 0.4541859231025376, + "ndcg_exponential@10": 0.5099104815026472, + "precision@1": 0.38461538461538464, + "precision@5": 0.19230769230769226, + "precision@10": 0.13076923076923078, + "recall@1": 0.23621794871794868, + "recall@5": 0.5487179487179487, + "recall@10": 0.7033882783882782, + "f1@1": 0.2715483234714004, + "f1@5": 0.25723492194080433, + "f1@10": 0.2026450736292365, + "map@1": 0.38461538461538464, + "map@5": 0.3914850427350427, + "map@10": 0.415758547008547, + "mrr@1": 0.38461538461538464, + "mrr@5": 0.491025641025641, + "mrr@10": 0.5144993894993896, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.38, - "hit@5": 0.623, - "hit@10": 0.697, - "ndcg_linear@1": 0.38, - "ndcg_linear@5": 0.5087323881529896, - "ndcg_linear@10": 0.5328292310353233, - "ndcg_exponential@1": 0.38, - "ndcg_exponential@5": 0.5087323881529896, - "ndcg_exponential@10": 0.5328292310353233, - "precision@1": 0.38, - "precision@5": 0.12459999999999999, - "precision@10": 0.06969999999999998, - "recall@1": 0.38, - "recall@5": 0.623, - "recall@10": 0.697, - "f1@1": 0.38, - "f1@5": 0.20766666666666675, - "f1@10": 0.12672727272727274, - "map@1": 0.38, - "map@5": 0.4706833333333333, - "map@10": 0.4807186507936508, - "mrr@1": 0.38, - "mrr@5": 0.4706833333333333, - "mrr@10": 0.4807186507936508, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.36875, + "hit@5": 0.7171875, + "hit@10": 0.8203125, + "ndcg_linear@1": 0.36875, + "ndcg_linear@5": 0.4426897295015239, + "ndcg_linear@10": 0.4850950127175607, + "ndcg_exponential@1": 0.36875, + "ndcg_exponential@5": 0.4426897295015239, + "ndcg_exponential@10": 0.4850950127175607, + "precision@1": 0.36875, + "precision@5": 0.203125, + "precision@10": 0.13968750000000002, + "recall@1": 0.23270833333333335, + "recall@5": 0.49864999124649856, + "recall@10": 0.6150380777310923, + "f1@1": 0.2621073457792208, + "f1@5": 0.25601551313270066, + "f1@10": 0.20543311717898116, + "map@1": 0.36875, + "map@5": 0.3719822048611111, + "map@10": 0.3923365021789966, + "mrr@1": 0.36875, + "mrr@5": 0.497265625, + "mrr@10": 0.5111972966269841, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.428, - "hit@5": 0.692, - "hit@10": 0.786, - "ndcg_linear@1": 0.428, - "ndcg_linear@5": 0.5672834391090181, - "ndcg_linear@10": 0.5981163502873708, - "ndcg_exponential@1": 0.428, - "ndcg_exponential@5": 0.5672834391090181, - "ndcg_exponential@10": 0.5981163502873708, - "precision@1": 0.428, - "precision@5": 0.13839999999999997, - "precision@10": 0.07859999999999999, - "recall@1": 0.428, - "recall@5": 0.692, - "recall@10": 0.786, - "f1@1": 0.428, - "f1@5": 0.23066666666666674, - "f1@10": 0.14290909090909093, - "map@1": 0.428, - "map@5": 0.5259333333333333, - "map@10": 0.5389079365079364, - "mrr@1": 0.428, - "mrr@5": 0.5259333333333333, - "mrr@10": 0.5389079365079364, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.4525862068965517, + "hit@5": 0.8103448275862069, + "hit@10": 0.9267241379310345, + "ndcg_linear@1": 0.4525862068965517, + "ndcg_linear@5": 0.42995014841633666, + "ndcg_linear@10": 0.43467280891274934, + "ndcg_exponential@1": 0.4525862068965517, + "ndcg_exponential@5": 0.4299501484163365, + "ndcg_exponential@10": 0.43467280891274934, + "precision@1": 0.4525862068965517, + "precision@5": 0.38448275862068965, + "precision@10": 0.33103448275862074, + "recall@1": 0.049016392011628045, + "recall@5": 0.22829352926193736, + "recall@10": 0.36167397868493684, + "f1@1": 0.08115811949451532, + "f1@5": 0.22694280671921516, + "f1@10": 0.27660068249123476, + "map@1": 0.4525862068965517, + "map@5": 0.3282327586206897, + "map@10": 0.2973695391710324, + "mrr@1": 0.4525862068965517, + "mrr@5": 0.5951867816091955, + "mrr@10": 0.6120929118773947, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.4692982456140351, + "hit@5": 0.7719298245614035, + "hit@10": 0.8640350877192983, + "ndcg_linear@1": 0.4692982456140351, + "ndcg_linear@5": 0.466179429204456, + "ndcg_linear@10": 0.5017617250757785, + "ndcg_exponential@1": 0.4692982456140351, + "ndcg_exponential@5": 0.466179429204456, + "ndcg_exponential@10": 0.5017617250757785, + "precision@1": 0.4692982456140351, + "precision@5": 0.21929824561403508, + "precision@10": 0.15131578947368426, + "recall@1": 0.2380325814536341, + "recall@5": 0.4968671679197995, + "recall@10": 0.6020624477861319, + "f1@1": 0.28556649037592774, + "f1@5": 0.26698957296533893, + "f1@10": 0.21278941819922756, + "map@1": 0.4692982456140351, + "map@5": 0.38164961013645227, + "map@10": 0.39600202763854087, + "mrr@1": 0.4692982456140351, + "mrr@5": 0.5818713450292398, + "mrr@10": 0.594714216095795, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.409, - "hit@5": 0.638, - "hit@10": 0.713, - "ndcg_linear@1": 0.409, - "ndcg_linear@5": 0.5335348498920724, - "ndcg_linear@10": 0.5576824580140078, - "ndcg_exponential@1": 0.409, - "ndcg_exponential@5": 0.5335348498920724, - "ndcg_exponential@10": 0.5576824580140078, - "precision@1": 0.409, - "precision@5": 0.12760000000000002, - "precision@10": 0.0713, - "recall@1": 0.409, - "recall@5": 0.638, - "recall@10": 0.713, - "f1@1": 0.409, - "f1@5": 0.21266666666666673, - "f1@10": 0.12963636363636366, - "map@1": 0.409, - "map@5": 0.49854999999999994, - "map@10": 0.5084662698412699, - "mrr@1": 0.409, - "mrr@5": 0.49854999999999994, - "mrr@10": 0.5084662698412699, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.7022058823529411, + "hit@5": 0.821078431372549, + "hit@10": 0.8615196078431373, + "ndcg_linear@1": 0.7022058823529411, + "ndcg_linear@5": 0.7687028213602233, + "ndcg_linear@10": 0.7817454060088397, + "ndcg_exponential@1": 0.7022058823529411, + "ndcg_exponential@5": 0.7687028213602233, + "ndcg_exponential@10": 0.7817454060088397, + "precision@1": 0.7022058823529411, + "precision@5": 0.1642156862745098, + "precision@10": 0.08615196078431371, + "recall@1": 0.7022058823529411, + "recall@5": 0.821078431372549, + "recall@10": 0.8615196078431373, + "f1@1": 0.7022058823529411, + "f1@5": 0.2736928104575164, + "f1@10": 0.15663992869875223, + "map@1": 0.7022058823529411, + "map@5": 0.7510212418300654, + "map@10": 0.7563827614379085, + "mrr@1": 0.7022058823529411, + "mrr@5": 0.7510212418300654, + "mrr@10": 0.7563827614379085, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.6, - "hit@5": 0.831, - "hit@10": 0.888, - "ndcg_linear@1": 0.6, - "ndcg_linear@5": 0.728875610371988, - "ndcg_linear@10": 0.7473187422682429, - "ndcg_exponential@1": 0.6, - "ndcg_exponential@5": 0.728875610371988, - "ndcg_exponential@10": 0.7473187422682429, - "precision@1": 0.6, - "precision@5": 0.16620000000000001, - "precision@10": 0.08880000000000002, - "recall@1": 0.6, - "recall@5": 0.831, - "recall@10": 0.888, - "f1@1": 0.6, - "f1@5": 0.277, - "f1@10": 0.16145454545454543, - "map@1": 0.6, - "map@5": 0.6944166666666668, - "map@10": 0.7020325396825396, - "mrr@1": 0.6, - "mrr@5": 0.6944166666666668, - "mrr@10": 0.7020325396825396, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.7301587301587301, + "hit@5": 0.9206349206349206, + "hit@10": 0.9365079365079365, + "ndcg_linear@1": 0.7301587301587301, + "ndcg_linear@5": 0.8441010759070336, + "ndcg_linear@10": 0.849392081198039, + "ndcg_exponential@1": 0.7301587301587301, + "ndcg_exponential@5": 0.8441010759070336, + "ndcg_exponential@10": 0.849392081198039, + "precision@1": 0.7301587301587301, + "precision@5": 0.18412698412698403, + "precision@10": 0.0936507936507936, + "recall@1": 0.7301587301587301, + "recall@5": 0.9206349206349206, + "recall@10": 0.9365079365079365, + "f1@1": 0.7301587301587301, + "f1@5": 0.3068783068783068, + "f1@10": 0.1702741702741703, + "map@1": 0.7301587301587301, + "map@5": 0.8174603174603174, + "map@10": 0.8197278911564626, + "mrr@1": 0.7301587301587301, + "mrr@5": 0.8174603174603174, + "mrr@10": 0.8197278911564626, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.14780600461893764, - "hit@5": 0.6327944572748267, - "hit@10": 1.0, - "ndcg_linear@1": 0.14780600461893764, - "ndcg_linear@5": 0.3858298763211173, - "ndcg_linear@10": 0.5069598731135728, - "ndcg_exponential@1": 0.14780600461893764, - "ndcg_exponential@5": 0.3858298763211173, - "ndcg_exponential@10": 0.5069598731135728, - "precision@1": 0.14780600461893764, - "precision@5": 0.12655889145496535, - "precision@10": 0.1, - "recall@1": 0.14780600461893764, - "recall@5": 0.6327944572748267, - "recall@10": 1.0, - "f1@1": 0.14780600461893764, - "f1@5": 0.21093148575827564, - "f1@10": 0.1818181818181818, - "map@1": 0.14780600461893764, - "map@5": 0.30546574287913775, - "map@10": 0.3568477583489131, - "mrr@1": 0.14780600461893764, - "mrr@5": 0.30546574287913775, - "mrr@10": 0.3568477583489131, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.5922165820642978, + "hit@5": 0.8206429780033841, + "hit@10": 0.8663282571912013, + "ndcg_linear@1": 0.5922165820642978, + "ndcg_linear@5": 0.717924714040525, + "ndcg_linear@10": 0.7327402059542735, + "ndcg_exponential@1": 0.5922165820642978, + "ndcg_exponential@5": 0.717924714040525, + "ndcg_exponential@10": 0.7327402059542735, + "precision@1": 0.5922165820642978, + "precision@5": 0.16412859560067683, + "precision@10": 0.08663282571912012, + "recall@1": 0.5922165820642978, + "recall@5": 0.8206429780033841, + "recall@10": 0.8663282571912013, + "f1@1": 0.5922165820642978, + "f1@5": 0.2735476593344614, + "f1@10": 0.15751422858021846, + "map@1": 0.5922165820642978, + "map@5": 0.6833051325437113, + "map@10": 0.6894455187602395, + "mrr@1": 0.5922165820642978, + "mrr@5": 0.6833051325437113, + "mrr@10": 0.6894455187602395, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.337, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.337, - "ndcg_linear@5": 0.7011998180757394, - "ndcg_linear@10": 0.7011998180757394, - "ndcg_exponential@1": 0.337, - "ndcg_exponential@5": 0.7011998180757394, - "ndcg_exponential@10": 0.7011998180757394, - "precision@1": 0.337, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.337, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.337, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.337, - "map@5": 0.6002625, - "map@10": 0.6002625, - "mrr@1": 0.337, - "mrr@5": 0.6002625, - "mrr@10": 0.6002625, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.8794964028776978, + "hit@5": 0.9748201438848921, + "hit@10": 0.9802158273381295, + "ndcg_linear@1": 0.8794964028776978, + "ndcg_linear@5": 0.9150078016078801, + "ndcg_linear@10": 0.9207838812109942, + "ndcg_exponential@1": 0.8794964028776978, + "ndcg_exponential@5": 0.9150078016078801, + "ndcg_exponential@10": 0.9207838812109942, + "precision@1": 0.8794964028776978, + "precision@5": 0.2384892086330935, + "precision@10": 0.12176258992805757, + "recall@1": 0.7598920863309353, + "recall@5": 0.954136690647482, + "recall@10": 0.9694244604316546, + "f1@1": 0.7997601918465228, + "f1@5": 0.37478588557725256, + "f1@10": 0.2138652714192283, + "map@1": 0.8794964028776978, + "map@5": 0.8905575539568344, + "map@10": 0.894137547105173, + "mrr@1": 0.8794964028776978, + "mrr@5": 0.9226618705035972, + "mrr@10": 0.9233213429256595, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.7465181058495822, + "hit@5": 0.9415041782729805, + "hit@10": 0.9665738161559888, + "ndcg_linear@1": 0.7465181058495822, + "ndcg_linear@5": 0.8565495992280632, + "ndcg_linear@10": 0.8647865140019099, + "ndcg_exponential@1": 0.7465181058495822, + "ndcg_exponential@5": 0.8565495992280632, + "ndcg_exponential@10": 0.8647865140019099, + "precision@1": 0.7465181058495822, + "precision@5": 0.18830083565459607, + "precision@10": 0.09665738161559885, + "recall@1": 0.7465181058495822, + "recall@5": 0.9415041782729805, + "recall@10": 0.9665738161559888, + "f1@1": 0.7465181058495822, + "f1@5": 0.31383472609099355, + "f1@10": 0.1757406938465435, + "map@1": 0.7465181058495822, + "map@5": 0.827669452181987, + "map@10": 0.8311491356059602, + "mrr@1": 0.7465181058495822, + "mrr@5": 0.827669452181987, + "mrr@10": 0.8311491356059602, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.30703703703703705, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.30703703703703705, - "ndcg_linear@5": 0.6745941548531916, - "ndcg_linear@10": 0.6745941548531916, - "ndcg_exponential@1": 0.30703703703703705, - "ndcg_exponential@5": 0.6745941548531916, - "ndcg_exponential@10": 0.6745941548531916, - "precision@1": 0.30703703703703705, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.30703703703703705, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.30703703703703705, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.30703703703703705, - "map@5": 0.5658024691358025, - "map@10": 0.5658024691358025, - "mrr@1": 0.30703703703703705, - "mrr@5": 0.5658024691358025, - "mrr@10": 0.5658024691358025, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.5052143684820394, + "hit@5": 0.7937427578215527, + "hit@10": 0.8887601390498262, + "ndcg_linear@1": 0.5052143684820394, + "ndcg_linear@5": 0.6604877574932249, + "ndcg_linear@10": 0.6916336549342924, + "ndcg_exponential@1": 0.5052143684820394, + "ndcg_exponential@5": 0.6604877574932249, + "ndcg_exponential@10": 0.6916336549342924, + "precision@1": 0.5052143684820394, + "precision@5": 0.15874855156431056, + "precision@10": 0.08887601390498262, + "recall@1": 0.5052143684820394, + "recall@5": 0.7937427578215527, + "recall@10": 0.8887601390498262, + "f1@1": 0.5052143684820394, + "f1@5": 0.26458091927385097, + "f1@10": 0.1615927525545139, + "map@1": 0.5052143684820394, + "map@5": 0.6159907300115876, + "map@10": 0.629089646673656, + "mrr@1": 0.5052143684820394, + "mrr@5": 0.6159907300115876, + "mrr@10": 0.629089646673656, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.20924801494628678, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.20924801494628678, - "ndcg_linear@5": 0.6098815656327321, - "ndcg_linear@10": 0.6098815656327321, - "ndcg_exponential@1": 0.20924801494628678, - "ndcg_exponential@5": 0.6098815656327321, - "ndcg_exponential@10": 0.6098815656327321, - "precision@1": 0.20924801494628678, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.20924801494628678, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.20924801494628678, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.20924801494628678, - "map@5": 0.4819554725206289, - "map@10": 0.4819554725206289, - "mrr@1": 0.20924801494628678, - "mrr@5": 0.4817803207224038, - "mrr@10": 0.4817803207224038, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.03502626970227671, + "hit@5": 0.3975481611208406, + "hit@10": 0.5674255691768827, + "ndcg_linear@1": 0.03502626970227671, + "ndcg_linear@5": 0.21944808482459283, + "ndcg_linear@10": 0.2752564930475152, + "ndcg_exponential@1": 0.03502626970227671, + "ndcg_exponential@5": 0.21944808482459285, + "ndcg_exponential@10": 0.2752564930475153, + "precision@1": 0.03502626970227671, + "precision@5": 0.07950963222416814, + "precision@10": 0.05674255691768828, + "recall@1": 0.03502626970227671, + "recall@5": 0.3975481611208406, + "recall@10": 0.5674255691768827, + "f1@1": 0.03502626970227671, + "f1@5": 0.13251605370694688, + "f1@10": 0.10316828530488775, + "map@1": 0.03502626970227671, + "map@5": 0.16103327495621714, + "map@10": 0.184575236983293, + "mrr@1": 0.03502626970227671, + "mrr@5": 0.16103327495621714, + "mrr@10": 0.184575236983293, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.34, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.34, - "ndcg_linear@5": 0.6804792280158065, - "ndcg_linear@10": 0.6804792280158065, - "ndcg_exponential@1": 0.34, - "ndcg_exponential@5": 0.6804792280158065, - "ndcg_exponential@10": 0.6804792280158065, - "precision@1": 0.34, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.34, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.34, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.34, - "map@5": 0.5750333333333333, - "map@10": 0.5750333333333333, - "mrr@1": 0.34, - "mrr@5": 0.5750333333333333, - "mrr@10": 0.5750333333333333, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.9903677758318739, + "hit@5": 0.9973730297723292, + "hit@10": 0.9982486865148862, + "ndcg_linear@1": 0.6762988908347927, + "ndcg_linear@5": 0.8018406693375459, + "ndcg_linear@10": 0.767343136404217, + "ndcg_exponential@1": 0.4519639729797348, + "ndcg_exponential@5": 0.6874729261482034, + "ndcg_exponential@10": 0.6978420362198057, + "precision@1": 0.9903677758318739, + "precision@5": 0.8740805604203151, + "precision@10": 0.7643607705779335, + "recall@1": 0.05845834634908238, + "recall@5": 0.2559977615058973, + "recall@10": 0.4381450224554127, + "f1@1": 0.1096528022965413, + "f1@5": 0.388099043607817, + "f1@10": 0.5446572421004792, + "map@1": 0.9903677758318739, + "map@5": 0.8628546409807355, + "map@10": 0.7436259056064535, + "mrr@1": 0.9903677758318739, + "mrr@5": 0.9936077057793345, + "mrr@10": 0.9937327995996997, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.523, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.523, - "ndcg_linear@5": 0.8239534924535853, - "ndcg_linear@10": 0.8239534924535853, - "ndcg_exponential@1": 0.523, - "ndcg_exponential@5": 0.8239534924535853, - "ndcg_exponential@10": 0.8239534924535853, - "precision@1": 0.523, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.523, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.523, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.523, - "map@5": 0.7615, - "map@10": 0.7615, - "mrr@1": 0.523, - "mrr@5": 0.7615, - "mrr@10": 0.7615, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.05250596658711217, + "hit@5": 0.24224343675417662, + "hit@10": 0.33890214797136037, + "ndcg_linear@1": 0.05250596658711217, + "ndcg_linear@5": 0.11893466223008214, + "ndcg_linear@10": 0.15008296554980238, + "ndcg_exponential@1": 0.05250596658711217, + "ndcg_exponential@5": 0.11893466223008214, + "ndcg_exponential@10": 0.15008296554980238, + "precision@1": 0.05250596658711217, + "precision@5": 0.056324582338902154, + "precision@10": 0.04307875894988067, + "recall@1": 0.028838504375497214, + "recall@5": 0.1757174110694397, + "recall@10": 0.2610182975338106, + "f1@1": 0.03586377822654672, + "f1@5": 0.08071268195513599, + "f1@10": 0.07038494595369645, + "map@1": 0.05250596658711217, + "map@5": 0.08618237867939538, + "map@10": 0.10006671765099569, + "mrr@1": 0.05250596658711217, + "mrr@5": 0.12171837708830548, + "mrr@10": 0.13489174906239346, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.6157517899761337, + "hit@5": 0.7482100238663485, + "hit@10": 0.7947494033412887, + "ndcg_linear@1": 0.4287987271280827, + "ndcg_linear@5": 0.43709371712705786, + "ndcg_linear@10": 0.4147091530579679, + "ndcg_exponential@1": 0.2952608250937606, + "ndcg_exponential@5": 0.3633856276662273, + "ndcg_exponential@10": 0.36808988125041325, + "precision@1": 0.6157517899761337, + "precision@5": 0.4797136038186158, + "precision@10": 0.41563245823389017, + "recall@1": 0.01970187328257191, + "recall@5": 0.07488813025872536, + "recall@10": 0.12545835862481708, + "f1@1": 0.03781026144552173, + "f1@5": 0.12459954660237356, + "f1@10": 0.18139093061449052, + "map@1": 0.6157517899761337, + "map@5": 0.44125696101829753, + "map@10": 0.36061324308738785, + "mrr@1": 0.6157517899761337, + "mrr@5": 0.6687947494033414, + "mrr@10": 0.6751861006932606, + "num_pred": 838, + "num_data": 838 } } }