{ "syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 1.0, "ndcg_at_3": 1.0, "ndcg_at_5": 1.0, "ndcg_at_10": 1.0, "ndcg_at_20": 1.0, "ndcg_at_50": 1.0, "ndcg_at_100": 1.0, "map_at_1": 1.0, "map_at_3": 1.0, "map_at_5": 1.0, "map_at_10": 1.0, "map_at_20": 1.0, "map_at_50": 1.0, "map_at_100": 1.0, "recall_at_1": 1.0, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 1.0, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 1.0, "mrr_at_3": 1.0, "mrr_at_5": 1.0, "mrr_at_10": 1.0, "mrr_at_20": 1.0, "mrr_at_50": 1.0, "mrr_at_100": 1.0, "naucs_at_1_max": null, "naucs_at_1_std": null, "naucs_at_1_diff1": null, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "tabfquad_test_subsampled": { "ndcg_at_1": 0.88214, "ndcg_at_3": 0.92083, "ndcg_at_5": 0.92959, "ndcg_at_10": 0.92959, "ndcg_at_20": 0.93232, "ndcg_at_50": 0.93664, "ndcg_at_100": 0.93664, "map_at_1": 0.88214, "map_at_3": 0.9119, "map_at_5": 0.91673, "map_at_10": 0.91673, "map_at_20": 0.91749, "map_at_50": 0.9182, "map_at_100": 0.9182, "recall_at_1": 0.88214, "recall_at_3": 0.94643, "recall_at_5": 0.96786, "recall_at_10": 0.96786, "recall_at_20": 0.97857, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88214, "precision_at_3": 0.31548, "precision_at_5": 0.19357, "precision_at_10": 0.09679, "precision_at_20": 0.04893, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8821428571428571, "mrr_at_3": 0.911904761904762, "mrr_at_5": 0.9167261904761906, "mrr_at_10": 0.9167261904761906, "mrr_at_20": 0.9174988553113552, "mrr_at_50": 0.9182272238973882, "mrr_at_100": 0.9182272238973882, "naucs_at_1_max": 0.5205091937765203, "naucs_at_1_std": 0.17098692376526314, "naucs_at_1_diff1": 0.8923303409058108, "naucs_at_3_max": 0.6315592903828198, "naucs_at_3_std": 0.39726112667289276, "naucs_at_3_diff1": 0.8994708994709021, "naucs_at_5_max": 0.8410104782653828, "naucs_at_5_std": 0.6781823840647401, "naucs_at_5_diff1": 0.8778400248988462, "naucs_at_10_max": 0.8410104782653828, "naucs_at_10_std": 0.6781823840647401, "naucs_at_10_diff1": 0.8778400248988462, "naucs_at_20_max": 0.8358232181761658, "naucs_at_20_std": 0.5635698723934072, "naucs_at_20_diff1": 0.9128540305010935, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "shiftproject_test": { "ndcg_at_1": 0.74, "ndcg_at_3": 0.86595, "ndcg_at_5": 0.87843, "ndcg_at_10": 0.885, "ndcg_at_20": 0.885, "ndcg_at_50": 0.885, "ndcg_at_100": 0.885, "map_at_1": 0.74, "map_at_3": 0.83667, "map_at_5": 0.84367, "map_at_10": 0.84644, "map_at_20": 0.84644, "map_at_50": 0.84644, "map_at_100": 0.84644, "recall_at_1": 0.74, "recall_at_3": 0.95, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.74, "precision_at_3": 0.31667, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.76, "mrr_at_3": 0.8483333333333333, "mrr_at_5": 0.8553333333333333, "mrr_at_10": 0.8581111111111112, "mrr_at_20": 0.8581111111111112, "mrr_at_50": 0.8581111111111112, "mrr_at_100": 0.8581111111111112, "naucs_at_1_max": 0.03039211710798386, "naucs_at_1_std": -0.4475690361299212, "naucs_at_1_diff1": 0.8377194760958598, "naucs_at_3_max": 0.22334267040149194, "naucs_at_3_std": -0.2550887021475216, "naucs_at_3_diff1": 0.6671335200746928, "naucs_at_5_max": 0.4960317460317504, "naucs_at_5_std": 0.07936507936508234, "naucs_at_5_diff1": 0.9346405228758136, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.92, "ndcg_at_3": 0.96286, "ndcg_at_5": 0.96672, "ndcg_at_10": 0.96672, "ndcg_at_20": 0.96672, "ndcg_at_50": 0.96672, "ndcg_at_100": 0.96672, "map_at_1": 0.92, "map_at_3": 0.95333, "map_at_5": 0.95533, "map_at_10": 0.95533, "map_at_20": 0.95533, "map_at_50": 0.95533, "map_at_100": 0.95533, "recall_at_1": 0.92, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.9533333333333335, "mrr_at_5": 0.9553333333333334, "mrr_at_10": 0.9553333333333334, "mrr_at_20": 0.9553333333333334, "mrr_at_50": 0.9553333333333334, "mrr_at_100": 0.9553333333333334, "naucs_at_1_max": 0.5717787114845921, "naucs_at_1_std": 0.13211951447245573, "naucs_at_1_diff1": 0.9162581699346404, "naucs_at_3_max": 1.0, "naucs_at_3_std": -0.1713352007469878, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "infovqa_test_subsampled": { "ndcg_at_1": 0.90688, "ndcg_at_3": 0.9344, "ndcg_at_5": 0.93789, "ndcg_at_10": 0.94192, "ndcg_at_20": 0.94343, "ndcg_at_50": 0.94506, "ndcg_at_100": 0.94608, "map_at_1": 0.90688, "map_at_3": 0.9278, "map_at_5": 0.92982, "map_at_10": 0.93155, "map_at_20": 0.93195, "map_at_50": 0.93222, "map_at_100": 0.93232, "recall_at_1": 0.90688, "recall_at_3": 0.95344, "recall_at_5": 0.96154, "recall_at_10": 0.97368, "recall_at_20": 0.97976, "recall_at_50": 0.98785, "recall_at_100": 0.99393, "precision_at_1": 0.90688, "precision_at_3": 0.31781, "precision_at_5": 0.19231, "precision_at_10": 0.09737, "precision_at_20": 0.04899, "precision_at_50": 0.01976, "precision_at_100": 0.00994, "mrr_at_1": 0.9089068825910931, "mrr_at_3": 0.9274628879892036, "mrr_at_5": 0.9313090418353576, "mrr_at_10": 0.9324055330634279, "mrr_at_20": 0.9328226194396553, "mrr_at_50": 0.9330837473646573, "mrr_at_100": 0.9331803487251008, "naucs_at_1_max": 0.684485676449017, "naucs_at_1_std": -0.034356881205015385, "naucs_at_1_diff1": 0.9485356955914271, "naucs_at_3_max": 0.8100214035543719, "naucs_at_3_std": 0.09195903861300392, "naucs_at_3_diff1": 0.9659300554613642, "naucs_at_5_max": 0.8399269173431374, "naucs_at_5_std": 0.3023006641912601, "naucs_at_5_diff1": 0.9587574355584939, "naucs_at_10_max": 0.9091549335326204, "naucs_at_10_std": 0.5518694240613073, "naucs_at_10_diff1": 0.9497686715135494, "naucs_at_20_max": 0.9461062354112437, "naucs_at_20_std": 0.7048540732016966, "naucs_at_20_diff1": 0.9477594183740975, "naucs_at_50_max": 0.9319439680295378, "naucs_at_50_std": 0.7613218810068237, "naucs_at_50_diff1": 0.9564661819783937, "naucs_at_100_max": 0.8638879360590712, "naucs_at_100_std": 0.5661775800352391, "naucs_at_100_diff1": 0.9564661819784259 }, "syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.97, "ndcg_at_3": 0.98262, "ndcg_at_5": 0.98693, "ndcg_at_10": 0.98693, "ndcg_at_20": 0.98693, "ndcg_at_50": 0.98693, "ndcg_at_100": 0.98693, "map_at_1": 0.97, "map_at_3": 0.98, "map_at_5": 0.9825, "map_at_10": 0.9825, "map_at_20": 0.9825, "map_at_50": 0.9825, "map_at_100": 0.9825, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.985, "mrr_at_5": 0.9875, "mrr_at_10": 0.9875, "mrr_at_20": 0.9875, "mrr_at_50": 0.9875, "mrr_at_100": 0.9875, "naucs_at_1_max": 0.7152194211017764, "naucs_at_1_std": -0.14519140989729698, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.5541549953314585, "naucs_at_3_std": 0.35807656395889226, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_energy_test": { "ndcg_at_1": 0.93, "ndcg_at_3": 0.96024, "ndcg_at_5": 0.96024, "ndcg_at_10": 0.96325, "ndcg_at_20": 0.96325, "ndcg_at_50": 0.96529, "ndcg_at_100": 0.96529, "map_at_1": 0.93, "map_at_3": 0.95333, "map_at_5": 0.95333, "map_at_10": 0.95444, "map_at_20": 0.95444, "map_at_50": 0.95479, "map_at_100": 0.95479, "recall_at_1": 0.93, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9633333333333333, "mrr_at_5": 0.9633333333333333, "mrr_at_10": 0.9647619047619047, "mrr_at_20": 0.9647619047619047, "mrr_at_50": 0.9651190476190474, "mrr_at_100": 0.9651190476190474, "naucs_at_1_max": 0.16613311991463284, "naucs_at_1_std": -0.8660130718954275, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.4561157796451816, "naucs_at_3_std": -1.7399626517273692, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.45611577964519334, "naucs_at_5_std": -1.7399626517273414, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.5541549953314738, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.5541549953314738, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "arxivqa_test_subsampled": { "ndcg_at_1": 0.852, "ndcg_at_3": 0.88502, "ndcg_at_5": 0.89329, "ndcg_at_10": 0.90249, "ndcg_at_20": 0.90596, "ndcg_at_50": 0.90954, "ndcg_at_100": 0.91052, "map_at_1": 0.852, "map_at_3": 0.87767, "map_at_5": 0.88227, "map_at_10": 0.88615, "map_at_20": 0.88707, "map_at_50": 0.88764, "map_at_100": 0.88773, "recall_at_1": 0.852, "recall_at_3": 0.906, "recall_at_5": 0.926, "recall_at_10": 0.954, "recall_at_20": 0.968, "recall_at_50": 0.986, "recall_at_100": 0.992, "precision_at_1": 0.852, "precision_at_3": 0.302, "precision_at_5": 0.1852, "precision_at_10": 0.0954, "precision_at_20": 0.0484, "precision_at_50": 0.01972, "precision_at_100": 0.00992, "mrr_at_1": 0.848, "mrr_at_3": 0.8773333333333334, "mrr_at_5": 0.8805333333333333, "mrr_at_10": 0.8846658730158731, "mrr_at_20": 0.8852919413919415, "mrr_at_50": 0.8859760655763251, "mrr_at_100": 0.8860616142188591, "naucs_at_1_max": 0.7993319630010277, "naucs_at_1_std": -0.22024797744222174, "naucs_at_1_diff1": 0.942749097425356, "naucs_at_3_max": 0.8195363251683654, "naucs_at_3_std": -0.03467628186026151, "naucs_at_3_diff1": 0.9137811152829927, "naucs_at_5_max": 0.849925555807911, "naucs_at_5_std": 0.13232139702728, "naucs_at_5_diff1": 0.9081434375552027, "naucs_at_10_max": 0.9036861121260074, "naucs_at_10_std": 0.22725611983923233, "naucs_at_10_diff1": 0.9481386757601598, "naucs_at_20_max": 0.9139239028944934, "naucs_at_20_std": 0.3862336601307174, "naucs_at_20_diff1": 0.9499591503267956, "naucs_at_50_max": 0.9626517273576126, "naucs_at_50_std": 0.7269574496465168, "naucs_at_50_diff1": 0.981325863678799, "naucs_at_100_max": 0.9346405228758211, "naucs_at_100_std": 0.5875350140056017, "naucs_at_100_diff1": 0.9673202614378978 }, "tatdqa_test": { "ndcg_at_1": 0.7181, "ndcg_at_3": 0.79929, "ndcg_at_5": 0.8165, "ndcg_at_10": 0.8306, "ndcg_at_20": 0.83575, "ndcg_at_50": 0.83865, "ndcg_at_100": 0.84014, "map_at_1": 0.7181, "map_at_3": 0.77947, "map_at_5": 0.78913, "map_at_10": 0.79492, "map_at_20": 0.79637, "map_at_50": 0.79684, "map_at_100": 0.79697, "recall_at_1": 0.7181, "recall_at_3": 0.85662, "recall_at_5": 0.89793, "recall_at_10": 0.94168, "recall_at_20": 0.96173, "recall_at_50": 0.97631, "recall_at_100": 0.98542, "precision_at_1": 0.7181, "precision_at_3": 0.28554, "precision_at_5": 0.17959, "precision_at_10": 0.09417, "precision_at_20": 0.04809, "precision_at_50": 0.01953, "precision_at_100": 0.00985, "mrr_at_1": 0.7144592952612394, "mrr_at_3": 0.7782503037667065, "mrr_at_5": 0.7879708383961115, "mrr_at_10": 0.7937686840633371, "mrr_at_20": 0.7953080049252534, "mrr_at_50": 0.7957658867171664, "mrr_at_100": 0.7958829747123382, "naucs_at_1_max": 0.26480172996152473, "naucs_at_1_std": -0.2910939798616077, "naucs_at_1_diff1": 0.8226089243817, "naucs_at_3_max": 0.2836306744382603, "naucs_at_3_std": -0.2425266495199308, "naucs_at_3_diff1": 0.7433439016425463, "naucs_at_5_max": 0.3340537303336632, "naucs_at_5_std": -0.16558602572353986, "naucs_at_5_diff1": 0.6954888652470635, "naucs_at_10_max": 0.4610181387271507, "naucs_at_10_std": 0.06304388076961981, "naucs_at_10_diff1": 0.6863193747748649, "naucs_at_20_max": 0.5671613152812367, "naucs_at_20_std": 0.32459556489147223, "naucs_at_20_diff1": 0.6703229684339075, "naucs_at_50_max": 0.6599121428507998, "naucs_at_50_std": 0.5026653248908133, "naucs_at_50_diff1": 0.6633001445724545, "naucs_at_100_max": 0.605271276246271, "naucs_at_100_std": 0.4253037050547079, "naucs_at_100_diff1": 0.6371839984422268 }, "docvqa_test_subsampled": { "ndcg_at_1": 0.54767, "ndcg_at_3": 0.61094, "ndcg_at_5": 0.63059, "ndcg_at_10": 0.65164, "ndcg_at_20": 0.66158, "ndcg_at_50": 0.67508, "ndcg_at_100": 0.68269, "map_at_1": 0.54767, "map_at_3": 0.5946, "map_at_5": 0.60525, "map_at_10": 0.61407, "map_at_20": 0.61673, "map_at_50": 0.61902, "map_at_100": 0.61971, "recall_at_1": 0.54767, "recall_at_3": 0.65854, "recall_at_5": 0.70732, "recall_at_10": 0.77162, "recall_at_20": 0.81153, "recall_at_50": 0.87805, "recall_at_100": 0.92461, "precision_at_1": 0.54767, "precision_at_3": 0.21951, "precision_at_5": 0.14146, "precision_at_10": 0.07716, "precision_at_20": 0.04058, "precision_at_50": 0.01756, "precision_at_100": 0.00925, "mrr_at_1": 0.5365853658536586, "mrr_at_3": 0.5864745011086476, "mrr_at_5": 0.5981152993348114, "mrr_at_10": 0.6069440045049799, "mrr_at_20": 0.6100419544214033, "mrr_at_50": 0.6121130002051095, "mrr_at_100": 0.6127512907121658, "naucs_at_1_max": 0.5857531689741774, "naucs_at_1_std": 0.06216122332092845, "naucs_at_1_diff1": 0.878278564197993, "naucs_at_3_max": 0.5141987790210335, "naucs_at_3_std": 0.05930889116083822, "naucs_at_3_diff1": 0.8264615764155229, "naucs_at_5_max": 0.49165696501987366, "naucs_at_5_std": 0.05690890861421505, "naucs_at_5_diff1": 0.799920782349014, "naucs_at_10_max": 0.4488716182693942, "naucs_at_10_std": 0.16018934235908214, "naucs_at_10_diff1": 0.785204886495734, "naucs_at_20_max": 0.3726284567565117, "naucs_at_20_std": 0.19462128087094807, "naucs_at_20_diff1": 0.7820678233644752, "naucs_at_50_max": 0.41854245341334345, "naucs_at_50_std": 0.5079871796073263, "naucs_at_50_diff1": 0.7542202790434573, "naucs_at_100_max": 0.462863548345596, "naucs_at_100_std": 0.7287339158082113, "naucs_at_100_diff1": 0.7094241075282547 } }