| {"arxivqa_test_subsampled": {"ndcg_at_1": 0.898, "ndcg_at_3": 0.92371, "ndcg_at_5": 0.9318, "ndcg_at_10": 0.93447, "ndcg_at_20": 0.93956, "ndcg_at_100": 0.94144, "ndcg_at_1000": 0.94169, "map_at_1": 0.898, "map_at_3": 0.918, "map_at_5": 0.9224, "map_at_10": 0.92355, "map_at_20": 0.92498, "map_at_100": 0.92524, "map_at_1000": 0.92525, "recall_at_1": 0.898, "recall_at_3": 0.94, "recall_at_5": 0.96, "recall_at_10": 0.968, "recall_at_20": 0.988, "recall_at_100": 0.998, "recall_at_1000": 1.0, "precision_at_1": 0.898, "precision_at_3": 0.31333, "precision_at_5": 0.192, "precision_at_10": 0.0968, "precision_at_20": 0.0494, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "mrr_at_1": 0.9, "mrr_at_3": 0.919, "mrr_at_5": 0.9233999999999999, "mrr_at_10": 0.9243999999999999, "mrr_at_20": 0.9259168496106575, "mrr_at_100": 0.9262595842745325, "mrr_at_1000": 0.9262680949128302, "naucs_at_1_max": 0.7899084084359674, "naucs_at_1_std": -0.05284411079091805, "naucs_at_1_diff1": 0.9286907362199668, "naucs_at_3_max": 0.877528789293499, "naucs_at_3_std": 0.022984749455341305, "naucs_at_3_diff1": 0.938453159041396, "naucs_at_5_max": 0.8712651727357632, "naucs_at_5_std": -0.14498132586368048, "naucs_at_5_diff1": 0.9468954248366002, "naucs_at_10_max": 0.8751167133520102, "naucs_at_10_std": -0.12683823529412844, "naucs_at_10_diff1": 0.9499591503267956, "naucs_at_20_max": 0.9782135076252653, "naucs_at_20_std": -0.012060379707453646, "naucs_at_20_diff1": 0.9564270152505441, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.56319, "ndcg_at_3": 0.63707, "ndcg_at_5": 0.65645, "ndcg_at_10": 0.66914, "ndcg_at_20": 0.68323, "ndcg_at_100": 0.69828, "ndcg_at_1000": 0.7092, "map_at_1": 0.56319, "map_at_3": 0.61826, "map_at_5": 0.62912, "map_at_10": 0.63424, "map_at_20": 0.63815, "map_at_100": 0.64018, "map_at_1000": 0.64061, "recall_at_1": 0.56319, "recall_at_3": 0.6918, "recall_at_5": 0.73836, "recall_at_10": 0.77827, "recall_at_20": 0.8337, "recall_at_100": 0.91574, "recall_at_1000": 1.0, "precision_at_1": 0.56319, "precision_at_3": 0.2306, "precision_at_5": 0.14767, "precision_at_10": 0.07783, "precision_at_20": 0.04169, "precision_at_100": 0.00916, "precision_at_1000": 0.001, "mrr_at_1": 0.5587583148558758, "mrr_at_3": 0.6145602365114562, "mrr_at_5": 0.6263118994826311, "mrr_at_10": 0.6312814556716995, "mrr_at_20": 0.6344677162848683, "mrr_at_100": 0.6368185594184747, "mrr_at_1000": 0.6372276029808464, "naucs_at_1_max": 0.4955028063271212, "naucs_at_1_std": 0.13834714245445492, "naucs_at_1_diff1": 0.874582814697347, "naucs_at_3_max": 0.4387271034051574, "naucs_at_3_std": 0.18484588189907436, "naucs_at_3_diff1": 0.7875362331468366, "naucs_at_5_max": 0.40797185094589483, "naucs_at_5_std": 0.33451687872869357, "naucs_at_5_diff1": 0.7797450509845094, "naucs_at_10_max": 0.3519077449526463, "naucs_at_10_std": 0.436541459053009, "naucs_at_10_diff1": 0.7700892280462437, "naucs_at_20_max": 0.27510679111583564, "naucs_at_20_std": 0.5246755736075187, "naucs_at_20_diff1": 0.7557956281490941, "naucs_at_100_max": 0.14673712974932374, "naucs_at_100_std": 0.868870193792784, "naucs_at_100_diff1": 0.7602063422350581, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "infovqa_test_subsampled": {"ndcg_at_1": 0.89271, "ndcg_at_3": 0.92332, "ndcg_at_5": 0.93011, "ndcg_at_10": 0.93463, "ndcg_at_20": 0.93571, "ndcg_at_100": 0.93872, "ndcg_at_1000": 0.93952, "map_at_1": 0.89271, "map_at_3": 0.91633, "map_at_5": 0.92018, "map_at_10": 0.922, "map_at_20": 0.92233, "map_at_100": 0.92274, "map_at_1000": 0.92277, "recall_at_1": 0.89271, "recall_at_3": 0.94332, "recall_at_5": 0.95951, "recall_at_10": 0.97368, "recall_at_20": 0.97773, "recall_at_100": 0.99393, "recall_at_1000": 1.0, "precision_at_1": 0.89271, "precision_at_3": 0.31444, "precision_at_5": 0.1919, "precision_at_10": 0.09737, "precision_at_20": 0.04889, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "mrr_at_1": 0.8927125506072875, "mrr_at_3": 0.915991902834008, "mrr_at_5": 0.9202429149797569, "mrr_at_10": 0.92152978600347, "mrr_at_20": 0.9220243572217254, "mrr_at_100": 0.9224499564127232, "mrr_at_1000": 0.9224803027234936, "naucs_at_1_max": 0.630222668714959, "naucs_at_1_std": -0.16437919548224778, "naucs_at_1_diff1": 0.945449894337394, "naucs_at_3_max": 0.7571077766020037, "naucs_at_3_std": -0.07810407547393516, "naucs_at_3_diff1": 0.9409220870784822, "naucs_at_5_max": 0.8436958283279918, "naucs_at_5_std": 0.1321783849886454, "naucs_at_5_diff1": 0.9461062354112442, "naucs_at_10_max": 0.8223529961696429, "naucs_at_10_std": 0.1964949763377099, "naucs_at_10_diff1": 0.9170865160173047, "naucs_at_20_max": 0.8964958869122784, "naucs_at_20_std": 0.4419658243478526, "naucs_at_20_diff1": 0.927259949634811, "naucs_at_100_max": 0.9564661819784259, "naucs_at_100_std": 0.39174816052979483, "naucs_at_100_diff1": 0.9564661819784259, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.875, "ndcg_at_3": 0.89707, "ndcg_at_5": 0.90306, "ndcg_at_10": 0.91077, "ndcg_at_20": 0.91552, "ndcg_at_100": 0.92202, "ndcg_at_1000": 0.92202, "map_at_1": 0.875, "map_at_3": 0.89226, "map_at_5": 0.89565, "map_at_10": 0.89862, "map_at_20": 0.90003, "map_at_100": 0.90114, "map_at_1000": 0.90114, "recall_at_1": 0.875, "recall_at_3": 0.91071, "recall_at_5": 0.925, "recall_at_10": 0.95, "recall_at_20": 0.96786, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.875, "precision_at_3": 0.30357, "precision_at_5": 0.185, "precision_at_10": 0.095, "precision_at_20": 0.04839, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.8714285714285714, "mrr_at_3": 0.8904761904761905, "mrr_at_5": 0.8938690476190476, "mrr_at_10": 0.8968934240362811, "mrr_at_20": 0.8983128379556949, "mrr_at_100": 0.8994147218051182, "mrr_at_1000": 0.8994147218051182, "naucs_at_1_max": 0.6103821394329553, "naucs_at_1_std": 0.5049034378852207, "naucs_at_1_diff1": 0.8316258046842889, "naucs_at_3_max": 0.6529971988795523, "naucs_at_3_std": 0.5739495798319322, "naucs_at_3_diff1": 0.8558356676003747, "naucs_at_5_max": 0.6348984038059658, "naucs_at_5_std": 0.5578231292517003, "naucs_at_5_diff1": 0.8478280200969256, "naucs_at_10_max": 0.7962518340669607, "naucs_at_10_std": 0.7304588502067524, "naucs_at_10_diff1": 0.8619447779111657, "naucs_at_20_max": 0.8451602863367568, "naucs_at_20_std": 0.7526195663450606, "naucs_at_20_diff1": 0.8451602863367568, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tatdqa_test": {"ndcg_at_1": 0.68591, "ndcg_at_3": 0.78261, "ndcg_at_5": 0.80502, "ndcg_at_10": 0.81862, "ndcg_at_20": 0.82317, "ndcg_at_100": 0.82826, "ndcg_at_1000": 0.82941, "map_at_1": 0.68591, "map_at_3": 0.75891, "map_at_5": 0.77158, "map_at_10": 0.77733, "map_at_20": 0.77863, "map_at_100": 0.77929, "map_at_1000": 0.77935, "recall_at_1": 0.68591, "recall_at_3": 0.85115, "recall_at_5": 0.90462, "recall_at_10": 0.94593, "recall_at_20": 0.96355, "recall_at_100": 0.99149, "recall_at_1000": 1.0, "precision_at_1": 0.68591, "precision_at_3": 0.28372, "precision_at_5": 0.18092, "precision_at_10": 0.09459, "precision_at_20": 0.04818, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "mrr_at_1": 0.6792223572296476, "mrr_at_3": 0.7560753341433779, "mrr_at_5": 0.768195625759417, "mrr_at_10": 0.7743227931107642, "mrr_at_20": 0.7755453925088823, "mrr_at_100": 0.776194844308675, "mrr_at_1000": 0.7762476549706333, "naucs_at_1_max": 0.2606010753831049, "naucs_at_1_std": -0.18473191791144652, "naucs_at_1_diff1": 0.8262201682503303, "naucs_at_3_max": 0.26218905367154793, "naucs_at_3_std": -0.19512597148163965, "naucs_at_3_diff1": 0.7261950096473103, "naucs_at_5_max": 0.3116145973112506, "naucs_at_5_std": -0.06603484394073734, "naucs_at_5_diff1": 0.6744441292997129, "naucs_at_10_max": 0.4173503769044982, "naucs_at_10_std": 0.15513110190627194, "naucs_at_10_diff1": 0.6614583125618199, "naucs_at_20_max": 0.4965273279990691, "naucs_at_20_std": 0.2918864185658315, "naucs_at_20_diff1": 0.6989831888228779, "naucs_at_100_max": 0.44210430745767804, "naucs_at_100_std": 0.3792107382178212, "naucs_at_100_diff1": 0.6977475231013419, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.74, "ndcg_at_3": 0.84202, "ndcg_at_5": 0.85837, "ndcg_at_10": 0.86798, "ndcg_at_20": 0.87077, "ndcg_at_100": 0.87245, "ndcg_at_1000": 0.87245, "map_at_1": 0.74, "map_at_3": 0.81833, "map_at_5": 0.82733, "map_at_10": 0.83125, "map_at_20": 0.83216, "map_at_100": 0.83233, "map_at_1000": 0.83233, "recall_at_1": 0.74, "recall_at_3": 0.91, "recall_at_5": 0.95, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.74, "precision_at_3": 0.30333, "precision_at_5": 0.19, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.74, "mrr_at_3": 0.82, "mrr_at_5": 0.8320000000000001, "mrr_at_10": 0.8350952380952381, "mrr_at_20": 0.8360043290043291, "mrr_at_100": 0.8361797676008202, "mrr_at_1000": 0.8361797676008202, "naucs_at_1_max": -0.012874579295244135, "naucs_at_1_std": -0.39635456794128376, "naucs_at_1_diff1": 0.8253720449292407, "naucs_at_3_max": 0.3808486357505968, "naucs_at_3_std": -0.23451602863367743, "naucs_at_3_diff1": 0.7072310405643747, "naucs_at_5_max": 0.5085901027077518, "naucs_at_5_std": -0.05760971055088009, "naucs_at_5_diff1": 0.8323996265172763, "naucs_at_10_max": 0.27544351073763346, "naucs_at_10_std": -0.3968253968253954, "naucs_at_10_diff1": 0.8692810457516353, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.99, "ndcg_at_3": 0.99, "ndcg_at_5": 0.99431, "ndcg_at_10": 0.99431, "ndcg_at_20": 0.99431, "ndcg_at_100": 0.99431, "ndcg_at_1000": 0.99431, "map_at_1": 0.99, "map_at_3": 0.99, "map_at_5": 0.9925, "map_at_10": 0.9925, "map_at_20": 0.9925, "map_at_100": 0.9925, "map_at_1000": 0.9925, "recall_at_1": 0.99, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.99, "mrr_at_3": 0.99, "mrr_at_5": 0.9925, "mrr_at_10": 0.9925, "mrr_at_20": 0.9925, "mrr_at_100": 0.9925, "mrr_at_1000": 0.9925, "naucs_at_1_max": 1.0, "naucs_at_1_std": 0.7222222222222201, "naucs_at_1_diff1": 0.8692810457516276, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.7222222222222157, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.93, "ndcg_at_3": 0.96024, "ndcg_at_5": 0.96024, "ndcg_at_10": 0.96325, "ndcg_at_20": 0.96325, "ndcg_at_100": 0.9652, "ndcg_at_1000": 0.9652, "map_at_1": 0.93, "map_at_3": 0.95333, "map_at_5": 0.95333, "map_at_10": 0.95444, "map_at_20": 0.95444, "map_at_100": 0.95474, "map_at_1000": 0.95474, "recall_at_1": 0.93, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.95, "mrr_at_3": 0.9633333333333334, "mrr_at_5": 0.9633333333333334, "mrr_at_10": 0.9647619047619048, "mrr_at_20": 0.9647619047619048, "mrr_at_100": 0.9650560224089637, "mrr_at_1000": 0.9650560224089637, "naucs_at_1_max": 0.625917033480058, "naucs_at_1_std": -0.5802987861811411, "naucs_at_1_diff1": 0.9813258636788056, "naucs_at_3_max": 0.7770774976657261, "naucs_at_3_std": -1.4458450046685478, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7770774976657324, "naucs_at_5_std": -1.445845004668519, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.93, "ndcg_at_3": 0.97286, "ndcg_at_5": 0.97286, "ndcg_at_10": 0.97286, "ndcg_at_20": 0.97286, "ndcg_at_100": 0.97286, "ndcg_at_1000": 0.97286, "map_at_1": 0.93, "map_at_3": 0.96333, "map_at_5": 0.96333, "map_at_10": 0.96333, "map_at_20": 0.96333, "map_at_100": 0.96333, "map_at_1000": 0.96333, "recall_at_1": 0.93, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.94, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9683333333333333, "mrr_at_10": 0.9683333333333333, "mrr_at_20": 0.9683333333333333, "mrr_at_100": 0.9683333333333333, "mrr_at_1000": 0.9683333333333333, "naucs_at_1_max": 0.6949446445244779, "naucs_at_1_std": 0.41516606642656995, "naucs_at_1_diff1": 0.8989595838335339, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98762, "ndcg_at_5": 0.98762, "ndcg_at_10": 0.98762, "ndcg_at_20": 0.98762, "ndcg_at_100": 0.98762, "ndcg_at_1000": 0.98762, "map_at_1": 0.97, "map_at_3": 0.98333, "map_at_5": 0.98333, "map_at_10": 0.98333, "map_at_20": 0.98333, "map_at_100": 0.98333, "map_at_1000": 0.98333, "recall_at_1": 0.97, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.97, "mrr_at_3": 0.9833333333333334, "mrr_at_5": 0.9833333333333334, "mrr_at_10": 0.9833333333333334, "mrr_at_20": 0.9833333333333334, "mrr_at_100": 0.9833333333333334, "mrr_at_1000": 0.9833333333333334, "naucs_at_1_max": 0.6591970121381903, "naucs_at_1_std": -0.20028011204482288, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |