| {"vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.66, "ndcg_at_3": 0.71012, "ndcg_at_5": 0.72638, "ndcg_at_10": 0.73988, "ndcg_at_20": 0.7531, "ndcg_at_100": 0.76637, "ndcg_at_1000": 0.77288, "map_at_1": 0.66, "map_at_3": 0.69833, "map_at_5": 0.70723, "map_at_10": 0.71276, "map_at_20": 0.71641, "map_at_100": 0.71808, "map_at_1000": 0.71839, "recall_at_1": 0.66, "recall_at_3": 0.744, "recall_at_5": 0.784, "recall_at_10": 0.826, "recall_at_20": 0.878, "recall_at_100": 0.952, "recall_at_1000": 1.0, "precision_at_1": 0.66, "precision_at_3": 0.248, "precision_at_5": 0.1568, "precision_at_10": 0.0826, "precision_at_20": 0.0439, "precision_at_100": 0.00952, "precision_at_1000": 0.001, "mrr_at_1": 0.658, "mrr_at_3": 0.697, "mrr_at_5": 0.7062999999999998, "mrr_at_10": 0.711679365079365, "mrr_at_20": 0.715144602207218, "mrr_at_100": 0.7168165983768064, "mrr_at_1000": 0.7171178855275856, "naucs_at_1_max": 0.7385708121743169, "naucs_at_1_std": -0.016684921504198747, "naucs_at_1_diff1": 0.8764977264429615, "naucs_at_3_max": 0.7171890142742163, "naucs_at_3_std": 0.0014413837283801306, "naucs_at_3_diff1": 0.8231163455378698, "naucs_at_5_max": 0.6995333433689601, "naucs_at_5_std": -0.037054982688545776, "naucs_at_5_diff1": 0.8149320713533046, "naucs_at_10_max": 0.682716456242827, "naucs_at_10_std": -0.03228519154062102, "naucs_at_10_diff1": 0.7960953214314127, "naucs_at_20_max": 0.6569500399742892, "naucs_at_20_std": -0.004577448229376444, "naucs_at_20_diff1": 0.7636970732548469, "naucs_at_100_max": 0.6544117647058815, "naucs_at_100_std": 0.031609866168690236, "naucs_at_100_diff1": 0.7494553376906276, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.35255, "ndcg_at_3": 0.43512, "ndcg_at_5": 0.4562, "ndcg_at_10": 0.47553, "ndcg_at_20": 0.49272, "ndcg_at_100": 0.5165, "ndcg_at_1000": 0.54161, "map_at_1": 0.35255, "map_at_3": 0.41463, "map_at_5": 0.42639, "map_at_10": 0.43434, "map_at_20": 0.43925, "map_at_100": 0.44238, "map_at_1000": 0.4433, "recall_at_1": 0.35255, "recall_at_3": 0.49446, "recall_at_5": 0.54545, "recall_at_10": 0.60532, "recall_at_20": 0.67184, "recall_at_100": 0.80266, "recall_at_1000": 1.0, "precision_at_1": 0.35255, "precision_at_3": 0.16482, "precision_at_5": 0.10909, "precision_at_10": 0.06053, "precision_at_20": 0.03359, "precision_at_100": 0.00803, "precision_at_1000": 0.001, "mrr_at_1": 0.35698447893569846, "mrr_at_3": 0.4161123429416113, "mrr_at_5": 0.42886178861788615, "mrr_at_10": 0.4363697955161369, "mrr_at_20": 0.44159612220354433, "mrr_at_100": 0.44462088540450384, "mrr_at_1000": 0.44554137996111115, "naucs_at_1_max": 0.6419929263025314, "naucs_at_1_std": -0.021897961771829243, "naucs_at_1_diff1": 0.7795928377598447, "naucs_at_3_max": 0.659701421177501, "naucs_at_3_std": 0.022288103939816777, "naucs_at_3_diff1": 0.6995028697301144, "naucs_at_5_max": 0.6716555716611096, "naucs_at_5_std": 0.014219104761796688, "naucs_at_5_diff1": 0.7037331573171028, "naucs_at_10_max": 0.6330510884467964, "naucs_at_10_std": 0.020934876205970728, "naucs_at_10_diff1": 0.6246045647421675, "naucs_at_20_max": 0.6251996582923346, "naucs_at_20_std": 0.04147502689765851, "naucs_at_20_diff1": 0.5989999988952245, "naucs_at_100_max": 0.6272739458730339, "naucs_at_100_std": 0.13211715215371614, "naucs_at_100_diff1": 0.5664109833036436, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.75506, "ndcg_at_3": 0.80887, "ndcg_at_5": 0.82247, "ndcg_at_10": 0.83356, "ndcg_at_20": 0.84099, "ndcg_at_100": 0.8487, "ndcg_at_1000": 0.85106, "map_at_1": 0.75506, "map_at_3": 0.79656, "map_at_5": 0.80425, "map_at_10": 0.80881, "map_at_20": 0.81098, "map_at_100": 0.81198, "map_at_1000": 0.81208, "recall_at_1": 0.75506, "recall_at_3": 0.84413, "recall_at_5": 0.87652, "recall_at_10": 0.91093, "recall_at_20": 0.93927, "recall_at_100": 0.98178, "recall_at_1000": 1.0, "precision_at_1": 0.75506, "precision_at_3": 0.28138, "precision_at_5": 0.1753, "precision_at_10": 0.09109, "precision_at_20": 0.04696, "precision_at_100": 0.00982, "precision_at_1000": 0.001, "mrr_at_1": 0.7550607287449392, "mrr_at_3": 0.7968960863697705, "mrr_at_5": 0.8039811066126854, "mrr_at_10": 0.8086899942163099, "mrr_at_20": 0.8106877920047721, "mrr_at_100": 0.8116880057675456, "mrr_at_1000": 0.8117814231366781, "naucs_at_1_max": 0.48291111024989514, "naucs_at_1_std": -0.10313080159854918, "naucs_at_1_diff1": 0.817365009834527, "naucs_at_3_max": 0.5377100892275435, "naucs_at_3_std": 0.052978040966377885, "naucs_at_3_diff1": 0.7738851460598186, "naucs_at_5_max": 0.6512824505628394, "naucs_at_5_std": 0.2510392819379365, "naucs_at_5_diff1": 0.764215706684777, "naucs_at_10_max": 0.7901821869098035, "naucs_at_10_std": 0.4987334858766418, "naucs_at_10_diff1": 0.7844747956477697, "naucs_at_20_max": 0.7586867491927254, "naucs_at_20_std": 0.4825853972298199, "naucs_at_20_diff1": 0.748988136752461, "naucs_at_100_max": 0.7646511507177958, "naucs_at_100_std": 0.6225373240060292, "naucs_at_100_diff1": 0.6577400793442638, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.70714, "ndcg_at_3": 0.77814, "ndcg_at_5": 0.79459, "ndcg_at_10": 0.80528, "ndcg_at_20": 0.81338, "ndcg_at_100": 0.82664, "ndcg_at_1000": 0.82664, "map_at_1": 0.70714, "map_at_3": 0.76071, "map_at_5": 0.77, "map_at_10": 0.77458, "map_at_20": 0.77679, "map_at_100": 0.77885, "map_at_1000": 0.77885, "recall_at_1": 0.70714, "recall_at_3": 0.82857, "recall_at_5": 0.86786, "recall_at_10": 0.9, "recall_at_20": 0.93214, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.70714, "precision_at_3": 0.27619, "precision_at_5": 0.17357, "precision_at_10": 0.09, "precision_at_20": 0.04661, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.7107142857142857, "mrr_at_3": 0.7648809523809527, "mrr_at_5": 0.7722023809523811, "mrr_at_10": 0.7767828798185944, "mrr_at_20": 0.7791727347913773, "mrr_at_100": 0.7810664757173029, "mrr_at_1000": 0.7810664757173029, "naucs_at_1_max": 0.6915330786298528, "naucs_at_1_std": 0.24860503892761912, "naucs_at_1_diff1": 0.8727401630627436, "naucs_at_3_max": 0.658585006583279, "naucs_at_3_std": 0.36226547070441156, "naucs_at_3_diff1": 0.7699144173798553, "naucs_at_5_max": 0.632196940078713, "naucs_at_5_std": 0.33106575963718865, "naucs_at_5_diff1": 0.7243985716892118, "naucs_at_10_max": 0.6453414699213036, "naucs_at_10_std": 0.3285814325730309, "naucs_at_10_diff1": 0.7146525276777389, "naucs_at_20_max": 0.619465329991645, "naucs_at_20_std": 0.38466263698461656, "naucs_at_20_diff1": 0.7717332547053887, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/tatdqa_test": {"ndcg_at_1": 0.35905, "ndcg_at_3": 0.45955, "ndcg_at_5": 0.48875, "ndcg_at_10": 0.52138, "ndcg_at_20": 0.54604, "ndcg_at_100": 0.57257, "ndcg_at_1000": 0.58023, "map_at_1": 0.35905, "map_at_3": 0.43449, "map_at_5": 0.45065, "map_at_10": 0.46436, "map_at_20": 0.47117, "map_at_100": 0.47479, "map_at_1000": 0.47515, "recall_at_1": 0.35905, "recall_at_3": 0.5322, "recall_at_5": 0.60328, "recall_at_10": 0.70292, "recall_at_20": 0.80012, "recall_at_100": 0.94411, "recall_at_1000": 1.0, "precision_at_1": 0.35905, "precision_at_3": 0.1774, "precision_at_5": 0.12066, "precision_at_10": 0.07029, "precision_at_20": 0.04001, "precision_at_100": 0.00944, "precision_at_1000": 0.001, "mrr_at_1": 0.3560145808019441, "mrr_at_3": 0.43428513568246285, "mrr_at_5": 0.4502632644795469, "mrr_at_10": 0.46368541341202363, "mrr_at_20": 0.47041433878780475, "mrr_at_100": 0.47411079560392455, "mrr_at_1000": 0.474477002509729, "naucs_at_1_max": 0.23229903852025777, "naucs_at_1_std": -0.13316330891699873, "naucs_at_1_diff1": 0.6193305202910026, "naucs_at_3_max": 0.24897300081072463, "naucs_at_3_std": -0.1120890167788737, "naucs_at_3_diff1": 0.505537994218999, "naucs_at_5_max": 0.24581092556331594, "naucs_at_5_std": -0.08749552257576938, "naucs_at_5_diff1": 0.4750192401430266, "naucs_at_10_max": 0.2524311375141825, "naucs_at_10_std": -0.07024076554764465, "naucs_at_10_diff1": 0.4563406313127248, "naucs_at_20_max": 0.25065629081252544, "naucs_at_20_std": -0.05899110281173882, "naucs_at_20_diff1": 0.44774417576741526, "naucs_at_100_max": 0.2529620466202078, "naucs_at_100_std": -0.10310328044369356, "naucs_at_100_diff1": 0.44211198162984516, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/shiftproject_test": {"ndcg_at_1": 0.54, "ndcg_at_3": 0.6594, "ndcg_at_5": 0.69342, "ndcg_at_10": 0.72014, "ndcg_at_20": 0.72547, "ndcg_at_100": 0.73729, "ndcg_at_1000": 0.73869, "map_at_1": 0.54, "map_at_3": 0.62833, "map_at_5": 0.64783, "map_at_10": 0.65939, "map_at_20": 0.66099, "map_at_100": 0.6629, "map_at_1000": 0.66297, "recall_at_1": 0.54, "recall_at_3": 0.75, "recall_at_5": 0.83, "recall_at_10": 0.91, "recall_at_20": 0.93, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.54, "precision_at_3": 0.25, "precision_at_5": 0.166, "precision_at_10": 0.091, "precision_at_20": 0.0465, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.55, "mrr_at_3": 0.6333333333333334, "mrr_at_5": 0.6548333333333334, "mrr_at_10": 0.6648611111111111, "mrr_at_20": 0.6665394327894327, "mrr_at_100": 0.6684732775052104, "mrr_at_1000": 0.6685452199512534, "naucs_at_1_max": -0.05591675696701972, "naucs_at_1_std": -0.29843572004112157, "naucs_at_1_diff1": 0.506265455252702, "naucs_at_3_max": 0.14568288854003233, "naucs_at_3_std": -0.17789638932495938, "naucs_at_3_diff1": 0.3948299319727896, "naucs_at_5_max": 0.3704359435769431, "naucs_at_5_std": -0.05212747431357655, "naucs_at_5_diff1": 0.37644395425785043, "naucs_at_10_max": 0.33696441539578875, "naucs_at_10_std": -0.1554103122730556, "naucs_at_10_diff1": 0.406473700591347, "naucs_at_20_max": 0.33093237294917927, "naucs_at_20_std": -0.32052821128451325, "naucs_at_20_diff1": 0.3642790449513133, "naucs_at_100_max": 0.554154995331464, "naucs_at_100_std": 0.35807656395892007, "naucs_at_100_diff1": 0.7222222222222041, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.93, "ndcg_at_3": 0.96655, "ndcg_at_5": 0.96655, "ndcg_at_10": 0.9697, "ndcg_at_20": 0.9697, "ndcg_at_100": 0.9697, "ndcg_at_1000": 0.9697, "map_at_1": 0.93, "map_at_3": 0.95833, "map_at_5": 0.95833, "map_at_10": 0.95958, "map_at_20": 0.95958, "map_at_100": 0.95958, "map_at_1000": 0.95958, "recall_at_1": 0.93, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.94, "mrr_at_3": 0.9633333333333334, "mrr_at_5": 0.9633333333333334, "mrr_at_10": 0.9645833333333335, "mrr_at_20": 0.9645833333333335, "mrr_at_100": 0.9645833333333335, "mrr_at_1000": 0.9645833333333335, "naucs_at_1_max": 0.715953047885822, "naucs_at_1_std": -0.44497799119648057, "naucs_at_1_diff1": 0.8499399759903964, "naucs_at_3_max": 0.8692810457516356, "naucs_at_3_std": -1.1517273576097802, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.8692810457516413, "naucs_at_5_std": -1.1517273576097316, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.91762, "ndcg_at_5": 0.92623, "ndcg_at_10": 0.92979, "ndcg_at_20": 0.93743, "ndcg_at_100": 0.93743, "ndcg_at_1000": 0.93743, "map_at_1": 0.89, "map_at_3": 0.91, "map_at_5": 0.915, "map_at_10": 0.91667, "map_at_20": 0.91879, "map_at_100": 0.91879, "map_at_1000": 0.91879, "recall_at_1": 0.89, "recall_at_3": 0.94, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.31333, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.9116666666666666, "mrr_at_5": 0.9166666666666665, "mrr_at_10": 0.9183333333333333, "mrr_at_20": 0.9204973262032086, "mrr_at_100": 0.9204973262032086, "mrr_at_1000": 0.9204973262032086, "naucs_at_1_max": 0.6191048878962272, "naucs_at_1_std": -0.39472553904303725, "naucs_at_1_diff1": 0.9248346361996395, "naucs_at_3_max": 0.7149081854964202, "naucs_at_3_std": -0.18199502023031355, "naucs_at_3_diff1": 0.9101307189542478, "naucs_at_5_max": 0.6050420168067251, "naucs_at_5_std": -0.5536881419234266, "naucs_at_5_diff1": 0.967320261437907, "naucs_at_10_max": 0.5659819483348872, "naucs_at_10_std": -0.9229691876750711, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.84, "ndcg_at_3": 0.90678, "ndcg_at_5": 0.9197, "ndcg_at_10": 0.9197, "ndcg_at_20": 0.9246, "ndcg_at_100": 0.9246, "ndcg_at_1000": 0.9246, "map_at_1": 0.84, "map_at_3": 0.89167, "map_at_5": 0.89917, "map_at_10": 0.89917, "map_at_20": 0.90042, "map_at_100": 0.90042, "map_at_1000": 0.90042, "recall_at_1": 0.84, "recall_at_3": 0.95, "recall_at_5": 0.98, "recall_at_10": 0.98, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.84, "precision_at_3": 0.31667, "precision_at_5": 0.196, "precision_at_10": 0.098, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.84, "mrr_at_3": 0.8916666666666667, "mrr_at_5": 0.8991666666666666, "mrr_at_10": 0.8991666666666666, "mrr_at_20": 0.9005241327300151, "mrr_at_100": 0.9005241327300151, "mrr_at_1000": 0.9005241327300151, "naucs_at_1_max": 0.47717231222385836, "naucs_at_1_std": -0.15381075110456469, "naucs_at_1_diff1": 0.871287432498772, "naucs_at_3_max": 0.6462184873949546, "naucs_at_3_std": 0.08478057889822578, "naucs_at_3_diff1": 0.7637721755368763, "naucs_at_5_max": 0.7770774976657324, "naucs_at_5_std": 0.21825396825397442, "naucs_at_5_diff1": 0.7957516339869297, "naucs_at_10_max": 0.7770774976657324, "naucs_at_10_std": 0.21825396825397442, "naucs_at_10_diff1": 0.7957516339869297, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.9, "ndcg_at_3": 0.96047, "ndcg_at_5": 0.96047, "ndcg_at_10": 0.96047, "ndcg_at_20": 0.96047, "ndcg_at_100": 0.96047, "ndcg_at_1000": 0.96047, "map_at_1": 0.9, "map_at_3": 0.94667, "map_at_5": 0.94667, "map_at_10": 0.94667, "map_at_20": 0.94667, "map_at_100": 0.94667, "map_at_1000": 0.94667, "recall_at_1": 0.9, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.91, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9516666666666667, "mrr_at_10": 0.9516666666666667, "mrr_at_20": 0.9516666666666667, "mrr_at_100": 0.9516666666666667, "mrr_at_1000": 0.9516666666666667, "naucs_at_1_max": 0.5098972922502337, "naucs_at_1_std": -0.22072829131652666, "naucs_at_1_diff1": 0.8884220354808596, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |