diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,3 +1,8669 @@ ---- -license: gemma ---- +--- +model-index: +- name: google/Gemma-Embeddings-v1.0 + results: + - dataset: + config: en + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 94.6269 + - type: f1 + value: 91.9315 + - type: f1_weighted + value: 94.77029999999999 + - type: ap + value: 77.8258 + - type: ap_weighted + value: 77.8258 + - type: main_score + value: 94.6269 + task: + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification (default) + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 97.0382 + - type: f1 + value: 97.0377 + - type: f1_weighted + value: 97.0377 + - type: ap + value: 95.8721 + - type: ap_weighted + value: 95.8721 + - type: main_score + value: 97.0382 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 65.30799999999999 + - type: f1 + value: 64.4521 + - type: f1_weighted + value: 64.4521 + - type: main_score + value: 65.30799999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB ArguAna (default) + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana + metrics: + - type: ndcg_at_1 + value: 69.844 + - type: ndcg_at_3 + value: 82.047 + - type: ndcg_at_5 + value: 83.734 + - type: ndcg_at_10 + value: 84.821 + - type: ndcg_at_20 + value: 85.051 + - type: ndcg_at_100 + value: 85.231 + - type: ndcg_at_1000 + value: 85.231 + - type: map_at_1 + value: 69.844 + - type: map_at_3 + value: 79.125 + - type: map_at_5 + value: 80.071 + - type: map_at_10 + value: 80.537 + - type: map_at_20 + value: 80.598 + - type: map_at_100 + value: 80.626 + - type: map_at_1000 + value: 80.626 + - type: recall_at_1 + value: 69.844 + - type: recall_at_3 + value: 90.469 + - type: recall_at_5 + value: 94.523 + - type: recall_at_10 + value: 97.795 + - type: recall_at_20 + value: 98.72 + - type: recall_at_100 + value: 99.644 + - type: recall_at_1000 + value: 99.644 + - type: precision_at_1 + value: 69.844 + - type: precision_at_3 + value: 30.156 + - type: precision_at_5 + value: 18.905 + - type: precision_at_10 + value: 9.78 + - type: precision_at_20 + value: 4.936 + - type: precision_at_100 + value: 0.996 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 70.0569 + - type: mrr_at_3 + value: 79.20819999999999 + - type: mrr_at_5 + value: 80.1541 + - type: mrr_at_10 + value: 80.6206 + - type: mrr_at_20 + value: 80.6819 + - type: mrr_at_100 + value: 80.7099 + - type: mrr_at_1000 + value: 80.7099 + - type: nauc_ndcg_at_1_max + value: 4.4853 + - type: nauc_ndcg_at_1_std + value: -32.4139 + - type: nauc_ndcg_at_1_diff1 + value: 54.961000000000006 + - type: nauc_ndcg_at_3_max + value: 10.9114 + - type: nauc_ndcg_at_3_std + value: -33.466499999999996 + - type: nauc_ndcg_at_3_diff1 + value: 50.809599999999996 + - type: nauc_ndcg_at_5_max + value: 8.2551 + - type: nauc_ndcg_at_5_std + value: -33.0891 + - type: nauc_ndcg_at_5_diff1 + value: 50.942600000000006 + - type: nauc_ndcg_at_10_max + value: 8.9955 + - type: nauc_ndcg_at_10_std + value: -33.372 + - type: nauc_ndcg_at_10_diff1 + value: 52.88420000000001 + - type: nauc_ndcg_at_20_max + value: 8.0304 + - type: nauc_ndcg_at_20_std + value: -33.2286 + - type: nauc_ndcg_at_20_diff1 + value: 52.56459999999999 + - type: nauc_ndcg_at_100_max + value: 7.7877 + - type: nauc_ndcg_at_100_std + value: -32.5506 + - type: nauc_ndcg_at_100_diff1 + value: 52.207800000000006 + - type: nauc_ndcg_at_1000_max + value: 7.7877 + - type: nauc_ndcg_at_1000_std + value: -32.5506 + - type: nauc_ndcg_at_1000_diff1 + value: 52.207800000000006 + - type: nauc_map_at_1_max + value: 4.4853 + - type: nauc_map_at_1_std + value: -32.4139 + - type: nauc_map_at_1_diff1 + value: 54.961000000000006 + - type: nauc_map_at_3_max + value: 8.5875 + - type: nauc_map_at_3_std + value: -33.1539 + - type: nauc_map_at_3_diff1 + value: 51.7761 + - type: nauc_map_at_5_max + value: 7.2324 + - type: nauc_map_at_5_std + value: -32.9639 + - type: nauc_map_at_5_diff1 + value: 51.9064 + - type: nauc_map_at_10_max + value: 7.4474 + - type: nauc_map_at_10_std + value: -33.0762 + - type: nauc_map_at_10_diff1 + value: 52.580400000000004 + - type: nauc_map_at_20_max + value: 7.2379999999999995 + - type: nauc_map_at_20_std + value: -33.056400000000004 + - type: nauc_map_at_20_diff1 + value: 52.5111 + - type: nauc_map_at_100_max + value: 7.210800000000001 + - type: nauc_map_at_100_std + value: -32.9841 + - type: nauc_map_at_100_diff1 + value: 52.469100000000005 + - type: nauc_map_at_1000_max + value: 7.210800000000001 + - type: nauc_map_at_1000_std + value: -32.9841 + - type: nauc_map_at_1000_diff1 + value: 52.469100000000005 + - type: nauc_recall_at_1_max + value: 4.4853 + - type: nauc_recall_at_1_std + value: -32.4139 + - type: nauc_recall_at_1_diff1 + value: 54.961000000000006 + - type: nauc_recall_at_3_max + value: 24.187 + - type: nauc_recall_at_3_std + value: -35.2013 + - type: nauc_recall_at_3_diff1 + value: 45.690599999999996 + - type: nauc_recall_at_5_max + value: 16.9677 + - type: nauc_recall_at_5_std + value: -34.041700000000006 + - type: nauc_recall_at_5_diff1 + value: 42.5248 + - type: nauc_recall_at_10_max + value: 43.9168 + - type: nauc_recall_at_10_std + value: -39.8657 + - type: nauc_recall_at_10_diff1 + value: 66.1909 + - type: nauc_recall_at_20_max + value: 29.317300000000003 + - type: nauc_recall_at_20_std + value: -37.4268 + - type: nauc_recall_at_20_diff1 + value: 62.67660000000001 + - type: nauc_recall_at_100_max + value: 37.0551 + - type: nauc_recall_at_100_std + value: 85.8517 + - type: nauc_recall_at_100_diff1 + value: 21.2768 + - type: nauc_recall_at_1000_max + value: 37.0551 + - type: nauc_recall_at_1000_std + value: 85.8517 + - type: nauc_recall_at_1000_diff1 + value: 21.2768 + - type: nauc_precision_at_1_max + value: 4.4853 + - type: nauc_precision_at_1_std + value: -32.4139 + - type: nauc_precision_at_1_diff1 + value: 54.961000000000006 + - type: nauc_precision_at_3_max + value: 24.187 + - type: nauc_precision_at_3_std + value: -35.2013 + - type: nauc_precision_at_3_diff1 + value: 45.690599999999996 + - type: nauc_precision_at_5_max + value: 16.9677 + - type: nauc_precision_at_5_std + value: -34.041700000000006 + - type: nauc_precision_at_5_diff1 + value: 42.5248 + - type: nauc_precision_at_10_max + value: 43.9168 + - type: nauc_precision_at_10_std + value: -39.8657 + - type: nauc_precision_at_10_diff1 + value: 66.1909 + - type: nauc_precision_at_20_max + value: 29.317300000000003 + - type: nauc_precision_at_20_std + value: -37.4268 + - type: nauc_precision_at_20_diff1 + value: 62.67660000000001 + - type: nauc_precision_at_100_max + value: 37.0551 + - type: nauc_precision_at_100_std + value: 85.8517 + - type: nauc_precision_at_100_diff1 + value: 21.2768 + - type: nauc_precision_at_1000_max + value: 37.0551 + - type: nauc_precision_at_1000_std + value: 85.8517 + - type: nauc_precision_at_1000_diff1 + value: 21.2768 + - type: nauc_mrr_at_1_max + value: 4.6327 + - type: nauc_mrr_at_1_std + value: -32.4116 + - type: nauc_mrr_at_1_diff1 + value: 54.4129 + - type: nauc_mrr_at_3_max + value: 8.6301 + - type: nauc_mrr_at_3_std + value: -33.264700000000005 + - type: nauc_mrr_at_3_diff1 + value: 51.452 + - type: nauc_mrr_at_5_max + value: 7.273899999999999 + - type: nauc_mrr_at_5_std + value: -33.0802 + - type: nauc_mrr_at_5_diff1 + value: 51.5652 + - type: nauc_mrr_at_10_max + value: 7.4876 + - type: nauc_mrr_at_10_std + value: -33.2021 + - type: nauc_mrr_at_10_diff1 + value: 52.2296 + - type: nauc_mrr_at_20_max + value: 7.277699999999999 + - type: nauc_mrr_at_20_std + value: -33.1827 + - type: nauc_mrr_at_20_diff1 + value: 52.15880000000001 + - type: nauc_mrr_at_100_max + value: 7.249999999999999 + - type: nauc_mrr_at_100_std + value: -33.110299999999995 + - type: nauc_mrr_at_100_diff1 + value: 52.1158 + - type: nauc_mrr_at_1000_max + value: 7.249999999999999 + - type: nauc_mrr_at_1000_std + value: -33.110299999999995 + - type: nauc_mrr_at_1000_diff1 + value: 52.1158 + - type: main_score + value: 84.821 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringS2S (default) + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: v_measure + value: 50.022299999999994 + - type: v_measure_std + value: 14.4899 + - type: main_score + value: 50.022299999999994 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions (default) + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: map + value: 65.6388 + - type: mrr + value: 79.3677 + - type: nAUC_map_max + value: 31.682900000000004 + - type: nAUC_map_std + value: 20.0437 + - type: nAUC_map_diff1 + value: 8.7821 + - type: nAUC_mrr_max + value: 44.033 + - type: nAUC_mrr_std + value: 33.0875 + - type: nAUC_mrr_diff1 + value: 17.7949 + - type: main_score + value: 65.6388 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES (default) + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: pearson + value: 89.9755 + - type: spearman + value: 89.8099 + - type: cosine_pearson + value: 89.9755 + - type: cosine_spearman + value: 89.8099 + - type: manhattan_pearson + value: 87.7735 + - type: manhattan_spearman + value: 89.57310000000001 + - type: euclidean_pearson + value: 87.708 + - type: euclidean_spearman + value: 89.8099 + - type: main_score + value: 89.8099 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification (default) + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 93.16879999999999 + - type: f1 + value: 93.1524 + - type: f1_weighted + value: 93.1524 + - type: main_score + value: 93.16879999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P (default) + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: v_measure + value: 54.024499999999996 + - type: v_measure_std + value: 1.0512000000000001 + - type: main_score + value: 54.024499999999996 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S (default) + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: v_measure + value: 50.925799999999995 + - type: v_measure_std + value: 1.024 + - type: main_score + value: 50.925799999999995 + task: + type: Clustering + - dataset: + config: default + name: MTEB CQADupstackAndroidRetrieval (default) + revision: f46a197baaae43b4f621051089b82a364682dfeb + split: test + type: mteb/cqadupstack-android + metrics: + - type: ndcg_at_1 + value: 47.067 + - type: ndcg_at_3 + value: 53.561 + - type: ndcg_at_5 + value: 56.497 + - type: ndcg_at_10 + value: 59.916000000000004 + - type: ndcg_at_20 + value: 61.814 + - type: ndcg_at_100 + value: 64.34 + - type: ndcg_at_1000 + value: 65.45299999999999 + - type: map_at_1 + value: 38.668 + - type: map_at_3 + value: 47.897 + - type: map_at_5 + value: 50.56699999999999 + - type: map_at_10 + value: 52.737 + - type: map_at_20 + value: 53.581 + - type: map_at_100 + value: 54.254000000000005 + - type: map_at_1000 + value: 54.339000000000006 + - type: recall_at_1 + value: 38.668 + - type: recall_at_3 + value: 56.269999999999996 + - type: recall_at_5 + value: 64.259 + - type: recall_at_10 + value: 74.05199999999999 + - type: recall_at_20 + value: 80.569 + - type: recall_at_100 + value: 91.43900000000001 + - type: recall_at_1000 + value: 98.257 + - type: precision_at_1 + value: 47.067 + - type: precision_at_3 + value: 25.799 + - type: precision_at_5 + value: 18.826999999999998 + - type: precision_at_10 + value: 11.716999999999999 + - type: precision_at_20 + value: 6.81 + - type: precision_at_100 + value: 1.7579999999999998 + - type: precision_at_1000 + value: 0.208 + - type: mrr_at_1 + value: 47.0672 + - type: mrr_at_3 + value: 55.555600000000005 + - type: mrr_at_5 + value: 57.257999999999996 + - type: mrr_at_10 + value: 58.383300000000006 + - type: mrr_at_20 + value: 58.7298 + - type: mrr_at_100 + value: 58.9092 + - type: mrr_at_1000 + value: 58.93659999999999 + - type: nauc_ndcg_at_1_max + value: 32.7003 + - type: nauc_ndcg_at_1_std + value: -9.8787 + - type: nauc_ndcg_at_1_diff1 + value: 53.543 + - type: nauc_ndcg_at_3_max + value: 32.6312 + - type: nauc_ndcg_at_3_std + value: -8.7036 + - type: nauc_ndcg_at_3_diff1 + value: 52.727599999999995 + - type: nauc_ndcg_at_5_max + value: 32.8095 + - type: nauc_ndcg_at_5_std + value: -6.3161 + - type: nauc_ndcg_at_5_diff1 + value: 51.804399999999994 + - type: nauc_ndcg_at_10_max + value: 32.1689 + - type: nauc_ndcg_at_10_std + value: -8.144 + - type: nauc_ndcg_at_10_diff1 + value: 51.0188 + - type: nauc_ndcg_at_20_max + value: 32.5967 + - type: nauc_ndcg_at_20_std + value: -7.3793 + - type: nauc_ndcg_at_20_diff1 + value: 51.698100000000004 + - type: nauc_ndcg_at_100_max + value: 33.3382 + - type: nauc_ndcg_at_100_std + value: -6.383900000000001 + - type: nauc_ndcg_at_100_diff1 + value: 51.465700000000005 + - type: nauc_ndcg_at_1000_max + value: 32.7849 + - type: nauc_ndcg_at_1000_std + value: -7.0913 + - type: nauc_ndcg_at_1000_diff1 + value: 51.4944 + - type: nauc_map_at_1_max + value: 26.2488 + - type: nauc_map_at_1_std + value: -11.1918 + - type: nauc_map_at_1_diff1 + value: 55.12629999999999 + - type: nauc_map_at_3_max + value: 30.157600000000002 + - type: nauc_map_at_3_std + value: -10.802299999999999 + - type: nauc_map_at_3_diff1 + value: 54.13440000000001 + - type: nauc_map_at_5_max + value: 31.088500000000003 + - type: nauc_map_at_5_std + value: -9.0711 + - type: nauc_map_at_5_diff1 + value: 53.729000000000006 + - type: nauc_map_at_10_max + value: 31.3165 + - type: nauc_map_at_10_std + value: -9.6771 + - type: nauc_map_at_10_diff1 + value: 53.3998 + - type: nauc_map_at_20_max + value: 31.5896 + - type: nauc_map_at_20_std + value: -9.163499999999999 + - type: nauc_map_at_20_diff1 + value: 53.436499999999995 + - type: nauc_map_at_100_max + value: 31.7416 + - type: nauc_map_at_100_std + value: -8.9088 + - type: nauc_map_at_100_diff1 + value: 53.213699999999996 + - type: nauc_map_at_1000_max + value: 31.7308 + - type: nauc_map_at_1000_std + value: -8.9222 + - type: nauc_map_at_1000_diff1 + value: 53.1991 + - type: nauc_recall_at_1_max + value: 26.2488 + - type: nauc_recall_at_1_std + value: -11.1918 + - type: nauc_recall_at_1_diff1 + value: 55.12629999999999 + - type: nauc_recall_at_3_max + value: 29.987799999999996 + - type: nauc_recall_at_3_std + value: -8.8979 + - type: nauc_recall_at_3_diff1 + value: 50.1606 + - type: nauc_recall_at_5_max + value: 30.7548 + - type: nauc_recall_at_5_std + value: -2.5221 + - type: nauc_recall_at_5_diff1 + value: 46.5351 + - type: nauc_recall_at_10_max + value: 27.4456 + - type: nauc_recall_at_10_std + value: -7.7719 + - type: nauc_recall_at_10_diff1 + value: 41.0327 + - type: nauc_recall_at_20_max + value: 30.598799999999997 + - type: nauc_recall_at_20_std + value: -0.7229 + - type: nauc_recall_at_20_diff1 + value: 43.335499999999996 + - type: nauc_recall_at_100_max + value: 44.4764 + - type: nauc_recall_at_100_std + value: 20.4865 + - type: nauc_recall_at_100_diff1 + value: 42.634100000000004 + - type: nauc_recall_at_1000_max + value: 44.5522 + - type: nauc_recall_at_1000_std + value: 53.301 + - type: nauc_recall_at_1000_diff1 + value: 39.488 + - type: nauc_precision_at_1_max + value: 32.7003 + - type: nauc_precision_at_1_std + value: -9.8787 + - type: nauc_precision_at_1_diff1 + value: 53.543 + - type: nauc_precision_at_3_max + value: 30.4913 + - type: nauc_precision_at_3_std + value: -2.7105 + - type: nauc_precision_at_3_diff1 + value: 28.8688 + - type: nauc_precision_at_5_max + value: 25.876900000000003 + - type: nauc_precision_at_5_std + value: 4.6525 + - type: nauc_precision_at_5_diff1 + value: 16.154 + - type: nauc_precision_at_10_max + value: 17.2851 + - type: nauc_precision_at_10_std + value: 4.2126 + - type: nauc_precision_at_10_diff1 + value: 2.6613 + - type: nauc_precision_at_20_max + value: 10.5899 + - type: nauc_precision_at_20_std + value: 6.668699999999999 + - type: nauc_precision_at_20_diff1 + value: -6.13 + - type: nauc_precision_at_100_max + value: 1.0815 + - type: nauc_precision_at_100_std + value: 7.1370000000000005 + - type: nauc_precision_at_100_diff1 + value: -17.5759 + - type: nauc_precision_at_1000_max + value: -5.915 + - type: nauc_precision_at_1000_std + value: 1.6254000000000002 + - type: nauc_precision_at_1000_diff1 + value: -21.4134 + - type: nauc_mrr_at_1_max + value: 32.7003 + - type: nauc_mrr_at_1_std + value: -9.8787 + - type: nauc_mrr_at_1_diff1 + value: 53.543 + - type: nauc_mrr_at_3_max + value: 33.9338 + - type: nauc_mrr_at_3_std + value: -7.9868999999999994 + - type: nauc_mrr_at_3_diff1 + value: 52.6479 + - type: nauc_mrr_at_5_max + value: 33.9982 + - type: nauc_mrr_at_5_std + value: -6.827500000000001 + - type: nauc_mrr_at_5_diff1 + value: 51.5701 + - type: nauc_mrr_at_10_max + value: 33.3568 + - type: nauc_mrr_at_10_std + value: -7.606300000000001 + - type: nauc_mrr_at_10_diff1 + value: 51.202400000000004 + - type: nauc_mrr_at_20_max + value: 33.4329 + - type: nauc_mrr_at_20_std + value: -7.5066 + - type: nauc_mrr_at_20_diff1 + value: 51.4203 + - type: nauc_mrr_at_100_max + value: 33.508700000000005 + - type: nauc_mrr_at_100_std + value: -7.455100000000001 + - type: nauc_mrr_at_100_diff1 + value: 51.442699999999995 + - type: nauc_mrr_at_1000_max + value: 33.4885 + - type: nauc_mrr_at_1000_std + value: -7.474200000000001 + - type: nauc_mrr_at_1000_diff1 + value: 51.4415 + - type: main_score + value: 59.916000000000004 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackEnglishRetrieval (default) + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 + split: test + type: mteb/cqadupstack-english + metrics: + - type: ndcg_at_1 + value: 50.127 + - type: ndcg_at_3 + value: 55.615 + - type: ndcg_at_5 + value: 57.462 + - type: ndcg_at_10 + value: 59.40800000000001 + - type: ndcg_at_20 + value: 60.870000000000005 + - type: ndcg_at_100 + value: 63.321000000000005 + - type: ndcg_at_1000 + value: 64.716 + - type: map_at_1 + value: 39.722 + - type: map_at_3 + value: 49.721 + - type: map_at_5 + value: 51.762 + - type: map_at_10 + value: 53.205999999999996 + - type: map_at_20 + value: 53.989 + - type: map_at_100 + value: 54.675 + - type: map_at_1000 + value: 54.791000000000004 + - type: recall_at_1 + value: 39.722 + - type: recall_at_3 + value: 57.428999999999995 + - type: recall_at_5 + value: 63.041000000000004 + - type: recall_at_10 + value: 69.61 + - type: recall_at_20 + value: 74.785 + - type: recall_at_100 + value: 85.83 + - type: recall_at_1000 + value: 94.017 + - type: precision_at_1 + value: 50.127 + - type: precision_at_3 + value: 27.389000000000003 + - type: precision_at_5 + value: 19.223000000000003 + - type: precision_at_10 + value: 11.286999999999999 + - type: precision_at_20 + value: 6.551 + - type: precision_at_100 + value: 1.7239999999999998 + - type: precision_at_1000 + value: 0.211 + - type: mrr_at_1 + value: 50.1274 + - type: mrr_at_3 + value: 57.6539 + - type: mrr_at_5 + value: 58.695299999999996 + - type: mrr_at_10 + value: 59.3822 + - type: mrr_at_20 + value: 59.689899999999994 + - type: mrr_at_100 + value: 59.9139 + - type: mrr_at_1000 + value: 59.9405 + - type: nauc_ndcg_at_1_max + value: 40.3466 + - type: nauc_ndcg_at_1_std + value: -13.8013 + - type: nauc_ndcg_at_1_diff1 + value: 57.7384 + - type: nauc_ndcg_at_3_max + value: 44.8558 + - type: nauc_ndcg_at_3_std + value: -17.1649 + - type: nauc_ndcg_at_3_diff1 + value: 56.2083 + - type: nauc_ndcg_at_5_max + value: 45.3495 + - type: nauc_ndcg_at_5_std + value: -17.1787 + - type: nauc_ndcg_at_5_diff1 + value: 55.2946 + - type: nauc_ndcg_at_10_max + value: 45.5771 + - type: nauc_ndcg_at_10_std + value: -17.194200000000002 + - type: nauc_ndcg_at_10_diff1 + value: 55.22899999999999 + - type: nauc_ndcg_at_20_max + value: 46.1671 + - type: nauc_ndcg_at_20_std + value: -15.8971 + - type: nauc_ndcg_at_20_diff1 + value: 55.4699 + - type: nauc_ndcg_at_100_max + value: 46.665600000000005 + - type: nauc_ndcg_at_100_std + value: -14.2615 + - type: nauc_ndcg_at_100_diff1 + value: 55.521699999999996 + - type: nauc_ndcg_at_1000_max + value: 46.5416 + - type: nauc_ndcg_at_1000_std + value: -13.414100000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 55.6847 + - type: nauc_map_at_1_max + value: 32.0258 + - type: nauc_map_at_1_std + value: -23.0677 + - type: nauc_map_at_1_diff1 + value: 58.34819999999999 + - type: nauc_map_at_3_max + value: 39.663199999999996 + - type: nauc_map_at_3_std + value: -23.261599999999998 + - type: nauc_map_at_3_diff1 + value: 56.930400000000006 + - type: nauc_map_at_5_max + value: 41.2777 + - type: nauc_map_at_5_std + value: -21.776200000000003 + - type: nauc_map_at_5_diff1 + value: 56.3832 + - type: nauc_map_at_10_max + value: 42.4307 + - type: nauc_map_at_10_std + value: -20.6698 + - type: nauc_map_at_10_diff1 + value: 56.3658 + - type: nauc_map_at_20_max + value: 43.146 + - type: nauc_map_at_20_std + value: -19.5408 + - type: nauc_map_at_20_diff1 + value: 56.432300000000005 + - type: nauc_map_at_100_max + value: 43.6798 + - type: nauc_map_at_100_std + value: -18.5361 + - type: nauc_map_at_100_diff1 + value: 56.4087 + - type: nauc_map_at_1000_max + value: 43.7037 + - type: nauc_map_at_1000_std + value: -18.3693 + - type: nauc_map_at_1000_diff1 + value: 56.4316 + - type: nauc_recall_at_1_max + value: 32.0258 + - type: nauc_recall_at_1_std + value: -23.0677 + - type: nauc_recall_at_1_diff1 + value: 58.34819999999999 + - type: nauc_recall_at_3_max + value: 41.504400000000004 + - type: nauc_recall_at_3_std + value: -23.471 + - type: nauc_recall_at_3_diff1 + value: 53.0711 + - type: nauc_recall_at_5_max + value: 43.6923 + - type: nauc_recall_at_5_std + value: -21.831999999999997 + - type: nauc_recall_at_5_diff1 + value: 50.1672 + - type: nauc_recall_at_10_max + value: 45.426100000000005 + - type: nauc_recall_at_10_std + value: -20.4132 + - type: nauc_recall_at_10_diff1 + value: 48.4065 + - type: nauc_recall_at_20_max + value: 49.0579 + - type: nauc_recall_at_20_std + value: -14.5552 + - type: nauc_recall_at_20_diff1 + value: 48.341499999999996 + - type: nauc_recall_at_100_max + value: 54.8657 + - type: nauc_recall_at_100_std + value: 0.1297 + - type: nauc_recall_at_100_diff1 + value: 46.576699999999995 + - type: nauc_recall_at_1000_max + value: 65.1502 + - type: nauc_recall_at_1000_std + value: 28.880699999999997 + - type: nauc_recall_at_1000_diff1 + value: 47.525099999999995 + - type: nauc_precision_at_1_max + value: 40.3466 + - type: nauc_precision_at_1_std + value: -13.8013 + - type: nauc_precision_at_1_diff1 + value: 57.7384 + - type: nauc_precision_at_3_max + value: 40.9044 + - type: nauc_precision_at_3_std + value: 3.1708 + - type: nauc_precision_at_3_diff1 + value: 27.9298 + - type: nauc_precision_at_5_max + value: 36.598000000000006 + - type: nauc_precision_at_5_std + value: 12.392 + - type: nauc_precision_at_5_diff1 + value: 15.7846 + - type: nauc_precision_at_10_max + value: 31.3687 + - type: nauc_precision_at_10_std + value: 20.7438 + - type: nauc_precision_at_10_diff1 + value: 6.7331 + - type: nauc_precision_at_20_max + value: 26.1811 + - type: nauc_precision_at_20_std + value: 28.4518 + - type: nauc_precision_at_20_diff1 + value: 0.30010000000000003 + - type: nauc_precision_at_100_max + value: 16.9808 + - type: nauc_precision_at_100_std + value: 38.7882 + - type: nauc_precision_at_100_diff1 + value: -8.8537 + - type: nauc_precision_at_1000_max + value: 7.2884 + - type: nauc_precision_at_1000_std + value: 39.2072 + - type: nauc_precision_at_1000_diff1 + value: -13.0202 + - type: nauc_mrr_at_1_max + value: 40.3466 + - type: nauc_mrr_at_1_std + value: -13.8013 + - type: nauc_mrr_at_1_diff1 + value: 57.7384 + - type: nauc_mrr_at_3_max + value: 45.2742 + - type: nauc_mrr_at_3_std + value: -12.6802 + - type: nauc_mrr_at_3_diff1 + value: 56.8512 + - type: nauc_mrr_at_5_max + value: 45.3012 + - type: nauc_mrr_at_5_std + value: -12.7147 + - type: nauc_mrr_at_5_diff1 + value: 56.2424 + - type: nauc_mrr_at_10_max + value: 45.1963 + - type: nauc_mrr_at_10_std + value: -12.7254 + - type: nauc_mrr_at_10_diff1 + value: 56.119699999999995 + - type: nauc_mrr_at_20_max + value: 45.2288 + - type: nauc_mrr_at_20_std + value: -12.5913 + - type: nauc_mrr_at_20_diff1 + value: 56.1426 + - type: nauc_mrr_at_100_max + value: 45.2468 + - type: nauc_mrr_at_100_std + value: -12.496500000000001 + - type: nauc_mrr_at_100_diff1 + value: 56.1812 + - type: nauc_mrr_at_1000_max + value: 45.2427 + - type: nauc_mrr_at_1000_std + value: -12.4903 + - type: nauc_mrr_at_1000_diff1 + value: 56.189299999999996 + - type: main_score + value: 59.40800000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGamingRetrieval (default) + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 + split: test + type: mteb/cqadupstack-gaming + metrics: + - type: ndcg_at_1 + value: 53.856 + - type: ndcg_at_3 + value: 62.492000000000004 + - type: ndcg_at_5 + value: 65.41 + - type: ndcg_at_10 + value: 68.134 + - type: ndcg_at_20 + value: 69.646 + - type: ndcg_at_100 + value: 71.184 + - type: ndcg_at_1000 + value: 71.663 + - type: map_at_1 + value: 47.236 + - type: map_at_3 + value: 58.36 + - type: map_at_5 + value: 60.484 + - type: map_at_10 + value: 61.978 + - type: map_at_20 + value: 62.57899999999999 + - type: map_at_100 + value: 62.900999999999996 + - type: map_at_1000 + value: 62.929 + - type: recall_at_1 + value: 47.236 + - type: recall_at_3 + value: 68.065 + - type: recall_at_5 + value: 75.155 + - type: recall_at_10 + value: 82.80499999999999 + - type: recall_at_20 + value: 88.215 + - type: recall_at_100 + value: 95.332 + - type: recall_at_1000 + value: 98.614 + - type: precision_at_1 + value: 53.856 + - type: precision_at_3 + value: 27.941 + - type: precision_at_5 + value: 19.034000000000002 + - type: precision_at_10 + value: 10.821 + - type: precision_at_20 + value: 5.947 + - type: precision_at_100 + value: 1.325 + - type: precision_at_1000 + value: 0.13899999999999998 + - type: mrr_at_1 + value: 53.855799999999995 + - type: mrr_at_3 + value: 62.6541 + - type: mrr_at_5 + value: 64.1243 + - type: mrr_at_10 + value: 65.084 + - type: mrr_at_20 + value: 65.3879 + - type: mrr_at_100 + value: 65.5377 + - type: mrr_at_1000 + value: 65.5496 + - type: nauc_ndcg_at_1_max + value: 33.8654 + - type: nauc_ndcg_at_1_std + value: -5.9437999999999995 + - type: nauc_ndcg_at_1_diff1 + value: 56.8669 + - type: nauc_ndcg_at_3_max + value: 34.058899999999994 + - type: nauc_ndcg_at_3_std + value: -5.7744 + - type: nauc_ndcg_at_3_diff1 + value: 52.8014 + - type: nauc_ndcg_at_5_max + value: 35.2914 + - type: nauc_ndcg_at_5_std + value: -4.482 + - type: nauc_ndcg_at_5_diff1 + value: 52.4343 + - type: nauc_ndcg_at_10_max + value: 36.458600000000004 + - type: nauc_ndcg_at_10_std + value: -2.3942 + - type: nauc_ndcg_at_10_diff1 + value: 52.9498 + - type: nauc_ndcg_at_20_max + value: 36.6183 + - type: nauc_ndcg_at_20_std + value: -1.8568000000000002 + - type: nauc_ndcg_at_20_diff1 + value: 52.5903 + - type: nauc_ndcg_at_100_max + value: 37.0184 + - type: nauc_ndcg_at_100_std + value: -0.7801 + - type: nauc_ndcg_at_100_diff1 + value: 53.011399999999995 + - type: nauc_ndcg_at_1000_max + value: 36.6608 + - type: nauc_ndcg_at_1000_std + value: -1.3958 + - type: nauc_ndcg_at_1000_diff1 + value: 53.0578 + - type: nauc_map_at_1_max + value: 25.787599999999998 + - type: nauc_map_at_1_std + value: -10.3742 + - type: nauc_map_at_1_diff1 + value: 56.4662 + - type: nauc_map_at_3_max + value: 31.4446 + - type: nauc_map_at_3_std + value: -8.140799999999999 + - type: nauc_map_at_3_diff1 + value: 53.8682 + - type: nauc_map_at_5_max + value: 32.8035 + - type: nauc_map_at_5_std + value: -6.8225999999999996 + - type: nauc_map_at_5_diff1 + value: 53.5451 + - type: nauc_map_at_10_max + value: 33.7173 + - type: nauc_map_at_10_std + value: -5.5325 + - type: nauc_map_at_10_diff1 + value: 53.6678 + - type: nauc_map_at_20_max + value: 34.2438 + - type: nauc_map_at_20_std + value: -4.8891 + - type: nauc_map_at_20_diff1 + value: 53.656000000000006 + - type: nauc_map_at_100_max + value: 34.473 + - type: nauc_map_at_100_std + value: -4.5106 + - type: nauc_map_at_100_diff1 + value: 53.7077 + - type: nauc_map_at_1000_max + value: 34.476600000000005 + - type: nauc_map_at_1000_std + value: -4.517 + - type: nauc_map_at_1000_diff1 + value: 53.7143 + - type: nauc_recall_at_1_max + value: 25.787599999999998 + - type: nauc_recall_at_1_std + value: -10.3742 + - type: nauc_recall_at_1_diff1 + value: 56.4662 + - type: nauc_recall_at_3_max + value: 32.044200000000004 + - type: nauc_recall_at_3_std + value: -7.696400000000001 + - type: nauc_recall_at_3_diff1 + value: 48.9202 + - type: nauc_recall_at_5_max + value: 34.389199999999995 + - type: nauc_recall_at_5_std + value: -4.2582 + - type: nauc_recall_at_5_diff1 + value: 46.0109 + - type: nauc_recall_at_10_max + value: 39.5274 + - type: nauc_recall_at_10_std + value: 3.9919999999999995 + - type: nauc_recall_at_10_diff1 + value: 46.383 + - type: nauc_recall_at_20_max + value: 43.5902 + - type: nauc_recall_at_20_std + value: 9.3885 + - type: nauc_recall_at_20_diff1 + value: 42.6035 + - type: nauc_recall_at_100_max + value: 61.5485 + - type: nauc_recall_at_100_std + value: 41.3982 + - type: nauc_recall_at_100_diff1 + value: 44.1753 + - type: nauc_recall_at_1000_max + value: 71.4815 + - type: nauc_recall_at_1000_std + value: 57.354400000000005 + - type: nauc_recall_at_1000_diff1 + value: 34.8468 + - type: nauc_precision_at_1_max + value: 33.8654 + - type: nauc_precision_at_1_std + value: -5.9437999999999995 + - type: nauc_precision_at_1_diff1 + value: 56.8669 + - type: nauc_precision_at_3_max + value: 33.655 + - type: nauc_precision_at_3_std + value: 7.826099999999999 + - type: nauc_precision_at_3_diff1 + value: 24.9975 + - type: nauc_precision_at_5_max + value: 32.9241 + - type: nauc_precision_at_5_std + value: 15.4324 + - type: nauc_precision_at_5_diff1 + value: 14.079 + - type: nauc_precision_at_10_max + value: 31.067600000000002 + - type: nauc_precision_at_10_std + value: 24.4877 + - type: nauc_precision_at_10_diff1 + value: 3.3716999999999997 + - type: nauc_precision_at_20_max + value: 28.786299999999997 + - type: nauc_precision_at_20_std + value: 29.323300000000003 + - type: nauc_precision_at_20_diff1 + value: -4.0988 + - type: nauc_precision_at_100_max + value: 23.4199 + - type: nauc_precision_at_100_std + value: 33.4154 + - type: nauc_precision_at_100_diff1 + value: -11.519400000000001 + - type: nauc_precision_at_1000_max + value: 19.2315 + - type: nauc_precision_at_1000_std + value: 31.391999999999996 + - type: nauc_precision_at_1000_diff1 + value: -14.5617 + - type: nauc_mrr_at_1_max + value: 33.8654 + - type: nauc_mrr_at_1_std + value: -5.9437999999999995 + - type: nauc_mrr_at_1_diff1 + value: 56.8669 + - type: nauc_mrr_at_3_max + value: 35.8396 + - type: nauc_mrr_at_3_std + value: -3.4635 + - type: nauc_mrr_at_3_diff1 + value: 53.6524 + - type: nauc_mrr_at_5_max + value: 36.0956 + - type: nauc_mrr_at_5_std + value: -3.0328999999999997 + - type: nauc_mrr_at_5_diff1 + value: 53.4449 + - type: nauc_mrr_at_10_max + value: 36.3936 + - type: nauc_mrr_at_10_std + value: -2.5233 + - type: nauc_mrr_at_10_diff1 + value: 53.858399999999996 + - type: nauc_mrr_at_20_max + value: 36.2638 + - type: nauc_mrr_at_20_std + value: -2.6908000000000003 + - type: nauc_mrr_at_20_diff1 + value: 53.805099999999996 + - type: nauc_mrr_at_100_max + value: 36.2945 + - type: nauc_mrr_at_100_std + value: -2.6416 + - type: nauc_mrr_at_100_diff1 + value: 53.8698 + - type: nauc_mrr_at_1000_max + value: 36.2806 + - type: nauc_mrr_at_1000_std + value: -2.6593 + - type: nauc_mrr_at_1000_diff1 + value: 53.8679 + - type: main_score + value: 68.134 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGisRetrieval (default) + revision: 5003b3064772da1887988e05400cf3806fe491f2 + split: test + type: mteb/cqadupstack-gis + metrics: + - type: ndcg_at_1 + value: 31.525 + - type: ndcg_at_3 + value: 40.165 + - type: ndcg_at_5 + value: 43.321 + - type: ndcg_at_10 + value: 46.778 + - type: ndcg_at_20 + value: 49.025 + - type: ndcg_at_100 + value: 51.461999999999996 + - type: ndcg_at_1000 + value: 52.935 + - type: map_at_1 + value: 29.044999999999998 + - type: map_at_3 + value: 36.976 + - type: map_at_5 + value: 38.853 + - type: map_at_10 + value: 40.388000000000005 + - type: map_at_20 + value: 41.082 + - type: map_at_100 + value: 41.486000000000004 + - type: map_at_1000 + value: 41.551 + - type: recall_at_1 + value: 29.044999999999998 + - type: recall_at_3 + value: 46.601 + - type: recall_at_5 + value: 54.062 + - type: recall_at_10 + value: 64.291 + - type: recall_at_20 + value: 72.531 + - type: recall_at_100 + value: 84.578 + - type: recall_at_1000 + value: 95.358 + - type: precision_at_1 + value: 31.525 + - type: precision_at_3 + value: 17.213 + - type: precision_at_5 + value: 12.203 + - type: precision_at_10 + value: 7.412000000000001 + - type: precision_at_20 + value: 4.266 + - type: precision_at_100 + value: 1.019 + - type: precision_at_1000 + value: 0.11800000000000001 + - type: mrr_at_1 + value: 31.525399999999998 + - type: mrr_at_3 + value: 39.529199999999996 + - type: mrr_at_5 + value: 41.3089 + - type: mrr_at_10 + value: 42.6025 + - type: mrr_at_20 + value: 43.1771 + - type: mrr_at_100 + value: 43.4611 + - type: mrr_at_1000 + value: 43.5085 + - type: nauc_ndcg_at_1_max + value: 22.6602 + - type: nauc_ndcg_at_1_std + value: -9.5981 + - type: nauc_ndcg_at_1_diff1 + value: 45.3169 + - type: nauc_ndcg_at_3_max + value: 19.7847 + - type: nauc_ndcg_at_3_std + value: -8.7083 + - type: nauc_ndcg_at_3_diff1 + value: 40.4401 + - type: nauc_ndcg_at_5_max + value: 20.457700000000003 + - type: nauc_ndcg_at_5_std + value: -8.6845 + - type: nauc_ndcg_at_5_diff1 + value: 38.7015 + - type: nauc_ndcg_at_10_max + value: 21.0795 + - type: nauc_ndcg_at_10_std + value: -6.5691 + - type: nauc_ndcg_at_10_diff1 + value: 37.966699999999996 + - type: nauc_ndcg_at_20_max + value: 21.4852 + - type: nauc_ndcg_at_20_std + value: -5.904800000000001 + - type: nauc_ndcg_at_20_diff1 + value: 38.0953 + - type: nauc_ndcg_at_100_max + value: 22.5439 + - type: nauc_ndcg_at_100_std + value: -5.1345 + - type: nauc_ndcg_at_100_diff1 + value: 38.7969 + - type: nauc_ndcg_at_1000_max + value: 22.3039 + - type: nauc_ndcg_at_1000_std + value: -5.9468 + - type: nauc_ndcg_at_1000_diff1 + value: 39.0377 + - type: nauc_map_at_1_max + value: 18.7633 + - type: nauc_map_at_1_std + value: -10.276 + - type: nauc_map_at_1_diff1 + value: 46.593 + - type: nauc_map_at_3_max + value: 19.0896 + - type: nauc_map_at_3_std + value: -9.214 + - type: nauc_map_at_3_diff1 + value: 41.980000000000004 + - type: nauc_map_at_5_max + value: 19.7273 + - type: nauc_map_at_5_std + value: -9.2142 + - type: nauc_map_at_5_diff1 + value: 41.073 + - type: nauc_map_at_10_max + value: 20.039 + - type: nauc_map_at_10_std + value: -8.3819 + - type: nauc_map_at_10_diff1 + value: 40.7757 + - type: nauc_map_at_20_max + value: 20.227600000000002 + - type: nauc_map_at_20_std + value: -8.2044 + - type: nauc_map_at_20_diff1 + value: 40.8699 + - type: nauc_map_at_100_max + value: 20.3876 + - type: nauc_map_at_100_std + value: -8.1094 + - type: nauc_map_at_100_diff1 + value: 40.9925 + - type: nauc_map_at_1000_max + value: 20.397299999999998 + - type: nauc_map_at_1000_std + value: -8.1295 + - type: nauc_map_at_1000_diff1 + value: 40.996300000000005 + - type: nauc_recall_at_1_max + value: 18.7633 + - type: nauc_recall_at_1_std + value: -10.276 + - type: nauc_recall_at_1_diff1 + value: 46.593 + - type: nauc_recall_at_3_max + value: 17.8827 + - type: nauc_recall_at_3_std + value: -7.2757000000000005 + - type: nauc_recall_at_3_diff1 + value: 35.817 + - type: nauc_recall_at_5_max + value: 18.8334 + - type: nauc_recall_at_5_std + value: -7.2427 + - type: nauc_recall_at_5_diff1 + value: 31.0566 + - type: nauc_recall_at_10_max + value: 20.1305 + - type: nauc_recall_at_10_std + value: -0.271 + - type: nauc_recall_at_10_diff1 + value: 27.4127 + - type: nauc_recall_at_20_max + value: 21.438 + - type: nauc_recall_at_20_std + value: 3.8486 + - type: nauc_recall_at_20_diff1 + value: 25.983099999999997 + - type: nauc_recall_at_100_max + value: 31.620900000000002 + - type: nauc_recall_at_100_std + value: 17.457700000000003 + - type: nauc_recall_at_100_diff1 + value: 26.546300000000002 + - type: nauc_recall_at_1000_max + value: 35.1108 + - type: nauc_recall_at_1000_std + value: 25.8201 + - type: nauc_recall_at_1000_diff1 + value: 15.2005 + - type: nauc_precision_at_1_max + value: 22.6602 + - type: nauc_precision_at_1_std + value: -9.5981 + - type: nauc_precision_at_1_diff1 + value: 45.3169 + - type: nauc_precision_at_3_max + value: 22.344 + - type: nauc_precision_at_3_std + value: -7.0357 + - type: nauc_precision_at_3_diff1 + value: 33.298100000000005 + - type: nauc_precision_at_5_max + value: 24.8904 + - type: nauc_precision_at_5_std + value: -5.7215 + - type: nauc_precision_at_5_diff1 + value: 27.1231 + - type: nauc_precision_at_10_max + value: 25.3317 + - type: nauc_precision_at_10_std + value: 2.7272000000000003 + - type: nauc_precision_at_10_diff1 + value: 19.3335 + - type: nauc_precision_at_20_max + value: 24.5711 + - type: nauc_precision_at_20_std + value: 6.5833 + - type: nauc_precision_at_20_diff1 + value: 13.7149 + - type: nauc_precision_at_100_max + value: 24.0549 + - type: nauc_precision_at_100_std + value: 12.7275 + - type: nauc_precision_at_100_diff1 + value: 5.2654 + - type: nauc_precision_at_1000_max + value: 17.191000000000003 + - type: nauc_precision_at_1000_std + value: 9.1143 + - type: nauc_precision_at_1000_diff1 + value: -5.5666 + - type: nauc_mrr_at_1_max + value: 22.6602 + - type: nauc_mrr_at_1_std + value: -9.5981 + - type: nauc_mrr_at_1_diff1 + value: 45.3169 + - type: nauc_mrr_at_3_max + value: 22.5354 + - type: nauc_mrr_at_3_std + value: -8.6094 + - type: nauc_mrr_at_3_diff1 + value: 40.982800000000005 + - type: nauc_mrr_at_5_max + value: 22.828699999999998 + - type: nauc_mrr_at_5_std + value: -8.6655 + - type: nauc_mrr_at_5_diff1 + value: 40.0766 + - type: nauc_mrr_at_10_max + value: 23.035600000000002 + - type: nauc_mrr_at_10_std + value: -7.864 + - type: nauc_mrr_at_10_diff1 + value: 39.8871 + - type: nauc_mrr_at_20_max + value: 23.0969 + - type: nauc_mrr_at_20_std + value: -7.6975 + - type: nauc_mrr_at_20_diff1 + value: 39.9707 + - type: nauc_mrr_at_100_max + value: 23.191200000000002 + - type: nauc_mrr_at_100_std + value: -7.6803 + - type: nauc_mrr_at_100_diff1 + value: 40.0729 + - type: nauc_mrr_at_1000_max + value: 23.1807 + - type: nauc_mrr_at_1000_std + value: -7.707 + - type: nauc_mrr_at_1000_diff1 + value: 40.0782 + - type: main_score + value: 46.778 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackMathematicaRetrieval (default) + revision: 90fceea13679c63fe563ded68f3b6f06e50061de + split: test + type: mteb/cqadupstack-mathematica + metrics: + - type: ndcg_at_1 + value: 26.617 + - type: ndcg_at_3 + value: 33.623999999999995 + - type: ndcg_at_5 + value: 36.981 + - type: ndcg_at_10 + value: 39.602 + - type: ndcg_at_20 + value: 42.059000000000005 + - type: ndcg_at_100 + value: 45.248 + - type: ndcg_at_1000 + value: 47.384 + - type: map_at_1 + value: 21.018 + - type: map_at_3 + value: 29.529 + - type: map_at_5 + value: 31.666 + - type: map_at_10 + value: 32.952 + - type: map_at_20 + value: 33.794000000000004 + - type: map_at_100 + value: 34.317 + - type: map_at_1000 + value: 34.416999999999994 + - type: recall_at_1 + value: 21.018 + - type: recall_at_3 + value: 38.624 + - type: recall_at_5 + value: 47.014 + - type: recall_at_10 + value: 54.668000000000006 + - type: recall_at_20 + value: 63.302 + - type: recall_at_100 + value: 78.487 + - type: recall_at_1000 + value: 93.118 + - type: precision_at_1 + value: 26.617 + - type: precision_at_3 + value: 16.915 + - type: precision_at_5 + value: 12.537 + - type: precision_at_10 + value: 7.5 + - type: precision_at_20 + value: 4.484 + - type: precision_at_100 + value: 1.172 + - type: precision_at_1000 + value: 0.148 + - type: mrr_at_1 + value: 26.616899999999998 + - type: mrr_at_3 + value: 34.8051 + - type: mrr_at_5 + value: 36.9755 + - type: mrr_at_10 + value: 38.000499999999995 + - type: mrr_at_20 + value: 38.452999999999996 + - type: mrr_at_100 + value: 38.817299999999996 + - type: mrr_at_1000 + value: 38.873200000000004 + - type: nauc_ndcg_at_1_max + value: 29.749799999999997 + - type: nauc_ndcg_at_1_std + value: -2.3403 + - type: nauc_ndcg_at_1_diff1 + value: 41.9574 + - type: nauc_ndcg_at_3_max + value: 29.7929 + - type: nauc_ndcg_at_3_std + value: -1.0050999999999999 + - type: nauc_ndcg_at_3_diff1 + value: 36.471900000000005 + - type: nauc_ndcg_at_5_max + value: 29.6171 + - type: nauc_ndcg_at_5_std + value: -1.0074 + - type: nauc_ndcg_at_5_diff1 + value: 34.5702 + - type: nauc_ndcg_at_10_max + value: 30.7265 + - type: nauc_ndcg_at_10_std + value: 0.46430000000000005 + - type: nauc_ndcg_at_10_diff1 + value: 35.1612 + - type: nauc_ndcg_at_20_max + value: 31.698999999999998 + - type: nauc_ndcg_at_20_std + value: 1.657 + - type: nauc_ndcg_at_20_diff1 + value: 35.131 + - type: nauc_ndcg_at_100_max + value: 31.717499999999998 + - type: nauc_ndcg_at_100_std + value: 2.4316999999999998 + - type: nauc_ndcg_at_100_diff1 + value: 35.1483 + - type: nauc_ndcg_at_1000_max + value: 31.390099999999997 + - type: nauc_ndcg_at_1000_std + value: 2.2651999999999997 + - type: nauc_ndcg_at_1000_diff1 + value: 35.9287 + - type: nauc_map_at_1_max + value: 27.181 + - type: nauc_map_at_1_std + value: -1.923 + - type: nauc_map_at_1_diff1 + value: 41.3209 + - type: nauc_map_at_3_max + value: 28.718100000000003 + - type: nauc_map_at_3_std + value: -1.8913 + - type: nauc_map_at_3_diff1 + value: 37.3018 + - type: nauc_map_at_5_max + value: 28.751900000000003 + - type: nauc_map_at_5_std + value: -1.9649 + - type: nauc_map_at_5_diff1 + value: 36.3067 + - type: nauc_map_at_10_max + value: 29.4293 + - type: nauc_map_at_10_std + value: -1.1372 + - type: nauc_map_at_10_diff1 + value: 36.7561 + - type: nauc_map_at_20_max + value: 29.788500000000003 + - type: nauc_map_at_20_std + value: -0.7448 + - type: nauc_map_at_20_diff1 + value: 36.7633 + - type: nauc_map_at_100_max + value: 29.859799999999996 + - type: nauc_map_at_100_std + value: -0.6194 + - type: nauc_map_at_100_diff1 + value: 36.8069 + - type: nauc_map_at_1000_max + value: 29.8362 + - type: nauc_map_at_1000_std + value: -0.6232 + - type: nauc_map_at_1000_diff1 + value: 36.835499999999996 + - type: nauc_recall_at_1_max + value: 27.181 + - type: nauc_recall_at_1_std + value: -1.923 + - type: nauc_recall_at_1_diff1 + value: 41.3209 + - type: nauc_recall_at_3_max + value: 28.5155 + - type: nauc_recall_at_3_std + value: -0.131 + - type: nauc_recall_at_3_diff1 + value: 31.5708 + - type: nauc_recall_at_5_max + value: 27.0032 + - type: nauc_recall_at_5_std + value: -0.7121 + - type: nauc_recall_at_5_diff1 + value: 26.3405 + - type: nauc_recall_at_10_max + value: 29.665200000000002 + - type: nauc_recall_at_10_std + value: 3.1462999999999997 + - type: nauc_recall_at_10_diff1 + value: 27.2852 + - type: nauc_recall_at_20_max + value: 33.2976 + - type: nauc_recall_at_20_std + value: 7.6558 + - type: nauc_recall_at_20_diff1 + value: 26.5332 + - type: nauc_recall_at_100_max + value: 33.5446 + - type: nauc_recall_at_100_std + value: 16.308600000000002 + - type: nauc_recall_at_100_diff1 + value: 22.561700000000002 + - type: nauc_recall_at_1000_max + value: 35.5524 + - type: nauc_recall_at_1000_std + value: 38.9644 + - type: nauc_recall_at_1000_diff1 + value: 27.861900000000002 + - type: nauc_precision_at_1_max + value: 29.749799999999997 + - type: nauc_precision_at_1_std + value: -2.3403 + - type: nauc_precision_at_1_diff1 + value: 41.9574 + - type: nauc_precision_at_3_max + value: 28.370099999999997 + - type: nauc_precision_at_3_std + value: 1.0373 + - type: nauc_precision_at_3_diff1 + value: 28.8024 + - type: nauc_precision_at_5_max + value: 27.184599999999996 + - type: nauc_precision_at_5_std + value: 2.5995999999999997 + - type: nauc_precision_at_5_diff1 + value: 22.8208 + - type: nauc_precision_at_10_max + value: 26.372600000000002 + - type: nauc_precision_at_10_std + value: 7.833600000000001 + - type: nauc_precision_at_10_diff1 + value: 19.8669 + - type: nauc_precision_at_20_max + value: 23.1904 + - type: nauc_precision_at_20_std + value: 10.5558 + - type: nauc_precision_at_20_diff1 + value: 14.5559 + - type: nauc_precision_at_100_max + value: 13.3218 + - type: nauc_precision_at_100_std + value: 11.7868 + - type: nauc_precision_at_100_diff1 + value: 4.2146 + - type: nauc_precision_at_1000_max + value: 0.7887 + - type: nauc_precision_at_1000_std + value: 5.9056 + - type: nauc_precision_at_1000_diff1 + value: -3.2767999999999997 + - type: nauc_mrr_at_1_max + value: 29.749799999999997 + - type: nauc_mrr_at_1_std + value: -2.3403 + - type: nauc_mrr_at_1_diff1 + value: 41.9574 + - type: nauc_mrr_at_3_max + value: 31.509500000000003 + - type: nauc_mrr_at_3_std + value: -0.41859999999999997 + - type: nauc_mrr_at_3_diff1 + value: 38.6987 + - type: nauc_mrr_at_5_max + value: 31.5247 + - type: nauc_mrr_at_5_std + value: -0.2595 + - type: nauc_mrr_at_5_diff1 + value: 37.5028 + - type: nauc_mrr_at_10_max + value: 31.7081 + - type: nauc_mrr_at_10_std + value: -0.0492 + - type: nauc_mrr_at_10_diff1 + value: 37.6581 + - type: nauc_mrr_at_20_max + value: 31.932 + - type: nauc_mrr_at_20_std + value: 0.2097 + - type: nauc_mrr_at_20_diff1 + value: 37.7422 + - type: nauc_mrr_at_100_max + value: 31.949699999999996 + - type: nauc_mrr_at_100_std + value: 0.1865 + - type: nauc_mrr_at_100_diff1 + value: 37.8221 + - type: nauc_mrr_at_1000_max + value: 31.9386 + - type: nauc_mrr_at_1000_std + value: 0.1795 + - type: nauc_mrr_at_1000_diff1 + value: 37.8506 + - type: main_score + value: 39.602 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackPhysicsRetrieval (default) + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 + split: test + type: mteb/cqadupstack-physics + metrics: + - type: ndcg_at_1 + value: 46.006 + - type: ndcg_at_3 + value: 51.910999999999994 + - type: ndcg_at_5 + value: 54.86299999999999 + - type: ndcg_at_10 + value: 57.135000000000005 + - type: ndcg_at_20 + value: 59.422 + - type: ndcg_at_100 + value: 62.474 + - type: ndcg_at_1000 + value: 63.532 + - type: map_at_1 + value: 37.16 + - type: map_at_3 + value: 46.947 + - type: map_at_5 + value: 49.295 + - type: map_at_10 + value: 50.662 + - type: map_at_20 + value: 51.53 + - type: map_at_100 + value: 52.149 + - type: map_at_1000 + value: 52.224000000000004 + - type: recall_at_1 + value: 37.16 + - type: recall_at_3 + value: 55.249 + - type: recall_at_5 + value: 63.234 + - type: recall_at_10 + value: 70.231 + - type: recall_at_20 + value: 77.9 + - type: recall_at_100 + value: 91.509 + - type: recall_at_1000 + value: 97.711 + - type: precision_at_1 + value: 46.006 + - type: precision_at_3 + value: 25.024 + - type: precision_at_5 + value: 17.671 + - type: precision_at_10 + value: 10.212 + - type: precision_at_20 + value: 5.914 + - type: precision_at_100 + value: 1.513 + - type: precision_at_1000 + value: 0.17500000000000002 + - type: mrr_at_1 + value: 46.0058 + - type: mrr_at_3 + value: 54.154599999999995 + - type: mrr_at_5 + value: 55.8101 + - type: mrr_at_10 + value: 56.6384 + - type: mrr_at_20 + value: 57.1217 + - type: mrr_at_100 + value: 57.3844 + - type: mrr_at_1000 + value: 57.404599999999995 + - type: nauc_ndcg_at_1_max + value: 25.011400000000002 + - type: nauc_ndcg_at_1_std + value: -10.9453 + - type: nauc_ndcg_at_1_diff1 + value: 52.5635 + - type: nauc_ndcg_at_3_max + value: 20.5699 + - type: nauc_ndcg_at_3_std + value: -14.1374 + - type: nauc_ndcg_at_3_diff1 + value: 50.095 + - type: nauc_ndcg_at_5_max + value: 20.6937 + - type: nauc_ndcg_at_5_std + value: -14.7377 + - type: nauc_ndcg_at_5_diff1 + value: 49.6968 + - type: nauc_ndcg_at_10_max + value: 21.0545 + - type: nauc_ndcg_at_10_std + value: -14.100999999999999 + - type: nauc_ndcg_at_10_diff1 + value: 49.2876 + - type: nauc_ndcg_at_20_max + value: 22.1813 + - type: nauc_ndcg_at_20_std + value: -13.619700000000002 + - type: nauc_ndcg_at_20_diff1 + value: 49.7752 + - type: nauc_ndcg_at_100_max + value: 23.765800000000002 + - type: nauc_ndcg_at_100_std + value: -11.5192 + - type: nauc_ndcg_at_100_diff1 + value: 49.8519 + - type: nauc_ndcg_at_1000_max + value: 23.2792 + - type: nauc_ndcg_at_1000_std + value: -11.7505 + - type: nauc_ndcg_at_1000_diff1 + value: 49.8422 + - type: nauc_map_at_1_max + value: 17.0234 + - type: nauc_map_at_1_std + value: -14.726600000000001 + - type: nauc_map_at_1_diff1 + value: 55.854000000000006 + - type: nauc_map_at_3_max + value: 18.4476 + - type: nauc_map_at_3_std + value: -14.8542 + - type: nauc_map_at_3_diff1 + value: 51.5951 + - type: nauc_map_at_5_max + value: 19.3995 + - type: nauc_map_at_5_std + value: -14.9116 + - type: nauc_map_at_5_diff1 + value: 51.081900000000005 + - type: nauc_map_at_10_max + value: 19.8911 + - type: nauc_map_at_10_std + value: -14.354700000000001 + - type: nauc_map_at_10_diff1 + value: 50.6725 + - type: nauc_map_at_20_max + value: 20.2847 + - type: nauc_map_at_20_std + value: -14.099999999999998 + - type: nauc_map_at_20_diff1 + value: 50.82020000000001 + - type: nauc_map_at_100_max + value: 20.6892 + - type: nauc_map_at_100_std + value: -13.6554 + - type: nauc_map_at_100_diff1 + value: 50.7695 + - type: nauc_map_at_1000_max + value: 20.6883 + - type: nauc_map_at_1000_std + value: -13.6632 + - type: nauc_map_at_1000_diff1 + value: 50.7647 + - type: nauc_recall_at_1_max + value: 17.0234 + - type: nauc_recall_at_1_std + value: -14.726600000000001 + - type: nauc_recall_at_1_diff1 + value: 55.854000000000006 + - type: nauc_recall_at_3_max + value: 16.1844 + - type: nauc_recall_at_3_std + value: -17.0942 + - type: nauc_recall_at_3_diff1 + value: 47.6143 + - type: nauc_recall_at_5_max + value: 17.1338 + - type: nauc_recall_at_5_std + value: -17.636499999999998 + - type: nauc_recall_at_5_diff1 + value: 44.345600000000005 + - type: nauc_recall_at_10_max + value: 18.972 + - type: nauc_recall_at_10_std + value: -15.596099999999998 + - type: nauc_recall_at_10_diff1 + value: 41.552499999999995 + - type: nauc_recall_at_20_max + value: 23.8339 + - type: nauc_recall_at_20_std + value: -14.122699999999998 + - type: nauc_recall_at_20_diff1 + value: 42.7171 + - type: nauc_recall_at_100_max + value: 43.231 + - type: nauc_recall_at_100_std + value: 8.0154 + - type: nauc_recall_at_100_diff1 + value: 42.7817 + - type: nauc_recall_at_1000_max + value: 53.58540000000001 + - type: nauc_recall_at_1000_std + value: 37.0029 + - type: nauc_recall_at_1000_diff1 + value: 44.239200000000004 + - type: nauc_precision_at_1_max + value: 25.011400000000002 + - type: nauc_precision_at_1_std + value: -10.9453 + - type: nauc_precision_at_1_diff1 + value: 52.5635 + - type: nauc_precision_at_3_max + value: 22.2424 + - type: nauc_precision_at_3_std + value: -5.4350000000000005 + - type: nauc_precision_at_3_diff1 + value: 23.4114 + - type: nauc_precision_at_5_max + value: 21.3318 + - type: nauc_precision_at_5_std + value: -2.8209999999999997 + - type: nauc_precision_at_5_diff1 + value: 14.0476 + - type: nauc_precision_at_10_max + value: 19.2971 + - type: nauc_precision_at_10_std + value: 2.5547 + - type: nauc_precision_at_10_diff1 + value: 4.0724 + - type: nauc_precision_at_20_max + value: 17.6513 + - type: nauc_precision_at_20_std + value: 6.0579 + - type: nauc_precision_at_20_diff1 + value: -3.1468999999999996 + - type: nauc_precision_at_100_max + value: 14.8878 + - type: nauc_precision_at_100_std + value: 13.919200000000002 + - type: nauc_precision_at_100_diff1 + value: -17.358999999999998 + - type: nauc_precision_at_1000_max + value: 8.6286 + - type: nauc_precision_at_1000_std + value: 11.5922 + - type: nauc_precision_at_1000_diff1 + value: -22.1277 + - type: nauc_mrr_at_1_max + value: 25.011400000000002 + - type: nauc_mrr_at_1_std + value: -10.9453 + - type: nauc_mrr_at_1_diff1 + value: 52.5635 + - type: nauc_mrr_at_3_max + value: 23.816000000000003 + - type: nauc_mrr_at_3_std + value: -12.188400000000001 + - type: nauc_mrr_at_3_diff1 + value: 51.1699 + - type: nauc_mrr_at_5_max + value: 23.7135 + - type: nauc_mrr_at_5_std + value: -12.1816 + - type: nauc_mrr_at_5_diff1 + value: 50.339 + - type: nauc_mrr_at_10_max + value: 23.9975 + - type: nauc_mrr_at_10_std + value: -11.7119 + - type: nauc_mrr_at_10_diff1 + value: 50.32489999999999 + - type: nauc_mrr_at_20_max + value: 24.2972 + - type: nauc_mrr_at_20_std + value: -11.6891 + - type: nauc_mrr_at_20_diff1 + value: 50.4005 + - type: nauc_mrr_at_100_max + value: 24.3557 + - type: nauc_mrr_at_100_std + value: -11.5637 + - type: nauc_mrr_at_100_diff1 + value: 50.454100000000004 + - type: nauc_mrr_at_1000_max + value: 24.334400000000002 + - type: nauc_mrr_at_1000_std + value: -11.574900000000001 + - type: nauc_mrr_at_1000_diff1 + value: 50.45269999999999 + - type: main_score + value: 57.135000000000005 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackProgrammersRetrieval (default) + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 + split: test + type: mteb/cqadupstack-programmers + metrics: + - type: ndcg_at_1 + value: 40.753 + - type: ndcg_at_3 + value: 47.27 + - type: ndcg_at_5 + value: 50.385999999999996 + - type: ndcg_at_10 + value: 53.565 + - type: ndcg_at_20 + value: 55.967999999999996 + - type: ndcg_at_100 + value: 58.763 + - type: ndcg_at_1000 + value: 60.02499999999999 + - type: map_at_1 + value: 33.005 + - type: map_at_3 + value: 42.314 + - type: map_at_5 + value: 44.856 + - type: map_at_10 + value: 46.633 + - type: map_at_20 + value: 47.494 + - type: map_at_100 + value: 48.064 + - type: map_at_1000 + value: 48.14 + - type: recall_at_1 + value: 33.005 + - type: recall_at_3 + value: 50.73800000000001 + - type: recall_at_5 + value: 59.047000000000004 + - type: recall_at_10 + value: 68.27600000000001 + - type: recall_at_20 + value: 76.75800000000001 + - type: recall_at_100 + value: 89.505 + - type: recall_at_1000 + value: 97.636 + - type: precision_at_1 + value: 40.753 + - type: precision_at_3 + value: 22.945 + - type: precision_at_5 + value: 16.644000000000002 + - type: precision_at_10 + value: 10.057 + - type: precision_at_20 + value: 5.862 + - type: precision_at_100 + value: 1.467 + - type: precision_at_1000 + value: 0.173 + - type: mrr_at_1 + value: 40.7534 + - type: mrr_at_3 + value: 49.048700000000004 + - type: mrr_at_5 + value: 50.9209 + - type: mrr_at_10 + value: 52.0898 + - type: mrr_at_20 + value: 52.605599999999995 + - type: mrr_at_100 + value: 52.85300000000001 + - type: mrr_at_1000 + value: 52.8799 + - type: nauc_ndcg_at_1_max + value: 34.4441 + - type: nauc_ndcg_at_1_std + value: -7.0414 + - type: nauc_ndcg_at_1_diff1 + value: 45.8482 + - type: nauc_ndcg_at_3_max + value: 31.577699999999997 + - type: nauc_ndcg_at_3_std + value: -6.3458 + - type: nauc_ndcg_at_3_diff1 + value: 40.919200000000004 + - type: nauc_ndcg_at_5_max + value: 32.2014 + - type: nauc_ndcg_at_5_std + value: -5.2417 + - type: nauc_ndcg_at_5_diff1 + value: 40.288000000000004 + - type: nauc_ndcg_at_10_max + value: 34.2368 + - type: nauc_ndcg_at_10_std + value: -4.5674 + - type: nauc_ndcg_at_10_diff1 + value: 40.5809 + - type: nauc_ndcg_at_20_max + value: 35.1035 + - type: nauc_ndcg_at_20_std + value: -3.9905000000000004 + - type: nauc_ndcg_at_20_diff1 + value: 41.1355 + - type: nauc_ndcg_at_100_max + value: 35.7455 + - type: nauc_ndcg_at_100_std + value: -3.2745 + - type: nauc_ndcg_at_100_diff1 + value: 41.4431 + - type: nauc_ndcg_at_1000_max + value: 35.1084 + - type: nauc_ndcg_at_1000_std + value: -4.0846 + - type: nauc_ndcg_at_1000_diff1 + value: 41.755900000000004 + - type: nauc_map_at_1_max + value: 28.055200000000003 + - type: nauc_map_at_1_std + value: -11.2817 + - type: nauc_map_at_1_diff1 + value: 45.1938 + - type: nauc_map_at_3_max + value: 29.7864 + - type: nauc_map_at_3_std + value: -8.1494 + - type: nauc_map_at_3_diff1 + value: 41.788 + - type: nauc_map_at_5_max + value: 30.809199999999997 + - type: nauc_map_at_5_std + value: -7.012599999999999 + - type: nauc_map_at_5_diff1 + value: 41.554 + - type: nauc_map_at_10_max + value: 32.2321 + - type: nauc_map_at_10_std + value: -6.3894 + - type: nauc_map_at_10_diff1 + value: 41.8427 + - type: nauc_map_at_20_max + value: 32.7711 + - type: nauc_map_at_20_std + value: -6.0764 + - type: nauc_map_at_20_diff1 + value: 42.1419 + - type: nauc_map_at_100_max + value: 33.0054 + - type: nauc_map_at_100_std + value: -5.8844 + - type: nauc_map_at_100_diff1 + value: 42.3068 + - type: nauc_map_at_1000_max + value: 32.9949 + - type: nauc_map_at_1000_std + value: -5.9162 + - type: nauc_map_at_1000_diff1 + value: 42.3228 + - type: nauc_recall_at_1_max + value: 28.055200000000003 + - type: nauc_recall_at_1_std + value: -11.2817 + - type: nauc_recall_at_1_diff1 + value: 45.1938 + - type: nauc_recall_at_3_max + value: 27.1828 + - type: nauc_recall_at_3_std + value: -6.9705 + - type: nauc_recall_at_3_diff1 + value: 35.2147 + - type: nauc_recall_at_5_max + value: 28.0093 + - type: nauc_recall_at_5_std + value: -2.9148 + - type: nauc_recall_at_5_diff1 + value: 32.376599999999996 + - type: nauc_recall_at_10_max + value: 33.3355 + - type: nauc_recall_at_10_std + value: -0.4752 + - type: nauc_recall_at_10_diff1 + value: 32.5726 + - type: nauc_recall_at_20_max + value: 35.9026 + - type: nauc_recall_at_20_std + value: 3.1338 + - type: nauc_recall_at_20_diff1 + value: 32.1894 + - type: nauc_recall_at_100_max + value: 45.4995 + - type: nauc_recall_at_100_std + value: 18.2978 + - type: nauc_recall_at_100_diff1 + value: 29.535 + - type: nauc_recall_at_1000_max + value: 42.8817 + - type: nauc_recall_at_1000_std + value: 34.7251 + - type: nauc_recall_at_1000_diff1 + value: 33.1814 + - type: nauc_precision_at_1_max + value: 34.4441 + - type: nauc_precision_at_1_std + value: -7.0414 + - type: nauc_precision_at_1_diff1 + value: 45.8482 + - type: nauc_precision_at_3_max + value: 30.514000000000003 + - type: nauc_precision_at_3_std + value: 2.968 + - type: nauc_precision_at_3_diff1 + value: 25.0624 + - type: nauc_precision_at_5_max + value: 30.268 + - type: nauc_precision_at_5_std + value: 7.8429 + - type: nauc_precision_at_5_diff1 + value: 18.8704 + - type: nauc_precision_at_10_max + value: 31.6838 + - type: nauc_precision_at_10_std + value: 11.9131 + - type: nauc_precision_at_10_diff1 + value: 14.0232 + - type: nauc_precision_at_20_max + value: 28.375099999999996 + - type: nauc_precision_at_20_std + value: 13.497700000000002 + - type: nauc_precision_at_20_diff1 + value: 10.795 + - type: nauc_precision_at_100_max + value: 20.1953 + - type: nauc_precision_at_100_std + value: 14.4028 + - type: nauc_precision_at_100_diff1 + value: 4.6725 + - type: nauc_precision_at_1000_max + value: 11.3706 + - type: nauc_precision_at_1000_std + value: 9.1752 + - type: nauc_precision_at_1000_diff1 + value: 1.302 + - type: nauc_mrr_at_1_max + value: 34.4441 + - type: nauc_mrr_at_1_std + value: -7.0414 + - type: nauc_mrr_at_1_diff1 + value: 45.8482 + - type: nauc_mrr_at_3_max + value: 34.760799999999996 + - type: nauc_mrr_at_3_std + value: -5.7082 + - type: nauc_mrr_at_3_diff1 + value: 41.8373 + - type: nauc_mrr_at_5_max + value: 35.0958 + - type: nauc_mrr_at_5_std + value: -4.7876 + - type: nauc_mrr_at_5_diff1 + value: 41.574299999999994 + - type: nauc_mrr_at_10_max + value: 35.5072 + - type: nauc_mrr_at_10_std + value: -4.820399999999999 + - type: nauc_mrr_at_10_diff1 + value: 41.9727 + - type: nauc_mrr_at_20_max + value: 35.6201 + - type: nauc_mrr_at_20_std + value: -4.7524 + - type: nauc_mrr_at_20_diff1 + value: 42.2289 + - type: nauc_mrr_at_100_max + value: 35.6408 + - type: nauc_mrr_at_100_std + value: -4.7266 + - type: nauc_mrr_at_100_diff1 + value: 42.2145 + - type: nauc_mrr_at_1000_max + value: 35.6255 + - type: nauc_mrr_at_1000_std + value: -4.7333 + - type: nauc_mrr_at_1000_diff1 + value: 42.221399999999996 + - type: main_score + value: 53.565 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackRetrieval (default) + revision: CQADupstackRetrieval_is_a_combined_dataset + split: test + type: CQADupstackRetrieval_is_a_combined_dataset + metrics: + - type: main_score + value: 51.03358333333333 + - type: ndcg_at_10 + value: 51.03358333333333 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackStatsRetrieval (default) + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a + split: test + type: mteb/cqadupstack-stats + metrics: + - type: ndcg_at_1 + value: 34.355999999999995 + - type: ndcg_at_3 + value: 39.660000000000004 + - type: ndcg_at_5 + value: 42.625 + - type: ndcg_at_10 + value: 45.717 + - type: ndcg_at_20 + value: 47.738 + - type: ndcg_at_100 + value: 50.586 + - type: ndcg_at_1000 + value: 52.317 + - type: map_at_1 + value: 30.009999999999998 + - type: map_at_3 + value: 36.597 + - type: map_at_5 + value: 38.507999999999996 + - type: map_at_10 + value: 40.034 + - type: map_at_20 + value: 40.633 + - type: map_at_100 + value: 41.089 + - type: map_at_1000 + value: 41.166000000000004 + - type: recall_at_1 + value: 30.009999999999998 + - type: recall_at_3 + value: 43.646 + - type: recall_at_5 + value: 50.763000000000005 + - type: recall_at_10 + value: 60.218 + - type: recall_at_20 + value: 67.756 + - type: recall_at_100 + value: 81.78 + - type: recall_at_1000 + value: 94.179 + - type: precision_at_1 + value: 34.355999999999995 + - type: precision_at_3 + value: 17.28 + - type: precision_at_5 + value: 12.454 + - type: precision_at_10 + value: 7.485 + - type: precision_at_20 + value: 4.287 + - type: precision_at_100 + value: 1.0670000000000002 + - type: precision_at_1000 + value: 0.128 + - type: mrr_at_1 + value: 34.355799999999995 + - type: mrr_at_3 + value: 40.0562 + - type: mrr_at_5 + value: 41.8124 + - type: mrr_at_10 + value: 42.998799999999996 + - type: mrr_at_20 + value: 43.5177 + - type: mrr_at_100 + value: 43.8815 + - type: mrr_at_1000 + value: 43.928200000000004 + - type: nauc_ndcg_at_1_max + value: 22.8762 + - type: nauc_ndcg_at_1_std + value: -7.6788 + - type: nauc_ndcg_at_1_diff1 + value: 57.015499999999996 + - type: nauc_ndcg_at_3_max + value: 22.8095 + - type: nauc_ndcg_at_3_std + value: -5.3355 + - type: nauc_ndcg_at_3_diff1 + value: 49.9449 + - type: nauc_ndcg_at_5_max + value: 25.366100000000003 + - type: nauc_ndcg_at_5_std + value: -3.8400999999999996 + - type: nauc_ndcg_at_5_diff1 + value: 49.0563 + - type: nauc_ndcg_at_10_max + value: 23.7052 + - type: nauc_ndcg_at_10_std + value: -4.4089 + - type: nauc_ndcg_at_10_diff1 + value: 47.130300000000005 + - type: nauc_ndcg_at_20_max + value: 24.2726 + - type: nauc_ndcg_at_20_std + value: -3.8846 + - type: nauc_ndcg_at_20_diff1 + value: 47.5163 + - type: nauc_ndcg_at_100_max + value: 25.487 + - type: nauc_ndcg_at_100_std + value: -2.1590000000000003 + - type: nauc_ndcg_at_100_diff1 + value: 47.8372 + - type: nauc_ndcg_at_1000_max + value: 25.2363 + - type: nauc_ndcg_at_1000_std + value: -2.5404 + - type: nauc_ndcg_at_1000_diff1 + value: 48.7815 + - type: nauc_map_at_1_max + value: 18.9891 + - type: nauc_map_at_1_std + value: -9.9207 + - type: nauc_map_at_1_diff1 + value: 55.4997 + - type: nauc_map_at_3_max + value: 21.235699999999998 + - type: nauc_map_at_3_std + value: -7.048 + - type: nauc_map_at_3_diff1 + value: 51.2863 + - type: nauc_map_at_5_max + value: 23.0436 + - type: nauc_map_at_5_std + value: -6.1008 + - type: nauc_map_at_5_diff1 + value: 50.779799999999994 + - type: nauc_map_at_10_max + value: 22.4576 + - type: nauc_map_at_10_std + value: -6.3836 + - type: nauc_map_at_10_diff1 + value: 49.8457 + - type: nauc_map_at_20_max + value: 22.599800000000002 + - type: nauc_map_at_20_std + value: -6.2443 + - type: nauc_map_at_20_diff1 + value: 49.9702 + - type: nauc_map_at_100_max + value: 22.8352 + - type: nauc_map_at_100_std + value: -5.9363 + - type: nauc_map_at_100_diff1 + value: 50.0868 + - type: nauc_map_at_1000_max + value: 22.8394 + - type: nauc_map_at_1000_std + value: -5.934699999999999 + - type: nauc_map_at_1000_diff1 + value: 50.1389 + - type: nauc_recall_at_1_max + value: 18.9891 + - type: nauc_recall_at_1_std + value: -9.9207 + - type: nauc_recall_at_1_diff1 + value: 55.4997 + - type: nauc_recall_at_3_max + value: 22.3469 + - type: nauc_recall_at_3_std + value: -3.1021 + - type: nauc_recall_at_3_diff1 + value: 44.217600000000004 + - type: nauc_recall_at_5_max + value: 29.2041 + - type: nauc_recall_at_5_std + value: 1.013 + - type: nauc_recall_at_5_diff1 + value: 41.4239 + - type: nauc_recall_at_10_max + value: 23.7313 + - type: nauc_recall_at_10_std + value: 0.3575 + - type: nauc_recall_at_10_diff1 + value: 34.661500000000004 + - type: nauc_recall_at_20_max + value: 25.496999999999996 + - type: nauc_recall_at_20_std + value: 3.1315000000000004 + - type: nauc_recall_at_20_diff1 + value: 34.2149 + - type: nauc_recall_at_100_max + value: 35.957 + - type: nauc_recall_at_100_std + value: 21.1095 + - type: nauc_recall_at_100_diff1 + value: 27.4781 + - type: nauc_recall_at_1000_max + value: 45.015699999999995 + - type: nauc_recall_at_1000_std + value: 45.8094 + - type: nauc_recall_at_1000_diff1 + value: 22.481499999999997 + - type: nauc_precision_at_1_max + value: 22.8762 + - type: nauc_precision_at_1_std + value: -7.6788 + - type: nauc_precision_at_1_diff1 + value: 57.015499999999996 + - type: nauc_precision_at_3_max + value: 24.8891 + - type: nauc_precision_at_3_std + value: -0.9313 + - type: nauc_precision_at_3_diff1 + value: 40.6115 + - type: nauc_precision_at_5_max + value: 28.7576 + - type: nauc_precision_at_5_std + value: 2.9669 + - type: nauc_precision_at_5_diff1 + value: 35.298 + - type: nauc_precision_at_10_max + value: 23.8354 + - type: nauc_precision_at_10_std + value: 3.2748 + - type: nauc_precision_at_10_diff1 + value: 24.2013 + - type: nauc_precision_at_20_max + value: 24.089199999999998 + - type: nauc_precision_at_20_std + value: 5.7543 + - type: nauc_precision_at_20_diff1 + value: 20.718 + - type: nauc_precision_at_100_max + value: 22.074199999999998 + - type: nauc_precision_at_100_std + value: 12.0253 + - type: nauc_precision_at_100_diff1 + value: 10.3669 + - type: nauc_precision_at_1000_max + value: 12.845799999999999 + - type: nauc_precision_at_1000_std + value: 8.9314 + - type: nauc_precision_at_1000_diff1 + value: 4.3847 + - type: nauc_mrr_at_1_max + value: 22.8762 + - type: nauc_mrr_at_1_std + value: -7.6788 + - type: nauc_mrr_at_1_diff1 + value: 57.015499999999996 + - type: nauc_mrr_at_3_max + value: 24.8244 + - type: nauc_mrr_at_3_std + value: -5.184699999999999 + - type: nauc_mrr_at_3_diff1 + value: 52.567 + - type: nauc_mrr_at_5_max + value: 25.9477 + - type: nauc_mrr_at_5_std + value: -4.3008999999999995 + - type: nauc_mrr_at_5_diff1 + value: 52.0231 + - type: nauc_mrr_at_10_max + value: 25.164599999999997 + - type: nauc_mrr_at_10_std + value: -4.3651 + - type: nauc_mrr_at_10_diff1 + value: 51.3857 + - type: nauc_mrr_at_20_max + value: 25.210500000000003 + - type: nauc_mrr_at_20_std + value: -4.3703 + - type: nauc_mrr_at_20_diff1 + value: 51.4896 + - type: nauc_mrr_at_100_max + value: 25.3392 + - type: nauc_mrr_at_100_std + value: -4.174300000000001 + - type: nauc_mrr_at_100_diff1 + value: 51.6015 + - type: nauc_mrr_at_1000_max + value: 25.3401 + - type: nauc_mrr_at_1000_std + value: -4.1697 + - type: nauc_mrr_at_1000_diff1 + value: 51.623799999999996 + - type: main_score + value: 45.717 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackTexRetrieval (default) + revision: 46989137a86843e03a6195de44b09deda022eec7 + split: test + type: mteb/cqadupstack-tex + metrics: + - type: ndcg_at_1 + value: 26.807 + - type: ndcg_at_3 + value: 32.134 + - type: ndcg_at_5 + value: 34.37 + - type: ndcg_at_10 + value: 37.219 + - type: ndcg_at_20 + value: 39.412000000000006 + - type: ndcg_at_100 + value: 42.775 + - type: ndcg_at_1000 + value: 45.174 + - type: map_at_1 + value: 21.89 + - type: map_at_3 + value: 28.498 + - type: map_at_5 + value: 30.076999999999998 + - type: map_at_10 + value: 31.455 + - type: map_at_20 + value: 32.173 + - type: map_at_100 + value: 32.738 + - type: map_at_1000 + value: 32.86 + - type: recall_at_1 + value: 21.89 + - type: recall_at_3 + value: 35.674 + - type: recall_at_5 + value: 41.582 + - type: recall_at_10 + value: 49.988 + - type: recall_at_20 + value: 58.012 + - type: recall_at_100 + value: 74.357 + - type: recall_at_1000 + value: 91.003 + - type: precision_at_1 + value: 26.807 + - type: precision_at_3 + value: 15.359 + - type: precision_at_5 + value: 11.067 + - type: precision_at_10 + value: 6.912999999999999 + - type: precision_at_20 + value: 4.14 + - type: precision_at_100 + value: 1.137 + - type: precision_at_1000 + value: 0.152 + - type: mrr_at_1 + value: 26.806600000000003 + - type: mrr_at_3 + value: 33.276 + - type: mrr_at_5 + value: 34.685100000000006 + - type: mrr_at_10 + value: 35.8652 + - type: mrr_at_20 + value: 36.3975 + - type: mrr_at_100 + value: 36.7734 + - type: mrr_at_1000 + value: 36.8373 + - type: nauc_ndcg_at_1_max + value: 22.2824 + - type: nauc_ndcg_at_1_std + value: -1.1636 + - type: nauc_ndcg_at_1_diff1 + value: 44.0723 + - type: nauc_ndcg_at_3_max + value: 23.5119 + - type: nauc_ndcg_at_3_std + value: 0.2747 + - type: nauc_ndcg_at_3_diff1 + value: 37.7517 + - type: nauc_ndcg_at_5_max + value: 23.494200000000003 + - type: nauc_ndcg_at_5_std + value: 0.5172 + - type: nauc_ndcg_at_5_diff1 + value: 35.808800000000005 + - type: nauc_ndcg_at_10_max + value: 23.9137 + - type: nauc_ndcg_at_10_std + value: 1.2572 + - type: nauc_ndcg_at_10_diff1 + value: 35.3517 + - type: nauc_ndcg_at_20_max + value: 24.147299999999998 + - type: nauc_ndcg_at_20_std + value: 1.7857999999999998 + - type: nauc_ndcg_at_20_diff1 + value: 34.904 + - type: nauc_ndcg_at_100_max + value: 24.677 + - type: nauc_ndcg_at_100_std + value: 3.3762 + - type: nauc_ndcg_at_100_diff1 + value: 35.476400000000005 + - type: nauc_ndcg_at_1000_max + value: 24.9518 + - type: nauc_ndcg_at_1000_std + value: 3.3005 + - type: nauc_ndcg_at_1000_diff1 + value: 35.9856 + - type: nauc_map_at_1_max + value: 18.5395 + - type: nauc_map_at_1_std + value: -1.8748 + - type: nauc_map_at_1_diff1 + value: 43.2271 + - type: nauc_map_at_3_max + value: 21.956300000000002 + - type: nauc_map_at_3_std + value: -0.3228 + - type: nauc_map_at_3_diff1 + value: 39.0086 + - type: nauc_map_at_5_max + value: 22.2144 + - type: nauc_map_at_5_std + value: -0.1749 + - type: nauc_map_at_5_diff1 + value: 37.7466 + - type: nauc_map_at_10_max + value: 22.621 + - type: nauc_map_at_10_std + value: 0.11750000000000001 + - type: nauc_map_at_10_diff1 + value: 37.5604 + - type: nauc_map_at_20_max + value: 22.744 + - type: nauc_map_at_20_std + value: 0.3284 + - type: nauc_map_at_20_diff1 + value: 37.4046 + - type: nauc_map_at_100_max + value: 22.9403 + - type: nauc_map_at_100_std + value: 0.594 + - type: nauc_map_at_100_diff1 + value: 37.519999999999996 + - type: nauc_map_at_1000_max + value: 22.983 + - type: nauc_map_at_1000_std + value: 0.6118 + - type: nauc_map_at_1000_diff1 + value: 37.5586 + - type: nauc_recall_at_1_max + value: 18.5395 + - type: nauc_recall_at_1_std + value: -1.8748 + - type: nauc_recall_at_1_diff1 + value: 43.2271 + - type: nauc_recall_at_3_max + value: 22.7927 + - type: nauc_recall_at_3_std + value: 1.0538 + - type: nauc_recall_at_3_diff1 + value: 33.2051 + - type: nauc_recall_at_5_max + value: 22.7185 + - type: nauc_recall_at_5_std + value: 1.3141 + - type: nauc_recall_at_5_diff1 + value: 28.321099999999998 + - type: nauc_recall_at_10_max + value: 23.3274 + - type: nauc_recall_at_10_std + value: 3.3770000000000002 + - type: nauc_recall_at_10_diff1 + value: 26.0137 + - type: nauc_recall_at_20_max + value: 23.8623 + - type: nauc_recall_at_20_std + value: 5.5042 + - type: nauc_recall_at_20_diff1 + value: 23.5772 + - type: nauc_recall_at_100_max + value: 26.5351 + - type: nauc_recall_at_100_std + value: 17.011100000000003 + - type: nauc_recall_at_100_diff1 + value: 23.150399999999998 + - type: nauc_recall_at_1000_max + value: 35.7909 + - type: nauc_recall_at_1000_std + value: 33.4656 + - type: nauc_recall_at_1000_diff1 + value: 19.8029 + - type: nauc_precision_at_1_max + value: 22.2824 + - type: nauc_precision_at_1_std + value: -1.1636 + - type: nauc_precision_at_1_diff1 + value: 44.0723 + - type: nauc_precision_at_3_max + value: 27.798099999999998 + - type: nauc_precision_at_3_std + value: 2.538 + - type: nauc_precision_at_3_diff1 + value: 30.9728 + - type: nauc_precision_at_5_max + value: 26.5049 + - type: nauc_precision_at_5_std + value: 2.7146 + - type: nauc_precision_at_5_diff1 + value: 24.1766 + - type: nauc_precision_at_10_max + value: 26.168799999999997 + - type: nauc_precision_at_10_std + value: 4.5483 + - type: nauc_precision_at_10_diff1 + value: 19.7263 + - type: nauc_precision_at_20_max + value: 24.2909 + - type: nauc_precision_at_20_std + value: 5.985399999999999 + - type: nauc_precision_at_20_diff1 + value: 14.394699999999998 + - type: nauc_precision_at_100_max + value: 20.945700000000002 + - type: nauc_precision_at_100_std + value: 9.717099999999999 + - type: nauc_precision_at_100_diff1 + value: 10.1707 + - type: nauc_precision_at_1000_max + value: 17.9958 + - type: nauc_precision_at_1000_std + value: 6.352399999999999 + - type: nauc_precision_at_1000_diff1 + value: 6.671100000000001 + - type: nauc_mrr_at_1_max + value: 22.2824 + - type: nauc_mrr_at_1_std + value: -1.1636 + - type: nauc_mrr_at_1_diff1 + value: 44.0723 + - type: nauc_mrr_at_3_max + value: 24.4906 + - type: nauc_mrr_at_3_std + value: 0.5277 + - type: nauc_mrr_at_3_diff1 + value: 39.3446 + - type: nauc_mrr_at_5_max + value: 24.3708 + - type: nauc_mrr_at_5_std + value: 0.5988 + - type: nauc_mrr_at_5_diff1 + value: 38.5081 + - type: nauc_mrr_at_10_max + value: 24.5065 + - type: nauc_mrr_at_10_std + value: 0.9650000000000001 + - type: nauc_mrr_at_10_diff1 + value: 38.4531 + - type: nauc_mrr_at_20_max + value: 24.577099999999998 + - type: nauc_mrr_at_20_std + value: 0.9927999999999999 + - type: nauc_mrr_at_20_diff1 + value: 38.3527 + - type: nauc_mrr_at_100_max + value: 24.593999999999998 + - type: nauc_mrr_at_100_std + value: 1.1214 + - type: nauc_mrr_at_100_diff1 + value: 38.4554 + - type: nauc_mrr_at_1000_max + value: 24.5991 + - type: nauc_mrr_at_1000_std + value: 1.1217 + - type: nauc_mrr_at_1000_diff1 + value: 38.4672 + - type: main_score + value: 37.219 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackUnixRetrieval (default) + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 + split: test + type: mteb/cqadupstack-unix + metrics: + - type: ndcg_at_1 + value: 41.884 + - type: ndcg_at_3 + value: 47.415 + - type: ndcg_at_5 + value: 50.442 + - type: ndcg_at_10 + value: 53.733 + - type: ndcg_at_20 + value: 55.527 + - type: ndcg_at_100 + value: 58.12199999999999 + - type: ndcg_at_1000 + value: 59.540000000000006 + - type: map_at_1 + value: 35.569 + - type: map_at_3 + value: 43.517 + - type: map_at_5 + value: 45.673 + - type: map_at_10 + value: 47.373 + - type: map_at_20 + value: 47.997 + - type: map_at_100 + value: 48.449999999999996 + - type: map_at_1000 + value: 48.524 + - type: recall_at_1 + value: 35.569 + - type: recall_at_3 + value: 51.43600000000001 + - type: recall_at_5 + value: 59.229 + - type: recall_at_10 + value: 68.675 + - type: recall_at_20 + value: 74.935 + - type: recall_at_100 + value: 87.12100000000001 + - type: recall_at_1000 + value: 96.389 + - type: precision_at_1 + value: 41.884 + - type: precision_at_3 + value: 21.735 + - type: precision_at_5 + value: 15.354000000000001 + - type: precision_at_10 + value: 9.207 + - type: precision_at_20 + value: 5.159 + - type: precision_at_100 + value: 1.2510000000000001 + - type: precision_at_1000 + value: 0.147 + - type: mrr_at_1 + value: 41.8843 + - type: mrr_at_3 + value: 48.8029 + - type: mrr_at_5 + value: 50.5053 + - type: mrr_at_10 + value: 51.6938 + - type: mrr_at_20 + value: 52.0562 + - type: mrr_at_100 + value: 52.3425 + - type: mrr_at_1000 + value: 52.3775 + - type: nauc_ndcg_at_1_max + value: 39.1822 + - type: nauc_ndcg_at_1_std + value: -10.6489 + - type: nauc_ndcg_at_1_diff1 + value: 53.662299999999995 + - type: nauc_ndcg_at_3_max + value: 39.4505 + - type: nauc_ndcg_at_3_std + value: -10.6853 + - type: nauc_ndcg_at_3_diff1 + value: 48.5749 + - type: nauc_ndcg_at_5_max + value: 38.7618 + - type: nauc_ndcg_at_5_std + value: -10.972800000000001 + - type: nauc_ndcg_at_5_diff1 + value: 47.846 + - type: nauc_ndcg_at_10_max + value: 38.9284 + - type: nauc_ndcg_at_10_std + value: -10.6664 + - type: nauc_ndcg_at_10_diff1 + value: 46.9536 + - type: nauc_ndcg_at_20_max + value: 39.760400000000004 + - type: nauc_ndcg_at_20_std + value: -9.5981 + - type: nauc_ndcg_at_20_diff1 + value: 47.6581 + - type: nauc_ndcg_at_100_max + value: 40.1241 + - type: nauc_ndcg_at_100_std + value: -8.7613 + - type: nauc_ndcg_at_100_diff1 + value: 47.8645 + - type: nauc_ndcg_at_1000_max + value: 39.8779 + - type: nauc_ndcg_at_1000_std + value: -9.0252 + - type: nauc_ndcg_at_1000_diff1 + value: 48.0716 + - type: nauc_map_at_1_max + value: 33.7108 + - type: nauc_map_at_1_std + value: -11.0197 + - type: nauc_map_at_1_diff1 + value: 51.6481 + - type: nauc_map_at_3_max + value: 37.4784 + - type: nauc_map_at_3_std + value: -11.2809 + - type: nauc_map_at_3_diff1 + value: 49.408 + - type: nauc_map_at_5_max + value: 37.6673 + - type: nauc_map_at_5_std + value: -11.2829 + - type: nauc_map_at_5_diff1 + value: 48.89 + - type: nauc_map_at_10_max + value: 37.9209 + - type: nauc_map_at_10_std + value: -11.2194 + - type: nauc_map_at_10_diff1 + value: 48.2784 + - type: nauc_map_at_20_max + value: 38.2975 + - type: nauc_map_at_20_std + value: -10.8997 + - type: nauc_map_at_20_diff1 + value: 48.547000000000004 + - type: nauc_map_at_100_max + value: 38.352799999999995 + - type: nauc_map_at_100_std + value: -10.7712 + - type: nauc_map_at_100_diff1 + value: 48.5685 + - type: nauc_map_at_1000_max + value: 38.3309 + - type: nauc_map_at_1000_std + value: -10.7669 + - type: nauc_map_at_1000_diff1 + value: 48.5663 + - type: nauc_recall_at_1_max + value: 33.7108 + - type: nauc_recall_at_1_std + value: -11.0197 + - type: nauc_recall_at_1_diff1 + value: 51.6481 + - type: nauc_recall_at_3_max + value: 37.8568 + - type: nauc_recall_at_3_std + value: -10.046 + - type: nauc_recall_at_3_diff1 + value: 44.973200000000006 + - type: nauc_recall_at_5_max + value: 36.5963 + - type: nauc_recall_at_5_std + value: -10.656 + - type: nauc_recall_at_5_diff1 + value: 41.8226 + - type: nauc_recall_at_10_max + value: 36.905300000000004 + - type: nauc_recall_at_10_std + value: -9.5656 + - type: nauc_recall_at_10_diff1 + value: 37.8973 + - type: nauc_recall_at_20_max + value: 40.465 + - type: nauc_recall_at_20_std + value: -4.2909999999999995 + - type: nauc_recall_at_20_diff1 + value: 40.2965 + - type: nauc_recall_at_100_max + value: 47.295500000000004 + - type: nauc_recall_at_100_std + value: 6.931900000000001 + - type: nauc_recall_at_100_diff1 + value: 39.684599999999996 + - type: nauc_recall_at_1000_max + value: 64.6766 + - type: nauc_recall_at_1000_std + value: 32.608399999999996 + - type: nauc_recall_at_1000_diff1 + value: 41.2191 + - type: nauc_precision_at_1_max + value: 39.1822 + - type: nauc_precision_at_1_std + value: -10.6489 + - type: nauc_precision_at_1_diff1 + value: 53.662299999999995 + - type: nauc_precision_at_3_max + value: 37.938 + - type: nauc_precision_at_3_std + value: -7.1814 + - type: nauc_precision_at_3_diff1 + value: 33.5813 + - type: nauc_precision_at_5_max + value: 33.5192 + - type: nauc_precision_at_5_std + value: -5.5998 + - type: nauc_precision_at_5_diff1 + value: 24.4701 + - type: nauc_precision_at_10_max + value: 27.776600000000002 + - type: nauc_precision_at_10_std + value: -4.016900000000001 + - type: nauc_precision_at_10_diff1 + value: 13.019400000000001 + - type: nauc_precision_at_20_max + value: 25.036199999999997 + - type: nauc_precision_at_20_std + value: 0.1629 + - type: nauc_precision_at_20_diff1 + value: 9.332 + - type: nauc_precision_at_100_max + value: 14.1849 + - type: nauc_precision_at_100_std + value: 6.534800000000001 + - type: nauc_precision_at_100_diff1 + value: -3.1784 + - type: nauc_precision_at_1000_max + value: 0.3891 + - type: nauc_precision_at_1000_std + value: 4.8176 + - type: nauc_precision_at_1000_diff1 + value: -13.1996 + - type: nauc_mrr_at_1_max + value: 39.1822 + - type: nauc_mrr_at_1_std + value: -10.6489 + - type: nauc_mrr_at_1_diff1 + value: 53.662299999999995 + - type: nauc_mrr_at_3_max + value: 40.5435 + - type: nauc_mrr_at_3_std + value: -9.9119 + - type: nauc_mrr_at_3_diff1 + value: 50.5792 + - type: nauc_mrr_at_5_max + value: 40.5036 + - type: nauc_mrr_at_5_std + value: -10.0048 + - type: nauc_mrr_at_5_diff1 + value: 50.1912 + - type: nauc_mrr_at_10_max + value: 40.367 + - type: nauc_mrr_at_10_std + value: -10.0094 + - type: nauc_mrr_at_10_diff1 + value: 49.914500000000004 + - type: nauc_mrr_at_20_max + value: 40.487 + - type: nauc_mrr_at_20_std + value: -9.8134 + - type: nauc_mrr_at_20_diff1 + value: 50.068900000000006 + - type: nauc_mrr_at_100_max + value: 40.4627 + - type: nauc_mrr_at_100_std + value: -9.7388 + - type: nauc_mrr_at_100_diff1 + value: 50.094300000000004 + - type: nauc_mrr_at_1000_max + value: 40.4524 + - type: nauc_mrr_at_1000_std + value: -9.748700000000001 + - type: nauc_mrr_at_1000_diff1 + value: 50.1065 + - type: main_score + value: 53.733 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWebmastersRetrieval (default) + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: mteb/cqadupstack-webmasters + metrics: + - type: ndcg_at_1 + value: 37.945 + - type: ndcg_at_3 + value: 44.157000000000004 + - type: ndcg_at_5 + value: 46.88 + - type: ndcg_at_10 + value: 50.208 + - type: ndcg_at_20 + value: 52.536 + - type: ndcg_at_100 + value: 55.711999999999996 + - type: ndcg_at_1000 + value: 57.340999999999994 + - type: map_at_1 + value: 31.174000000000003 + - type: map_at_3 + value: 39.391 + - type: map_at_5 + value: 41.333 + - type: map_at_10 + value: 43.246 + - type: map_at_20 + value: 44.21 + - type: map_at_100 + value: 45.013 + - type: map_at_1000 + value: 45.221000000000004 + - type: recall_at_1 + value: 31.174000000000003 + - type: recall_at_3 + value: 47.085 + - type: recall_at_5 + value: 54.237 + - type: recall_at_10 + value: 63.611 + - type: recall_at_20 + value: 72.473 + - type: recall_at_100 + value: 87.45100000000001 + - type: recall_at_1000 + value: 97.429 + - type: precision_at_1 + value: 37.945 + - type: precision_at_3 + value: 20.751 + - type: precision_at_5 + value: 15.02 + - type: precision_at_10 + value: 9.722999999999999 + - type: precision_at_20 + value: 5.988 + - type: precision_at_100 + value: 1.818 + - type: precision_at_1000 + value: 0.256 + - type: mrr_at_1 + value: 37.9447 + - type: mrr_at_3 + value: 45.3228 + - type: mrr_at_5 + value: 47.0224 + - type: mrr_at_10 + value: 48.234 + - type: mrr_at_20 + value: 48.7403 + - type: mrr_at_100 + value: 49.059999999999995 + - type: mrr_at_1000 + value: 49.0914 + - type: nauc_ndcg_at_1_max + value: 26.172 + - type: nauc_ndcg_at_1_std + value: -9.07 + - type: nauc_ndcg_at_1_diff1 + value: 46.664899999999996 + - type: nauc_ndcg_at_3_max + value: 23.9966 + - type: nauc_ndcg_at_3_std + value: -11.0207 + - type: nauc_ndcg_at_3_diff1 + value: 43.539 + - type: nauc_ndcg_at_5_max + value: 24.9051 + - type: nauc_ndcg_at_5_std + value: -9.9938 + - type: nauc_ndcg_at_5_diff1 + value: 44.5711 + - type: nauc_ndcg_at_10_max + value: 27.603 + - type: nauc_ndcg_at_10_std + value: -8.339599999999999 + - type: nauc_ndcg_at_10_diff1 + value: 45.121 + - type: nauc_ndcg_at_20_max + value: 27.1764 + - type: nauc_ndcg_at_20_std + value: -7.295400000000001 + - type: nauc_ndcg_at_20_diff1 + value: 43.925 + - type: nauc_ndcg_at_100_max + value: 28.0352 + - type: nauc_ndcg_at_100_std + value: -6.6677 + - type: nauc_ndcg_at_100_diff1 + value: 43.6903 + - type: nauc_ndcg_at_1000_max + value: 27.104400000000002 + - type: nauc_ndcg_at_1000_std + value: -6.9685 + - type: nauc_ndcg_at_1000_diff1 + value: 43.952000000000005 + - type: nauc_map_at_1_max + value: 20.5331 + - type: nauc_map_at_1_std + value: -12.358600000000001 + - type: nauc_map_at_1_diff1 + value: 48.4715 + - type: nauc_map_at_3_max + value: 21.4883 + - type: nauc_map_at_3_std + value: -12.6776 + - type: nauc_map_at_3_diff1 + value: 44.2352 + - type: nauc_map_at_5_max + value: 22.3393 + - type: nauc_map_at_5_std + value: -11.6253 + - type: nauc_map_at_5_diff1 + value: 44.4847 + - type: nauc_map_at_10_max + value: 24.371399999999998 + - type: nauc_map_at_10_std + value: -10.5509 + - type: nauc_map_at_10_diff1 + value: 45.3059 + - type: nauc_map_at_20_max + value: 24.4314 + - type: nauc_map_at_20_std + value: -10.012799999999999 + - type: nauc_map_at_20_diff1 + value: 45.1512 + - type: nauc_map_at_100_max + value: 24.672900000000002 + - type: nauc_map_at_100_std + value: -9.637500000000001 + - type: nauc_map_at_100_diff1 + value: 45.31 + - type: nauc_map_at_1000_max + value: 24.432499999999997 + - type: nauc_map_at_1000_std + value: -9.5451 + - type: nauc_map_at_1000_diff1 + value: 45.3162 + - type: nauc_recall_at_1_max + value: 20.5331 + - type: nauc_recall_at_1_std + value: -12.358600000000001 + - type: nauc_recall_at_1_diff1 + value: 48.4715 + - type: nauc_recall_at_3_max + value: 19.8608 + - type: nauc_recall_at_3_std + value: -12.6162 + - type: nauc_recall_at_3_diff1 + value: 39.216699999999996 + - type: nauc_recall_at_5_max + value: 22.131700000000002 + - type: nauc_recall_at_5_std + value: -9.728100000000001 + - type: nauc_recall_at_5_diff1 + value: 39.307900000000004 + - type: nauc_recall_at_10_max + value: 32.0438 + - type: nauc_recall_at_10_std + value: -3.6334999999999997 + - type: nauc_recall_at_10_diff1 + value: 39.2567 + - type: nauc_recall_at_20_max + value: 32.0439 + - type: nauc_recall_at_20_std + value: 2.7743 + - type: nauc_recall_at_20_diff1 + value: 32.6522 + - type: nauc_recall_at_100_max + value: 47.1356 + - type: nauc_recall_at_100_std + value: 12.581700000000001 + - type: nauc_recall_at_100_diff1 + value: 25.913700000000002 + - type: nauc_recall_at_1000_max + value: 59.09799999999999 + - type: nauc_recall_at_1000_std + value: 47.4747 + - type: nauc_recall_at_1000_diff1 + value: -1.6067999999999998 + - type: nauc_precision_at_1_max + value: 26.172 + - type: nauc_precision_at_1_std + value: -9.07 + - type: nauc_precision_at_1_diff1 + value: 46.664899999999996 + - type: nauc_precision_at_3_max + value: 24.7472 + - type: nauc_precision_at_3_std + value: -5.6165 + - type: nauc_precision_at_3_diff1 + value: 29.5543 + - type: nauc_precision_at_5_max + value: 26.334000000000003 + - type: nauc_precision_at_5_std + value: 0.8363 + - type: nauc_precision_at_5_diff1 + value: 26.732899999999997 + - type: nauc_precision_at_10_max + value: 26.837100000000003 + - type: nauc_precision_at_10_std + value: 8.7927 + - type: nauc_precision_at_10_diff1 + value: 20.8763 + - type: nauc_precision_at_20_max + value: 18.232200000000002 + - type: nauc_precision_at_20_std + value: 11.752600000000001 + - type: nauc_precision_at_20_diff1 + value: 11.7568 + - type: nauc_precision_at_100_max + value: 2.1069 + - type: nauc_precision_at_100_std + value: 14.2173 + - type: nauc_precision_at_100_diff1 + value: 0.9792000000000001 + - type: nauc_precision_at_1000_max + value: -12.2237 + - type: nauc_precision_at_1000_std + value: 9.9255 + - type: nauc_precision_at_1000_diff1 + value: -5.8681 + - type: nauc_mrr_at_1_max + value: 26.172 + - type: nauc_mrr_at_1_std + value: -9.07 + - type: nauc_mrr_at_1_diff1 + value: 46.664899999999996 + - type: nauc_mrr_at_3_max + value: 25.629800000000003 + - type: nauc_mrr_at_3_std + value: -10.238800000000001 + - type: nauc_mrr_at_3_diff1 + value: 44.330799999999996 + - type: nauc_mrr_at_5_max + value: 26.7314 + - type: nauc_mrr_at_5_std + value: -9.589 + - type: nauc_mrr_at_5_diff1 + value: 45.0557 + - type: nauc_mrr_at_10_max + value: 27.4486 + - type: nauc_mrr_at_10_std + value: -8.8187 + - type: nauc_mrr_at_10_diff1 + value: 44.6457 + - type: nauc_mrr_at_20_max + value: 27.270100000000003 + - type: nauc_mrr_at_20_std + value: -8.6464 + - type: nauc_mrr_at_20_diff1 + value: 44.4286 + - type: nauc_mrr_at_100_max + value: 27.284399999999998 + - type: nauc_mrr_at_100_std + value: -8.664299999999999 + - type: nauc_mrr_at_100_diff1 + value: 44.4562 + - type: nauc_mrr_at_1000_max + value: 27.27 + - type: nauc_mrr_at_1000_std + value: -8.6626 + - type: nauc_mrr_at_1000_diff1 + value: 44.465900000000005 + - type: main_score + value: 50.208 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWordpressRetrieval (default) + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack-wordpress + metrics: + - type: ndcg_at_1 + value: 27.911 + - type: ndcg_at_3 + value: 34.677 + - type: ndcg_at_5 + value: 38.315 + - type: ndcg_at_10 + value: 40.988 + - type: ndcg_at_20 + value: 42.99 + - type: ndcg_at_100 + value: 46.389 + - type: ndcg_at_1000 + value: 48.172 + - type: map_at_1 + value: 25.456 + - type: map_at_3 + value: 31.837 + - type: map_at_5 + value: 34.097 + - type: map_at_10 + value: 35.326 + - type: map_at_20 + value: 35.918 + - type: map_at_100 + value: 36.434 + - type: map_at_1000 + value: 36.513 + - type: recall_at_1 + value: 25.456 + - type: recall_at_3 + value: 39.892 + - type: recall_at_5 + value: 48.524 + - type: recall_at_10 + value: 56.254000000000005 + - type: recall_at_20 + value: 63.783 + - type: recall_at_100 + value: 81.164 + - type: recall_at_1000 + value: 93.89 + - type: precision_at_1 + value: 27.911 + - type: precision_at_3 + value: 14.849 + - type: precision_at_5 + value: 11.054 + - type: precision_at_10 + value: 6.543 + - type: precision_at_20 + value: 3.762 + - type: precision_at_100 + value: 0.9820000000000001 + - type: precision_at_1000 + value: 0.126 + - type: mrr_at_1 + value: 27.9113 + - type: mrr_at_3 + value: 34.5348 + - type: mrr_at_5 + value: 36.411 + - type: mrr_at_10 + value: 37.4528 + - type: mrr_at_20 + value: 37.992399999999996 + - type: mrr_at_100 + value: 38.4368 + - type: mrr_at_1000 + value: 38.4845 + - type: nauc_ndcg_at_1_max + value: 20.846999999999998 + - type: nauc_ndcg_at_1_std + value: -4.2672 + - type: nauc_ndcg_at_1_diff1 + value: 44.0328 + - type: nauc_ndcg_at_3_max + value: 22.7709 + - type: nauc_ndcg_at_3_std + value: -4.4297 + - type: nauc_ndcg_at_3_diff1 + value: 39.555099999999996 + - type: nauc_ndcg_at_5_max + value: 24.9005 + - type: nauc_ndcg_at_5_std + value: -2.1591 + - type: nauc_ndcg_at_5_diff1 + value: 37.3692 + - type: nauc_ndcg_at_10_max + value: 24.909100000000002 + - type: nauc_ndcg_at_10_std + value: -0.384 + - type: nauc_ndcg_at_10_diff1 + value: 37.2953 + - type: nauc_ndcg_at_20_max + value: 25.519399999999997 + - type: nauc_ndcg_at_20_std + value: 0.2725 + - type: nauc_ndcg_at_20_diff1 + value: 37.1091 + - type: nauc_ndcg_at_100_max + value: 25.6145 + - type: nauc_ndcg_at_100_std + value: 0.8262999999999999 + - type: nauc_ndcg_at_100_diff1 + value: 36.5502 + - type: nauc_ndcg_at_1000_max + value: 24.5673 + - type: nauc_ndcg_at_1000_std + value: 0.060899999999999996 + - type: nauc_ndcg_at_1000_diff1 + value: 36.9253 + - type: nauc_map_at_1_max + value: 19.8422 + - type: nauc_map_at_1_std + value: -5.319100000000001 + - type: nauc_map_at_1_diff1 + value: 44.1229 + - type: nauc_map_at_3_max + value: 21.9723 + - type: nauc_map_at_3_std + value: -5.1189 + - type: nauc_map_at_3_diff1 + value: 40.771 + - type: nauc_map_at_5_max + value: 23.4629 + - type: nauc_map_at_5_std + value: -3.5612 + - type: nauc_map_at_5_diff1 + value: 39.307700000000004 + - type: nauc_map_at_10_max + value: 23.519499999999997 + - type: nauc_map_at_10_std + value: -2.8228 + - type: nauc_map_at_10_diff1 + value: 39.4316 + - type: nauc_map_at_20_max + value: 23.6993 + - type: nauc_map_at_20_std + value: -2.5308 + - type: nauc_map_at_20_diff1 + value: 39.2955 + - type: nauc_map_at_100_max + value: 23.674799999999998 + - type: nauc_map_at_100_std + value: -2.4657999999999998 + - type: nauc_map_at_100_diff1 + value: 39.1997 + - type: nauc_map_at_1000_max + value: 23.629 + - type: nauc_map_at_1000_std + value: -2.4773 + - type: nauc_map_at_1000_diff1 + value: 39.1866 + - type: nauc_recall_at_1_max + value: 19.8422 + - type: nauc_recall_at_1_std + value: -5.319100000000001 + - type: nauc_recall_at_1_diff1 + value: 44.1229 + - type: nauc_recall_at_3_max + value: 23.5368 + - type: nauc_recall_at_3_std + value: -4.4474 + - type: nauc_recall_at_3_diff1 + value: 36.3819 + - type: nauc_recall_at_5_max + value: 28.0457 + - type: nauc_recall_at_5_std + value: 0.7798 + - type: nauc_recall_at_5_diff1 + value: 31.097599999999996 + - type: nauc_recall_at_10_max + value: 27.5608 + - type: nauc_recall_at_10_std + value: 5.9596 + - type: nauc_recall_at_10_diff1 + value: 29.6752 + - type: nauc_recall_at_20_max + value: 30.1434 + - type: nauc_recall_at_20_std + value: 8.7057 + - type: nauc_recall_at_20_diff1 + value: 28.402500000000003 + - type: nauc_recall_at_100_max + value: 35.001 + - type: nauc_recall_at_100_std + value: 18.8733 + - type: nauc_recall_at_100_diff1 + value: 18.171499999999998 + - type: nauc_recall_at_1000_max + value: 24.1775 + - type: nauc_recall_at_1000_std + value: 23.6246 + - type: nauc_recall_at_1000_diff1 + value: 9.8065 + - type: nauc_precision_at_1_max + value: 20.846999999999998 + - type: nauc_precision_at_1_std + value: -4.2672 + - type: nauc_precision_at_1_diff1 + value: 44.0328 + - type: nauc_precision_at_3_max + value: 25.306600000000003 + - type: nauc_precision_at_3_std + value: -1.959 + - type: nauc_precision_at_3_diff1 + value: 36.350500000000004 + - type: nauc_precision_at_5_max + value: 28.2705 + - type: nauc_precision_at_5_std + value: 5.4924 + - type: nauc_precision_at_5_diff1 + value: 28.198099999999997 + - type: nauc_precision_at_10_max + value: 26.6247 + - type: nauc_precision_at_10_std + value: 11.3267 + - type: nauc_precision_at_10_diff1 + value: 25.2188 + - type: nauc_precision_at_20_max + value: 27.254499999999997 + - type: nauc_precision_at_20_std + value: 15.3152 + - type: nauc_precision_at_20_diff1 + value: 19.916 + - type: nauc_precision_at_100_max + value: 20.3749 + - type: nauc_precision_at_100_std + value: 20.8664 + - type: nauc_precision_at_100_diff1 + value: 3.8397 + - type: nauc_precision_at_1000_max + value: -12.1998 + - type: nauc_precision_at_1000_std + value: 2.7227 + - type: nauc_precision_at_1000_diff1 + value: -18.4254 + - type: nauc_mrr_at_1_max + value: 20.846999999999998 + - type: nauc_mrr_at_1_std + value: -4.2672 + - type: nauc_mrr_at_1_diff1 + value: 44.0328 + - type: nauc_mrr_at_3_max + value: 22.907 + - type: nauc_mrr_at_3_std + value: -3.8749 + - type: nauc_mrr_at_3_diff1 + value: 40.1759 + - type: nauc_mrr_at_5_max + value: 23.819499999999998 + - type: nauc_mrr_at_5_std + value: -2.5065 + - type: nauc_mrr_at_5_diff1 + value: 39.2975 + - type: nauc_mrr_at_10_max + value: 23.8817 + - type: nauc_mrr_at_10_std + value: -1.6466999999999998 + - type: nauc_mrr_at_10_diff1 + value: 39.1727 + - type: nauc_mrr_at_20_max + value: 24 + - type: nauc_mrr_at_20_std + value: -1.5741 + - type: nauc_mrr_at_20_diff1 + value: 39.1967 + - type: nauc_mrr_at_100_max + value: 23.811799999999998 + - type: nauc_mrr_at_100_std + value: -1.6327 + - type: nauc_mrr_at_100_diff1 + value: 39.0917 + - type: nauc_mrr_at_1000_max + value: 23.7897 + - type: nauc_mrr_at_1000_std + value: -1.6494000000000002 + - type: nauc_mrr_at_1000_diff1 + value: 39.1019 + - type: main_score + value: 40.988 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER (default) + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: ndcg_at_1 + value: 45.668 + - type: ndcg_at_3 + value: 38.864 + - type: ndcg_at_5 + value: 41.327000000000005 + - type: ndcg_at_10 + value: 45.04 + - type: ndcg_at_20 + value: 47.542 + - type: ndcg_at_100 + value: 50.183 + - type: ndcg_at_1000 + value: 52.129000000000005 + - type: map_at_1 + value: 20.186 + - type: map_at_3 + value: 29.237000000000002 + - type: map_at_5 + value: 32.458999999999996 + - type: map_at_10 + value: 34.713 + - type: map_at_20 + value: 35.759 + - type: map_at_100 + value: 36.351 + - type: map_at_1000 + value: 36.455 + - type: recall_at_1 + value: 20.186 + - type: recall_at_3 + value: 34.772 + - type: recall_at_5 + value: 42.491 + - type: recall_at_10 + value: 50.611 + - type: recall_at_20 + value: 57.595 + - type: recall_at_100 + value: 67.374 + - type: recall_at_1000 + value: 78.244 + - type: precision_at_1 + value: 45.668 + - type: precision_at_3 + value: 29.316 + - type: precision_at_5 + value: 22.306 + - type: precision_at_10 + value: 13.668 + - type: precision_at_20 + value: 7.925 + - type: precision_at_100 + value: 1.9109999999999998 + - type: precision_at_1000 + value: 0.22899999999999998 + - type: mrr_at_1 + value: 45.6678 + - type: mrr_at_3 + value: 55.7438 + - type: mrr_at_5 + value: 57.3398 + - type: mrr_at_10 + value: 58.032799999999995 + - type: mrr_at_20 + value: 58.3472 + - type: mrr_at_100 + value: 58.4846 + - type: mrr_at_1000 + value: 58.504400000000004 + - type: nauc_ndcg_at_1_max + value: 39.312599999999996 + - type: nauc_ndcg_at_1_std + value: 13.444600000000001 + - type: nauc_ndcg_at_1_diff1 + value: 31.551499999999997 + - type: nauc_ndcg_at_3_max + value: 40.7886 + - type: nauc_ndcg_at_3_std + value: 11.7545 + - type: nauc_ndcg_at_3_diff1 + value: 24.758399999999998 + - type: nauc_ndcg_at_5_max + value: 41.4458 + - type: nauc_ndcg_at_5_std + value: 12.7212 + - type: nauc_ndcg_at_5_diff1 + value: 23.8522 + - type: nauc_ndcg_at_10_max + value: 41.6993 + - type: nauc_ndcg_at_10_std + value: 14.6038 + - type: nauc_ndcg_at_10_diff1 + value: 23.8755 + - type: nauc_ndcg_at_20_max + value: 41.4782 + - type: nauc_ndcg_at_20_std + value: 17.1696 + - type: nauc_ndcg_at_20_diff1 + value: 23.877200000000002 + - type: nauc_ndcg_at_100_max + value: 41.652499999999996 + - type: nauc_ndcg_at_100_std + value: 19.2863 + - type: nauc_ndcg_at_100_diff1 + value: 23.9355 + - type: nauc_ndcg_at_1000_max + value: 41.7572 + - type: nauc_ndcg_at_1000_std + value: 19.889200000000002 + - type: nauc_ndcg_at_1000_diff1 + value: 24.0865 + - type: nauc_map_at_1_max + value: 34.5948 + - type: nauc_map_at_1_std + value: 9.3331 + - type: nauc_map_at_1_diff1 + value: 33.4788 + - type: nauc_map_at_3_max + value: 39.2329 + - type: nauc_map_at_3_std + value: 11.0441 + - type: nauc_map_at_3_diff1 + value: 26.2025 + - type: nauc_map_at_5_max + value: 40.1248 + - type: nauc_map_at_5_std + value: 12.484 + - type: nauc_map_at_5_diff1 + value: 24.7156 + - type: nauc_map_at_10_max + value: 40.6486 + - type: nauc_map_at_10_std + value: 13.386400000000002 + - type: nauc_map_at_10_diff1 + value: 24.726100000000002 + - type: nauc_map_at_20_max + value: 40.6126 + - type: nauc_map_at_20_std + value: 14.5582 + - type: nauc_map_at_20_diff1 + value: 24.6569 + - type: nauc_map_at_100_max + value: 40.7502 + - type: nauc_map_at_100_std + value: 15.082899999999999 + - type: nauc_map_at_100_diff1 + value: 24.5925 + - type: nauc_map_at_1000_max + value: 40.745 + - type: nauc_map_at_1000_std + value: 15.1392 + - type: nauc_map_at_1000_diff1 + value: 24.6006 + - type: nauc_recall_at_1_max + value: 34.5948 + - type: nauc_recall_at_1_std + value: 9.3331 + - type: nauc_recall_at_1_diff1 + value: 33.4788 + - type: nauc_recall_at_3_max + value: 38.5191 + - type: nauc_recall_at_3_std + value: 9.8077 + - type: nauc_recall_at_3_diff1 + value: 21.4604 + - type: nauc_recall_at_5_max + value: 38.1356 + - type: nauc_recall_at_5_std + value: 11.158 + - type: nauc_recall_at_5_diff1 + value: 17.6417 + - type: nauc_recall_at_10_max + value: 36.6836 + - type: nauc_recall_at_10_std + value: 14.6125 + - type: nauc_recall_at_10_diff1 + value: 16.9109 + - type: nauc_recall_at_20_max + value: 34.7404 + - type: nauc_recall_at_20_std + value: 20.89 + - type: nauc_recall_at_20_diff1 + value: 16.233 + - type: nauc_recall_at_100_max + value: 33.6466 + - type: nauc_recall_at_100_std + value: 28.839399999999998 + - type: nauc_recall_at_100_diff1 + value: 15.2031 + - type: nauc_recall_at_1000_max + value: 33.4333 + - type: nauc_recall_at_1000_std + value: 35.3876 + - type: nauc_recall_at_1000_diff1 + value: 14.2567 + - type: nauc_precision_at_1_max + value: 39.312599999999996 + - type: nauc_precision_at_1_std + value: 13.444600000000001 + - type: nauc_precision_at_1_diff1 + value: 31.551499999999997 + - type: nauc_precision_at_3_max + value: 38.6969 + - type: nauc_precision_at_3_std + value: 11.604000000000001 + - type: nauc_precision_at_3_diff1 + value: 12.2982 + - type: nauc_precision_at_5_max + value: 34.0346 + - type: nauc_precision_at_5_std + value: 13.222700000000001 + - type: nauc_precision_at_5_diff1 + value: 7.2342 + - type: nauc_precision_at_10_max + value: 29.3584 + - type: nauc_precision_at_10_std + value: 16.1479 + - type: nauc_precision_at_10_diff1 + value: 5.3597 + - type: nauc_precision_at_20_max + value: 23.502799999999997 + - type: nauc_precision_at_20_std + value: 21.465799999999998 + - type: nauc_precision_at_20_diff1 + value: 2.835 + - type: nauc_precision_at_100_max + value: 16.001 + - type: nauc_precision_at_100_std + value: 26.1729 + - type: nauc_precision_at_100_diff1 + value: -1.1341999999999999 + - type: nauc_precision_at_1000_max + value: 6.7147 + - type: nauc_precision_at_1000_std + value: 25.3562 + - type: nauc_precision_at_1000_diff1 + value: -5.8931 + - type: nauc_mrr_at_1_max + value: 39.312599999999996 + - type: nauc_mrr_at_1_std + value: 13.444600000000001 + - type: nauc_mrr_at_1_diff1 + value: 31.551499999999997 + - type: nauc_mrr_at_3_max + value: 41.599799999999995 + - type: nauc_mrr_at_3_std + value: 13.084499999999998 + - type: nauc_mrr_at_3_diff1 + value: 27.8827 + - type: nauc_mrr_at_5_max + value: 41.7667 + - type: nauc_mrr_at_5_std + value: 13.2025 + - type: nauc_mrr_at_5_diff1 + value: 27.8692 + - type: nauc_mrr_at_10_max + value: 41.6294 + - type: nauc_mrr_at_10_std + value: 13.9039 + - type: nauc_mrr_at_10_diff1 + value: 27.9569 + - type: nauc_mrr_at_20_max + value: 41.6353 + - type: nauc_mrr_at_20_std + value: 13.9752 + - type: nauc_mrr_at_20_diff1 + value: 28.0767 + - type: nauc_mrr_at_100_max + value: 41.6002 + - type: nauc_mrr_at_100_std + value: 14.0432 + - type: nauc_mrr_at_100_diff1 + value: 28.1348 + - type: nauc_mrr_at_1000_max + value: 41.5999 + - type: nauc_mrr_at_1000_std + value: 14.043 + - type: nauc_mrr_at_1000_diff1 + value: 28.1343 + - type: main_score + value: 45.04 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia (default) + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: ndcg_at_1 + value: 65.625 + - type: ndcg_at_3 + value: 57.938 + - type: ndcg_at_5 + value: 55.498999999999995 + - type: ndcg_at_10 + value: 52.577 + - type: ndcg_at_20 + value: 52.566 + - type: ndcg_at_100 + value: 58.352000000000004 + - type: ndcg_at_1000 + value: 64.887 + - type: map_at_1 + value: 10.327 + - type: map_at_3 + value: 17.702 + - type: map_at_5 + value: 21.409 + - type: map_at_10 + value: 25.832 + - type: map_at_20 + value: 31.006 + - type: map_at_100 + value: 38.357 + - type: map_at_1000 + value: 40.194 + - type: recall_at_1 + value: 10.327 + - type: recall_at_3 + value: 18.999 + - type: recall_at_5 + value: 24.297 + - type: recall_at_10 + value: 31.435000000000002 + - type: recall_at_20 + value: 41.801 + - type: recall_at_100 + value: 64.751 + - type: recall_at_1000 + value: 86.043 + - type: precision_at_1 + value: 76.5 + - type: precision_at_3 + value: 61.833000000000006 + - type: precision_at_5 + value: 53.55 + - type: precision_at_10 + value: 41.8 + - type: precision_at_20 + value: 32.963 + - type: precision_at_100 + value: 13.498 + - type: precision_at_1000 + value: 2.357 + - type: mrr_at_1 + value: 76.5 + - type: mrr_at_3 + value: 82.8333 + - type: mrr_at_5 + value: 83.4458 + - type: mrr_at_10 + value: 83.6805 + - type: mrr_at_20 + value: 83.7449 + - type: mrr_at_100 + value: 83.8219 + - type: mrr_at_1000 + value: 83.8283 + - type: nauc_ndcg_at_1_max + value: 51.568400000000004 + - type: nauc_ndcg_at_1_std + value: 30.5435 + - type: nauc_ndcg_at_1_diff1 + value: 49.4987 + - type: nauc_ndcg_at_3_max + value: 43.4883 + - type: nauc_ndcg_at_3_std + value: 31.5687 + - type: nauc_ndcg_at_3_diff1 + value: 30.1514 + - type: nauc_ndcg_at_5_max + value: 42.2335 + - type: nauc_ndcg_at_5_std + value: 32.3902 + - type: nauc_ndcg_at_5_diff1 + value: 31.9359 + - type: nauc_ndcg_at_10_max + value: 42.0877 + - type: nauc_ndcg_at_10_std + value: 31.6409 + - type: nauc_ndcg_at_10_diff1 + value: 34.9684 + - type: nauc_ndcg_at_20_max + value: 39.1372 + - type: nauc_ndcg_at_20_std + value: 27.4368 + - type: nauc_ndcg_at_20_diff1 + value: 34.865899999999996 + - type: nauc_ndcg_at_100_max + value: 42.838300000000004 + - type: nauc_ndcg_at_100_std + value: 35.3636 + - type: nauc_ndcg_at_100_diff1 + value: 36.2467 + - type: nauc_ndcg_at_1000_max + value: 48.1669 + - type: nauc_ndcg_at_1000_std + value: 43.3838 + - type: nauc_ndcg_at_1000_diff1 + value: 36.2397 + - type: nauc_map_at_1_max + value: -4.0852 + - type: nauc_map_at_1_std + value: -20.336299999999998 + - type: nauc_map_at_1_diff1 + value: 37.5075 + - type: nauc_map_at_3_max + value: 5.606 + - type: nauc_map_at_3_std + value: -15.477599999999999 + - type: nauc_map_at_3_diff1 + value: 30.1676 + - type: nauc_map_at_5_max + value: 9.9675 + - type: nauc_map_at_5_std + value: -10.4882 + - type: nauc_map_at_5_diff1 + value: 29.8808 + - type: nauc_map_at_10_max + value: 16.0247 + - type: nauc_map_at_10_std + value: -1.3446 + - type: nauc_map_at_10_diff1 + value: 30.4367 + - type: nauc_map_at_20_max + value: 23.0361 + - type: nauc_map_at_20_std + value: 8.992899999999999 + - type: nauc_map_at_20_diff1 + value: 30.1643 + - type: nauc_map_at_100_max + value: 31.816699999999997 + - type: nauc_map_at_100_std + value: 25.555099999999996 + - type: nauc_map_at_100_diff1 + value: 30.549 + - type: nauc_map_at_1000_max + value: 33.242399999999996 + - type: nauc_map_at_1000_std + value: 28.1767 + - type: nauc_map_at_1000_diff1 + value: 30.0242 + - type: nauc_recall_at_1_max + value: -4.0852 + - type: nauc_recall_at_1_std + value: -20.336299999999998 + - type: nauc_recall_at_1_diff1 + value: 37.5075 + - type: nauc_recall_at_3_max + value: 2.3935 + - type: nauc_recall_at_3_std + value: -16.4596 + - type: nauc_recall_at_3_diff1 + value: 26.9506 + - type: nauc_recall_at_5_max + value: 5.1899 + - type: nauc_recall_at_5_std + value: -12.879399999999999 + - type: nauc_recall_at_5_diff1 + value: 25.2065 + - type: nauc_recall_at_10_max + value: 11.216 + - type: nauc_recall_at_10_std + value: -5.339 + - type: nauc_recall_at_10_diff1 + value: 26.0229 + - type: nauc_recall_at_20_max + value: 17.707800000000002 + - type: nauc_recall_at_20_std + value: 3.9654000000000003 + - type: nauc_recall_at_20_diff1 + value: 27.145200000000003 + - type: nauc_recall_at_100_max + value: 31.8321 + - type: nauc_recall_at_100_std + value: 31.219599999999996 + - type: nauc_recall_at_100_diff1 + value: 27.9692 + - type: nauc_recall_at_1000_max + value: 52.7876 + - type: nauc_recall_at_1000_std + value: 52.9031 + - type: nauc_recall_at_1000_diff1 + value: 33.1839 + - type: nauc_precision_at_1_max + value: 61.8036 + - type: nauc_precision_at_1_std + value: 44.4747 + - type: nauc_precision_at_1_diff1 + value: 53.412800000000004 + - type: nauc_precision_at_3_max + value: 43.5783 + - type: nauc_precision_at_3_std + value: 43.266799999999996 + - type: nauc_precision_at_3_diff1 + value: 8.7252 + - type: nauc_precision_at_5_max + value: 41.7952 + - type: nauc_precision_at_5_std + value: 45.880900000000004 + - type: nauc_precision_at_5_diff1 + value: 7.077400000000001 + - type: nauc_precision_at_10_max + value: 38.8324 + - type: nauc_precision_at_10_std + value: 50.418099999999995 + - type: nauc_precision_at_10_diff1 + value: 4.1962 + - type: nauc_precision_at_20_max + value: 35.4474 + - type: nauc_precision_at_20_std + value: 49.4221 + - type: nauc_precision_at_20_diff1 + value: 1.1421000000000001 + - type: nauc_precision_at_100_max + value: 26.096700000000002 + - type: nauc_precision_at_100_std + value: 43.0639 + - type: nauc_precision_at_100_diff1 + value: -4.6077 + - type: nauc_precision_at_1000_max + value: 4.3174 + - type: nauc_precision_at_1000_std + value: 19.775599999999997 + - type: nauc_precision_at_1000_diff1 + value: -15.1778 + - type: nauc_mrr_at_1_max + value: 61.8036 + - type: nauc_mrr_at_1_std + value: 44.4747 + - type: nauc_mrr_at_1_diff1 + value: 53.412800000000004 + - type: nauc_mrr_at_3_max + value: 61.1576 + - type: nauc_mrr_at_3_std + value: 49.4501 + - type: nauc_mrr_at_3_diff1 + value: 48.682900000000004 + - type: nauc_mrr_at_5_max + value: 60.728 + - type: nauc_mrr_at_5_std + value: 48.776399999999995 + - type: nauc_mrr_at_5_diff1 + value: 48.9195 + - type: nauc_mrr_at_10_max + value: 60.7957 + - type: nauc_mrr_at_10_std + value: 48.849199999999996 + - type: nauc_mrr_at_10_diff1 + value: 48.6244 + - type: nauc_mrr_at_20_max + value: 60.879099999999994 + - type: nauc_mrr_at_20_std + value: 48.715599999999995 + - type: nauc_mrr_at_20_diff1 + value: 48.6482 + - type: nauc_mrr_at_100_max + value: 60.7809 + - type: nauc_mrr_at_100_std + value: 48.5439 + - type: nauc_mrr_at_100_diff1 + value: 48.869099999999996 + - type: nauc_mrr_at_1000_max + value: 60.7977 + - type: nauc_mrr_at_1000_std + value: 48.5617 + - type: nauc_mrr_at_1000_diff1 + value: 48.875099999999996 + - type: main_score + value: 52.577 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification (default) + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 92.855 + - type: f1 + value: 89.1999 + - type: f1_weighted + value: 92.9881 + - type: main_score + value: 92.855 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER (default) + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: ndcg_at_1 + value: 91.089 + - type: ndcg_at_3 + value: 92.536 + - type: ndcg_at_5 + value: 93.135 + - type: ndcg_at_10 + value: 93.57900000000001 + - type: ndcg_at_20 + value: 93.828 + - type: ndcg_at_100 + value: 94.072 + - type: ndcg_at_1000 + value: 94.195 + - type: map_at_1 + value: 84.598 + - type: map_at_3 + value: 90.347 + - type: map_at_5 + value: 90.928 + - type: map_at_10 + value: 91.25 + - type: map_at_20 + value: 91.36800000000001 + - type: map_at_100 + value: 91.432 + - type: map_at_1000 + value: 91.44 + - type: recall_at_1 + value: 84.598 + - type: recall_at_3 + value: 94.30199999999999 + - type: recall_at_5 + value: 95.86099999999999 + - type: recall_at_10 + value: 97.07900000000001 + - type: recall_at_20 + value: 97.816 + - type: recall_at_100 + value: 98.775 + - type: recall_at_1000 + value: 99.49 + - type: precision_at_1 + value: 91.089 + - type: precision_at_3 + value: 34.833 + - type: precision_at_5 + value: 21.482 + - type: precision_at_10 + value: 11.020000000000001 + - type: precision_at_20 + value: 5.614 + - type: precision_at_100 + value: 1.151 + - type: precision_at_1000 + value: 0.117 + - type: mrr_at_1 + value: 91.0891 + - type: mrr_at_3 + value: 94.56700000000001 + - type: mrr_at_5 + value: 94.7537 + - type: mrr_at_10 + value: 94.8075 + - type: mrr_at_20 + value: 94.8157 + - type: mrr_at_100 + value: 94.8214 + - type: mrr_at_1000 + value: 94.82169999999999 + - type: nauc_ndcg_at_1_max + value: 27.069399999999998 + - type: nauc_ndcg_at_1_std + value: -14.5981 + - type: nauc_ndcg_at_1_diff1 + value: 77.91120000000001 + - type: nauc_ndcg_at_3_max + value: 21.5811 + - type: nauc_ndcg_at_3_std + value: -4.1468 + - type: nauc_ndcg_at_3_diff1 + value: 48.83 + - type: nauc_ndcg_at_5_max + value: 20.523 + - type: nauc_ndcg_at_5_std + value: -3.3154999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 47.5873 + - type: nauc_ndcg_at_10_max + value: 20.2836 + - type: nauc_ndcg_at_10_std + value: -2.5668 + - type: nauc_ndcg_at_10_diff1 + value: 48.6967 + - type: nauc_ndcg_at_20_max + value: 21.810499999999998 + - type: nauc_ndcg_at_20_std + value: -2.731 + - type: nauc_ndcg_at_20_diff1 + value: 50.4818 + - type: nauc_ndcg_at_100_max + value: 22.7895 + - type: nauc_ndcg_at_100_std + value: -3.3550000000000004 + - type: nauc_ndcg_at_100_diff1 + value: 52.141099999999994 + - type: nauc_ndcg_at_1000_max + value: 22.8887 + - type: nauc_ndcg_at_1000_std + value: -3.8968000000000003 + - type: nauc_ndcg_at_1000_diff1 + value: 53.1322 + - type: nauc_map_at_1_max + value: 17.4165 + - type: nauc_map_at_1_std + value: -13.8024 + - type: nauc_map_at_1_diff1 + value: 55.0895 + - type: nauc_map_at_3_max + value: 18.6504 + - type: nauc_map_at_3_std + value: -5.1091999999999995 + - type: nauc_map_at_3_diff1 + value: 46.7271 + - type: nauc_map_at_5_max + value: 18.9415 + - type: nauc_map_at_5_std + value: -4.5544 + - type: nauc_map_at_5_diff1 + value: 47.0325 + - type: nauc_map_at_10_max + value: 19.3631 + - type: nauc_map_at_10_std + value: -4.2128 + - type: nauc_map_at_10_diff1 + value: 47.8632 + - type: nauc_map_at_20_max + value: 19.9518 + - type: nauc_map_at_20_std + value: -4.1866 + - type: nauc_map_at_20_diff1 + value: 48.464600000000004 + - type: nauc_map_at_100_max + value: 20.1926 + - type: nauc_map_at_100_std + value: -4.2646999999999995 + - type: nauc_map_at_100_diff1 + value: 48.7761 + - type: nauc_map_at_1000_max + value: 20.2031 + - type: nauc_map_at_1000_std + value: -4.2917 + - type: nauc_map_at_1000_diff1 + value: 48.8186 + - type: nauc_recall_at_1_max + value: 17.4165 + - type: nauc_recall_at_1_std + value: -13.8024 + - type: nauc_recall_at_1_diff1 + value: 55.0895 + - type: nauc_recall_at_3_max + value: 13.7634 + - type: nauc_recall_at_3_std + value: 4.8161000000000005 + - type: nauc_recall_at_3_diff1 + value: 23.3279 + - type: nauc_recall_at_5_max + value: 11.2744 + - type: nauc_recall_at_5_std + value: 9.3473 + - type: nauc_recall_at_5_diff1 + value: 13.1573 + - type: nauc_recall_at_10_max + value: 7.927199999999999 + - type: nauc_recall_at_10_std + value: 16.963900000000002 + - type: nauc_recall_at_10_diff1 + value: 7.453 + - type: nauc_recall_at_20_max + value: 15.133 + - type: nauc_recall_at_20_std + value: 22.0635 + - type: nauc_recall_at_20_diff1 + value: 8.630799999999999 + - type: nauc_recall_at_100_max + value: 24.5063 + - type: nauc_recall_at_100_std + value: 29.017799999999998 + - type: nauc_recall_at_100_diff1 + value: 7.1233 + - type: nauc_recall_at_1000_max + value: 29.046 + - type: nauc_recall_at_1000_std + value: 41.5053 + - type: nauc_recall_at_1000_diff1 + value: 8.9752 + - type: nauc_precision_at_1_max + value: 27.069399999999998 + - type: nauc_precision_at_1_std + value: -14.5981 + - type: nauc_precision_at_1_diff1 + value: 77.91120000000001 + - type: nauc_precision_at_3_max + value: 4.7452000000000005 + - type: nauc_precision_at_3_std + value: 18.5957 + - type: nauc_precision_at_3_diff1 + value: -11.627 + - type: nauc_precision_at_5_max + value: 2.5 + - type: nauc_precision_at_5_std + value: 17.3486 + - type: nauc_precision_at_5_diff1 + value: -16.4117 + - type: nauc_precision_at_10_max + value: 2.2216 + - type: nauc_precision_at_10_std + value: 15.543899999999999 + - type: nauc_precision_at_10_diff1 + value: -15.697700000000001 + - type: nauc_precision_at_20_max + value: 4.5785 + - type: nauc_precision_at_20_std + value: 13.3715 + - type: nauc_precision_at_20_diff1 + value: -13.305900000000001 + - type: nauc_precision_at_100_max + value: 5.5239 + - type: nauc_precision_at_100_std + value: 10.3968 + - type: nauc_precision_at_100_diff1 + value: -11.649700000000001 + - type: nauc_precision_at_1000_max + value: 4.2727 + - type: nauc_precision_at_1000_std + value: 7.7141 + - type: nauc_precision_at_1000_diff1 + value: -10.2325 + - type: nauc_mrr_at_1_max + value: 27.069399999999998 + - type: nauc_mrr_at_1_std + value: -14.5981 + - type: nauc_mrr_at_1_diff1 + value: 77.91120000000001 + - type: nauc_mrr_at_3_max + value: 30.462600000000002 + - type: nauc_mrr_at_3_std + value: -10.8943 + - type: nauc_mrr_at_3_diff1 + value: 76.82 + - type: nauc_mrr_at_5_max + value: 30.1114 + - type: nauc_mrr_at_5_std + value: -11.483799999999999 + - type: nauc_mrr_at_5_diff1 + value: 76.5938 + - type: nauc_mrr_at_10_max + value: 29.8093 + - type: nauc_mrr_at_10_std + value: -11.4619 + - type: nauc_mrr_at_10_diff1 + value: 76.7031 + - type: nauc_mrr_at_20_max + value: 29.817700000000002 + - type: nauc_mrr_at_20_std + value: -11.5811 + - type: nauc_mrr_at_20_diff1 + value: 76.7699 + - type: nauc_mrr_at_100_max + value: 29.8109 + - type: nauc_mrr_at_100_std + value: -11.6356 + - type: nauc_mrr_at_100_diff1 + value: 76.7814 + - type: nauc_mrr_at_1000_max + value: 29.810599999999997 + - type: nauc_mrr_at_1000_std + value: -11.638 + - type: nauc_mrr_at_1000_diff1 + value: 76.7821 + - type: main_score + value: 93.57900000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 (default) + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: ndcg_at_1 + value: 58.48799999999999 + - type: ndcg_at_3 + value: 56.16100000000001 + - type: ndcg_at_5 + value: 57.511 + - type: ndcg_at_10 + value: 60.284000000000006 + - type: ndcg_at_20 + value: 63.104000000000006 + - type: ndcg_at_100 + value: 66.61399999999999 + - type: ndcg_at_1000 + value: 68.08 + - type: map_at_1 + value: 31.047000000000004 + - type: map_at_3 + value: 45.858 + - type: map_at_5 + value: 49.452 + - type: map_at_10 + value: 52.19200000000001 + - type: map_at_20 + value: 53.488 + - type: map_at_100 + value: 54.367 + - type: map_at_1000 + value: 54.484 + - type: recall_at_1 + value: 31.047000000000004 + - type: recall_at_3 + value: 51.278 + - type: recall_at_5 + value: 58.619 + - type: recall_at_10 + value: 67.388 + - type: recall_at_20 + value: 76.058 + - type: recall_at_100 + value: 89.872 + - type: recall_at_1000 + value: 98.104 + - type: precision_at_1 + value: 58.48799999999999 + - type: precision_at_3 + value: 37.397000000000006 + - type: precision_at_5 + value: 27.315 + - type: precision_at_10 + value: 16.636 + - type: precision_at_20 + value: 9.506 + - type: precision_at_100 + value: 2.31 + - type: precision_at_1000 + value: 0.258 + - type: mrr_at_1 + value: 58.4877 + - type: mrr_at_3 + value: 65.3035 + - type: mrr_at_5 + value: 66.5381 + - type: mrr_at_10 + value: 67.3128 + - type: mrr_at_20 + value: 67.6732 + - type: mrr_at_100 + value: 67.8703 + - type: mrr_at_1000 + value: 67.8843 + - type: nauc_ndcg_at_1_max + value: 22.2154 + - type: nauc_ndcg_at_1_std + value: -6.8969000000000005 + - type: nauc_ndcg_at_1_diff1 + value: 63.343 + - type: nauc_ndcg_at_3_max + value: 18.290100000000002 + - type: nauc_ndcg_at_3_std + value: -4.3137 + - type: nauc_ndcg_at_3_diff1 + value: 49.6392 + - type: nauc_ndcg_at_5_max + value: 15.2734 + - type: nauc_ndcg_at_5_std + value: -4.8328999999999995 + - type: nauc_ndcg_at_5_diff1 + value: 50.128099999999996 + - type: nauc_ndcg_at_10_max + value: 14.333499999999999 + - type: nauc_ndcg_at_10_std + value: -4.4392000000000005 + - type: nauc_ndcg_at_10_diff1 + value: 50.4035 + - type: nauc_ndcg_at_20_max + value: 16.0761 + - type: nauc_ndcg_at_20_std + value: -1.917 + - type: nauc_ndcg_at_20_diff1 + value: 51.334900000000005 + - type: nauc_ndcg_at_100_max + value: 18.3939 + - type: nauc_ndcg_at_100_std + value: -0.16199999999999998 + - type: nauc_ndcg_at_100_diff1 + value: 51.565099999999994 + - type: nauc_ndcg_at_1000_max + value: 19.3296 + - type: nauc_ndcg_at_1000_std + value: -2.0654 + - type: nauc_ndcg_at_1000_diff1 + value: 51.78620000000001 + - type: nauc_map_at_1_max + value: 1.4908 + - type: nauc_map_at_1_std + value: -9.4582 + - type: nauc_map_at_1_diff1 + value: 53.4035 + - type: nauc_map_at_3_max + value: 8.225100000000001 + - type: nauc_map_at_3_std + value: -8.0511 + - type: nauc_map_at_3_diff1 + value: 49.9005 + - type: nauc_map_at_5_max + value: 11.188099999999999 + - type: nauc_map_at_5_std + value: -7.1714 + - type: nauc_map_at_5_diff1 + value: 49.3836 + - type: nauc_map_at_10_max + value: 12.885299999999999 + - type: nauc_map_at_10_std + value: -6.292000000000001 + - type: nauc_map_at_10_diff1 + value: 49.1492 + - type: nauc_map_at_20_max + value: 13.8849 + - type: nauc_map_at_20_std + value: -5.256 + - type: nauc_map_at_20_diff1 + value: 49.5846 + - type: nauc_map_at_100_max + value: 14.6337 + - type: nauc_map_at_100_std + value: -4.7753 + - type: nauc_map_at_100_diff1 + value: 49.6103 + - type: nauc_map_at_1000_max + value: 14.6885 + - type: nauc_map_at_1000_std + value: -4.8452 + - type: nauc_map_at_1000_diff1 + value: 49.6053 + - type: nauc_recall_at_1_max + value: 1.4908 + - type: nauc_recall_at_1_std + value: -9.4582 + - type: nauc_recall_at_1_diff1 + value: 53.4035 + - type: nauc_recall_at_3_max + value: 4.301 + - type: nauc_recall_at_3_std + value: -5.7848999999999995 + - type: nauc_recall_at_3_diff1 + value: 43.4693 + - type: nauc_recall_at_5_max + value: 5.289 + - type: nauc_recall_at_5_std + value: -4.2011 + - type: nauc_recall_at_5_diff1 + value: 41.1386 + - type: nauc_recall_at_10_max + value: 4.936999999999999 + - type: nauc_recall_at_10_std + value: -2.048 + - type: nauc_recall_at_10_diff1 + value: 39.4644 + - type: nauc_recall_at_20_max + value: 7.1711 + - type: nauc_recall_at_20_std + value: 8.978800000000001 + - type: nauc_recall_at_20_diff1 + value: 40.2059 + - type: nauc_recall_at_100_max + value: 10.020199999999999 + - type: nauc_recall_at_100_std + value: 37.0448 + - type: nauc_recall_at_100_diff1 + value: 34.5356 + - type: nauc_recall_at_1000_max + value: 30.9022 + - type: nauc_recall_at_1000_std + value: 42.3465 + - type: nauc_recall_at_1000_diff1 + value: 34.7997 + - type: nauc_precision_at_1_max + value: 22.2154 + - type: nauc_precision_at_1_std + value: -6.8969000000000005 + - type: nauc_precision_at_1_diff1 + value: 63.343 + - type: nauc_precision_at_3_max + value: 27.120499999999996 + - type: nauc_precision_at_3_std + value: 2.8301 + - type: nauc_precision_at_3_diff1 + value: 21.6329 + - type: nauc_precision_at_5_max + value: 28.3782 + - type: nauc_precision_at_5_std + value: 4.8704 + - type: nauc_precision_at_5_diff1 + value: 12.8683 + - type: nauc_precision_at_10_max + value: 27.403899999999997 + - type: nauc_precision_at_10_std + value: 8.1265 + - type: nauc_precision_at_10_diff1 + value: 5.0926 + - type: nauc_precision_at_20_max + value: 29.383300000000002 + - type: nauc_precision_at_20_std + value: 12.908100000000001 + - type: nauc_precision_at_20_diff1 + value: 0.6472 + - type: nauc_precision_at_100_max + value: 30.294500000000003 + - type: nauc_precision_at_100_std + value: 15.93 + - type: nauc_precision_at_100_diff1 + value: -8.704 + - type: nauc_precision_at_1000_max + value: 29.9313 + - type: nauc_precision_at_1000_std + value: 10.1372 + - type: nauc_precision_at_1000_diff1 + value: -13.424800000000001 + - type: nauc_mrr_at_1_max + value: 22.2154 + - type: nauc_mrr_at_1_std + value: -6.8969000000000005 + - type: nauc_mrr_at_1_diff1 + value: 63.343 + - type: nauc_mrr_at_3_max + value: 23.3901 + - type: nauc_mrr_at_3_std + value: -4.6844 + - type: nauc_mrr_at_3_diff1 + value: 60.8869 + - type: nauc_mrr_at_5_max + value: 22.615299999999998 + - type: nauc_mrr_at_5_std + value: -4.5552 + - type: nauc_mrr_at_5_diff1 + value: 60.522 + - type: nauc_mrr_at_10_max + value: 22.7886 + - type: nauc_mrr_at_10_std + value: -4.4885 + - type: nauc_mrr_at_10_diff1 + value: 60.4902 + - type: nauc_mrr_at_20_max + value: 22.9083 + - type: nauc_mrr_at_20_std + value: -4.1969 + - type: nauc_mrr_at_20_diff1 + value: 60.547799999999995 + - type: nauc_mrr_at_100_max + value: 23.0224 + - type: nauc_mrr_at_100_std + value: -4.151 + - type: nauc_mrr_at_100_diff1 + value: 60.581399999999995 + - type: nauc_mrr_at_1000_max + value: 23.0223 + - type: nauc_mrr_at_1000_std + value: -4.1821 + - type: nauc_mrr_at_1000_diff1 + value: 60.5878 + - type: main_score + value: 60.284000000000006 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA (default) + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: ndcg_at_1 + value: 92.086 + - type: ndcg_at_3 + value: 84.129 + - type: ndcg_at_5 + value: 86.128 + - type: ndcg_at_10 + value: 87.473 + - type: ndcg_at_20 + value: 88.273 + - type: ndcg_at_100 + value: 89.067 + - type: ndcg_at_1000 + value: 89.467 + - type: map_at_1 + value: 46.043 + - type: map_at_3 + value: 79.89 + - type: map_at_5 + value: 81.625 + - type: map_at_10 + value: 82.485 + - type: map_at_20 + value: 82.83 + - type: map_at_100 + value: 83.00699999999999 + - type: map_at_1000 + value: 83.03 + - type: recall_at_1 + value: 46.043 + - type: recall_at_3 + value: 83.011 + - type: recall_at_5 + value: 86.935 + - type: recall_at_10 + value: 90.304 + - type: recall_at_20 + value: 92.86999999999999 + - type: recall_at_100 + value: 96.32 + - type: recall_at_1000 + value: 98.933 + - type: precision_at_1 + value: 92.086 + - type: precision_at_3 + value: 55.340999999999994 + - type: precision_at_5 + value: 34.774 + - type: precision_at_10 + value: 18.061 + - type: precision_at_20 + value: 9.286999999999999 + - type: precision_at_100 + value: 1.926 + - type: precision_at_1000 + value: 0.198 + - type: mrr_at_1 + value: 92.0864 + - type: mrr_at_3 + value: 94.4452 + - type: mrr_at_5 + value: 94.6255 + - type: mrr_at_10 + value: 94.7333 + - type: mrr_at_20 + value: 94.76440000000001 + - type: mrr_at_100 + value: 94.7801 + - type: mrr_at_1000 + value: 94.7809 + - type: nauc_ndcg_at_1_max + value: 37.6408 + - type: nauc_ndcg_at_1_std + value: -7.9706 + - type: nauc_ndcg_at_1_diff1 + value: 64.2193 + - type: nauc_ndcg_at_3_max + value: 35.579 + - type: nauc_ndcg_at_3_std + value: 4.5917 + - type: nauc_ndcg_at_3_diff1 + value: -7.6203 + - type: nauc_ndcg_at_5_max + value: 37.7564 + - type: nauc_ndcg_at_5_std + value: 7.9123 + - type: nauc_ndcg_at_5_diff1 + value: -6.2265 + - type: nauc_ndcg_at_10_max + value: 38.8436 + - type: nauc_ndcg_at_10_std + value: 9.86 + - type: nauc_ndcg_at_10_diff1 + value: -5.3233999999999995 + - type: nauc_ndcg_at_20_max + value: 39.0612 + - type: nauc_ndcg_at_20_std + value: 11.0778 + - type: nauc_ndcg_at_20_diff1 + value: -4.0485 + - type: nauc_ndcg_at_100_max + value: 38.9758 + - type: nauc_ndcg_at_100_std + value: 10.9792 + - type: nauc_ndcg_at_100_diff1 + value: -2.8198999999999996 + - type: nauc_ndcg_at_1000_max + value: 38.366099999999996 + - type: nauc_ndcg_at_1000_std + value: 9.4395 + - type: nauc_ndcg_at_1000_diff1 + value: -2.1656 + - type: nauc_map_at_1_max + value: 37.6408 + - type: nauc_map_at_1_std + value: -7.9706 + - type: nauc_map_at_1_diff1 + value: 64.2193 + - type: nauc_map_at_3_max + value: 33.882 + - type: nauc_map_at_3_std + value: 3.9527 + - type: nauc_map_at_3_diff1 + value: -12.516 + - type: nauc_map_at_5_max + value: 35.452099999999994 + - type: nauc_map_at_5_std + value: 6.228899999999999 + - type: nauc_map_at_5_diff1 + value: -11.5097 + - type: nauc_map_at_10_max + value: 35.961999999999996 + - type: nauc_map_at_10_std + value: 7.000000000000001 + - type: nauc_map_at_10_diff1 + value: -11.0337 + - type: nauc_map_at_20_max + value: 35.9944 + - type: nauc_map_at_20_std + value: 7.3074 + - type: nauc_map_at_20_diff1 + value: -10.6965 + - type: nauc_map_at_100_max + value: 35.970600000000005 + - type: nauc_map_at_100_std + value: 7.279299999999999 + - type: nauc_map_at_100_diff1 + value: -10.5362 + - type: nauc_map_at_1000_max + value: 35.9476 + - type: nauc_map_at_1000_std + value: 7.2231000000000005 + - type: nauc_map_at_1000_diff1 + value: -10.5154 + - type: nauc_recall_at_1_max + value: 37.6408 + - type: nauc_recall_at_1_std + value: -7.9706 + - type: nauc_recall_at_1_diff1 + value: 64.2193 + - type: nauc_recall_at_3_max + value: 35.9731 + - type: nauc_recall_at_3_std + value: 8.0627 + - type: nauc_recall_at_3_diff1 + value: -18.9248 + - type: nauc_recall_at_5_max + value: 40.184799999999996 + - type: nauc_recall_at_5_std + value: 15.5623 + - type: nauc_recall_at_5_diff1 + value: -18.8156 + - type: nauc_recall_at_10_max + value: 43.8976 + - type: nauc_recall_at_10_std + value: 23.7287 + - type: nauc_recall_at_10_diff1 + value: -19.8106 + - type: nauc_recall_at_20_max + value: 46.7029 + - type: nauc_recall_at_20_std + value: 34.2093 + - type: nauc_recall_at_20_diff1 + value: -18.305 + - type: nauc_recall_at_100_max + value: 53.403999999999996 + - type: nauc_recall_at_100_std + value: 53.4122 + - type: nauc_recall_at_100_diff1 + value: -16.8661 + - type: nauc_recall_at_1000_max + value: 56.882299999999994 + - type: nauc_recall_at_1000_std + value: 70.0182 + - type: nauc_recall_at_1000_diff1 + value: -17.042099999999998 + - type: nauc_precision_at_1_max + value: 37.6408 + - type: nauc_precision_at_1_std + value: -7.9706 + - type: nauc_precision_at_1_diff1 + value: 64.2193 + - type: nauc_precision_at_3_max + value: 35.9731 + - type: nauc_precision_at_3_std + value: 8.0627 + - type: nauc_precision_at_3_diff1 + value: -18.9248 + - type: nauc_precision_at_5_max + value: 40.184799999999996 + - type: nauc_precision_at_5_std + value: 15.5623 + - type: nauc_precision_at_5_diff1 + value: -18.8156 + - type: nauc_precision_at_10_max + value: 43.8976 + - type: nauc_precision_at_10_std + value: 23.7287 + - type: nauc_precision_at_10_diff1 + value: -19.8106 + - type: nauc_precision_at_20_max + value: 46.7029 + - type: nauc_precision_at_20_std + value: 34.2093 + - type: nauc_precision_at_20_diff1 + value: -18.305 + - type: nauc_precision_at_100_max + value: 53.403999999999996 + - type: nauc_precision_at_100_std + value: 53.4122 + - type: nauc_precision_at_100_diff1 + value: -16.8661 + - type: nauc_precision_at_1000_max + value: 56.882299999999994 + - type: nauc_precision_at_1000_std + value: 70.0182 + - type: nauc_precision_at_1000_diff1 + value: -17.042099999999998 + - type: nauc_mrr_at_1_max + value: 37.6408 + - type: nauc_mrr_at_1_std + value: -7.9706 + - type: nauc_mrr_at_1_diff1 + value: 64.2193 + - type: nauc_mrr_at_3_max + value: 43.0267 + - type: nauc_mrr_at_3_std + value: -3.9602 + - type: nauc_mrr_at_3_diff1 + value: 64.7898 + - type: nauc_mrr_at_5_max + value: 42.548700000000004 + - type: nauc_mrr_at_5_std + value: -4.1829 + - type: nauc_mrr_at_5_diff1 + value: 64.81989999999999 + - type: nauc_mrr_at_10_max + value: 42.5037 + - type: nauc_mrr_at_10_std + value: -3.8122000000000003 + - type: nauc_mrr_at_10_diff1 + value: 64.84440000000001 + - type: nauc_mrr_at_20_max + value: 42.4425 + - type: nauc_mrr_at_20_std + value: -3.8257 + - type: nauc_mrr_at_20_diff1 + value: 64.8602 + - type: nauc_mrr_at_100_max + value: 42.3146 + - type: nauc_mrr_at_100_std + value: -3.9995999999999996 + - type: nauc_mrr_at_100_diff1 + value: 64.81660000000001 + - type: nauc_mrr_at_1000_max + value: 42.3073 + - type: nauc_mrr_at_1000_std + value: -4.0055 + - type: nauc_mrr_at_1000_diff1 + value: 64.81360000000001 + - type: main_score + value: 87.473 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification (default) + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 97.07679999999999 + - type: f1 + value: 97.07639999999999 + - type: f1_weighted + value: 97.07639999999999 + - type: ap + value: 95.4623 + - type: ap_weighted + value: 95.4623 + - type: main_score + value: 97.07679999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB MSMARCO (default) + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: ndcg_at_1 + value: 27.12 + - type: ndcg_at_3 + value: 39.287 + - type: ndcg_at_5 + value: 43.478 + - type: ndcg_at_10 + value: 47.396 + - type: ndcg_at_20 + value: 49.915 + - type: ndcg_at_100 + value: 52.410000000000004 + - type: ndcg_at_1000 + value: 53.20700000000001 + - type: map_at_1 + value: 26.391 + - type: map_at_3 + value: 36.016999999999996 + - type: map_at_5 + value: 38.385999999999996 + - type: map_at_10 + value: 40.058 + - type: map_at_20 + value: 40.772999999999996 + - type: map_at_100 + value: 41.15 + - type: map_at_1000 + value: 41.185 + - type: recall_at_1 + value: 26.391 + - type: recall_at_3 + value: 48.025 + - type: recall_at_5 + value: 58.036 + - type: recall_at_10 + value: 69.852 + - type: recall_at_20 + value: 79.605 + - type: recall_at_100 + value: 92.499 + - type: recall_at_1000 + value: 98.446 + - type: precision_at_1 + value: 27.12 + - type: precision_at_3 + value: 16.608999999999998 + - type: precision_at_5 + value: 12.089 + - type: precision_at_10 + value: 7.314 + - type: precision_at_20 + value: 4.18 + - type: precision_at_100 + value: 0.9780000000000001 + - type: precision_at_1000 + value: 0.105 + - type: mrr_at_1 + value: 27.120300000000004 + - type: mrr_at_3 + value: 36.728699999999996 + - type: mrr_at_5 + value: 39.0518 + - type: mrr_at_10 + value: 40.6561 + - type: mrr_at_20 + value: 41.3384 + - type: mrr_at_100 + value: 41.6825 + - type: mrr_at_1000 + value: 41.7118 + - type: nauc_ndcg_at_1_max + value: -1.7341 + - type: nauc_ndcg_at_1_std + value: -22.0897 + - type: nauc_ndcg_at_1_diff1 + value: 43.9115 + - type: nauc_ndcg_at_3_max + value: -2.2762000000000002 + - type: nauc_ndcg_at_3_std + value: -24.0353 + - type: nauc_ndcg_at_3_diff1 + value: 40.4042 + - type: nauc_ndcg_at_5_max + value: -2.1643 + - type: nauc_ndcg_at_5_std + value: -23.5777 + - type: nauc_ndcg_at_5_diff1 + value: 40.1946 + - type: nauc_ndcg_at_10_max + value: -1.6878 + - type: nauc_ndcg_at_10_std + value: -22.9484 + - type: nauc_ndcg_at_10_diff1 + value: 40.5053 + - type: nauc_ndcg_at_20_max + value: -1.0808 + - type: nauc_ndcg_at_20_std + value: -20.8231 + - type: nauc_ndcg_at_20_diff1 + value: 40.4996 + - type: nauc_ndcg_at_100_max + value: -1.387 + - type: nauc_ndcg_at_100_std + value: -19.6544 + - type: nauc_ndcg_at_100_diff1 + value: 40.808499999999995 + - type: nauc_ndcg_at_1000_max + value: -1.3396 + - type: nauc_ndcg_at_1000_std + value: -20.7437 + - type: nauc_ndcg_at_1000_diff1 + value: 40.8921 + - type: nauc_map_at_1_max + value: -1.7507000000000001 + - type: nauc_map_at_1_std + value: -22.192899999999998 + - type: nauc_map_at_1_diff1 + value: 44.0719 + - type: nauc_map_at_3_max + value: -2.1371 + - type: nauc_map_at_3_std + value: -23.7158 + - type: nauc_map_at_3_diff1 + value: 41.351 + - type: nauc_map_at_5_max + value: -2.1025 + - type: nauc_map_at_5_std + value: -23.5251 + - type: nauc_map_at_5_diff1 + value: 41.255399999999995 + - type: nauc_map_at_10_max + value: -1.9206 + - type: nauc_map_at_10_std + value: -23.2697 + - type: nauc_map_at_10_diff1 + value: 41.4134 + - type: nauc_map_at_20_max + value: -1.7760000000000002 + - type: nauc_map_at_20_std + value: -22.7164 + - type: nauc_map_at_20_diff1 + value: 41.4186 + - type: nauc_map_at_100_max + value: -1.8270000000000002 + - type: nauc_map_at_100_std + value: -22.551 + - type: nauc_map_at_100_diff1 + value: 41.4761 + - type: nauc_map_at_1000_max + value: -1.8245 + - type: nauc_map_at_1000_std + value: -22.5827 + - type: nauc_map_at_1000_diff1 + value: 41.4796 + - type: nauc_recall_at_1_max + value: -1.7507000000000001 + - type: nauc_recall_at_1_std + value: -22.192899999999998 + - type: nauc_recall_at_1_diff1 + value: 44.0719 + - type: nauc_recall_at_3_max + value: -2.5709 + - type: nauc_recall_at_3_std + value: -24.9526 + - type: nauc_recall_at_3_diff1 + value: 37.6496 + - type: nauc_recall_at_5_max + value: -2.2352 + - type: nauc_recall_at_5_std + value: -23.7151 + - type: nauc_recall_at_5_diff1 + value: 36.7421 + - type: nauc_recall_at_10_max + value: -0.4821 + - type: nauc_recall_at_10_std + value: -21.5386 + - type: nauc_recall_at_10_diff1 + value: 37.1132 + - type: nauc_recall_at_20_max + value: 3.5499 + - type: nauc_recall_at_20_std + value: -8.5039 + - type: nauc_recall_at_20_diff1 + value: 35.985299999999995 + - type: nauc_recall_at_100_max + value: 4.6888 + - type: nauc_recall_at_100_std + value: 30.0406 + - type: nauc_recall_at_100_diff1 + value: 34.8416 + - type: nauc_recall_at_1000_max + value: 30.544300000000003 + - type: nauc_recall_at_1000_std + value: 72.42269999999999 + - type: nauc_recall_at_1000_diff1 + value: 26.676299999999998 + - type: nauc_precision_at_1_max + value: -1.7341 + - type: nauc_precision_at_1_std + value: -22.0897 + - type: nauc_precision_at_1_diff1 + value: 43.9115 + - type: nauc_precision_at_3_max + value: -2.7643 + - type: nauc_precision_at_3_std + value: -24.537100000000002 + - type: nauc_precision_at_3_diff1 + value: 36.9028 + - type: nauc_precision_at_5_max + value: -2.4927 + - type: nauc_precision_at_5_std + value: -22.6954 + - type: nauc_precision_at_5_diff1 + value: 35.0569 + - type: nauc_precision_at_10_max + value: -1.3371 + - type: nauc_precision_at_10_std + value: -19.017 + - type: nauc_precision_at_10_diff1 + value: 33.0978 + - type: nauc_precision_at_20_max + value: 1.9426999999999999 + - type: nauc_precision_at_20_std + value: -5.3872 + - type: nauc_precision_at_20_diff1 + value: 28.509400000000003 + - type: nauc_precision_at_100_max + value: 2.8586 + - type: nauc_precision_at_100_std + value: 20.869 + - type: nauc_precision_at_100_diff1 + value: 13.559899999999999 + - type: nauc_precision_at_1000_max + value: 6.1333 + - type: nauc_precision_at_1000_std + value: 15.551400000000001 + - type: nauc_precision_at_1000_diff1 + value: -3.4235 + - type: nauc_mrr_at_1_max + value: -1.7341 + - type: nauc_mrr_at_1_std + value: -22.0897 + - type: nauc_mrr_at_1_diff1 + value: 43.9115 + - type: nauc_mrr_at_3_max + value: -2.1852 + - type: nauc_mrr_at_3_std + value: -23.5165 + - type: nauc_mrr_at_3_diff1 + value: 41.1678 + - type: nauc_mrr_at_5_max + value: -2.1132999999999997 + - type: nauc_mrr_at_5_std + value: -23.1653 + - type: nauc_mrr_at_5_diff1 + value: 41.0944 + - type: nauc_mrr_at_10_max + value: -1.8908 + - type: nauc_mrr_at_10_std + value: -22.8918 + - type: nauc_mrr_at_10_diff1 + value: 41.1907 + - type: nauc_mrr_at_20_max + value: -1.7221 + - type: nauc_mrr_at_20_std + value: -22.375 + - type: nauc_mrr_at_20_diff1 + value: 41.2234 + - type: nauc_mrr_at_100_max + value: -1.7874999999999999 + - type: nauc_mrr_at_100_std + value: -22.2616 + - type: nauc_mrr_at_100_diff1 + value: 41.286899999999996 + - type: nauc_mrr_at_1000_max + value: -1.7856 + - type: nauc_mrr_at_1000_std + value: -22.2926 + - type: nauc_mrr_at_1000_diff1 + value: 41.2906 + - type: main_score + value: 47.396 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 99.2157 + - type: f1 + value: 99.1286 + - type: f1_weighted + value: 99.21640000000001 + - type: main_score + value: 99.2157 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 94.5372 + - type: f1 + value: 78.7627 + - type: f1_weighted + value: 95.2685 + - type: main_score + value: 94.5372 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 82.0646 + - type: f1 + value: 80.2035 + - type: f1_weighted + value: 80.8017 + - type: main_score + value: 82.0646 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 87.5723 + - type: f1 + value: 86.2565 + - type: f1_weighted + value: 86.92020000000001 + - type: main_score + value: 87.5723 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P (default) + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: v_measure + value: 47.488200000000006 + - type: v_measure_std + value: 1.2606 + - type: main_score + value: 47.488200000000006 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S (default) + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: v_measure + value: 45.0597 + - type: v_measure_std + value: 1.5357 + - type: main_score + value: 45.0597 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking (default) + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: map + value: 30.8519 + - type: mrr + value: 32.1466 + - type: nAUC_map_max + value: -16.602800000000002 + - type: nAUC_map_std + value: -8.7712 + - type: nAUC_map_diff1 + value: 8.7311 + - type: nAUC_mrr_max + value: -11.0311 + - type: nAUC_mrr_std + value: -5.2932 + - type: nAUC_mrr_diff1 + value: 8.7991 + - type: main_score + value: 30.8519 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus (default) + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: ndcg_at_1 + value: 50.773999999999994 + - type: ndcg_at_3 + value: 46.766000000000005 + - type: ndcg_at_5 + value: 44.401 + - type: ndcg_at_10 + value: 40.955000000000005 + - type: ndcg_at_20 + value: 38.436 + - type: ndcg_at_100 + value: 37.101 + - type: ndcg_at_1000 + value: 45.458999999999996 + - type: map_at_1 + value: 6.7860000000000005 + - type: map_at_3 + value: 11.305 + - type: map_at_5 + value: 13.355 + - type: map_at_10 + value: 15.841 + - type: map_at_20 + value: 17.724 + - type: map_at_100 + value: 20.146 + - type: map_at_1000 + value: 21.664 + - type: recall_at_1 + value: 6.7860000000000005 + - type: recall_at_3 + value: 12.848 + - type: recall_at_5 + value: 16.059 + - type: recall_at_10 + value: 20.699 + - type: recall_at_20 + value: 25.349 + - type: recall_at_100 + value: 37.377 + - type: recall_at_1000 + value: 68.326 + - type: precision_at_1 + value: 52.322 + - type: precision_at_3 + value: 43.963 + - type: precision_at_5 + value: 38.7 + - type: precision_at_10 + value: 30.402 + - type: precision_at_20 + value: 22.415 + - type: precision_at_100 + value: 9.074 + - type: precision_at_1000 + value: 2.141 + - type: mrr_at_1 + value: 52.322 + - type: mrr_at_3 + value: 60.2167 + - type: mrr_at_5 + value: 61.161 + - type: mrr_at_10 + value: 61.6213 + - type: mrr_at_20 + value: 61.9851 + - type: mrr_at_100 + value: 62.1286 + - type: mrr_at_1000 + value: 62.16159999999999 + - type: nauc_ndcg_at_1_max + value: 47.306 + - type: nauc_ndcg_at_1_std + value: 15.371299999999998 + - type: nauc_ndcg_at_1_diff1 + value: 34.3673 + - type: nauc_ndcg_at_3_max + value: 49.5171 + - type: nauc_ndcg_at_3_std + value: 21.7163 + - type: nauc_ndcg_at_3_diff1 + value: 24.3249 + - type: nauc_ndcg_at_5_max + value: 50.1667 + - type: nauc_ndcg_at_5_std + value: 25.496799999999997 + - type: nauc_ndcg_at_5_diff1 + value: 21.0998 + - type: nauc_ndcg_at_10_max + value: 48.174499999999995 + - type: nauc_ndcg_at_10_std + value: 25.674799999999998 + - type: nauc_ndcg_at_10_diff1 + value: 19.2271 + - type: nauc_ndcg_at_20_max + value: 46.451100000000004 + - type: nauc_ndcg_at_20_std + value: 26.3454 + - type: nauc_ndcg_at_20_diff1 + value: 19.6892 + - type: nauc_ndcg_at_100_max + value: 47.394 + - type: nauc_ndcg_at_100_std + value: 29.3957 + - type: nauc_ndcg_at_100_diff1 + value: 22.639 + - type: nauc_ndcg_at_1000_max + value: 48.8094 + - type: nauc_ndcg_at_1000_std + value: 33.6209 + - type: nauc_ndcg_at_1000_diff1 + value: 24.0513 + - type: nauc_map_at_1_max + value: 22.2337 + - type: nauc_map_at_1_std + value: -15.3141 + - type: nauc_map_at_1_diff1 + value: 46.8412 + - type: nauc_map_at_3_max + value: 31.1176 + - type: nauc_map_at_3_std + value: -6.8641 + - type: nauc_map_at_3_diff1 + value: 38.2225 + - type: nauc_map_at_5_max + value: 34.1685 + - type: nauc_map_at_5_std + value: -2.7371 + - type: nauc_map_at_5_diff1 + value: 33.8161 + - type: nauc_map_at_10_max + value: 38.3438 + - type: nauc_map_at_10_std + value: 2.4334000000000002 + - type: nauc_map_at_10_diff1 + value: 29.9155 + - type: nauc_map_at_20_max + value: 41.6186 + - type: nauc_map_at_20_std + value: 8.1891 + - type: nauc_map_at_20_diff1 + value: 28.083999999999996 + - type: nauc_map_at_100_max + value: 43.8986 + - type: nauc_map_at_100_std + value: 14.971699999999998 + - type: nauc_map_at_100_diff1 + value: 25.7392 + - type: nauc_map_at_1000_max + value: 43.7337 + - type: nauc_map_at_1000_std + value: 17.3602 + - type: nauc_map_at_1000_diff1 + value: 24.3521 + - type: nauc_recall_at_1_max + value: 22.2337 + - type: nauc_recall_at_1_std + value: -15.3141 + - type: nauc_recall_at_1_diff1 + value: 46.8412 + - type: nauc_recall_at_3_max + value: 27.4814 + - type: nauc_recall_at_3_std + value: -6.2251 + - type: nauc_recall_at_3_diff1 + value: 33.189099999999996 + - type: nauc_recall_at_5_max + value: 27.6656 + - type: nauc_recall_at_5_std + value: -1.3779 + - type: nauc_recall_at_5_diff1 + value: 26.5088 + - type: nauc_recall_at_10_max + value: 29.8338 + - type: nauc_recall_at_10_std + value: 0.6765 + - type: nauc_recall_at_10_diff1 + value: 19.3518 + - type: nauc_recall_at_20_max + value: 29.566300000000002 + - type: nauc_recall_at_20_std + value: 6.649299999999999 + - type: nauc_recall_at_20_diff1 + value: 16.3787 + - type: nauc_recall_at_100_max + value: 29.775299999999998 + - type: nauc_recall_at_100_std + value: 19.5727 + - type: nauc_recall_at_100_diff1 + value: 13.4263 + - type: nauc_recall_at_1000_max + value: 15.575800000000001 + - type: nauc_recall_at_1000_std + value: 16.5073 + - type: nauc_recall_at_1000_diff1 + value: 9.413 + - type: nauc_precision_at_1_max + value: 47.6567 + - type: nauc_precision_at_1_std + value: 16.1159 + - type: nauc_precision_at_1_diff1 + value: 35.7474 + - type: nauc_precision_at_3_max + value: 45.9337 + - type: nauc_precision_at_3_std + value: 28.306700000000003 + - type: nauc_precision_at_3_diff1 + value: 12.9558 + - type: nauc_precision_at_5_max + value: 45.3828 + - type: nauc_precision_at_5_std + value: 34.0723 + - type: nauc_precision_at_5_diff1 + value: 3.936 + - type: nauc_precision_at_10_max + value: 40.2787 + - type: nauc_precision_at_10_std + value: 36.1164 + - type: nauc_precision_at_10_diff1 + value: -1.9665 + - type: nauc_precision_at_20_max + value: 33.8095 + - type: nauc_precision_at_20_std + value: 37.288 + - type: nauc_precision_at_20_diff1 + value: -4.3394 + - type: nauc_precision_at_100_max + value: 19.880200000000002 + - type: nauc_precision_at_100_std + value: 35.8879 + - type: nauc_precision_at_100_diff1 + value: -11.5763 + - type: nauc_precision_at_1000_max + value: 2.9351 + - type: nauc_precision_at_1000_std + value: 17.5752 + - type: nauc_precision_at_1000_diff1 + value: -13.2391 + - type: nauc_mrr_at_1_max + value: 47.6567 + - type: nauc_mrr_at_1_std + value: 16.1159 + - type: nauc_mrr_at_1_diff1 + value: 35.7474 + - type: nauc_mrr_at_3_max + value: 51.1154 + - type: nauc_mrr_at_3_std + value: 22.6976 + - type: nauc_mrr_at_3_diff1 + value: 35.0163 + - type: nauc_mrr_at_5_max + value: 50.6561 + - type: nauc_mrr_at_5_std + value: 23.716 + - type: nauc_mrr_at_5_diff1 + value: 34.965 + - type: nauc_mrr_at_10_max + value: 50.6931 + - type: nauc_mrr_at_10_std + value: 24.0343 + - type: nauc_mrr_at_10_diff1 + value: 34.5146 + - type: nauc_mrr_at_20_max + value: 50.7143 + - type: nauc_mrr_at_20_std + value: 24.1366 + - type: nauc_mrr_at_20_diff1 + value: 34.819 + - type: nauc_mrr_at_100_max + value: 50.76500000000001 + - type: nauc_mrr_at_100_std + value: 24.1494 + - type: nauc_mrr_at_100_diff1 + value: 34.7759 + - type: nauc_mrr_at_1000_max + value: 50.7421 + - type: nauc_mrr_at_1000_std + value: 24.110300000000002 + - type: nauc_mrr_at_1000_diff1 + value: 34.7687 + - type: main_score + value: 40.955000000000005 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ (default) + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: ndcg_at_1 + value: 56.518 + - type: ndcg_at_3 + value: 68.057 + - type: ndcg_at_5 + value: 71.497 + - type: ndcg_at_10 + value: 73.91499999999999 + - type: ndcg_at_20 + value: 74.994 + - type: ndcg_at_100 + value: 75.804 + - type: ndcg_at_1000 + value: 75.917 + - type: map_at_1 + value: 50.739000000000004 + - type: map_at_3 + value: 63.958000000000006 + - type: map_at_5 + value: 66.194 + - type: map_at_10 + value: 67.375 + - type: map_at_20 + value: 67.74 + - type: map_at_100 + value: 67.887 + - type: map_at_1000 + value: 67.893 + - type: recall_at_1 + value: 50.739000000000004 + - type: recall_at_3 + value: 76.364 + - type: recall_at_5 + value: 84.11800000000001 + - type: recall_at_10 + value: 91.037 + - type: recall_at_20 + value: 94.914 + - type: recall_at_100 + value: 98.84100000000001 + - type: recall_at_1000 + value: 99.643 + - type: precision_at_1 + value: 56.518 + - type: precision_at_3 + value: 29.809 + - type: precision_at_5 + value: 20.023 + - type: precision_at_10 + value: 10.943999999999999 + - type: precision_at_20 + value: 5.7459999999999996 + - type: precision_at_100 + value: 1.202 + - type: precision_at_1000 + value: 0.121 + - type: mrr_at_1 + value: 56.518 + - type: mrr_at_3 + value: 67.4392 + - type: mrr_at_5 + value: 68.9064 + - type: mrr_at_10 + value: 69.6792 + - type: mrr_at_20 + value: 69.8936 + - type: mrr_at_100 + value: 69.9803 + - type: mrr_at_1000 + value: 69.9841 + - type: nauc_ndcg_at_1_max + value: 21.104899999999997 + - type: nauc_ndcg_at_1_std + value: -8.8061 + - type: nauc_ndcg_at_1_diff1 + value: 51.5617 + - type: nauc_ndcg_at_3_max + value: 25.8231 + - type: nauc_ndcg_at_3_std + value: -11.3673 + - type: nauc_ndcg_at_3_diff1 + value: 48.4532 + - type: nauc_ndcg_at_5_max + value: 27.3793 + - type: nauc_ndcg_at_5_std + value: -10.9771 + - type: nauc_ndcg_at_5_diff1 + value: 48.3739 + - type: nauc_ndcg_at_10_max + value: 27.019 + - type: nauc_ndcg_at_10_std + value: -9.5004 + - type: nauc_ndcg_at_10_diff1 + value: 48.762 + - type: nauc_ndcg_at_20_max + value: 26.8793 + - type: nauc_ndcg_at_20_std + value: -9.1081 + - type: nauc_ndcg_at_20_diff1 + value: 48.971599999999995 + - type: nauc_ndcg_at_100_max + value: 26.188200000000002 + - type: nauc_ndcg_at_100_std + value: -8.8193 + - type: nauc_ndcg_at_100_diff1 + value: 49.160900000000005 + - type: nauc_ndcg_at_1000_max + value: 25.976 + - type: nauc_ndcg_at_1000_std + value: -9.037 + - type: nauc_ndcg_at_1000_diff1 + value: 49.032 + - type: nauc_map_at_1_max + value: 19.5507 + - type: nauc_map_at_1_std + value: -10.5558 + - type: nauc_map_at_1_diff1 + value: 51.809099999999994 + - type: nauc_map_at_3_max + value: 24.3671 + - type: nauc_map_at_3_std + value: -11.4169 + - type: nauc_map_at_3_diff1 + value: 49.2235 + - type: nauc_map_at_5_max + value: 25.221 + - type: nauc_map_at_5_std + value: -11.1358 + - type: nauc_map_at_5_diff1 + value: 49.161500000000004 + - type: nauc_map_at_10_max + value: 25.0963 + - type: nauc_map_at_10_std + value: -10.516300000000001 + - type: nauc_map_at_10_diff1 + value: 49.239 + - type: nauc_map_at_20_max + value: 25.065900000000003 + - type: nauc_map_at_20_std + value: -10.3531 + - type: nauc_map_at_20_diff1 + value: 49.278 + - type: nauc_map_at_100_max + value: 24.9721 + - type: nauc_map_at_100_std + value: -10.2936 + - type: nauc_map_at_100_diff1 + value: 49.2973 + - type: nauc_map_at_1000_max + value: 24.9646 + - type: nauc_map_at_1000_std + value: -10.3019 + - type: nauc_map_at_1000_diff1 + value: 49.2939 + - type: nauc_recall_at_1_max + value: 19.5507 + - type: nauc_recall_at_1_std + value: -10.5558 + - type: nauc_recall_at_1_diff1 + value: 51.809099999999994 + - type: nauc_recall_at_3_max + value: 29.2624 + - type: nauc_recall_at_3_std + value: -13.894400000000001 + - type: nauc_recall_at_3_diff1 + value: 44.7434 + - type: nauc_recall_at_5_max + value: 36.0211 + - type: nauc_recall_at_5_std + value: -14.130999999999998 + - type: nauc_recall_at_5_diff1 + value: 43.3309 + - type: nauc_recall_at_10_max + value: 39.385799999999996 + - type: nauc_recall_at_10_std + value: -6.685199999999999 + - type: nauc_recall_at_10_diff1 + value: 44.2087 + - type: nauc_recall_at_20_max + value: 47.641600000000004 + - type: nauc_recall_at_20_std + value: -0.281 + - type: nauc_recall_at_20_diff1 + value: 47.0697 + - type: nauc_recall_at_100_max + value: 64.6308 + - type: nauc_recall_at_100_std + value: 45.0589 + - type: nauc_recall_at_100_diff1 + value: 65.0598 + - type: nauc_recall_at_1000_max + value: 68.5287 + - type: nauc_recall_at_1000_std + value: 77.1208 + - type: nauc_recall_at_1000_diff1 + value: 49.7482 + - type: nauc_precision_at_1_max + value: 21.104899999999997 + - type: nauc_precision_at_1_std + value: -8.8061 + - type: nauc_precision_at_1_diff1 + value: 51.5617 + - type: nauc_precision_at_3_max + value: 21.184 + - type: nauc_precision_at_3_std + value: -3.5241000000000002 + - type: nauc_precision_at_3_diff1 + value: 19.3059 + - type: nauc_precision_at_5_max + value: 18.4921 + - type: nauc_precision_at_5_std + value: 1.0416999999999998 + - type: nauc_precision_at_5_diff1 + value: 7.2985999999999995 + - type: nauc_precision_at_10_max + value: 12.1251 + - type: nauc_precision_at_10_std + value: 7.9022 + - type: nauc_precision_at_10_diff1 + value: -3.3798000000000004 + - type: nauc_precision_at_20_max + value: 8.2779 + - type: nauc_precision_at_20_std + value: 10.8969 + - type: nauc_precision_at_20_diff1 + value: -10.1609 + - type: nauc_precision_at_100_max + value: 2.0527 + - type: nauc_precision_at_100_std + value: 14.127799999999999 + - type: nauc_precision_at_100_diff1 + value: -17.0174 + - type: nauc_precision_at_1000_max + value: 0.0936 + - type: nauc_precision_at_1000_std + value: 13.403 + - type: nauc_precision_at_1000_diff1 + value: -19.3205 + - type: nauc_mrr_at_1_max + value: 21.104899999999997 + - type: nauc_mrr_at_1_std + value: -8.8061 + - type: nauc_mrr_at_1_diff1 + value: 51.5617 + - type: nauc_mrr_at_3_max + value: 24.9568 + - type: nauc_mrr_at_3_std + value: -8.7933 + - type: nauc_mrr_at_3_diff1 + value: 48.821799999999996 + - type: nauc_mrr_at_5_max + value: 25.3627 + - type: nauc_mrr_at_5_std + value: -8.7224 + - type: nauc_mrr_at_5_diff1 + value: 48.9393 + - type: nauc_mrr_at_10_max + value: 25.1135 + - type: nauc_mrr_at_10_std + value: -8.3704 + - type: nauc_mrr_at_10_diff1 + value: 49.132999999999996 + - type: nauc_mrr_at_20_max + value: 25.015700000000002 + - type: nauc_mrr_at_20_std + value: -8.4009 + - type: nauc_mrr_at_20_diff1 + value: 49.2012 + - type: nauc_mrr_at_100_max + value: 24.9285 + - type: nauc_mrr_at_100_std + value: -8.3989 + - type: nauc_mrr_at_100_diff1 + value: 49.223099999999995 + - type: nauc_mrr_at_1000_max + value: 24.921599999999998 + - type: nauc_mrr_at_1000_std + value: -8.4031 + - type: nauc_mrr_at_1000_diff1 + value: 49.2186 + - type: main_score + value: 73.91499999999999 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval (default) + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: ndcg_at_1 + value: 84.99 + - type: ndcg_at_3 + value: 88.71900000000001 + - type: ndcg_at_5 + value: 89.997 + - type: ndcg_at_10 + value: 91.019 + - type: ndcg_at_20 + value: 91.532 + - type: ndcg_at_100 + value: 91.92399999999999 + - type: ndcg_at_1000 + value: 91.977 + - type: map_at_1 + value: 73.833 + - type: map_at_3 + value: 85.117 + - type: map_at_5 + value: 86.85000000000001 + - type: map_at_10 + value: 87.875 + - type: map_at_20 + value: 88.256 + - type: map_at_100 + value: 88.44300000000001 + - type: map_at_1000 + value: 88.455 + - type: recall_at_1 + value: 73.833 + - type: recall_at_3 + value: 89.934 + - type: recall_at_5 + value: 93.795 + - type: recall_at_10 + value: 96.799 + - type: recall_at_20 + value: 98.458 + - type: recall_at_100 + value: 99.79299999999999 + - type: recall_at_1000 + value: 99.98899999999999 + - type: precision_at_1 + value: 84.99 + - type: precision_at_3 + value: 38.897 + - type: precision_at_5 + value: 25.407999999999998 + - type: precision_at_10 + value: 13.766 + - type: precision_at_20 + value: 7.255000000000001 + - type: precision_at_100 + value: 1.543 + - type: precision_at_1000 + value: 0.157 + - type: mrr_at_1 + value: 85 + - type: mrr_at_3 + value: 89.485 + - type: mrr_at_5 + value: 90.0545 + - type: mrr_at_10 + value: 90.256 + - type: mrr_at_20 + value: 90.307 + - type: mrr_at_100 + value: 90.3212 + - type: mrr_at_1000 + value: 90.3214 + - type: nauc_ndcg_at_1_max + value: 33.0127 + - type: nauc_ndcg_at_1_std + value: -59.3688 + - type: nauc_ndcg_at_1_diff1 + value: 81.11880000000001 + - type: nauc_ndcg_at_3_max + value: 29.525800000000004 + - type: nauc_ndcg_at_3_std + value: -71.4482 + - type: nauc_ndcg_at_3_diff1 + value: 78.98849999999999 + - type: nauc_ndcg_at_5_max + value: 30.3419 + - type: nauc_ndcg_at_5_std + value: -73.92490000000001 + - type: nauc_ndcg_at_5_diff1 + value: 79.8099 + - type: nauc_ndcg_at_10_max + value: 31.262800000000002 + - type: nauc_ndcg_at_10_std + value: -71.8798 + - type: nauc_ndcg_at_10_diff1 + value: 80.01310000000001 + - type: nauc_ndcg_at_20_max + value: 31.8336 + - type: nauc_ndcg_at_20_std + value: -69.852 + - type: nauc_ndcg_at_20_diff1 + value: 79.9131 + - type: nauc_ndcg_at_100_max + value: 32.351400000000005 + - type: nauc_ndcg_at_100_std + value: -67.02420000000001 + - type: nauc_ndcg_at_100_diff1 + value: 79.8222 + - type: nauc_ndcg_at_1000_max + value: 32.3924 + - type: nauc_ndcg_at_1000_std + value: -66.57690000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 79.8063 + - type: nauc_map_at_1_max + value: 21.4243 + - type: nauc_map_at_1_std + value: -57.04900000000001 + - type: nauc_map_at_1_diff1 + value: 83.3378 + - type: nauc_map_at_3_max + value: 27.078799999999998 + - type: nauc_map_at_3_std + value: -73.0069 + - type: nauc_map_at_3_diff1 + value: 80.437 + - type: nauc_map_at_5_max + value: 28.931600000000003 + - type: nauc_map_at_5_std + value: -73.7017 + - type: nauc_map_at_5_diff1 + value: 80.2443 + - type: nauc_map_at_10_max + value: 30.246699999999997 + - type: nauc_map_at_10_std + value: -71.5712 + - type: nauc_map_at_10_diff1 + value: 80.0294 + - type: nauc_map_at_20_max + value: 30.6119 + - type: nauc_map_at_20_std + value: -70.0168 + - type: nauc_map_at_20_diff1 + value: 79.86619999999999 + - type: nauc_map_at_100_max + value: 30.778899999999997 + - type: nauc_map_at_100_std + value: -68.85860000000001 + - type: nauc_map_at_100_diff1 + value: 79.8048 + - type: nauc_map_at_1000_max + value: 30.798199999999998 + - type: nauc_map_at_1000_std + value: -68.77210000000001 + - type: nauc_map_at_1000_diff1 + value: 79.8039 + - type: nauc_recall_at_1_max + value: 21.4243 + - type: nauc_recall_at_1_std + value: -57.04900000000001 + - type: nauc_recall_at_1_diff1 + value: 83.3378 + - type: nauc_recall_at_3_max + value: 22.6679 + - type: nauc_recall_at_3_std + value: -86.9046 + - type: nauc_recall_at_3_diff1 + value: 77.608 + - type: nauc_recall_at_5_max + value: 24.4242 + - type: nauc_recall_at_5_std + value: -100.1963 + - type: nauc_recall_at_5_diff1 + value: 77.5562 + - type: nauc_recall_at_10_max + value: 26.995599999999996 + - type: nauc_recall_at_10_std + value: -110.56330000000001 + - type: nauc_recall_at_10_diff1 + value: 78.6007 + - type: nauc_recall_at_20_max + value: 27.3385 + - type: nauc_recall_at_20_std + value: -117.10199999999999 + - type: nauc_recall_at_20_diff1 + value: 77.7938 + - type: nauc_recall_at_100_max + value: 33.0847 + - type: nauc_recall_at_100_std + value: -110.4169 + - type: nauc_recall_at_100_diff1 + value: 76.4873 + - type: nauc_recall_at_1000_max + value: -16.532 + - type: nauc_recall_at_1000_std + value: -24.5592 + - type: nauc_recall_at_1000_diff1 + value: 72.233 + - type: nauc_precision_at_1_max + value: 33.0127 + - type: nauc_precision_at_1_std + value: -59.3688 + - type: nauc_precision_at_1_diff1 + value: 81.11880000000001 + - type: nauc_precision_at_3_max + value: 4.6178 + - type: nauc_precision_at_3_std + value: 8.1134 + - type: nauc_precision_at_3_diff1 + value: -27.1918 + - type: nauc_precision_at_5_max + value: 1.3161 + - type: nauc_precision_at_5_std + value: 21.8406 + - type: nauc_precision_at_5_diff1 + value: -37.5509 + - type: nauc_precision_at_10_max + value: -1.4878 + - type: nauc_precision_at_10_std + value: 35.5171 + - type: nauc_precision_at_10_diff1 + value: -43.601099999999995 + - type: nauc_precision_at_20_max + value: -3.0787999999999998 + - type: nauc_precision_at_20_std + value: 43.1194 + - type: nauc_precision_at_20_diff1 + value: -45.7438 + - type: nauc_precision_at_100_max + value: -4.3248 + - type: nauc_precision_at_100_std + value: 51.5534 + - type: nauc_precision_at_100_diff1 + value: -46.8655 + - type: nauc_precision_at_1000_max + value: -4.4053 + - type: nauc_precision_at_1000_std + value: 53.2738 + - type: nauc_precision_at_1000_diff1 + value: -46.8777 + - type: nauc_mrr_at_1_max + value: 32.994099999999996 + - type: nauc_mrr_at_1_std + value: -59.4653 + - type: nauc_mrr_at_1_diff1 + value: 81.0983 + - type: nauc_mrr_at_3_max + value: 33.381699999999995 + - type: nauc_mrr_at_3_std + value: -65.6011 + - type: nauc_mrr_at_3_diff1 + value: 80.3293 + - type: nauc_mrr_at_5_max + value: 33.5696 + - type: nauc_mrr_at_5_std + value: -65.3317 + - type: nauc_mrr_at_5_diff1 + value: 80.5711 + - type: nauc_mrr_at_10_max + value: 33.453500000000005 + - type: nauc_mrr_at_10_std + value: -64.90209999999999 + - type: nauc_mrr_at_10_diff1 + value: 80.5965 + - type: nauc_mrr_at_20_max + value: 33.414500000000004 + - type: nauc_mrr_at_20_std + value: -64.7197 + - type: nauc_mrr_at_20_diff1 + value: 80.5804 + - type: nauc_mrr_at_100_max + value: 33.4032 + - type: nauc_mrr_at_100_std + value: -64.6315 + - type: nauc_mrr_at_100_diff1 + value: 80.5771 + - type: nauc_mrr_at_1000_max + value: 33.4024 + - type: nauc_mrr_at_1000_std + value: -64.6301 + - type: nauc_mrr_at_1000_diff1 + value: 80.5769 + - type: main_score + value: 91.019 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering (default) + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: v_measure + value: 72.7484 + - type: v_measure_std + value: 2.9369 + - type: main_score + value: 72.7484 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P (default) + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: v_measure + value: 73.0078 + - type: v_measure_std + value: 12.3013 + - type: main_score + value: 73.0078 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS (default) + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: ndcg_at_1 + value: 31.3 + - type: ndcg_at_3 + value: 26.807 + - type: ndcg_at_5 + value: 24.29 + - type: ndcg_at_10 + value: 29.189999999999998 + - type: ndcg_at_20 + value: 33.212 + - type: ndcg_at_100 + value: 41.062 + - type: ndcg_at_1000 + value: 46.201 + - type: map_at_1 + value: 6.358 + - type: map_at_3 + value: 12.309000000000001 + - type: map_at_5 + value: 15.543000000000001 + - type: map_at_10 + value: 18.404999999999998 + - type: map_at_20 + value: 20.102 + - type: map_at_100 + value: 22.012 + - type: map_at_1000 + value: 22.391 + - type: recall_at_1 + value: 6.358 + - type: recall_at_3 + value: 15.482999999999999 + - type: recall_at_5 + value: 22.343 + - type: recall_at_10 + value: 31.378 + - type: recall_at_20 + value: 40.797 + - type: recall_at_100 + value: 66.122 + - type: recall_at_1000 + value: 90.878 + - type: precision_at_1 + value: 31.3 + - type: precision_at_3 + value: 25.467000000000002 + - type: precision_at_5 + value: 22.06 + - type: precision_at_10 + value: 15.479999999999999 + - type: precision_at_20 + value: 10.059999999999999 + - type: precision_at_100 + value: 3.2620000000000005 + - type: precision_at_1000 + value: 0.44799999999999995 + - type: mrr_at_1 + value: 31.3 + - type: mrr_at_3 + value: 41.5167 + - type: mrr_at_5 + value: 43.8317 + - type: mrr_at_10 + value: 45.236900000000006 + - type: mrr_at_20 + value: 45.894200000000005 + - type: mrr_at_100 + value: 46.2296 + - type: mrr_at_1000 + value: 46.247 + - type: nauc_ndcg_at_1_max + value: 22.6455 + - type: nauc_ndcg_at_1_std + value: -3.4214 + - type: nauc_ndcg_at_1_diff1 + value: 22.2194 + - type: nauc_ndcg_at_3_max + value: 26.16 + - type: nauc_ndcg_at_3_std + value: -4.1874 + - type: nauc_ndcg_at_3_diff1 + value: 15.6529 + - type: nauc_ndcg_at_5_max + value: 29.849500000000003 + - type: nauc_ndcg_at_5_std + value: -3.5488 + - type: nauc_ndcg_at_5_diff1 + value: 15.6251 + - type: nauc_ndcg_at_10_max + value: 30.462600000000002 + - type: nauc_ndcg_at_10_std + value: -2.4431000000000003 + - type: nauc_ndcg_at_10_diff1 + value: 13.424700000000001 + - type: nauc_ndcg_at_20_max + value: 32.0054 + - type: nauc_ndcg_at_20_std + value: -0.9074000000000001 + - type: nauc_ndcg_at_20_diff1 + value: 13.2326 + - type: nauc_ndcg_at_100_max + value: 33.604099999999995 + - type: nauc_ndcg_at_100_std + value: 3.8350000000000004 + - type: nauc_ndcg_at_100_diff1 + value: 12.7082 + - type: nauc_ndcg_at_1000_max + value: 32.5997 + - type: nauc_ndcg_at_1000_std + value: 3.2862 + - type: nauc_ndcg_at_1000_diff1 + value: 13.7365 + - type: nauc_map_at_1_max + value: 22.3207 + - type: nauc_map_at_1_std + value: -3.543 + - type: nauc_map_at_1_diff1 + value: 21.9335 + - type: nauc_map_at_3_max + value: 24.9035 + - type: nauc_map_at_3_std + value: -5.9363 + - type: nauc_map_at_3_diff1 + value: 15.101 + - type: nauc_map_at_5_max + value: 28.5337 + - type: nauc_map_at_5_std + value: -6.2807 + - type: nauc_map_at_5_diff1 + value: 14.9171 + - type: nauc_map_at_10_max + value: 29.496899999999997 + - type: nauc_map_at_10_std + value: -5.608 + - type: nauc_map_at_10_diff1 + value: 12.7308 + - type: nauc_map_at_20_max + value: 30.4348 + - type: nauc_map_at_20_std + value: -4.4265 + - type: nauc_map_at_20_diff1 + value: 12.4533 + - type: nauc_map_at_100_max + value: 31.244100000000003 + - type: nauc_map_at_100_std + value: -2.6229999999999998 + - type: nauc_map_at_100_diff1 + value: 12.2408 + - type: nauc_map_at_1000_max + value: 31.200699999999998 + - type: nauc_map_at_1000_std + value: -2.5584 + - type: nauc_map_at_1000_diff1 + value: 12.295499999999999 + - type: nauc_recall_at_1_max + value: 22.3207 + - type: nauc_recall_at_1_std + value: -3.543 + - type: nauc_recall_at_1_diff1 + value: 21.9335 + - type: nauc_recall_at_3_max + value: 26.617800000000003 + - type: nauc_recall_at_3_std + value: -4.601 + - type: nauc_recall_at_3_diff1 + value: 12.969800000000001 + - type: nauc_recall_at_5_max + value: 31.523 + - type: nauc_recall_at_5_std + value: -2.8593 + - type: nauc_recall_at_5_diff1 + value: 13.077 + - type: nauc_recall_at_10_max + value: 30.361 + - type: nauc_recall_at_10_std + value: -0.7305 + - type: nauc_recall_at_10_diff1 + value: 8.5364 + - type: nauc_recall_at_20_max + value: 31.821700000000003 + - type: nauc_recall_at_20_std + value: 2.5871999999999997 + - type: nauc_recall_at_20_diff1 + value: 7.7219 + - type: nauc_recall_at_100_max + value: 32.658500000000004 + - type: nauc_recall_at_100_std + value: 17.088 + - type: nauc_recall_at_100_diff1 + value: 4.2962 + - type: nauc_recall_at_1000_max + value: 28.8568 + - type: nauc_recall_at_1000_std + value: 30.724400000000003 + - type: nauc_recall_at_1000_diff1 + value: 5.7278 + - type: nauc_precision_at_1_max + value: 22.6455 + - type: nauc_precision_at_1_std + value: -3.4214 + - type: nauc_precision_at_1_diff1 + value: 22.2194 + - type: nauc_precision_at_3_max + value: 27.0287 + - type: nauc_precision_at_3_std + value: -4.2745999999999995 + - type: nauc_precision_at_3_diff1 + value: 13.2524 + - type: nauc_precision_at_5_max + value: 31.798199999999998 + - type: nauc_precision_at_5_std + value: -2.6458 + - type: nauc_precision_at_5_diff1 + value: 13.1913 + - type: nauc_precision_at_10_max + value: 30.442700000000002 + - type: nauc_precision_at_10_std + value: -0.7052 + - type: nauc_precision_at_10_diff1 + value: 8.698500000000001 + - type: nauc_precision_at_20_max + value: 31.8098 + - type: nauc_precision_at_20_std + value: 2.6527 + - type: nauc_precision_at_20_diff1 + value: 7.988199999999999 + - type: nauc_precision_at_100_max + value: 31.9799 + - type: nauc_precision_at_100_std + value: 16.4552 + - type: nauc_precision_at_100_diff1 + value: 4.6661 + - type: nauc_precision_at_1000_max + value: 26.039099999999998 + - type: nauc_precision_at_1000_std + value: 26.8761 + - type: nauc_precision_at_1000_diff1 + value: 5.564299999999999 + - type: nauc_mrr_at_1_max + value: 22.6455 + - type: nauc_mrr_at_1_std + value: -3.4214 + - type: nauc_mrr_at_1_diff1 + value: 22.2194 + - type: nauc_mrr_at_3_max + value: 25.827699999999997 + - type: nauc_mrr_at_3_std + value: -2.0878 + - type: nauc_mrr_at_3_diff1 + value: 19.0105 + - type: nauc_mrr_at_5_max + value: 26.479799999999997 + - type: nauc_mrr_at_5_std + value: -1.0343 + - type: nauc_mrr_at_5_diff1 + value: 19.4599 + - type: nauc_mrr_at_10_max + value: 26.3345 + - type: nauc_mrr_at_10_std + value: -1.0147 + - type: nauc_mrr_at_10_diff1 + value: 19.572 + - type: nauc_mrr_at_20_max + value: 26.4581 + - type: nauc_mrr_at_20_std + value: -1.0342 + - type: nauc_mrr_at_20_diff1 + value: 19.5204 + - type: nauc_mrr_at_100_max + value: 26.334200000000003 + - type: nauc_mrr_at_100_std + value: -1.0591 + - type: nauc_mrr_at_100_diff1 + value: 19.5134 + - type: nauc_mrr_at_1000_max + value: 26.3192 + - type: nauc_mrr_at_1000_std + value: -1.0868 + - type: nauc_mrr_at_1000_diff1 + value: 19.5308 + - type: main_score + value: 29.189999999999998 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R (default) + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: pearson + value: 82.9977 + - type: spearman + value: 82.7264 + - type: cosine_pearson + value: 82.9977 + - type: cosine_spearman + value: 82.7264 + - type: manhattan_pearson + value: 79.2844 + - type: manhattan_spearman + value: 82.706 + - type: euclidean_pearson + value: 79.30319999999999 + - type: euclidean_spearman + value: 82.7264 + - type: main_score + value: 82.7264 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 (default) + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: pearson + value: 86.691 + - type: spearman + value: 79.9669 + - type: cosine_pearson + value: 86.691 + - type: cosine_spearman + value: 79.9669 + - type: manhattan_pearson + value: 81.131 + - type: manhattan_spearman + value: 79.9913 + - type: euclidean_pearson + value: 81.13550000000001 + - type: euclidean_spearman + value: 79.9667 + - type: main_score + value: 79.9669 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 (default) + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: pearson + value: 85.1053 + - type: spearman + value: 83.94890000000001 + - type: cosine_pearson + value: 85.1053 + - type: cosine_spearman + value: 83.94890000000001 + - type: manhattan_pearson + value: 83.7957 + - type: manhattan_spearman + value: 83.8831 + - type: euclidean_pearson + value: 83.8318 + - type: euclidean_spearman + value: 83.94890000000001 + - type: main_score + value: 83.94890000000001 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 (default) + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: pearson + value: 84.23610000000001 + - type: spearman + value: 84.2503 + - type: cosine_pearson + value: 84.23610000000001 + - type: cosine_spearman + value: 84.2503 + - type: manhattan_pearson + value: 82.3061 + - type: manhattan_spearman + value: 84.2598 + - type: euclidean_pearson + value: 82.30330000000001 + - type: euclidean_spearman + value: 84.2503 + - type: main_score + value: 84.2503 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 (default) + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: pearson + value: 90.5569 + - type: spearman + value: 90.4496 + - type: cosine_pearson + value: 90.5569 + - type: cosine_spearman + value: 90.4496 + - type: manhattan_pearson + value: 88.5942 + - type: manhattan_spearman + value: 90.4286 + - type: euclidean_pearson + value: 88.6003 + - type: euclidean_spearman + value: 90.4496 + - type: main_score + value: 90.4496 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 (default) + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: pearson + value: 86.447 + - type: spearman + value: 86.4776 + - type: cosine_pearson + value: 86.447 + - type: cosine_spearman + value: 86.4776 + - type: manhattan_pearson + value: 85.3768 + - type: manhattan_spearman + value: 86.48599999999999 + - type: euclidean_pearson + value: 85.3792 + - type: euclidean_spearman + value: 86.4776 + - type: main_score + value: 86.4776 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 90.40820000000001 + - type: spearman + value: 89.7495 + - type: cosine_pearson + value: 90.40820000000001 + - type: cosine_spearman + value: 89.7495 + - type: manhattan_pearson + value: 88.20519999999999 + - type: manhattan_spearman + value: 89.62689999999999 + - type: euclidean_pearson + value: 88.268 + - type: euclidean_spearman + value: 89.7495 + - type: main_score + value: 89.7495 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 69.5732 + - type: spearman + value: 67.7261 + - type: cosine_pearson + value: 69.5732 + - type: cosine_spearman + value: 67.7261 + - type: manhattan_pearson + value: 69.7793 + - type: manhattan_spearman + value: 67.9213 + - type: euclidean_pearson + value: 69.6908 + - type: euclidean_spearman + value: 67.7261 + - type: main_score + value: 67.7261 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark (default) + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: pearson + value: 86.6914 + - type: spearman + value: 87.2151 + - type: cosine_pearson + value: 86.6914 + - type: cosine_spearman + value: 87.2151 + - type: manhattan_pearson + value: 85.8277 + - type: manhattan_spearman + value: 87.2492 + - type: euclidean_pearson + value: 85.79719999999999 + - type: euclidean_spearman + value: 87.2151 + - type: main_score + value: 87.2151 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR (default) + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: map + value: 89.137 + - type: mrr + value: 96.7541 + - type: nAUC_map_max + value: 52.1481 + - type: nAUC_map_std + value: 72.15859999999999 + - type: nAUC_map_diff1 + value: -10.389 + - type: nAUC_mrr_max + value: 85.25160000000001 + - type: nAUC_mrr_std + value: 87.73570000000001 + - type: nAUC_mrr_diff1 + value: 30.605300000000003 + - type: main_score + value: 89.137 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact (default) + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: ndcg_at_1 + value: 71.667 + - type: ndcg_at_3 + value: 79.611 + - type: ndcg_at_5 + value: 81.394 + - type: ndcg_at_10 + value: 83.279 + - type: ndcg_at_20 + value: 83.8 + - type: ndcg_at_100 + value: 84.233 + - type: ndcg_at_1000 + value: 84.316 + - type: map_at_1 + value: 68.57799999999999 + - type: map_at_3 + value: 76.639 + - type: map_at_5 + value: 78.168 + - type: map_at_10 + value: 79.148 + - type: map_at_20 + value: 79.31 + - type: map_at_100 + value: 79.36800000000001 + - type: map_at_1000 + value: 79.37100000000001 + - type: recall_at_1 + value: 68.57799999999999 + - type: recall_at_3 + value: 85.47200000000001 + - type: recall_at_5 + value: 89.839 + - type: recall_at_10 + value: 95 + - type: recall_at_20 + value: 97 + - type: recall_at_100 + value: 99.333 + - type: recall_at_1000 + value: 100 + - type: precision_at_1 + value: 71.667 + - type: precision_at_3 + value: 31 + - type: precision_at_5 + value: 20.067 + - type: precision_at_10 + value: 10.767 + - type: precision_at_20 + value: 5.5 + - type: precision_at_100 + value: 1.123 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: mrr_at_1 + value: 71.6667 + - type: mrr_at_3 + value: 78.2222 + - type: mrr_at_5 + value: 79.0222 + - type: mrr_at_10 + value: 79.7295 + - type: mrr_at_20 + value: 79.83879999999999 + - type: mrr_at_100 + value: 79.89739999999999 + - type: mrr_at_1000 + value: 79.9004 + - type: nauc_ndcg_at_1_max + value: 42.2431 + - type: nauc_ndcg_at_1_std + value: -2.0832 + - type: nauc_ndcg_at_1_diff1 + value: 76.9413 + - type: nauc_ndcg_at_3_max + value: 34.7709 + - type: nauc_ndcg_at_3_std + value: -6.3732999999999995 + - type: nauc_ndcg_at_3_diff1 + value: 74.6789 + - type: nauc_ndcg_at_5_max + value: 37.940400000000004 + - type: nauc_ndcg_at_5_std + value: -3.9581999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 75.22330000000001 + - type: nauc_ndcg_at_10_max + value: 41.6103 + - type: nauc_ndcg_at_10_std + value: -0.0314 + - type: nauc_ndcg_at_10_diff1 + value: 75.2945 + - type: nauc_ndcg_at_20_max + value: 42.524 + - type: nauc_ndcg_at_20_std + value: 0.2979 + - type: nauc_ndcg_at_20_diff1 + value: 75.4989 + - type: nauc_ndcg_at_100_max + value: 41.727399999999996 + - type: nauc_ndcg_at_100_std + value: -0.4197 + - type: nauc_ndcg_at_100_diff1 + value: 75.7163 + - type: nauc_ndcg_at_1000_max + value: 41.3855 + - type: nauc_ndcg_at_1000_std + value: -0.6131 + - type: nauc_ndcg_at_1000_diff1 + value: 75.618 + - type: nauc_map_at_1_max + value: 32.7432 + - type: nauc_map_at_1_std + value: -10.6948 + - type: nauc_map_at_1_diff1 + value: 77.2203 + - type: nauc_map_at_3_max + value: 32.7526 + - type: nauc_map_at_3_std + value: -7.8953 + - type: nauc_map_at_3_diff1 + value: 75.88380000000001 + - type: nauc_map_at_5_max + value: 36.868 + - type: nauc_map_at_5_std + value: -4.5381 + - type: nauc_map_at_5_diff1 + value: 75.5504 + - type: nauc_map_at_10_max + value: 39.0762 + - type: nauc_map_at_10_std + value: -2.1559 + - type: nauc_map_at_10_diff1 + value: 75.5037 + - type: nauc_map_at_20_max + value: 39.3914 + - type: nauc_map_at_20_std + value: -2.075 + - type: nauc_map_at_20_diff1 + value: 75.5527 + - type: nauc_map_at_100_max + value: 39.2883 + - type: nauc_map_at_100_std + value: -2.1987 + - type: nauc_map_at_100_diff1 + value: 75.57979999999999 + - type: nauc_map_at_1000_max + value: 39.278200000000005 + - type: nauc_map_at_1000_std + value: -2.1991 + - type: nauc_map_at_1000_diff1 + value: 75.5776 + - type: nauc_recall_at_1_max + value: 32.7432 + - type: nauc_recall_at_1_std + value: -10.6948 + - type: nauc_recall_at_1_diff1 + value: 77.2203 + - type: nauc_recall_at_3_max + value: 23.718500000000002 + - type: nauc_recall_at_3_std + value: -14.9527 + - type: nauc_recall_at_3_diff1 + value: 70.99849999999999 + - type: nauc_recall_at_5_max + value: 34.1278 + - type: nauc_recall_at_5_std + value: -8.9991 + - type: nauc_recall_at_5_diff1 + value: 72.9131 + - type: nauc_recall_at_10_max + value: 53.4174 + - type: nauc_recall_at_10_std + value: 10.591299999999999 + - type: nauc_recall_at_10_diff1 + value: 72.1148 + - type: nauc_recall_at_20_max + value: 74.4061 + - type: nauc_recall_at_20_std + value: 23.5605 + - type: nauc_recall_at_20_diff1 + value: 74.515 + - type: nauc_recall_at_100_max + value: 100 + - type: nauc_recall_at_100_std + value: 41.4332 + - type: nauc_recall_at_100_diff1 + value: 93.4641 + - type: nauc_recall_at_1000_max + value: null + - type: nauc_recall_at_1000_std + value: null + - type: nauc_recall_at_1000_diff1 + value: null + - type: nauc_precision_at_1_max + value: 42.2431 + - type: nauc_precision_at_1_std + value: -2.0832 + - type: nauc_precision_at_1_diff1 + value: 76.9413 + - type: nauc_precision_at_3_max + value: 31.2606 + - type: nauc_precision_at_3_std + value: 19.564300000000003 + - type: nauc_precision_at_3_diff1 + value: 27.538899999999998 + - type: nauc_precision_at_5_max + value: 36.896 + - type: nauc_precision_at_5_std + value: 32.9313 + - type: nauc_precision_at_5_diff1 + value: 5.233899999999999 + - type: nauc_precision_at_10_max + value: 40.0781 + - type: nauc_precision_at_10_std + value: 48.0555 + - type: nauc_precision_at_10_diff1 + value: -14.6074 + - type: nauc_precision_at_20_max + value: 39.3814 + - type: nauc_precision_at_20_std + value: 49.9845 + - type: nauc_precision_at_20_diff1 + value: -21.171 + - type: nauc_precision_at_100_max + value: 36.6046 + - type: nauc_precision_at_100_std + value: 53.1439 + - type: nauc_precision_at_100_diff1 + value: -30.216500000000003 + - type: nauc_precision_at_1000_max + value: 34.7361 + - type: nauc_precision_at_1000_std + value: 53.4891 + - type: nauc_precision_at_1000_diff1 + value: -33.8617 + - type: nauc_mrr_at_1_max + value: 42.2431 + - type: nauc_mrr_at_1_std + value: -2.0832 + - type: nauc_mrr_at_1_diff1 + value: 76.9413 + - type: nauc_mrr_at_3_max + value: 40.1861 + - type: nauc_mrr_at_3_std + value: -2.1431 + - type: nauc_mrr_at_3_diff1 + value: 75.3883 + - type: nauc_mrr_at_5_max + value: 40.9913 + - type: nauc_mrr_at_5_std + value: -1.6580000000000001 + - type: nauc_mrr_at_5_diff1 + value: 75.8294 + - type: nauc_mrr_at_10_max + value: 41.8035 + - type: nauc_mrr_at_10_std + value: -1.1311 + - type: nauc_mrr_at_10_diff1 + value: 75.9254 + - type: nauc_mrr_at_20_max + value: 41.9873 + - type: nauc_mrr_at_20_std + value: -1.1159000000000001 + - type: nauc_mrr_at_20_diff1 + value: 75.9764 + - type: nauc_mrr_at_100_max + value: 41.890699999999995 + - type: nauc_mrr_at_100_std + value: -1.239 + - type: nauc_mrr_at_100_diff1 + value: 76.00529999999999 + - type: nauc_mrr_at_1000_max + value: 41.8809 + - type: nauc_mrr_at_1000_std + value: -1.2392 + - type: nauc_mrr_at_1000_diff1 + value: 76.0031 + - type: main_score + value: 83.279 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions (default) + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: similarity_accuracy + value: 99.8644 + - type: similarity_accuracy_threshold + value: 85.02 + - type: similarity_f1 + value: 93.1875 + - type: similarity_f1_threshold + value: 85.02 + - type: similarity_precision + value: 92.6805 + - type: similarity_recall + value: 93.7 + - type: similarity_ap + value: 96.7847 + - type: cosine_accuracy + value: 99.8644 + - type: cosine_accuracy_threshold + value: 85.02 + - type: cosine_f1 + value: 93.1875 + - type: cosine_f1_threshold + value: 85.02 + - type: cosine_precision + value: 92.6805 + - type: cosine_recall + value: 93.7 + - type: cosine_ap + value: 96.7847 + - type: manhattan_accuracy + value: 99.8634 + - type: manhattan_accuracy_threshold + value: 2593.8221 + - type: manhattan_f1 + value: 93.1275 + - type: manhattan_f1_threshold + value: 2593.8221 + - type: manhattan_precision + value: 92.7579 + - type: manhattan_recall + value: 93.5 + - type: manhattan_ap + value: 96.806 + - type: euclidean_accuracy + value: 99.8644 + - type: euclidean_accuracy_threshold + value: 54.7358 + - type: euclidean_f1 + value: 93.1875 + - type: euclidean_f1_threshold + value: 54.7358 + - type: euclidean_precision + value: 92.6805 + - type: euclidean_recall + value: 93.7 + - type: euclidean_ap + value: 96.7847 + - type: dot_accuracy + value: 99.8644 + - type: dot_accuracy_threshold + value: 85.02 + - type: dot_f1 + value: 93.1875 + - type: dot_f1_threshold + value: 85.02 + - type: dot_precision + value: 92.6805 + - type: dot_recall + value: 93.7 + - type: dot_ap + value: 96.7847 + - type: max_accuracy + value: 99.8644 + - type: max_f1 + value: 93.1875 + - type: max_precision + value: 92.7579 + - type: max_recall + value: 93.7 + - type: max_ap + value: 96.806 + - type: main_score + value: 96.806 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering (default) + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: v_measure + value: 81.7075 + - type: v_measure_std + value: 2.4228 + - type: main_score + value: 81.7075 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P (default) + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: v_measure + value: 39.836 + - type: v_measure_std + value: 1.5339 + - type: main_score + value: 39.836 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions (default) + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: map + value: 62.9227 + - type: mrr + value: 64.1239 + - type: nAUC_map_max + value: 9.3055 + - type: nAUC_map_std + value: 3.2321000000000004 + - type: nAUC_map_diff1 + value: 45.2884 + - type: nAUC_mrr_max + value: 10.8913 + - type: nAUC_mrr_std + value: 4.1469 + - type: nAUC_mrr_diff1 + value: 45.280300000000004 + - type: main_score + value: 62.9227 + task: + type: Reranking + - dataset: + config: default + name: MTEB SummEval (default) + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: pearson + value: 45.5445 + - type: spearman + value: 40.5224 + - type: cosine_spearman + value: 40.5224 + - type: cosine_pearson + value: 45.5445 + - type: dot_spearman + value: 40.5224 + - type: dot_pearson + value: 45.5446 + - type: main_score + value: 40.5224 + task: + type: Summarization + - dataset: + config: default + name: MTEB TRECCOVID (default) + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: ndcg_at_1 + value: 86 + - type: ndcg_at_3 + value: 86.70400000000001 + - type: ndcg_at_5 + value: 84.654 + - type: ndcg_at_10 + value: 80.972 + - type: ndcg_at_20 + value: 76.783 + - type: ndcg_at_100 + value: 63.146 + - type: ndcg_at_1000 + value: 56.424 + - type: map_at_1 + value: 0.251 + - type: map_at_3 + value: 0.719 + - type: map_at_5 + value: 1.131 + - type: map_at_10 + value: 2.0820000000000003 + - type: map_at_20 + value: 3.746 + - type: map_at_100 + value: 13.020999999999999 + - type: map_at_1000 + value: 31.527 + - type: recall_at_1 + value: 0.251 + - type: recall_at_3 + value: 0.742 + - type: recall_at_5 + value: 1.179 + - type: recall_at_10 + value: 2.237 + - type: recall_at_20 + value: 4.144 + - type: recall_at_100 + value: 16.054 + - type: recall_at_1000 + value: 52.76 + - type: precision_at_1 + value: 92 + - type: precision_at_3 + value: 92 + - type: precision_at_5 + value: 88.8 + - type: precision_at_10 + value: 85 + - type: precision_at_20 + value: 79.7 + - type: precision_at_100 + value: 64.53999999999999 + - type: precision_at_1000 + value: 24.471999999999998 + - type: mrr_at_1 + value: 92 + - type: mrr_at_3 + value: 95.6667 + - type: mrr_at_5 + value: 95.6667 + - type: mrr_at_10 + value: 95.6667 + - type: mrr_at_20 + value: 95.6667 + - type: mrr_at_100 + value: 95.6667 + - type: mrr_at_1000 + value: 95.6667 + - type: nauc_ndcg_at_1_max + value: 7.0274 + - type: nauc_ndcg_at_1_std + value: 41.318 + - type: nauc_ndcg_at_1_diff1 + value: -46.5125 + - type: nauc_ndcg_at_3_max + value: 2.0167 + - type: nauc_ndcg_at_3_std + value: 36.144999999999996 + - type: nauc_ndcg_at_3_diff1 + value: -17.705199999999998 + - type: nauc_ndcg_at_5_max + value: -6.812 + - type: nauc_ndcg_at_5_std + value: 41.8996 + - type: nauc_ndcg_at_5_diff1 + value: -14.7154 + - type: nauc_ndcg_at_10_max + value: 15.1784 + - type: nauc_ndcg_at_10_std + value: 51.709799999999994 + - type: nauc_ndcg_at_10_diff1 + value: -5.0968 + - type: nauc_ndcg_at_20_max + value: 28.403200000000002 + - type: nauc_ndcg_at_20_std + value: 59.824299999999994 + - type: nauc_ndcg_at_20_diff1 + value: -14.036000000000001 + - type: nauc_ndcg_at_100_max + value: 35.4195 + - type: nauc_ndcg_at_100_std + value: 75.7747 + - type: nauc_ndcg_at_100_diff1 + value: -10.4627 + - type: nauc_ndcg_at_1000_max + value: 31.450400000000002 + - type: nauc_ndcg_at_1000_std + value: 78.85940000000001 + - type: nauc_ndcg_at_1000_diff1 + value: -1.3263 + - type: nauc_map_at_1_max + value: -3.8297 + - type: nauc_map_at_1_std + value: -10.6113 + - type: nauc_map_at_1_diff1 + value: 9.2146 + - type: nauc_map_at_3_max + value: -3.1165000000000003 + - type: nauc_map_at_3_std + value: -8.4396 + - type: nauc_map_at_3_diff1 + value: 14.183000000000002 + - type: nauc_map_at_5_max + value: -4.4023 + - type: nauc_map_at_5_std + value: -6.641500000000001 + - type: nauc_map_at_5_diff1 + value: 16.1186 + - type: nauc_map_at_10_max + value: 1.802 + - type: nauc_map_at_10_std + value: 0.9958 + - type: nauc_map_at_10_diff1 + value: 20.3485 + - type: nauc_map_at_20_max + value: 10.9146 + - type: nauc_map_at_20_std + value: 10.3413 + - type: nauc_map_at_20_diff1 + value: 14.7839 + - type: nauc_map_at_100_max + value: 25.633 + - type: nauc_map_at_100_std + value: 44.9724 + - type: nauc_map_at_100_diff1 + value: 6.572699999999999 + - type: nauc_map_at_1000_max + value: 33.8688 + - type: nauc_map_at_1000_std + value: 76.9255 + - type: nauc_map_at_1000_diff1 + value: -5.8205 + - type: nauc_recall_at_1_max + value: -3.8297 + - type: nauc_recall_at_1_std + value: -10.6113 + - type: nauc_recall_at_1_diff1 + value: 9.2146 + - type: nauc_recall_at_3_max + value: -6.209 + - type: nauc_recall_at_3_std + value: -11.3272 + - type: nauc_recall_at_3_diff1 + value: 16.497500000000002 + - type: nauc_recall_at_5_max + value: -7.6928 + - type: nauc_recall_at_5_std + value: -8.9985 + - type: nauc_recall_at_5_diff1 + value: 19.028100000000002 + - type: nauc_recall_at_10_max + value: -1.3407 + - type: nauc_recall_at_10_std + value: -2.5698 + - type: nauc_recall_at_10_diff1 + value: 21.570700000000002 + - type: nauc_recall_at_20_max + value: 6.866700000000001 + - type: nauc_recall_at_20_std + value: 5.7298 + - type: nauc_recall_at_20_diff1 + value: 16.050800000000002 + - type: nauc_recall_at_100_max + value: 16.4856 + - type: nauc_recall_at_100_std + value: 33.1774 + - type: nauc_recall_at_100_diff1 + value: 12.0273 + - type: nauc_recall_at_1000_max + value: 25.3677 + - type: nauc_recall_at_1000_std + value: 71.1541 + - type: nauc_recall_at_1000_diff1 + value: 0.796 + - type: nauc_precision_at_1_max + value: 57.236200000000004 + - type: nauc_precision_at_1_std + value: 47.7241 + - type: nauc_precision_at_1_diff1 + value: -57.8198 + - type: nauc_precision_at_3_max + value: 35.6953 + - type: nauc_precision_at_3_std + value: 31.414199999999997 + - type: nauc_precision_at_3_diff1 + value: -6.7696000000000005 + - type: nauc_precision_at_5_max + value: 1.699 + - type: nauc_precision_at_5_std + value: 37.6284 + - type: nauc_precision_at_5_diff1 + value: -4.9533000000000005 + - type: nauc_precision_at_10_max + value: 31.645400000000002 + - type: nauc_precision_at_10_std + value: 48.4684 + - type: nauc_precision_at_10_diff1 + value: 8.3324 + - type: nauc_precision_at_20_max + value: 45.7958 + - type: nauc_precision_at_20_std + value: 56.3558 + - type: nauc_precision_at_20_diff1 + value: -7.8348 + - type: nauc_precision_at_100_max + value: 40.1005 + - type: nauc_precision_at_100_std + value: 73.342 + - type: nauc_precision_at_100_diff1 + value: -7.284400000000001 + - type: nauc_precision_at_1000_max + value: 27.9268 + - type: nauc_precision_at_1000_std + value: 50.145799999999994 + - type: nauc_precision_at_1000_diff1 + value: -15.678700000000001 + - type: nauc_mrr_at_1_max + value: 57.236200000000004 + - type: nauc_mrr_at_1_std + value: 47.7241 + - type: nauc_mrr_at_1_diff1 + value: -57.8198 + - type: nauc_mrr_at_3_max + value: 53.7779 + - type: nauc_mrr_at_3_std + value: 51.74530000000001 + - type: nauc_mrr_at_3_diff1 + value: -49.1094 + - type: nauc_mrr_at_5_max + value: 53.7779 + - type: nauc_mrr_at_5_std + value: 51.74530000000001 + - type: nauc_mrr_at_5_diff1 + value: -49.1094 + - type: nauc_mrr_at_10_max + value: 53.7779 + - type: nauc_mrr_at_10_std + value: 51.74530000000001 + - type: nauc_mrr_at_10_diff1 + value: -49.1094 + - type: nauc_mrr_at_20_max + value: 53.7779 + - type: nauc_mrr_at_20_std + value: 51.74530000000001 + - type: nauc_mrr_at_20_diff1 + value: -49.1094 + - type: nauc_mrr_at_100_max + value: 53.7779 + - type: nauc_mrr_at_100_std + value: 51.74530000000001 + - type: nauc_mrr_at_100_diff1 + value: -49.1094 + - type: nauc_mrr_at_1000_max + value: 53.7779 + - type: nauc_mrr_at_1000_std + value: 51.74530000000001 + - type: nauc_mrr_at_1000_diff1 + value: -49.1094 + - type: main_score + value: 80.972 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 (default) + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: ndcg_at_1 + value: 45.918 + - type: ndcg_at_3 + value: 42.414 + - type: ndcg_at_5 + value: 36.911 + - type: ndcg_at_10 + value: 34.059 + - type: ndcg_at_20 + value: 33.789 + - type: ndcg_at_100 + value: 43.24 + - type: ndcg_at_1000 + value: 53.028 + - type: map_at_1 + value: 3.711 + - type: map_at_3 + value: 8.031 + - type: map_at_5 + value: 10.174999999999999 + - type: map_at_10 + value: 13.745 + - type: map_at_20 + value: 16.833000000000002 + - type: map_at_100 + value: 20.534 + - type: map_at_1000 + value: 21.929000000000002 + - type: recall_at_1 + value: 3.711 + - type: recall_at_3 + value: 9.289 + - type: recall_at_5 + value: 12.469 + - type: recall_at_10 + value: 20.31 + - type: recall_at_20 + value: 28.549999999999997 + - type: recall_at_100 + value: 50.132 + - type: recall_at_1000 + value: 78.636 + - type: precision_at_1 + value: 48.980000000000004 + - type: precision_at_3 + value: 43.537 + - type: precision_at_5 + value: 35.510000000000005 + - type: precision_at_10 + value: 29.592000000000002 + - type: precision_at_20 + value: 21.633 + - type: precision_at_100 + value: 8.265 + - type: precision_at_1000 + value: 1.478 + - type: mrr_at_1 + value: 48.9796 + - type: mrr_at_3 + value: 61.9048 + - type: mrr_at_5 + value: 62.8231 + - type: mrr_at_10 + value: 64.78620000000001 + - type: mrr_at_20 + value: 64.8882 + - type: mrr_at_100 + value: 64.9667 + - type: mrr_at_1000 + value: 64.9667 + - type: nauc_ndcg_at_1_max + value: -14.377 + - type: nauc_ndcg_at_1_std + value: -24.7998 + - type: nauc_ndcg_at_1_diff1 + value: -2.7112000000000003 + - type: nauc_ndcg_at_3_max + value: -25.411 + - type: nauc_ndcg_at_3_std + value: -21.4105 + - type: nauc_ndcg_at_3_diff1 + value: 11.6233 + - type: nauc_ndcg_at_5_max + value: -18.7583 + - type: nauc_ndcg_at_5_std + value: -12.3778 + - type: nauc_ndcg_at_5_diff1 + value: 2.0221 + - type: nauc_ndcg_at_10_max + value: -20.5164 + - type: nauc_ndcg_at_10_std + value: -15.9037 + - type: nauc_ndcg_at_10_diff1 + value: 4.8377 + - type: nauc_ndcg_at_20_max + value: -24.3335 + - type: nauc_ndcg_at_20_std + value: -15.4334 + - type: nauc_ndcg_at_20_diff1 + value: 5.2053 + - type: nauc_ndcg_at_100_max + value: -27.9931 + - type: nauc_ndcg_at_100_std + value: -0.267 + - type: nauc_ndcg_at_100_diff1 + value: 8.0295 + - type: nauc_ndcg_at_1000_max + value: -22.2584 + - type: nauc_ndcg_at_1000_std + value: 16.6679 + - type: nauc_ndcg_at_1000_diff1 + value: -0.8999999999999999 + - type: nauc_map_at_1_max + value: -19.5845 + - type: nauc_map_at_1_std + value: -33.0644 + - type: nauc_map_at_1_diff1 + value: -5.815300000000001 + - type: nauc_map_at_3_max + value: -28.4895 + - type: nauc_map_at_3_std + value: -32.191199999999995 + - type: nauc_map_at_3_diff1 + value: 9.8452 + - type: nauc_map_at_5_max + value: -17.3979 + - type: nauc_map_at_5_std + value: -21.3281 + - type: nauc_map_at_5_diff1 + value: -2.7651 + - type: nauc_map_at_10_max + value: -16.5472 + - type: nauc_map_at_10_std + value: -21.7069 + - type: nauc_map_at_10_diff1 + value: -1.7826000000000002 + - type: nauc_map_at_20_max + value: -18.6049 + - type: nauc_map_at_20_std + value: -17.8565 + - type: nauc_map_at_20_diff1 + value: -0.0181 + - type: nauc_map_at_100_max + value: -20.030800000000003 + - type: nauc_map_at_100_std + value: -8.6978 + - type: nauc_map_at_100_diff1 + value: 1.1159000000000001 + - type: nauc_map_at_1000_max + value: -18.5756 + - type: nauc_map_at_1000_std + value: -4.4186000000000005 + - type: nauc_map_at_1000_diff1 + value: -0.7358 + - type: nauc_recall_at_1_max + value: -19.5845 + - type: nauc_recall_at_1_std + value: -33.0644 + - type: nauc_recall_at_1_diff1 + value: -5.815300000000001 + - type: nauc_recall_at_3_max + value: -33.051199999999994 + - type: nauc_recall_at_3_std + value: -30.767099999999996 + - type: nauc_recall_at_3_diff1 + value: 11.7941 + - type: nauc_recall_at_5_max + value: -18.8571 + - type: nauc_recall_at_5_std + value: -17.8328 + - type: nauc_recall_at_5_diff1 + value: -5.9348 + - type: nauc_recall_at_10_max + value: -20.657700000000002 + - type: nauc_recall_at_10_std + value: -20.5083 + - type: nauc_recall_at_10_diff1 + value: 0.7172999999999999 + - type: nauc_recall_at_20_max + value: -21.78 + - type: nauc_recall_at_20_std + value: -12.2194 + - type: nauc_recall_at_20_diff1 + value: 2.4215 + - type: nauc_recall_at_100_max + value: -28.1499 + - type: nauc_recall_at_100_std + value: 12.5616 + - type: nauc_recall_at_100_diff1 + value: 6.282400000000001 + - type: nauc_recall_at_1000_max + value: -3.4448 + - type: nauc_recall_at_1000_std + value: 70.2153 + - type: nauc_recall_at_1000_diff1 + value: -20.1278 + - type: nauc_precision_at_1_max + value: -16.253600000000002 + - type: nauc_precision_at_1_std + value: -28.961100000000002 + - type: nauc_precision_at_1_diff1 + value: -4.5123999999999995 + - type: nauc_precision_at_3_max + value: -31.231399999999997 + - type: nauc_precision_at_3_std + value: -21.6787 + - type: nauc_precision_at_3_diff1 + value: 14.080799999999998 + - type: nauc_precision_at_5_max + value: -18.4843 + - type: nauc_precision_at_5_std + value: -4.0988 + - type: nauc_precision_at_5_diff1 + value: -2.3491 + - type: nauc_precision_at_10_max + value: -21.7679 + - type: nauc_precision_at_10_std + value: -2.7599 + - type: nauc_precision_at_10_diff1 + value: 10.6409 + - type: nauc_precision_at_20_max + value: -17.049300000000002 + - type: nauc_precision_at_20_std + value: 12.609200000000001 + - type: nauc_precision_at_20_diff1 + value: 11.3369 + - type: nauc_precision_at_100_max + value: -9.675699999999999 + - type: nauc_precision_at_100_std + value: 44.9955 + - type: nauc_precision_at_100_diff1 + value: 5.7501999999999995 + - type: nauc_precision_at_1000_max + value: 29.789500000000004 + - type: nauc_precision_at_1000_std + value: 58.205200000000005 + - type: nauc_precision_at_1000_diff1 + value: -22.6755 + - type: nauc_mrr_at_1_max + value: -16.253600000000002 + - type: nauc_mrr_at_1_std + value: -28.961100000000002 + - type: nauc_mrr_at_1_diff1 + value: -4.5123999999999995 + - type: nauc_mrr_at_3_max + value: -30.4084 + - type: nauc_mrr_at_3_std + value: -29.1267 + - type: nauc_mrr_at_3_diff1 + value: -2.9535 + - type: nauc_mrr_at_5_max + value: -31.6427 + - type: nauc_mrr_at_5_std + value: -27.5858 + - type: nauc_mrr_at_5_diff1 + value: -2.032 + - type: nauc_mrr_at_10_max + value: -31.1008 + - type: nauc_mrr_at_10_std + value: -27.338099999999997 + - type: nauc_mrr_at_10_diff1 + value: -0.1675 + - type: nauc_mrr_at_20_max + value: -30.7834 + - type: nauc_mrr_at_20_std + value: -27.6591 + - type: nauc_mrr_at_20_diff1 + value: -0.3828 + - type: nauc_mrr_at_100_max + value: -30.3645 + - type: nauc_mrr_at_100_std + value: -28.003 + - type: nauc_mrr_at_100_diff1 + value: -0.48979999999999996 + - type: nauc_mrr_at_1000_max + value: -30.3645 + - type: nauc_mrr_at_1000_std + value: -28.003 + - type: nauc_mrr_at_1000_diff1 + value: -0.48979999999999996 + - type: main_score + value: 34.059 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification (default) + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 95.1318 + - type: f1 + value: 84.8018 + - type: f1_weighted + value: 95.3488 + - type: ap + value: 54.4247 + - type: ap_weighted + value: 54.4247 + - type: main_score + value: 95.1318 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification (default) + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 81.4488 + - type: f1 + value: 81.77990000000001 + - type: f1_weighted + value: 81.4677 + - type: main_score + value: 81.4488 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering (default) + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: v_measure + value: 71.18560000000001 + - type: v_measure_std + value: 1.1396 + - type: main_score + value: 71.18560000000001 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 (default) + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: similarity_accuracy + value: 88.3293 + - type: similarity_accuracy_threshold + value: 89.5055 + - type: similarity_f1 + value: 72.9896 + - type: similarity_f1_threshold + value: 87.6934 + - type: similarity_precision + value: 69.5767 + - type: similarity_recall + value: 76.7546 + - type: similarity_ap + value: 80.33160000000001 + - type: cosine_accuracy + value: 88.3293 + - type: cosine_accuracy_threshold + value: 89.5055 + - type: cosine_f1 + value: 72.9896 + - type: cosine_f1_threshold + value: 87.6934 + - type: cosine_precision + value: 69.5767 + - type: cosine_recall + value: 76.7546 + - type: cosine_ap + value: 80.33160000000001 + - type: manhattan_accuracy + value: 88.29350000000001 + - type: manhattan_accuracy_threshold + value: 2182.2741 + - type: manhattan_f1 + value: 73.0484 + - type: manhattan_f1_threshold + value: 2329.781 + - type: manhattan_precision + value: 70.9245 + - type: manhattan_recall + value: 75.3034 + - type: manhattan_ap + value: 80.3871 + - type: euclidean_accuracy + value: 88.3293 + - type: euclidean_accuracy_threshold + value: 45.8136 + - type: euclidean_f1 + value: 72.9896 + - type: euclidean_f1_threshold + value: 49.6117 + - type: euclidean_precision + value: 69.5767 + - type: euclidean_recall + value: 76.7546 + - type: euclidean_ap + value: 80.33160000000001 + - type: dot_accuracy + value: 88.3293 + - type: dot_accuracy_threshold + value: 89.5055 + - type: dot_f1 + value: 72.9896 + - type: dot_f1_threshold + value: 87.6934 + - type: dot_precision + value: 69.5767 + - type: dot_recall + value: 76.7546 + - type: dot_ap + value: 80.33160000000001 + - type: max_accuracy + value: 88.3293 + - type: max_f1 + value: 73.0484 + - type: max_precision + value: 70.9245 + - type: max_recall + value: 76.7546 + - type: max_ap + value: 80.3871 + - type: main_score + value: 80.3871 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus (default) + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: similarity_accuracy + value: 89.5098 + - type: similarity_accuracy_threshold + value: 86.3375 + - type: similarity_f1 + value: 79.5103 + - type: similarity_f1_threshold + value: 85.09649999999999 + - type: similarity_precision + value: 77.381 + - type: similarity_recall + value: 81.76010000000001 + - type: similarity_ap + value: 87.07770000000001 + - type: cosine_accuracy + value: 89.5098 + - type: cosine_accuracy_threshold + value: 86.3375 + - type: cosine_f1 + value: 79.5103 + - type: cosine_f1_threshold + value: 85.09649999999999 + - type: cosine_precision + value: 77.381 + - type: cosine_recall + value: 81.76010000000001 + - type: cosine_ap + value: 87.07770000000001 + - type: manhattan_accuracy + value: 89.5195 + - type: manhattan_accuracy_threshold + value: 2522.3334999999997 + - type: manhattan_f1 + value: 79.4922 + - type: manhattan_f1_threshold + value: 2646.0447 + - type: manhattan_precision + value: 75.5303 + - type: manhattan_recall + value: 83.8928 + - type: manhattan_ap + value: 87.0889 + - type: euclidean_accuracy + value: 89.5098 + - type: euclidean_accuracy_threshold + value: 52.2734 + - type: euclidean_f1 + value: 79.5103 + - type: euclidean_f1_threshold + value: 54.595800000000004 + - type: euclidean_precision + value: 77.381 + - type: euclidean_recall + value: 81.76010000000001 + - type: euclidean_ap + value: 87.07770000000001 + - type: dot_accuracy + value: 89.5098 + - type: dot_accuracy_threshold + value: 86.3375 + - type: dot_f1 + value: 79.5103 + - type: dot_f1_threshold + value: 85.09649999999999 + - type: dot_precision + value: 77.381 + - type: dot_recall + value: 81.76010000000001 + - type: dot_ap + value: 87.07770000000001 + - type: max_accuracy + value: 89.5195 + - type: max_f1 + value: 79.5103 + - type: max_precision + value: 77.381 + - type: max_recall + value: 83.8928 + - type: max_ap + value: 87.0889 + - type: main_score + value: 87.0889 + task: + type: PairClassification +tags: +- mteb +license: gemma +language: +- en +base_model: +- google/gemma-2-9b-it +--- + +# Gemma Embeddings v1.0 + +GemmaEmbed is a dense-vector embedding model, trained especially for retrieval. As of December 12, 2024, GemmaEmbed achieves the #1 position overall on the MTEB leaderboard, with a score of 72.72. + +# Important Notes +* This is not an official Google product. +* This is a research project. + +# Results summary + +Results compared to BGE-EN-ICL on each Task + +Model | Total (56) |Classification (12) | Classification Pair (3) | STS (10) |Clustering (11) | Reranking (4) | Retrieval (15) | Summary (1) +-- | -- | -- | -- | -- | -- | -- | -- | -- +NV-Embed-v2 | 0.7231 | 0.9037 | 0.8867 | 0.8431 | 0.5846 | 0.6065 | 0.6265 | 0.3070 +bge-en-icl | 0.7167 | 0.8895 | 0.8814 | 0.8425 | 0.5789 | 0.5986 | 0.6216 | 0.3077 +Gemma-Embeddings-v1.0 | 0.7272 | 0.9000 | 0.8809 | 0.8423 | 0.5826 | 0.6214 | 0.6371 | 0.4052 + +# Model & Data + +Our base encoder model is [Gemma2 9B](https://huggingface.co/google/gemma-2-9b). + +We use the [BGE-EN-ICL training data](https://huggingface.co/datasets/cfli/bge-full-data). + +# Research Team + +* Nicholas Monath +* Michael Boratko +* Seungyeon Kim +* Andrew McCallum +* Rob Fergus +* Manzil Zaheer +