diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,2723 +1,2723 @@ ---- -tags: -- mteb -- Sentence Transformers -- sentence-similarity -- sentence-transformers -model-index: -- name: e5-large-v2 - results: - - task: - type: Classification - dataset: - type: mteb/amazon_counterfactual - name: MTEB AmazonCounterfactualClassification (en) - config: en - split: test - revision: e8379541af4e31359cca9fbcf4b00f2671dba205 - metrics: - - type: accuracy - value: 79.22388059701493 - - type: ap - value: 43.20816505595132 - - type: f1 - value: 73.27811303522058 - - task: - type: Classification - dataset: - type: mteb/amazon_polarity - name: MTEB AmazonPolarityClassification - config: default - split: test - revision: e2d317d38cd51312af73b3d32a06d1a08b442046 - metrics: - - type: accuracy - value: 93.748325 - - type: ap - value: 90.72534979701297 - - type: f1 - value: 93.73895874282185 - - task: - type: Classification - dataset: - type: mteb/amazon_reviews_multi - name: MTEB AmazonReviewsClassification (en) - config: en - split: test - revision: 1399c76144fd37290681b995c656ef9b2e06e26d - metrics: - - type: accuracy - value: 48.612 - - type: f1 - value: 47.61157345898393 - - task: - type: Retrieval - dataset: - type: arguana - name: MTEB ArguAna - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 23.541999999999998 - - type: map_at_10 - value: 38.208 - - type: map_at_100 - value: 39.417 - - type: map_at_1000 - value: 39.428999999999995 - - type: map_at_3 - value: 33.95 - - type: map_at_5 - value: 36.329 - - type: mrr_at_1 - value: 23.755000000000003 - - type: mrr_at_10 - value: 38.288 - - type: mrr_at_100 - value: 39.511 - - type: mrr_at_1000 - value: 39.523 - - type: mrr_at_3 - value: 34.009 - - type: mrr_at_5 - value: 36.434 - - type: ndcg_at_1 - value: 23.541999999999998 - - type: ndcg_at_10 - value: 46.417 - - type: ndcg_at_100 - value: 51.812000000000005 - - type: ndcg_at_1000 - value: 52.137 - - type: ndcg_at_3 - value: 37.528 - - type: ndcg_at_5 - value: 41.81 - - type: precision_at_1 - value: 23.541999999999998 - - type: precision_at_10 - value: 7.269 - - type: precision_at_100 - value: 0.9690000000000001 - - type: precision_at_1000 - value: 0.099 - - type: precision_at_3 - value: 15.979 - - type: precision_at_5 - value: 11.664 - - type: recall_at_1 - value: 23.541999999999998 - - type: recall_at_10 - value: 72.688 - - type: recall_at_100 - value: 96.871 - - type: recall_at_1000 - value: 99.431 - - type: recall_at_3 - value: 47.937000000000005 - - type: recall_at_5 - value: 58.321 - - task: - type: Clustering - dataset: - type: mteb/arxiv-clustering-p2p - name: MTEB ArxivClusteringP2P - config: default - split: test - revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d - metrics: - - type: v_measure - value: 45.546499570522094 - - task: - type: Clustering - dataset: - type: mteb/arxiv-clustering-s2s - name: MTEB ArxivClusteringS2S - config: default - split: test - revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 - metrics: - - type: v_measure - value: 41.01607489943561 - - task: - type: Reranking - dataset: - type: mteb/askubuntudupquestions-reranking - name: MTEB AskUbuntuDupQuestions - config: default - split: test - revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 - metrics: - - type: map - value: 59.616107510107774 - - type: mrr - value: 72.75106626214661 - - task: - type: STS - dataset: - type: mteb/biosses-sts - name: MTEB BIOSSES - config: default - split: test - revision: d3fb88f8f02e40887cd149695127462bbcf29b4a - metrics: - - type: cos_sim_pearson - value: 84.33018094733868 - - type: cos_sim_spearman - value: 83.60190492611737 - - type: euclidean_pearson - value: 82.1492450218961 - - type: euclidean_spearman - value: 82.70308926526991 - - type: manhattan_pearson - value: 81.93959600076842 - - type: manhattan_spearman - value: 82.73260801016369 - - task: - type: Classification - dataset: - type: mteb/banking77 - name: MTEB Banking77Classification - config: default - split: test - revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 - metrics: - - type: accuracy - value: 84.54545454545455 - - type: f1 - value: 84.49582530928923 - - task: - type: Clustering - dataset: - type: mteb/biorxiv-clustering-p2p - name: MTEB BiorxivClusteringP2P - config: default - split: test - revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 - metrics: - - type: v_measure - value: 37.362725540120096 - - task: - type: Clustering - dataset: - type: mteb/biorxiv-clustering-s2s - name: MTEB BiorxivClusteringS2S - config: default - split: test - revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 - metrics: - - type: v_measure - value: 34.849509608178145 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackAndroidRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 31.502999999999997 - - type: map_at_10 - value: 43.323 - - type: map_at_100 - value: 44.708999999999996 - - type: map_at_1000 - value: 44.838 - - type: map_at_3 - value: 38.987 - - type: map_at_5 - value: 41.516999999999996 - - type: mrr_at_1 - value: 38.769999999999996 - - type: mrr_at_10 - value: 49.13 - - type: mrr_at_100 - value: 49.697 - - type: mrr_at_1000 - value: 49.741 - - type: mrr_at_3 - value: 45.804 - - type: mrr_at_5 - value: 47.842 - - type: ndcg_at_1 - value: 38.769999999999996 - - type: ndcg_at_10 - value: 50.266999999999996 - - type: ndcg_at_100 - value: 54.967 - - type: ndcg_at_1000 - value: 56.976000000000006 - - type: ndcg_at_3 - value: 43.823 - - type: ndcg_at_5 - value: 47.12 - - type: precision_at_1 - value: 38.769999999999996 - - type: precision_at_10 - value: 10.057 - - type: precision_at_100 - value: 1.554 - - type: precision_at_1000 - value: 0.202 - - type: precision_at_3 - value: 21.125 - - type: precision_at_5 - value: 15.851 - - type: recall_at_1 - value: 31.502999999999997 - - type: recall_at_10 - value: 63.715999999999994 - - type: recall_at_100 - value: 83.61800000000001 - - type: recall_at_1000 - value: 96.63199999999999 - - type: recall_at_3 - value: 45.403 - - type: recall_at_5 - value: 54.481 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackEnglishRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 27.833000000000002 - - type: map_at_10 - value: 37.330999999999996 - - type: map_at_100 - value: 38.580999999999996 - - type: map_at_1000 - value: 38.708 - - type: map_at_3 - value: 34.713 - - type: map_at_5 - value: 36.104 - - type: mrr_at_1 - value: 35.223 - - type: mrr_at_10 - value: 43.419000000000004 - - type: mrr_at_100 - value: 44.198 - - type: mrr_at_1000 - value: 44.249 - - type: mrr_at_3 - value: 41.614000000000004 - - type: mrr_at_5 - value: 42.553000000000004 - - type: ndcg_at_1 - value: 35.223 - - type: ndcg_at_10 - value: 42.687999999999995 - - type: ndcg_at_100 - value: 47.447 - - type: ndcg_at_1000 - value: 49.701 - - type: ndcg_at_3 - value: 39.162 - - type: ndcg_at_5 - value: 40.557 - - type: precision_at_1 - value: 35.223 - - type: precision_at_10 - value: 7.962 - - type: precision_at_100 - value: 1.304 - - type: precision_at_1000 - value: 0.18 - - type: precision_at_3 - value: 19.023 - - type: precision_at_5 - value: 13.184999999999999 - - type: recall_at_1 - value: 27.833000000000002 - - type: recall_at_10 - value: 51.881 - - type: recall_at_100 - value: 72.04 - - type: recall_at_1000 - value: 86.644 - - type: recall_at_3 - value: 40.778 - - type: recall_at_5 - value: 45.176 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackGamingRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 38.175 - - type: map_at_10 - value: 51.174 - - type: map_at_100 - value: 52.26499999999999 - - type: map_at_1000 - value: 52.315999999999995 - - type: map_at_3 - value: 47.897 - - type: map_at_5 - value: 49.703 - - type: mrr_at_1 - value: 43.448 - - type: mrr_at_10 - value: 54.505 - - type: mrr_at_100 - value: 55.216 - - type: mrr_at_1000 - value: 55.242000000000004 - - type: mrr_at_3 - value: 51.98500000000001 - - type: mrr_at_5 - value: 53.434000000000005 - - type: ndcg_at_1 - value: 43.448 - - type: ndcg_at_10 - value: 57.282 - - type: ndcg_at_100 - value: 61.537 - - type: ndcg_at_1000 - value: 62.546 - - type: ndcg_at_3 - value: 51.73799999999999 - - type: ndcg_at_5 - value: 54.324 - - type: precision_at_1 - value: 43.448 - - type: precision_at_10 - value: 9.292 - - type: precision_at_100 - value: 1.233 - - type: precision_at_1000 - value: 0.136 - - type: precision_at_3 - value: 23.218 - - type: precision_at_5 - value: 15.887 - - type: recall_at_1 - value: 38.175 - - type: recall_at_10 - value: 72.00999999999999 - - type: recall_at_100 - value: 90.155 - - type: recall_at_1000 - value: 97.257 - - type: recall_at_3 - value: 57.133 - - type: recall_at_5 - value: 63.424 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackGisRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 22.405 - - type: map_at_10 - value: 30.043 - - type: map_at_100 - value: 31.191000000000003 - - type: map_at_1000 - value: 31.275 - - type: map_at_3 - value: 27.034000000000002 - - type: map_at_5 - value: 28.688000000000002 - - type: mrr_at_1 - value: 24.068 - - type: mrr_at_10 - value: 31.993 - - type: mrr_at_100 - value: 32.992 - - type: mrr_at_1000 - value: 33.050000000000004 - - type: mrr_at_3 - value: 28.964000000000002 - - type: mrr_at_5 - value: 30.653000000000002 - - type: ndcg_at_1 - value: 24.068 - - type: ndcg_at_10 - value: 35.198 - - type: ndcg_at_100 - value: 40.709 - - type: ndcg_at_1000 - value: 42.855 - - type: ndcg_at_3 - value: 29.139 - - type: ndcg_at_5 - value: 32.045 - - type: precision_at_1 - value: 24.068 - - type: precision_at_10 - value: 5.65 - - type: precision_at_100 - value: 0.885 - - type: precision_at_1000 - value: 0.11199999999999999 - - type: precision_at_3 - value: 12.279 - - type: precision_at_5 - value: 8.994 - - type: recall_at_1 - value: 22.405 - - type: recall_at_10 - value: 49.391 - - type: recall_at_100 - value: 74.53699999999999 - - type: recall_at_1000 - value: 90.605 - - type: recall_at_3 - value: 33.126 - - type: recall_at_5 - value: 40.073 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackMathematicaRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 13.309999999999999 - - type: map_at_10 - value: 20.688000000000002 - - type: map_at_100 - value: 22.022 - - type: map_at_1000 - value: 22.152 - - type: map_at_3 - value: 17.954 - - type: map_at_5 - value: 19.439 - - type: mrr_at_1 - value: 16.294 - - type: mrr_at_10 - value: 24.479 - - type: mrr_at_100 - value: 25.515 - - type: mrr_at_1000 - value: 25.593 - - type: mrr_at_3 - value: 21.642 - - type: mrr_at_5 - value: 23.189999999999998 - - type: ndcg_at_1 - value: 16.294 - - type: ndcg_at_10 - value: 25.833000000000002 - - type: ndcg_at_100 - value: 32.074999999999996 - - type: ndcg_at_1000 - value: 35.083 - - type: ndcg_at_3 - value: 20.493 - - type: ndcg_at_5 - value: 22.949 - - type: precision_at_1 - value: 16.294 - - type: precision_at_10 - value: 5.112 - - type: precision_at_100 - value: 0.96 - - type: precision_at_1000 - value: 0.134 - - type: precision_at_3 - value: 9.908999999999999 - - type: precision_at_5 - value: 7.587000000000001 - - type: recall_at_1 - value: 13.309999999999999 - - type: recall_at_10 - value: 37.851 - - type: recall_at_100 - value: 64.835 - - type: recall_at_1000 - value: 86.334 - - type: recall_at_3 - value: 23.493 - - type: recall_at_5 - value: 29.528 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackPhysicsRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 25.857999999999997 - - type: map_at_10 - value: 35.503 - - type: map_at_100 - value: 36.957 - - type: map_at_1000 - value: 37.065 - - type: map_at_3 - value: 32.275999999999996 - - type: map_at_5 - value: 34.119 - - type: mrr_at_1 - value: 31.954 - - type: mrr_at_10 - value: 40.851 - - type: mrr_at_100 - value: 41.863 - - type: mrr_at_1000 - value: 41.900999999999996 - - type: mrr_at_3 - value: 38.129999999999995 - - type: mrr_at_5 - value: 39.737 - - type: ndcg_at_1 - value: 31.954 - - type: ndcg_at_10 - value: 41.343999999999994 - - type: ndcg_at_100 - value: 47.397 - - type: ndcg_at_1000 - value: 49.501 - - type: ndcg_at_3 - value: 36.047000000000004 - - type: ndcg_at_5 - value: 38.639 - - type: precision_at_1 - value: 31.954 - - type: precision_at_10 - value: 7.68 - - type: precision_at_100 - value: 1.247 - - type: precision_at_1000 - value: 0.16199999999999998 - - type: precision_at_3 - value: 17.132 - - type: precision_at_5 - value: 12.589 - - type: recall_at_1 - value: 25.857999999999997 - - type: recall_at_10 - value: 53.43599999999999 - - type: recall_at_100 - value: 78.82400000000001 - - type: recall_at_1000 - value: 92.78999999999999 - - type: recall_at_3 - value: 38.655 - - type: recall_at_5 - value: 45.216 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackProgrammersRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 24.709 - - type: map_at_10 - value: 34.318 - - type: map_at_100 - value: 35.657 - - type: map_at_1000 - value: 35.783 - - type: map_at_3 - value: 31.326999999999998 - - type: map_at_5 - value: 33.021 - - type: mrr_at_1 - value: 30.137000000000004 - - type: mrr_at_10 - value: 39.093 - - type: mrr_at_100 - value: 39.992 - - type: mrr_at_1000 - value: 40.056999999999995 - - type: mrr_at_3 - value: 36.606 - - type: mrr_at_5 - value: 37.861 - - type: ndcg_at_1 - value: 30.137000000000004 - - type: ndcg_at_10 - value: 39.974 - - type: ndcg_at_100 - value: 45.647999999999996 - - type: ndcg_at_1000 - value: 48.259 - - type: ndcg_at_3 - value: 35.028 - - type: ndcg_at_5 - value: 37.175999999999995 - - type: precision_at_1 - value: 30.137000000000004 - - type: precision_at_10 - value: 7.363 - - type: precision_at_100 - value: 1.184 - - type: precision_at_1000 - value: 0.161 - - type: precision_at_3 - value: 16.857 - - type: precision_at_5 - value: 11.963 - - type: recall_at_1 - value: 24.709 - - type: recall_at_10 - value: 52.087 - - type: recall_at_100 - value: 76.125 - - type: recall_at_1000 - value: 93.82300000000001 - - type: recall_at_3 - value: 38.149 - - type: recall_at_5 - value: 43.984 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 23.40791666666667 - - type: map_at_10 - value: 32.458083333333335 - - type: map_at_100 - value: 33.691916666666664 - - type: map_at_1000 - value: 33.81191666666666 - - type: map_at_3 - value: 29.51625 - - type: map_at_5 - value: 31.168083333333335 - - type: mrr_at_1 - value: 27.96591666666666 - - type: mrr_at_10 - value: 36.528583333333344 - - type: mrr_at_100 - value: 37.404 - - type: mrr_at_1000 - value: 37.464333333333336 - - type: mrr_at_3 - value: 33.92883333333333 - - type: mrr_at_5 - value: 35.41933333333333 - - type: ndcg_at_1 - value: 27.96591666666666 - - type: ndcg_at_10 - value: 37.89141666666666 - - type: ndcg_at_100 - value: 43.23066666666666 - - type: ndcg_at_1000 - value: 45.63258333333333 - - type: ndcg_at_3 - value: 32.811249999999994 - - type: ndcg_at_5 - value: 35.22566666666667 - - type: precision_at_1 - value: 27.96591666666666 - - type: precision_at_10 - value: 6.834083333333332 - - type: precision_at_100 - value: 1.12225 - - type: precision_at_1000 - value: 0.15241666666666667 - - type: precision_at_3 - value: 15.264333333333335 - - type: precision_at_5 - value: 11.039416666666666 - - type: recall_at_1 - value: 23.40791666666667 - - type: recall_at_10 - value: 49.927083333333336 - - type: recall_at_100 - value: 73.44641666666668 - - type: recall_at_1000 - value: 90.19950000000001 - - type: recall_at_3 - value: 35.88341666666667 - - type: recall_at_5 - value: 42.061249999999994 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackStatsRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 19.592000000000002 - - type: map_at_10 - value: 26.895999999999997 - - type: map_at_100 - value: 27.921000000000003 - - type: map_at_1000 - value: 28.02 - - type: map_at_3 - value: 24.883 - - type: map_at_5 - value: 25.812 - - type: mrr_at_1 - value: 22.698999999999998 - - type: mrr_at_10 - value: 29.520999999999997 - - type: mrr_at_100 - value: 30.458000000000002 - - type: mrr_at_1000 - value: 30.526999999999997 - - type: mrr_at_3 - value: 27.633000000000003 - - type: mrr_at_5 - value: 28.483999999999998 - - type: ndcg_at_1 - value: 22.698999999999998 - - type: ndcg_at_10 - value: 31.061 - - type: ndcg_at_100 - value: 36.398 - - type: ndcg_at_1000 - value: 38.89 - - type: ndcg_at_3 - value: 27.149 - - type: ndcg_at_5 - value: 28.627000000000002 - - type: precision_at_1 - value: 22.698999999999998 - - type: precision_at_10 - value: 5.106999999999999 - - type: precision_at_100 - value: 0.857 - - type: precision_at_1000 - value: 0.11499999999999999 - - type: precision_at_3 - value: 11.963 - - type: precision_at_5 - value: 8.221 - - type: recall_at_1 - value: 19.592000000000002 - - type: recall_at_10 - value: 41.329 - - type: recall_at_100 - value: 66.094 - - type: recall_at_1000 - value: 84.511 - - type: recall_at_3 - value: 30.61 - - type: recall_at_5 - value: 34.213 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackTexRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 14.71 - - type: map_at_10 - value: 20.965 - - type: map_at_100 - value: 21.994 - - type: map_at_1000 - value: 22.133 - - type: map_at_3 - value: 18.741 - - type: map_at_5 - value: 19.951 - - type: mrr_at_1 - value: 18.307000000000002 - - type: mrr_at_10 - value: 24.66 - - type: mrr_at_100 - value: 25.540000000000003 - - type: mrr_at_1000 - value: 25.629 - - type: mrr_at_3 - value: 22.511 - - type: mrr_at_5 - value: 23.72 - - type: ndcg_at_1 - value: 18.307000000000002 - - type: ndcg_at_10 - value: 25.153 - - type: ndcg_at_100 - value: 30.229 - - type: ndcg_at_1000 - value: 33.623 - - type: ndcg_at_3 - value: 21.203 - - type: ndcg_at_5 - value: 23.006999999999998 - - type: precision_at_1 - value: 18.307000000000002 - - type: precision_at_10 - value: 4.725 - - type: precision_at_100 - value: 0.8659999999999999 - - type: precision_at_1000 - value: 0.133 - - type: precision_at_3 - value: 10.14 - - type: precision_at_5 - value: 7.481 - - type: recall_at_1 - value: 14.71 - - type: recall_at_10 - value: 34.087 - - type: recall_at_100 - value: 57.147999999999996 - - type: recall_at_1000 - value: 81.777 - - type: recall_at_3 - value: 22.996 - - type: recall_at_5 - value: 27.73 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackUnixRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 23.472 - - type: map_at_10 - value: 32.699 - - type: map_at_100 - value: 33.867000000000004 - - type: map_at_1000 - value: 33.967000000000006 - - type: map_at_3 - value: 29.718 - - type: map_at_5 - value: 31.345 - - type: mrr_at_1 - value: 28.265 - - type: mrr_at_10 - value: 36.945 - - type: mrr_at_100 - value: 37.794 - - type: mrr_at_1000 - value: 37.857 - - type: mrr_at_3 - value: 34.266000000000005 - - type: mrr_at_5 - value: 35.768 - - type: ndcg_at_1 - value: 28.265 - - type: ndcg_at_10 - value: 38.35 - - type: ndcg_at_100 - value: 43.739 - - type: ndcg_at_1000 - value: 46.087 - - type: ndcg_at_3 - value: 33.004 - - type: ndcg_at_5 - value: 35.411 - - type: precision_at_1 - value: 28.265 - - type: precision_at_10 - value: 6.715999999999999 - - type: precision_at_100 - value: 1.059 - - type: precision_at_1000 - value: 0.13799999999999998 - - type: precision_at_3 - value: 15.299 - - type: precision_at_5 - value: 10.951 - - type: recall_at_1 - value: 23.472 - - type: recall_at_10 - value: 51.413 - - type: recall_at_100 - value: 75.17 - - type: recall_at_1000 - value: 91.577 - - type: recall_at_3 - value: 36.651 - - type: recall_at_5 - value: 42.814 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackWebmastersRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 23.666 - - type: map_at_10 - value: 32.963 - - type: map_at_100 - value: 34.544999999999995 - - type: map_at_1000 - value: 34.792 - - type: map_at_3 - value: 29.74 - - type: map_at_5 - value: 31.5 - - type: mrr_at_1 - value: 29.051 - - type: mrr_at_10 - value: 38.013000000000005 - - type: mrr_at_100 - value: 38.997 - - type: mrr_at_1000 - value: 39.055 - - type: mrr_at_3 - value: 34.947 - - type: mrr_at_5 - value: 36.815 - - type: ndcg_at_1 - value: 29.051 - - type: ndcg_at_10 - value: 39.361000000000004 - - type: ndcg_at_100 - value: 45.186 - - type: ndcg_at_1000 - value: 47.867 - - type: ndcg_at_3 - value: 33.797 - - type: ndcg_at_5 - value: 36.456 - - type: precision_at_1 - value: 29.051 - - type: precision_at_10 - value: 7.668 - - type: precision_at_100 - value: 1.532 - - type: precision_at_1000 - value: 0.247 - - type: precision_at_3 - value: 15.876000000000001 - - type: precision_at_5 - value: 11.779 - - type: recall_at_1 - value: 23.666 - - type: recall_at_10 - value: 51.858000000000004 - - type: recall_at_100 - value: 77.805 - - type: recall_at_1000 - value: 94.504 - - type: recall_at_3 - value: 36.207 - - type: recall_at_5 - value: 43.094 - - task: - type: Retrieval - dataset: - type: BeIR/cqadupstack - name: MTEB CQADupstackWordpressRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 15.662 - - type: map_at_10 - value: 23.594 - - type: map_at_100 - value: 24.593999999999998 - - type: map_at_1000 - value: 24.694 - - type: map_at_3 - value: 20.925 - - type: map_at_5 - value: 22.817999999999998 - - type: mrr_at_1 - value: 17.375 - - type: mrr_at_10 - value: 25.734 - - type: mrr_at_100 - value: 26.586 - - type: mrr_at_1000 - value: 26.671 - - type: mrr_at_3 - value: 23.044 - - type: mrr_at_5 - value: 24.975 - - type: ndcg_at_1 - value: 17.375 - - type: ndcg_at_10 - value: 28.186 - - type: ndcg_at_100 - value: 33.436 - - type: ndcg_at_1000 - value: 36.203 - - type: ndcg_at_3 - value: 23.152 - - type: ndcg_at_5 - value: 26.397 - - type: precision_at_1 - value: 17.375 - - type: precision_at_10 - value: 4.677 - - type: precision_at_100 - value: 0.786 - - type: precision_at_1000 - value: 0.109 - - type: precision_at_3 - value: 10.351 - - type: precision_at_5 - value: 7.985 - - type: recall_at_1 - value: 15.662 - - type: recall_at_10 - value: 40.066 - - type: recall_at_100 - value: 65.006 - - type: recall_at_1000 - value: 85.94000000000001 - - type: recall_at_3 - value: 27.400000000000002 - - type: recall_at_5 - value: 35.002 - - task: - type: Retrieval - dataset: - type: climate-fever - name: MTEB ClimateFEVER - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 8.853 - - type: map_at_10 - value: 15.568000000000001 - - type: map_at_100 - value: 17.383000000000003 - - type: map_at_1000 - value: 17.584 - - type: map_at_3 - value: 12.561 - - type: map_at_5 - value: 14.056 - - type: mrr_at_1 - value: 18.958 - - type: mrr_at_10 - value: 28.288000000000004 - - type: mrr_at_100 - value: 29.432000000000002 - - type: mrr_at_1000 - value: 29.498 - - type: mrr_at_3 - value: 25.049 - - type: mrr_at_5 - value: 26.857 - - type: ndcg_at_1 - value: 18.958 - - type: ndcg_at_10 - value: 22.21 - - type: ndcg_at_100 - value: 29.596 - - type: ndcg_at_1000 - value: 33.583 - - type: ndcg_at_3 - value: 16.994999999999997 - - type: ndcg_at_5 - value: 18.95 - - type: precision_at_1 - value: 18.958 - - type: precision_at_10 - value: 7.192 - - type: precision_at_100 - value: 1.5 - - type: precision_at_1000 - value: 0.22399999999999998 - - type: precision_at_3 - value: 12.573 - - type: precision_at_5 - value: 10.202 - - type: recall_at_1 - value: 8.853 - - type: recall_at_10 - value: 28.087 - - type: recall_at_100 - value: 53.701 - - type: recall_at_1000 - value: 76.29899999999999 - - type: recall_at_3 - value: 15.913 - - type: recall_at_5 - value: 20.658 - - task: - type: Retrieval - dataset: - type: dbpedia-entity - name: MTEB DBPedia - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 9.077 - - type: map_at_10 - value: 20.788999999999998 - - type: map_at_100 - value: 30.429000000000002 - - type: map_at_1000 - value: 32.143 - - type: map_at_3 - value: 14.692 - - type: map_at_5 - value: 17.139 - - type: mrr_at_1 - value: 70.75 - - type: mrr_at_10 - value: 78.036 - - type: mrr_at_100 - value: 78.401 - - type: mrr_at_1000 - value: 78.404 - - type: mrr_at_3 - value: 76.75 - - type: mrr_at_5 - value: 77.47500000000001 - - type: ndcg_at_1 - value: 58.12500000000001 - - type: ndcg_at_10 - value: 44.015 - - type: ndcg_at_100 - value: 49.247 - - type: ndcg_at_1000 - value: 56.211999999999996 - - type: ndcg_at_3 - value: 49.151 - - type: ndcg_at_5 - value: 46.195 - - type: precision_at_1 - value: 70.75 - - type: precision_at_10 - value: 35.5 - - type: precision_at_100 - value: 11.355 - - type: precision_at_1000 - value: 2.1950000000000003 - - type: precision_at_3 - value: 53.083000000000006 - - type: precision_at_5 - value: 44.800000000000004 - - type: recall_at_1 - value: 9.077 - - type: recall_at_10 - value: 26.259 - - type: recall_at_100 - value: 56.547000000000004 - - type: recall_at_1000 - value: 78.551 - - type: recall_at_3 - value: 16.162000000000003 - - type: recall_at_5 - value: 19.753999999999998 - - task: - type: Classification - dataset: - type: mteb/emotion - name: MTEB EmotionClassification - config: default - split: test - revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 - metrics: - - type: accuracy - value: 49.44500000000001 - - type: f1 - value: 44.67067691783401 - - task: - type: Retrieval - dataset: - type: fever - name: MTEB FEVER - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 68.182 - - type: map_at_10 - value: 78.223 - - type: map_at_100 - value: 78.498 - - type: map_at_1000 - value: 78.512 - - type: map_at_3 - value: 76.71 - - type: map_at_5 - value: 77.725 - - type: mrr_at_1 - value: 73.177 - - type: mrr_at_10 - value: 82.513 - - type: mrr_at_100 - value: 82.633 - - type: mrr_at_1000 - value: 82.635 - - type: mrr_at_3 - value: 81.376 - - type: mrr_at_5 - value: 82.182 - - type: ndcg_at_1 - value: 73.177 - - type: ndcg_at_10 - value: 82.829 - - type: ndcg_at_100 - value: 83.84 - - type: ndcg_at_1000 - value: 84.07900000000001 - - type: ndcg_at_3 - value: 80.303 - - type: ndcg_at_5 - value: 81.846 - - type: precision_at_1 - value: 73.177 - - type: precision_at_10 - value: 10.241999999999999 - - type: precision_at_100 - value: 1.099 - - type: precision_at_1000 - value: 0.11399999999999999 - - type: precision_at_3 - value: 31.247999999999998 - - type: precision_at_5 - value: 19.697 - - type: recall_at_1 - value: 68.182 - - type: recall_at_10 - value: 92.657 - - type: recall_at_100 - value: 96.709 - - type: recall_at_1000 - value: 98.184 - - type: recall_at_3 - value: 85.9 - - type: recall_at_5 - value: 89.755 - - task: - type: Retrieval - dataset: - type: fiqa - name: MTEB FiQA2018 - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 21.108 - - type: map_at_10 - value: 33.342 - - type: map_at_100 - value: 35.281 - - type: map_at_1000 - value: 35.478 - - type: map_at_3 - value: 29.067 - - type: map_at_5 - value: 31.563000000000002 - - type: mrr_at_1 - value: 41.667 - - type: mrr_at_10 - value: 49.913000000000004 - - type: mrr_at_100 - value: 50.724000000000004 - - type: mrr_at_1000 - value: 50.766 - - type: mrr_at_3 - value: 47.504999999999995 - - type: mrr_at_5 - value: 49.033 - - type: ndcg_at_1 - value: 41.667 - - type: ndcg_at_10 - value: 41.144 - - type: ndcg_at_100 - value: 48.326 - - type: ndcg_at_1000 - value: 51.486 - - type: ndcg_at_3 - value: 37.486999999999995 - - type: ndcg_at_5 - value: 38.78 - - type: precision_at_1 - value: 41.667 - - type: precision_at_10 - value: 11.358 - - type: precision_at_100 - value: 1.873 - - type: precision_at_1000 - value: 0.244 - - type: precision_at_3 - value: 25 - - type: precision_at_5 - value: 18.519 - - type: recall_at_1 - value: 21.108 - - type: recall_at_10 - value: 47.249 - - type: recall_at_100 - value: 74.52 - - type: recall_at_1000 - value: 93.31 - - type: recall_at_3 - value: 33.271 - - type: recall_at_5 - value: 39.723000000000006 - - task: - type: Retrieval - dataset: - type: hotpotqa - name: MTEB HotpotQA - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 40.317 - - type: map_at_10 - value: 64.861 - - type: map_at_100 - value: 65.697 - - type: map_at_1000 - value: 65.755 - - type: map_at_3 - value: 61.258 - - type: map_at_5 - value: 63.590999999999994 - - type: mrr_at_1 - value: 80.635 - - type: mrr_at_10 - value: 86.528 - - type: mrr_at_100 - value: 86.66199999999999 - - type: mrr_at_1000 - value: 86.666 - - type: mrr_at_3 - value: 85.744 - - type: mrr_at_5 - value: 86.24300000000001 - - type: ndcg_at_1 - value: 80.635 - - type: ndcg_at_10 - value: 73.13199999999999 - - type: ndcg_at_100 - value: 75.927 - - type: ndcg_at_1000 - value: 76.976 - - type: ndcg_at_3 - value: 68.241 - - type: ndcg_at_5 - value: 71.071 - - type: precision_at_1 - value: 80.635 - - type: precision_at_10 - value: 15.326 - - type: precision_at_100 - value: 1.7500000000000002 - - type: precision_at_1000 - value: 0.189 - - type: precision_at_3 - value: 43.961 - - type: precision_at_5 - value: 28.599999999999998 - - type: recall_at_1 - value: 40.317 - - type: recall_at_10 - value: 76.631 - - type: recall_at_100 - value: 87.495 - - type: recall_at_1000 - value: 94.362 - - type: recall_at_3 - value: 65.94200000000001 - - type: recall_at_5 - value: 71.499 - - task: - type: Classification - dataset: - type: mteb/imdb - name: MTEB ImdbClassification - config: default - split: test - revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 - metrics: - - type: accuracy - value: 91.686 - - type: ap - value: 87.5577120393173 - - type: f1 - value: 91.6629447355139 - - task: - type: Retrieval - dataset: - type: msmarco - name: MTEB MSMARCO - config: default - split: dev - revision: None - metrics: - - type: map_at_1 - value: 23.702 - - type: map_at_10 - value: 36.414 - - type: map_at_100 - value: 37.561 - - type: map_at_1000 - value: 37.605 - - type: map_at_3 - value: 32.456 - - type: map_at_5 - value: 34.827000000000005 - - type: mrr_at_1 - value: 24.355 - - type: mrr_at_10 - value: 37.01 - - type: mrr_at_100 - value: 38.085 - - type: mrr_at_1000 - value: 38.123000000000005 - - type: mrr_at_3 - value: 33.117999999999995 - - type: mrr_at_5 - value: 35.452 - - type: ndcg_at_1 - value: 24.384 - - type: ndcg_at_10 - value: 43.456 - - type: ndcg_at_100 - value: 48.892 - - type: ndcg_at_1000 - value: 49.964 - - type: ndcg_at_3 - value: 35.475 - - type: ndcg_at_5 - value: 39.711 - - type: precision_at_1 - value: 24.384 - - type: precision_at_10 - value: 6.7940000000000005 - - type: precision_at_100 - value: 0.951 - - type: precision_at_1000 - value: 0.104 - - type: precision_at_3 - value: 15.052999999999999 - - type: precision_at_5 - value: 11.189 - - type: recall_at_1 - value: 23.702 - - type: recall_at_10 - value: 65.057 - - type: recall_at_100 - value: 90.021 - - type: recall_at_1000 - value: 98.142 - - type: recall_at_3 - value: 43.551 - - type: recall_at_5 - value: 53.738 - - task: - type: Classification - dataset: - type: mteb/mtop_domain - name: MTEB MTOPDomainClassification (en) - config: en - split: test - revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf - metrics: - - type: accuracy - value: 94.62380300957591 - - type: f1 - value: 94.49871222100734 - - task: - type: Classification - dataset: - type: mteb/mtop_intent - name: MTEB MTOPIntentClassification (en) - config: en - split: test - revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba - metrics: - - type: accuracy - value: 77.14090287277702 - - type: f1 - value: 60.32101258220515 - - task: - type: Classification - dataset: - type: mteb/amazon_massive_intent - name: MTEB MassiveIntentClassification (en) - config: en - split: test - revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 - metrics: - - type: accuracy - value: 73.84330867518494 - - type: f1 - value: 71.92248688515255 - - task: - type: Classification - dataset: - type: mteb/amazon_massive_scenario - name: MTEB MassiveScenarioClassification (en) - config: en - split: test - revision: 7d571f92784cd94a019292a1f45445077d0ef634 - metrics: - - type: accuracy - value: 78.10692669804976 - - type: f1 - value: 77.9904839122866 - - task: - type: Clustering - dataset: - type: mteb/medrxiv-clustering-p2p - name: MTEB MedrxivClusteringP2P - config: default - split: test - revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 - metrics: - - type: v_measure - value: 31.822988923078444 - - task: - type: Clustering - dataset: - type: mteb/medrxiv-clustering-s2s - name: MTEB MedrxivClusteringS2S - config: default - split: test - revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 - metrics: - - type: v_measure - value: 30.38394880253403 - - task: - type: Reranking - dataset: - type: mteb/mind_small - name: MTEB MindSmallReranking - config: default - split: test - revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 - metrics: - - type: map - value: 31.82504612539082 - - type: mrr - value: 32.84462298174977 - - task: - type: Retrieval - dataset: - type: nfcorpus - name: MTEB NFCorpus - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 6.029 - - type: map_at_10 - value: 14.088999999999999 - - type: map_at_100 - value: 17.601 - - type: map_at_1000 - value: 19.144 - - type: map_at_3 - value: 10.156 - - type: map_at_5 - value: 11.892 - - type: mrr_at_1 - value: 46.44 - - type: mrr_at_10 - value: 56.596999999999994 - - type: mrr_at_100 - value: 57.11000000000001 - - type: mrr_at_1000 - value: 57.14 - - type: mrr_at_3 - value: 54.334 - - type: mrr_at_5 - value: 55.774 - - type: ndcg_at_1 - value: 44.891999999999996 - - type: ndcg_at_10 - value: 37.134 - - type: ndcg_at_100 - value: 33.652 - - type: ndcg_at_1000 - value: 42.548 - - type: ndcg_at_3 - value: 41.851 - - type: ndcg_at_5 - value: 39.842 - - type: precision_at_1 - value: 46.44 - - type: precision_at_10 - value: 27.647 - - type: precision_at_100 - value: 8.309999999999999 - - type: precision_at_1000 - value: 2.146 - - type: precision_at_3 - value: 39.422000000000004 - - type: precision_at_5 - value: 34.675 - - type: recall_at_1 - value: 6.029 - - type: recall_at_10 - value: 18.907 - - type: recall_at_100 - value: 33.76 - - type: recall_at_1000 - value: 65.14999999999999 - - type: recall_at_3 - value: 11.584999999999999 - - type: recall_at_5 - value: 14.626 - - task: - type: Retrieval - dataset: - type: nq - name: MTEB NQ - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 39.373000000000005 - - type: map_at_10 - value: 55.836 - - type: map_at_100 - value: 56.611999999999995 - - type: map_at_1000 - value: 56.63 - - type: map_at_3 - value: 51.747 - - type: map_at_5 - value: 54.337999999999994 - - type: mrr_at_1 - value: 44.147999999999996 - - type: mrr_at_10 - value: 58.42699999999999 - - type: mrr_at_100 - value: 58.902 - - type: mrr_at_1000 - value: 58.914 - - type: mrr_at_3 - value: 55.156000000000006 - - type: mrr_at_5 - value: 57.291000000000004 - - type: ndcg_at_1 - value: 44.119 - - type: ndcg_at_10 - value: 63.444 - - type: ndcg_at_100 - value: 66.40599999999999 - - type: ndcg_at_1000 - value: 66.822 - - type: ndcg_at_3 - value: 55.962 - - type: ndcg_at_5 - value: 60.228 - - type: precision_at_1 - value: 44.119 - - type: precision_at_10 - value: 10.006 - - type: precision_at_100 - value: 1.17 - - type: precision_at_1000 - value: 0.121 - - type: precision_at_3 - value: 25.135 - - type: precision_at_5 - value: 17.59 - - type: recall_at_1 - value: 39.373000000000005 - - type: recall_at_10 - value: 83.78999999999999 - - type: recall_at_100 - value: 96.246 - - type: recall_at_1000 - value: 99.324 - - type: recall_at_3 - value: 64.71900000000001 - - type: recall_at_5 - value: 74.508 - - task: - type: Retrieval - dataset: - type: quora - name: MTEB QuoraRetrieval - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 69.199 - - type: map_at_10 - value: 82.892 - - type: map_at_100 - value: 83.578 - - type: map_at_1000 - value: 83.598 - - type: map_at_3 - value: 79.948 - - type: map_at_5 - value: 81.779 - - type: mrr_at_1 - value: 79.67 - - type: mrr_at_10 - value: 86.115 - - type: mrr_at_100 - value: 86.249 - - type: mrr_at_1000 - value: 86.251 - - type: mrr_at_3 - value: 85.08200000000001 - - type: mrr_at_5 - value: 85.783 - - type: ndcg_at_1 - value: 79.67 - - type: ndcg_at_10 - value: 86.839 - - type: ndcg_at_100 - value: 88.252 - - type: ndcg_at_1000 - value: 88.401 - - type: ndcg_at_3 - value: 83.86200000000001 - - type: ndcg_at_5 - value: 85.473 - - type: precision_at_1 - value: 79.67 - - type: precision_at_10 - value: 13.19 - - type: precision_at_100 - value: 1.521 - - type: precision_at_1000 - value: 0.157 - - type: precision_at_3 - value: 36.677 - - type: precision_at_5 - value: 24.118000000000002 - - type: recall_at_1 - value: 69.199 - - type: recall_at_10 - value: 94.321 - - type: recall_at_100 - value: 99.20400000000001 - - type: recall_at_1000 - value: 99.947 - - type: recall_at_3 - value: 85.787 - - type: recall_at_5 - value: 90.365 - - task: - type: Clustering - dataset: - type: mteb/reddit-clustering - name: MTEB RedditClustering - config: default - split: test - revision: 24640382cdbf8abc73003fb0fa6d111a705499eb - metrics: - - type: v_measure - value: 55.82810046856353 - - task: - type: Clustering - dataset: - type: mteb/reddit-clustering-p2p - name: MTEB RedditClusteringP2P - config: default - split: test - revision: 282350215ef01743dc01b456c7f5241fa8937f16 - metrics: - - type: v_measure - value: 63.38132611783628 - - task: - type: Retrieval - dataset: - type: scidocs - name: MTEB SCIDOCS - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 5.127000000000001 - - type: map_at_10 - value: 12.235 - - type: map_at_100 - value: 14.417 - - type: map_at_1000 - value: 14.75 - - type: map_at_3 - value: 8.906 - - type: map_at_5 - value: 10.591000000000001 - - type: mrr_at_1 - value: 25.2 - - type: mrr_at_10 - value: 35.879 - - type: mrr_at_100 - value: 36.935 - - type: mrr_at_1000 - value: 36.997 - - type: mrr_at_3 - value: 32.783 - - type: mrr_at_5 - value: 34.367999999999995 - - type: ndcg_at_1 - value: 25.2 - - type: ndcg_at_10 - value: 20.509 - - type: ndcg_at_100 - value: 28.67 - - type: ndcg_at_1000 - value: 34.42 - - type: ndcg_at_3 - value: 19.948 - - type: ndcg_at_5 - value: 17.166 - - type: precision_at_1 - value: 25.2 - - type: precision_at_10 - value: 10.440000000000001 - - type: precision_at_100 - value: 2.214 - - type: precision_at_1000 - value: 0.359 - - type: precision_at_3 - value: 18.533 - - type: precision_at_5 - value: 14.860000000000001 - - type: recall_at_1 - value: 5.127000000000001 - - type: recall_at_10 - value: 21.147 - - type: recall_at_100 - value: 44.946999999999996 - - type: recall_at_1000 - value: 72.89 - - type: recall_at_3 - value: 11.277 - - type: recall_at_5 - value: 15.042 - - task: - type: STS - dataset: - type: mteb/sickr-sts - name: MTEB SICK-R - config: default - split: test - revision: a6ea5a8cab320b040a23452cc28066d9beae2cee - metrics: - - type: cos_sim_pearson - value: 83.0373011786213 - - type: cos_sim_spearman - value: 79.27889560856613 - - type: euclidean_pearson - value: 80.31186315495655 - - type: euclidean_spearman - value: 79.41630415280811 - - type: manhattan_pearson - value: 80.31755140442013 - - type: manhattan_spearman - value: 79.43069870027611 - - task: - type: STS - dataset: - type: mteb/sts12-sts - name: MTEB STS12 - config: default - split: test - revision: a0d554a64d88156834ff5ae9920b964011b16384 - metrics: - - type: cos_sim_pearson - value: 84.8659751342045 - - type: cos_sim_spearman - value: 76.95377612997667 - - type: euclidean_pearson - value: 81.24552945497848 - - type: euclidean_spearman - value: 77.18236963555253 - - type: manhattan_pearson - value: 81.26477607759037 - - type: manhattan_spearman - value: 77.13821753062756 - - task: - type: STS - dataset: - type: mteb/sts13-sts - name: MTEB STS13 - config: default - split: test - revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca - metrics: - - type: cos_sim_pearson - value: 83.34597139044875 - - type: cos_sim_spearman - value: 84.124169425592 - - type: euclidean_pearson - value: 83.68590721511401 - - type: euclidean_spearman - value: 84.18846190846398 - - type: manhattan_pearson - value: 83.57630235061498 - - type: manhattan_spearman - value: 84.10244043726902 - - task: - type: STS - dataset: - type: mteb/sts14-sts - name: MTEB STS14 - config: default - split: test - revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 - metrics: - - type: cos_sim_pearson - value: 82.67641885599572 - - type: cos_sim_spearman - value: 80.46450725650428 - - type: euclidean_pearson - value: 81.61645042715865 - - type: euclidean_spearman - value: 80.61418394236874 - - type: manhattan_pearson - value: 81.55712034928871 - - type: manhattan_spearman - value: 80.57905670523951 - - task: - type: STS - dataset: - type: mteb/sts15-sts - name: MTEB STS15 - config: default - split: test - revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 - metrics: - - type: cos_sim_pearson - value: 88.86650310886782 - - type: cos_sim_spearman - value: 89.76081629222328 - - type: euclidean_pearson - value: 89.1530747029954 - - type: euclidean_spearman - value: 89.80990657280248 - - type: manhattan_pearson - value: 89.10640563278132 - - type: manhattan_spearman - value: 89.76282108434047 - - task: - type: STS - dataset: - type: mteb/sts16-sts - name: MTEB STS16 - config: default - split: test - revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 - metrics: - - type: cos_sim_pearson - value: 83.93864027911118 - - type: cos_sim_spearman - value: 85.47096193999023 - - type: euclidean_pearson - value: 85.03141840870533 - - type: euclidean_spearman - value: 85.43124029598181 - - type: manhattan_pearson - value: 84.99002664393512 - - type: manhattan_spearman - value: 85.39169195120834 - - task: - type: STS - dataset: - type: mteb/sts17-crosslingual-sts - name: MTEB STS17 (en-en) - config: en-en - split: test - revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d - metrics: - - type: cos_sim_pearson - value: 88.7045343749832 - - type: cos_sim_spearman - value: 89.03262221146677 - - type: euclidean_pearson - value: 89.56078218264365 - - type: euclidean_spearman - value: 89.17827006466868 - - type: manhattan_pearson - value: 89.52717595468582 - - type: manhattan_spearman - value: 89.15878115952923 - - task: - type: STS - dataset: - type: mteb/sts22-crosslingual-sts - name: MTEB STS22 (en) - config: en - split: test - revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 - metrics: - - type: cos_sim_pearson - value: 64.20191302875551 - - type: cos_sim_spearman - value: 64.11446552557646 - - type: euclidean_pearson - value: 64.6918197393619 - - type: euclidean_spearman - value: 63.440182631197764 - - type: manhattan_pearson - value: 64.55692904121835 - - type: manhattan_spearman - value: 63.424877742756266 - - task: - type: STS - dataset: - type: mteb/stsbenchmark-sts - name: MTEB STSBenchmark - config: default - split: test - revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 - metrics: - - type: cos_sim_pearson - value: 86.37793104662344 - - type: cos_sim_spearman - value: 87.7357802629067 - - type: euclidean_pearson - value: 87.4286301545109 - - type: euclidean_spearman - value: 87.78452920777421 - - type: manhattan_pearson - value: 87.42445169331255 - - type: manhattan_spearman - value: 87.78537677249598 - - task: - type: Reranking - dataset: - type: mteb/scidocs-reranking - name: MTEB SciDocsRR - config: default - split: test - revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab - metrics: - - type: map - value: 84.31465405081792 - - type: mrr - value: 95.7173781193389 - - task: - type: Retrieval - dataset: - type: scifact - name: MTEB SciFact - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 57.760999999999996 - - type: map_at_10 - value: 67.904 - - type: map_at_100 - value: 68.539 - - type: map_at_1000 - value: 68.562 - - type: map_at_3 - value: 65.415 - - type: map_at_5 - value: 66.788 - - type: mrr_at_1 - value: 60.333000000000006 - - type: mrr_at_10 - value: 68.797 - - type: mrr_at_100 - value: 69.236 - - type: mrr_at_1000 - value: 69.257 - - type: mrr_at_3 - value: 66.667 - - type: mrr_at_5 - value: 67.967 - - type: ndcg_at_1 - value: 60.333000000000006 - - type: ndcg_at_10 - value: 72.24199999999999 - - type: ndcg_at_100 - value: 74.86 - - type: ndcg_at_1000 - value: 75.354 - - type: ndcg_at_3 - value: 67.93400000000001 - - type: ndcg_at_5 - value: 70.02199999999999 - - type: precision_at_1 - value: 60.333000000000006 - - type: precision_at_10 - value: 9.533 - - type: precision_at_100 - value: 1.09 - - type: precision_at_1000 - value: 0.11299999999999999 - - type: precision_at_3 - value: 26.778000000000002 - - type: precision_at_5 - value: 17.467 - - type: recall_at_1 - value: 57.760999999999996 - - type: recall_at_10 - value: 84.383 - - type: recall_at_100 - value: 96.267 - - type: recall_at_1000 - value: 100 - - type: recall_at_3 - value: 72.628 - - type: recall_at_5 - value: 78.094 - - task: - type: PairClassification - dataset: - type: mteb/sprintduplicatequestions-pairclassification - name: MTEB SprintDuplicateQuestions - config: default - split: test - revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 - metrics: - - type: cos_sim_accuracy - value: 99.8029702970297 - - type: cos_sim_ap - value: 94.9210324173411 - - type: cos_sim_f1 - value: 89.8521162672106 - - type: cos_sim_precision - value: 91.67533818938605 - - type: cos_sim_recall - value: 88.1 - - type: dot_accuracy - value: 99.69504950495049 - - type: dot_ap - value: 90.4919719146181 - - type: dot_f1 - value: 84.72289156626506 - - type: dot_precision - value: 81.76744186046511 - - type: dot_recall - value: 87.9 - - type: euclidean_accuracy - value: 99.79702970297029 - - type: euclidean_ap - value: 94.87827463795753 - - type: euclidean_f1 - value: 89.55680081507896 - - type: euclidean_precision - value: 91.27725856697819 - - type: euclidean_recall - value: 87.9 - - type: manhattan_accuracy - value: 99.7990099009901 - - type: manhattan_ap - value: 94.87587025149682 - - type: manhattan_f1 - value: 89.76298537569339 - - type: manhattan_precision - value: 90.53916581892166 - - type: manhattan_recall - value: 89 - - type: max_accuracy - value: 99.8029702970297 - - type: max_ap - value: 94.9210324173411 - - type: max_f1 - value: 89.8521162672106 - - task: - type: Clustering - dataset: - type: mteb/stackexchange-clustering - name: MTEB StackExchangeClustering - config: default - split: test - revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 - metrics: - - type: v_measure - value: 65.92385753948724 - - task: - type: Clustering - dataset: - type: mteb/stackexchange-clustering-p2p - name: MTEB StackExchangeClusteringP2P - config: default - split: test - revision: 815ca46b2622cec33ccafc3735d572c266efdb44 - metrics: - - type: v_measure - value: 33.671756975431144 - - task: - type: Reranking - dataset: - type: mteb/stackoverflowdupquestions-reranking - name: MTEB StackOverflowDupQuestions - config: default - split: test - revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 - metrics: - - type: map - value: 50.677928036739004 - - type: mrr - value: 51.56413133435193 - - task: - type: Summarization - dataset: - type: mteb/summeval - name: MTEB SummEval - config: default - split: test - revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c - metrics: - - type: cos_sim_pearson - value: 30.523589340819683 - - type: cos_sim_spearman - value: 30.187407518823235 - - type: dot_pearson - value: 29.039713969699015 - - type: dot_spearman - value: 29.114740651155508 - - task: - type: Retrieval - dataset: - type: trec-covid - name: MTEB TRECCOVID - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 0.211 - - type: map_at_10 - value: 1.6199999999999999 - - type: map_at_100 - value: 8.658000000000001 - - type: map_at_1000 - value: 21.538 - - type: map_at_3 - value: 0.575 - - type: map_at_5 - value: 0.919 - - type: mrr_at_1 - value: 78 - - type: mrr_at_10 - value: 86.18599999999999 - - type: mrr_at_100 - value: 86.18599999999999 - - type: mrr_at_1000 - value: 86.18599999999999 - - type: mrr_at_3 - value: 85 - - type: mrr_at_5 - value: 85.9 - - type: ndcg_at_1 - value: 74 - - type: ndcg_at_10 - value: 66.542 - - type: ndcg_at_100 - value: 50.163999999999994 - - type: ndcg_at_1000 - value: 45.696999999999996 - - type: ndcg_at_3 - value: 71.531 - - type: ndcg_at_5 - value: 70.45 - - type: precision_at_1 - value: 78 - - type: precision_at_10 - value: 69.39999999999999 - - type: precision_at_100 - value: 51.06 - - type: precision_at_1000 - value: 20.022000000000002 - - type: precision_at_3 - value: 76 - - type: precision_at_5 - value: 74.8 - - type: recall_at_1 - value: 0.211 - - type: recall_at_10 - value: 1.813 - - type: recall_at_100 - value: 12.098 - - type: recall_at_1000 - value: 42.618 - - type: recall_at_3 - value: 0.603 - - type: recall_at_5 - value: 0.987 - - task: - type: Retrieval - dataset: - type: webis-touche2020 - name: MTEB Touche2020 - config: default - split: test - revision: None - metrics: - - type: map_at_1 - value: 2.2079999999999997 - - type: map_at_10 - value: 7.777000000000001 - - type: map_at_100 - value: 12.825000000000001 - - type: map_at_1000 - value: 14.196 - - type: map_at_3 - value: 4.285 - - type: map_at_5 - value: 6.177 - - type: mrr_at_1 - value: 30.612000000000002 - - type: mrr_at_10 - value: 42.635 - - type: mrr_at_100 - value: 43.955 - - type: mrr_at_1000 - value: 43.955 - - type: mrr_at_3 - value: 38.435 - - type: mrr_at_5 - value: 41.088 - - type: ndcg_at_1 - value: 28.571 - - type: ndcg_at_10 - value: 20.666999999999998 - - type: ndcg_at_100 - value: 31.840000000000003 - - type: ndcg_at_1000 - value: 43.191 - - type: ndcg_at_3 - value: 23.45 - - type: ndcg_at_5 - value: 22.994 - - type: precision_at_1 - value: 30.612000000000002 - - type: precision_at_10 - value: 17.959 - - type: precision_at_100 - value: 6.755 - - type: precision_at_1000 - value: 1.4200000000000002 - - type: precision_at_3 - value: 23.810000000000002 - - type: precision_at_5 - value: 23.673 - - type: recall_at_1 - value: 2.2079999999999997 - - type: recall_at_10 - value: 13.144 - - type: recall_at_100 - value: 42.491 - - type: recall_at_1000 - value: 77.04299999999999 - - type: recall_at_3 - value: 5.3469999999999995 - - type: recall_at_5 - value: 9.139 - - task: - type: Classification - dataset: - type: mteb/toxic_conversations_50k - name: MTEB ToxicConversationsClassification - config: default - split: test - revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c - metrics: - - type: accuracy - value: 70.9044 - - type: ap - value: 14.625783489340755 - - type: f1 - value: 54.814936562590546 - - task: - type: Classification - dataset: - type: mteb/tweet_sentiment_extraction - name: MTEB TweetSentimentExtractionClassification - config: default - split: test - revision: d604517c81ca91fe16a244d1248fc021f9ecee7a - metrics: - - type: accuracy - value: 60.94227504244483 - - type: f1 - value: 61.22516038508854 - - task: - type: Clustering - dataset: - type: mteb/twentynewsgroups-clustering - name: MTEB TwentyNewsgroupsClustering - config: default - split: test - revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 - metrics: - - type: v_measure - value: 49.602409155145864 - - task: - type: PairClassification - dataset: - type: mteb/twittersemeval2015-pairclassification - name: MTEB TwitterSemEval2015 - config: default - split: test - revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 - metrics: - - type: cos_sim_accuracy - value: 86.94641473445789 - - type: cos_sim_ap - value: 76.91572747061197 - - type: cos_sim_f1 - value: 70.14348097317529 - - type: cos_sim_precision - value: 66.53254437869822 - - type: cos_sim_recall - value: 74.1688654353562 - - type: dot_accuracy - value: 84.80061989628658 - - type: dot_ap - value: 70.7952548895177 - - type: dot_f1 - value: 65.44780728844965 - - type: dot_precision - value: 61.53310104529617 - - type: dot_recall - value: 69.89445910290237 - - type: euclidean_accuracy - value: 86.94641473445789 - - type: euclidean_ap - value: 76.80774009393652 - - type: euclidean_f1 - value: 70.30522503879979 - - type: euclidean_precision - value: 68.94977168949772 - - type: euclidean_recall - value: 71.71503957783642 - - type: manhattan_accuracy - value: 86.8629671574179 - - type: manhattan_ap - value: 76.76518632600317 - - type: manhattan_f1 - value: 70.16056518946692 - - type: manhattan_precision - value: 68.360450563204 - - type: manhattan_recall - value: 72.0580474934037 - - type: max_accuracy - value: 86.94641473445789 - - type: max_ap - value: 76.91572747061197 - - type: max_f1 - value: 70.30522503879979 - - task: - type: PairClassification - dataset: - type: mteb/twitterurlcorpus-pairclassification - name: MTEB TwitterURLCorpus - config: default - split: test - revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf - metrics: - - type: cos_sim_accuracy - value: 89.10428066907285 - - type: cos_sim_ap - value: 86.25114759921435 - - type: cos_sim_f1 - value: 78.37857884586856 - - type: cos_sim_precision - value: 75.60818546078993 - - type: cos_sim_recall - value: 81.35971666153372 - - type: dot_accuracy - value: 87.41995575736406 - - type: dot_ap - value: 81.51838010086782 - - type: dot_f1 - value: 74.77398015435503 - - type: dot_precision - value: 71.53002390662354 - - type: dot_recall - value: 78.32614721281182 - - type: euclidean_accuracy - value: 89.12368533395428 - - type: euclidean_ap - value: 86.33456799874504 - - type: euclidean_f1 - value: 78.45496750232127 - - type: euclidean_precision - value: 75.78388462366364 - - type: euclidean_recall - value: 81.32121958731136 - - type: manhattan_accuracy - value: 89.10622113556099 - - type: manhattan_ap - value: 86.31215061745333 - - type: manhattan_f1 - value: 78.40684906011539 - - type: manhattan_precision - value: 75.89536643366722 - - type: manhattan_recall - value: 81.09023714197721 - - type: max_accuracy - value: 89.12368533395428 - - type: max_ap - value: 86.33456799874504 - - type: max_f1 - value: 78.45496750232127 -language: -- en -license: mit ---- - -# E5-large-v2 - -[Text Embeddings by Weakly-Supervised Contrastive Pre-training](https://arxiv.org/pdf/2212.03533.pdf). -Liang Wang, Nan Yang, Xiaolong Huang, Binxing Jiao, Linjun Yang, Daxin Jiang, Rangan Majumder, Furu Wei, arXiv 2022 - -This model has 24 layers and the embedding size is 1024. - -## Usage - -Below is an example to encode queries and passages from the MS-MARCO passage ranking dataset. - -```python -import torch.nn.functional as F - -from torch import Tensor -from transformers import AutoTokenizer, AutoModel - - -def average_pool(last_hidden_states: Tensor, - attention_mask: Tensor) -> Tensor: - last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0) - return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] - - -# Each input text should start with "query: " or "passage: ". -# For tasks other than retrieval, you can simply use the "query: " prefix. -input_texts = ['query: how much protein should a female eat', - 'query: summit define', - "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", - "passage: Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."] - -tokenizer = AutoTokenizer.from_pretrained('intfloat/e5-large-v2') -model = AutoModel.from_pretrained('intfloat/e5-large-v2') - -# Tokenize the input texts -batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt') - -outputs = model(**batch_dict) -embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask']) - -# normalize embeddings -embeddings = F.normalize(embeddings, p=2, dim=1) -scores = (embeddings[:2] @ embeddings[2:].T) * 100 -print(scores.tolist()) -``` - -## Training Details - -Please refer to our paper at [https://arxiv.org/pdf/2212.03533.pdf](https://arxiv.org/pdf/2212.03533.pdf). - -## Benchmark Evaluation - -Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results -on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB benchmark](https://arxiv.org/abs/2210.07316). - -## Support for Sentence Transformers - -Below is an example for usage with sentence_transformers. -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('intfloat/e5-large-v2') -input_texts = [ - 'query: how much protein should a female eat', - 'query: summit define', - "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", - "passage: Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments." -] -embeddings = model.encode(input_texts, normalize_embeddings=True) -``` - -Package requirements - -`pip install sentence_transformers~=2.2.2` - -Contributors: [michaelfeil](https://huggingface.co/michaelfeil) - -## FAQ - -**1. Do I need to add the prefix "query: " and "passage: " to input texts?** - -Yes, this is how the model is trained, otherwise you will see a performance degradation. - -Here are some rules of thumb: -- Use "query: " and "passage: " correspondingly for asymmetric tasks such as passage retrieval in open QA, ad-hoc information retrieval. - -- Use "query: " prefix for symmetric tasks such as semantic similarity, paraphrase retrieval. - -- Use "query: " prefix if you want to use embeddings as features, such as linear probing classification, clustering. - -**2. Why are my reproduced results slightly different from reported in the model card?** - -Different versions of `transformers` and `pytorch` could cause negligible but non-zero performance differences. - -**3. Why does the cosine similarity scores distribute around 0.7 to 1.0?** - -This is a known and expected behavior as we use a low temperature 0.01 for InfoNCE contrastive loss. - -For text embedding tasks like text retrieval or semantic similarity, -what matters is the relative order of the scores instead of the absolute values, -so this should not be an issue. - -## Citation - -If you find our paper or models helpful, please consider cite as follows: - -``` -@article{wang2022text, - title={Text Embeddings by Weakly-Supervised Contrastive Pre-training}, - author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Jiao, Binxing and Yang, Linjun and Jiang, Daxin and Majumder, Rangan and Wei, Furu}, - journal={arXiv preprint arXiv:2212.03533}, - year={2022} -} -``` - -## Limitations - -This model only works for English texts. Long texts will be truncated to at most 512 tokens. +--- +tags: +- mteb +- Sentence Transformers +- sentence-similarity +- sentence-transformers +model-index: +- name: e5-large-v2 + results: + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en) + config: en + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 79.22388059701493 + - type: ap + value: 43.20816505595132 + - type: f1 + value: 73.27811303522058 + - task: + type: Classification + dataset: + type: mteb/amazon_polarity + name: MTEB AmazonPolarityClassification + config: default + split: test + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + metrics: + - type: accuracy + value: 93.748325 + - type: ap + value: 90.72534979701297 + - type: f1 + value: 93.73895874282185 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (en) + config: en + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 48.612 + - type: f1 + value: 47.61157345898393 + - task: + type: Retrieval + dataset: + type: arguana + name: MTEB ArguAna + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.541999999999998 + - type: map_at_10 + value: 38.208 + - type: map_at_100 + value: 39.417 + - type: map_at_1000 + value: 39.428999999999995 + - type: map_at_3 + value: 33.95 + - type: map_at_5 + value: 36.329 + - type: mrr_at_1 + value: 23.755000000000003 + - type: mrr_at_10 + value: 38.288 + - type: mrr_at_100 + value: 39.511 + - type: mrr_at_1000 + value: 39.523 + - type: mrr_at_3 + value: 34.009 + - type: mrr_at_5 + value: 36.434 + - type: ndcg_at_1 + value: 23.541999999999998 + - type: ndcg_at_10 + value: 46.417 + - type: ndcg_at_100 + value: 51.812000000000005 + - type: ndcg_at_1000 + value: 52.137 + - type: ndcg_at_3 + value: 37.528 + - type: ndcg_at_5 + value: 41.81 + - type: precision_at_1 + value: 23.541999999999998 + - type: precision_at_10 + value: 7.269 + - type: precision_at_100 + value: 0.9690000000000001 + - type: precision_at_1000 + value: 0.099 + - type: precision_at_3 + value: 15.979 + - type: precision_at_5 + value: 11.664 + - type: recall_at_1 + value: 23.541999999999998 + - type: recall_at_10 + value: 72.688 + - type: recall_at_100 + value: 96.871 + - type: recall_at_1000 + value: 99.431 + - type: recall_at_3 + value: 47.937000000000005 + - type: recall_at_5 + value: 58.321 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-p2p + name: MTEB ArxivClusteringP2P + config: default + split: test + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + metrics: + - type: v_measure + value: 45.546499570522094 + - task: + type: Clustering + dataset: + type: mteb/arxiv-clustering-s2s + name: MTEB ArxivClusteringS2S + config: default + split: test + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + metrics: + - type: v_measure + value: 41.01607489943561 + - task: + type: Reranking + dataset: + type: mteb/askubuntudupquestions-reranking + name: MTEB AskUbuntuDupQuestions + config: default + split: test + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + metrics: + - type: map + value: 59.616107510107774 + - type: mrr + value: 72.75106626214661 + - task: + type: STS + dataset: + type: mteb/biosses-sts + name: MTEB BIOSSES + config: default + split: test + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + metrics: + - type: cos_sim_pearson + value: 84.33018094733868 + - type: cos_sim_spearman + value: 83.60190492611737 + - type: euclidean_pearson + value: 82.1492450218961 + - type: euclidean_spearman + value: 82.70308926526991 + - type: manhattan_pearson + value: 81.93959600076842 + - type: manhattan_spearman + value: 82.73260801016369 + - task: + type: Classification + dataset: + type: mteb/banking77 + name: MTEB Banking77Classification + config: default + split: test + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + metrics: + - type: accuracy + value: 84.54545454545455 + - type: f1 + value: 84.49582530928923 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-p2p + name: MTEB BiorxivClusteringP2P + config: default + split: test + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + metrics: + - type: v_measure + value: 37.362725540120096 + - task: + type: Clustering + dataset: + type: mteb/biorxiv-clustering-s2s + name: MTEB BiorxivClusteringS2S + config: default + split: test + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + metrics: + - type: v_measure + value: 34.849509608178145 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackAndroidRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 31.502999999999997 + - type: map_at_10 + value: 43.323 + - type: map_at_100 + value: 44.708999999999996 + - type: map_at_1000 + value: 44.838 + - type: map_at_3 + value: 38.987 + - type: map_at_5 + value: 41.516999999999996 + - type: mrr_at_1 + value: 38.769999999999996 + - type: mrr_at_10 + value: 49.13 + - type: mrr_at_100 + value: 49.697 + - type: mrr_at_1000 + value: 49.741 + - type: mrr_at_3 + value: 45.804 + - type: mrr_at_5 + value: 47.842 + - type: ndcg_at_1 + value: 38.769999999999996 + - type: ndcg_at_10 + value: 50.266999999999996 + - type: ndcg_at_100 + value: 54.967 + - type: ndcg_at_1000 + value: 56.976000000000006 + - type: ndcg_at_3 + value: 43.823 + - type: ndcg_at_5 + value: 47.12 + - type: precision_at_1 + value: 38.769999999999996 + - type: precision_at_10 + value: 10.057 + - type: precision_at_100 + value: 1.554 + - type: precision_at_1000 + value: 0.202 + - type: precision_at_3 + value: 21.125 + - type: precision_at_5 + value: 15.851 + - type: recall_at_1 + value: 31.502999999999997 + - type: recall_at_10 + value: 63.715999999999994 + - type: recall_at_100 + value: 83.61800000000001 + - type: recall_at_1000 + value: 96.63199999999999 + - type: recall_at_3 + value: 45.403 + - type: recall_at_5 + value: 54.481 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackEnglishRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 27.833000000000002 + - type: map_at_10 + value: 37.330999999999996 + - type: map_at_100 + value: 38.580999999999996 + - type: map_at_1000 + value: 38.708 + - type: map_at_3 + value: 34.713 + - type: map_at_5 + value: 36.104 + - type: mrr_at_1 + value: 35.223 + - type: mrr_at_10 + value: 43.419000000000004 + - type: mrr_at_100 + value: 44.198 + - type: mrr_at_1000 + value: 44.249 + - type: mrr_at_3 + value: 41.614000000000004 + - type: mrr_at_5 + value: 42.553000000000004 + - type: ndcg_at_1 + value: 35.223 + - type: ndcg_at_10 + value: 42.687999999999995 + - type: ndcg_at_100 + value: 47.447 + - type: ndcg_at_1000 + value: 49.701 + - type: ndcg_at_3 + value: 39.162 + - type: ndcg_at_5 + value: 40.557 + - type: precision_at_1 + value: 35.223 + - type: precision_at_10 + value: 7.962 + - type: precision_at_100 + value: 1.304 + - type: precision_at_1000 + value: 0.18 + - type: precision_at_3 + value: 19.023 + - type: precision_at_5 + value: 13.184999999999999 + - type: recall_at_1 + value: 27.833000000000002 + - type: recall_at_10 + value: 51.881 + - type: recall_at_100 + value: 72.04 + - type: recall_at_1000 + value: 86.644 + - type: recall_at_3 + value: 40.778 + - type: recall_at_5 + value: 45.176 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGamingRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 38.175 + - type: map_at_10 + value: 51.174 + - type: map_at_100 + value: 52.26499999999999 + - type: map_at_1000 + value: 52.315999999999995 + - type: map_at_3 + value: 47.897 + - type: map_at_5 + value: 49.703 + - type: mrr_at_1 + value: 43.448 + - type: mrr_at_10 + value: 54.505 + - type: mrr_at_100 + value: 55.216 + - type: mrr_at_1000 + value: 55.242000000000004 + - type: mrr_at_3 + value: 51.98500000000001 + - type: mrr_at_5 + value: 53.434000000000005 + - type: ndcg_at_1 + value: 43.448 + - type: ndcg_at_10 + value: 57.282 + - type: ndcg_at_100 + value: 61.537 + - type: ndcg_at_1000 + value: 62.546 + - type: ndcg_at_3 + value: 51.73799999999999 + - type: ndcg_at_5 + value: 54.324 + - type: precision_at_1 + value: 43.448 + - type: precision_at_10 + value: 9.292 + - type: precision_at_100 + value: 1.233 + - type: precision_at_1000 + value: 0.136 + - type: precision_at_3 + value: 23.218 + - type: precision_at_5 + value: 15.887 + - type: recall_at_1 + value: 38.175 + - type: recall_at_10 + value: 72.00999999999999 + - type: recall_at_100 + value: 90.155 + - type: recall_at_1000 + value: 97.257 + - type: recall_at_3 + value: 57.133 + - type: recall_at_5 + value: 63.424 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackGisRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 22.405 + - type: map_at_10 + value: 30.043 + - type: map_at_100 + value: 31.191000000000003 + - type: map_at_1000 + value: 31.275 + - type: map_at_3 + value: 27.034000000000002 + - type: map_at_5 + value: 28.688000000000002 + - type: mrr_at_1 + value: 24.068 + - type: mrr_at_10 + value: 31.993 + - type: mrr_at_100 + value: 32.992 + - type: mrr_at_1000 + value: 33.050000000000004 + - type: mrr_at_3 + value: 28.964000000000002 + - type: mrr_at_5 + value: 30.653000000000002 + - type: ndcg_at_1 + value: 24.068 + - type: ndcg_at_10 + value: 35.198 + - type: ndcg_at_100 + value: 40.709 + - type: ndcg_at_1000 + value: 42.855 + - type: ndcg_at_3 + value: 29.139 + - type: ndcg_at_5 + value: 32.045 + - type: precision_at_1 + value: 24.068 + - type: precision_at_10 + value: 5.65 + - type: precision_at_100 + value: 0.885 + - type: precision_at_1000 + value: 0.11199999999999999 + - type: precision_at_3 + value: 12.279 + - type: precision_at_5 + value: 8.994 + - type: recall_at_1 + value: 22.405 + - type: recall_at_10 + value: 49.391 + - type: recall_at_100 + value: 74.53699999999999 + - type: recall_at_1000 + value: 90.605 + - type: recall_at_3 + value: 33.126 + - type: recall_at_5 + value: 40.073 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackMathematicaRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 13.309999999999999 + - type: map_at_10 + value: 20.688000000000002 + - type: map_at_100 + value: 22.022 + - type: map_at_1000 + value: 22.152 + - type: map_at_3 + value: 17.954 + - type: map_at_5 + value: 19.439 + - type: mrr_at_1 + value: 16.294 + - type: mrr_at_10 + value: 24.479 + - type: mrr_at_100 + value: 25.515 + - type: mrr_at_1000 + value: 25.593 + - type: mrr_at_3 + value: 21.642 + - type: mrr_at_5 + value: 23.189999999999998 + - type: ndcg_at_1 + value: 16.294 + - type: ndcg_at_10 + value: 25.833000000000002 + - type: ndcg_at_100 + value: 32.074999999999996 + - type: ndcg_at_1000 + value: 35.083 + - type: ndcg_at_3 + value: 20.493 + - type: ndcg_at_5 + value: 22.949 + - type: precision_at_1 + value: 16.294 + - type: precision_at_10 + value: 5.112 + - type: precision_at_100 + value: 0.96 + - type: precision_at_1000 + value: 0.134 + - type: precision_at_3 + value: 9.908999999999999 + - type: precision_at_5 + value: 7.587000000000001 + - type: recall_at_1 + value: 13.309999999999999 + - type: recall_at_10 + value: 37.851 + - type: recall_at_100 + value: 64.835 + - type: recall_at_1000 + value: 86.334 + - type: recall_at_3 + value: 23.493 + - type: recall_at_5 + value: 29.528 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackPhysicsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 25.857999999999997 + - type: map_at_10 + value: 35.503 + - type: map_at_100 + value: 36.957 + - type: map_at_1000 + value: 37.065 + - type: map_at_3 + value: 32.275999999999996 + - type: map_at_5 + value: 34.119 + - type: mrr_at_1 + value: 31.954 + - type: mrr_at_10 + value: 40.851 + - type: mrr_at_100 + value: 41.863 + - type: mrr_at_1000 + value: 41.900999999999996 + - type: mrr_at_3 + value: 38.129999999999995 + - type: mrr_at_5 + value: 39.737 + - type: ndcg_at_1 + value: 31.954 + - type: ndcg_at_10 + value: 41.343999999999994 + - type: ndcg_at_100 + value: 47.397 + - type: ndcg_at_1000 + value: 49.501 + - type: ndcg_at_3 + value: 36.047000000000004 + - type: ndcg_at_5 + value: 38.639 + - type: precision_at_1 + value: 31.954 + - type: precision_at_10 + value: 7.68 + - type: precision_at_100 + value: 1.247 + - type: precision_at_1000 + value: 0.16199999999999998 + - type: precision_at_3 + value: 17.132 + - type: precision_at_5 + value: 12.589 + - type: recall_at_1 + value: 25.857999999999997 + - type: recall_at_10 + value: 53.43599999999999 + - type: recall_at_100 + value: 78.82400000000001 + - type: recall_at_1000 + value: 92.78999999999999 + - type: recall_at_3 + value: 38.655 + - type: recall_at_5 + value: 45.216 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackProgrammersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 24.709 + - type: map_at_10 + value: 34.318 + - type: map_at_100 + value: 35.657 + - type: map_at_1000 + value: 35.783 + - type: map_at_3 + value: 31.326999999999998 + - type: map_at_5 + value: 33.021 + - type: mrr_at_1 + value: 30.137000000000004 + - type: mrr_at_10 + value: 39.093 + - type: mrr_at_100 + value: 39.992 + - type: mrr_at_1000 + value: 40.056999999999995 + - type: mrr_at_3 + value: 36.606 + - type: mrr_at_5 + value: 37.861 + - type: ndcg_at_1 + value: 30.137000000000004 + - type: ndcg_at_10 + value: 39.974 + - type: ndcg_at_100 + value: 45.647999999999996 + - type: ndcg_at_1000 + value: 48.259 + - type: ndcg_at_3 + value: 35.028 + - type: ndcg_at_5 + value: 37.175999999999995 + - type: precision_at_1 + value: 30.137000000000004 + - type: precision_at_10 + value: 7.363 + - type: precision_at_100 + value: 1.184 + - type: precision_at_1000 + value: 0.161 + - type: precision_at_3 + value: 16.857 + - type: precision_at_5 + value: 11.963 + - type: recall_at_1 + value: 24.709 + - type: recall_at_10 + value: 52.087 + - type: recall_at_100 + value: 76.125 + - type: recall_at_1000 + value: 93.82300000000001 + - type: recall_at_3 + value: 38.149 + - type: recall_at_5 + value: 43.984 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.40791666666667 + - type: map_at_10 + value: 32.458083333333335 + - type: map_at_100 + value: 33.691916666666664 + - type: map_at_1000 + value: 33.81191666666666 + - type: map_at_3 + value: 29.51625 + - type: map_at_5 + value: 31.168083333333335 + - type: mrr_at_1 + value: 27.96591666666666 + - type: mrr_at_10 + value: 36.528583333333344 + - type: mrr_at_100 + value: 37.404 + - type: mrr_at_1000 + value: 37.464333333333336 + - type: mrr_at_3 + value: 33.92883333333333 + - type: mrr_at_5 + value: 35.41933333333333 + - type: ndcg_at_1 + value: 27.96591666666666 + - type: ndcg_at_10 + value: 37.89141666666666 + - type: ndcg_at_100 + value: 43.23066666666666 + - type: ndcg_at_1000 + value: 45.63258333333333 + - type: ndcg_at_3 + value: 32.811249999999994 + - type: ndcg_at_5 + value: 35.22566666666667 + - type: precision_at_1 + value: 27.96591666666666 + - type: precision_at_10 + value: 6.834083333333332 + - type: precision_at_100 + value: 1.12225 + - type: precision_at_1000 + value: 0.15241666666666667 + - type: precision_at_3 + value: 15.264333333333335 + - type: precision_at_5 + value: 11.039416666666666 + - type: recall_at_1 + value: 23.40791666666667 + - type: recall_at_10 + value: 49.927083333333336 + - type: recall_at_100 + value: 73.44641666666668 + - type: recall_at_1000 + value: 90.19950000000001 + - type: recall_at_3 + value: 35.88341666666667 + - type: recall_at_5 + value: 42.061249999999994 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackStatsRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 19.592000000000002 + - type: map_at_10 + value: 26.895999999999997 + - type: map_at_100 + value: 27.921000000000003 + - type: map_at_1000 + value: 28.02 + - type: map_at_3 + value: 24.883 + - type: map_at_5 + value: 25.812 + - type: mrr_at_1 + value: 22.698999999999998 + - type: mrr_at_10 + value: 29.520999999999997 + - type: mrr_at_100 + value: 30.458000000000002 + - type: mrr_at_1000 + value: 30.526999999999997 + - type: mrr_at_3 + value: 27.633000000000003 + - type: mrr_at_5 + value: 28.483999999999998 + - type: ndcg_at_1 + value: 22.698999999999998 + - type: ndcg_at_10 + value: 31.061 + - type: ndcg_at_100 + value: 36.398 + - type: ndcg_at_1000 + value: 38.89 + - type: ndcg_at_3 + value: 27.149 + - type: ndcg_at_5 + value: 28.627000000000002 + - type: precision_at_1 + value: 22.698999999999998 + - type: precision_at_10 + value: 5.106999999999999 + - type: precision_at_100 + value: 0.857 + - type: precision_at_1000 + value: 0.11499999999999999 + - type: precision_at_3 + value: 11.963 + - type: precision_at_5 + value: 8.221 + - type: recall_at_1 + value: 19.592000000000002 + - type: recall_at_10 + value: 41.329 + - type: recall_at_100 + value: 66.094 + - type: recall_at_1000 + value: 84.511 + - type: recall_at_3 + value: 30.61 + - type: recall_at_5 + value: 34.213 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackTexRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 14.71 + - type: map_at_10 + value: 20.965 + - type: map_at_100 + value: 21.994 + - type: map_at_1000 + value: 22.133 + - type: map_at_3 + value: 18.741 + - type: map_at_5 + value: 19.951 + - type: mrr_at_1 + value: 18.307000000000002 + - type: mrr_at_10 + value: 24.66 + - type: mrr_at_100 + value: 25.540000000000003 + - type: mrr_at_1000 + value: 25.629 + - type: mrr_at_3 + value: 22.511 + - type: mrr_at_5 + value: 23.72 + - type: ndcg_at_1 + value: 18.307000000000002 + - type: ndcg_at_10 + value: 25.153 + - type: ndcg_at_100 + value: 30.229 + - type: ndcg_at_1000 + value: 33.623 + - type: ndcg_at_3 + value: 21.203 + - type: ndcg_at_5 + value: 23.006999999999998 + - type: precision_at_1 + value: 18.307000000000002 + - type: precision_at_10 + value: 4.725 + - type: precision_at_100 + value: 0.8659999999999999 + - type: precision_at_1000 + value: 0.133 + - type: precision_at_3 + value: 10.14 + - type: precision_at_5 + value: 7.481 + - type: recall_at_1 + value: 14.71 + - type: recall_at_10 + value: 34.087 + - type: recall_at_100 + value: 57.147999999999996 + - type: recall_at_1000 + value: 81.777 + - type: recall_at_3 + value: 22.996 + - type: recall_at_5 + value: 27.73 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackUnixRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.472 + - type: map_at_10 + value: 32.699 + - type: map_at_100 + value: 33.867000000000004 + - type: map_at_1000 + value: 33.967000000000006 + - type: map_at_3 + value: 29.718 + - type: map_at_5 + value: 31.345 + - type: mrr_at_1 + value: 28.265 + - type: mrr_at_10 + value: 36.945 + - type: mrr_at_100 + value: 37.794 + - type: mrr_at_1000 + value: 37.857 + - type: mrr_at_3 + value: 34.266000000000005 + - type: mrr_at_5 + value: 35.768 + - type: ndcg_at_1 + value: 28.265 + - type: ndcg_at_10 + value: 38.35 + - type: ndcg_at_100 + value: 43.739 + - type: ndcg_at_1000 + value: 46.087 + - type: ndcg_at_3 + value: 33.004 + - type: ndcg_at_5 + value: 35.411 + - type: precision_at_1 + value: 28.265 + - type: precision_at_10 + value: 6.715999999999999 + - type: precision_at_100 + value: 1.059 + - type: precision_at_1000 + value: 0.13799999999999998 + - type: precision_at_3 + value: 15.299 + - type: precision_at_5 + value: 10.951 + - type: recall_at_1 + value: 23.472 + - type: recall_at_10 + value: 51.413 + - type: recall_at_100 + value: 75.17 + - type: recall_at_1000 + value: 91.577 + - type: recall_at_3 + value: 36.651 + - type: recall_at_5 + value: 42.814 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWebmastersRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 23.666 + - type: map_at_10 + value: 32.963 + - type: map_at_100 + value: 34.544999999999995 + - type: map_at_1000 + value: 34.792 + - type: map_at_3 + value: 29.74 + - type: map_at_5 + value: 31.5 + - type: mrr_at_1 + value: 29.051 + - type: mrr_at_10 + value: 38.013000000000005 + - type: mrr_at_100 + value: 38.997 + - type: mrr_at_1000 + value: 39.055 + - type: mrr_at_3 + value: 34.947 + - type: mrr_at_5 + value: 36.815 + - type: ndcg_at_1 + value: 29.051 + - type: ndcg_at_10 + value: 39.361000000000004 + - type: ndcg_at_100 + value: 45.186 + - type: ndcg_at_1000 + value: 47.867 + - type: ndcg_at_3 + value: 33.797 + - type: ndcg_at_5 + value: 36.456 + - type: precision_at_1 + value: 29.051 + - type: precision_at_10 + value: 7.668 + - type: precision_at_100 + value: 1.532 + - type: precision_at_1000 + value: 0.247 + - type: precision_at_3 + value: 15.876000000000001 + - type: precision_at_5 + value: 11.779 + - type: recall_at_1 + value: 23.666 + - type: recall_at_10 + value: 51.858000000000004 + - type: recall_at_100 + value: 77.805 + - type: recall_at_1000 + value: 94.504 + - type: recall_at_3 + value: 36.207 + - type: recall_at_5 + value: 43.094 + - task: + type: Retrieval + dataset: + type: BeIR/cqadupstack + name: MTEB CQADupstackWordpressRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 15.662 + - type: map_at_10 + value: 23.594 + - type: map_at_100 + value: 24.593999999999998 + - type: map_at_1000 + value: 24.694 + - type: map_at_3 + value: 20.925 + - type: map_at_5 + value: 22.817999999999998 + - type: mrr_at_1 + value: 17.375 + - type: mrr_at_10 + value: 25.734 + - type: mrr_at_100 + value: 26.586 + - type: mrr_at_1000 + value: 26.671 + - type: mrr_at_3 + value: 23.044 + - type: mrr_at_5 + value: 24.975 + - type: ndcg_at_1 + value: 17.375 + - type: ndcg_at_10 + value: 28.186 + - type: ndcg_at_100 + value: 33.436 + - type: ndcg_at_1000 + value: 36.203 + - type: ndcg_at_3 + value: 23.152 + - type: ndcg_at_5 + value: 26.397 + - type: precision_at_1 + value: 17.375 + - type: precision_at_10 + value: 4.677 + - type: precision_at_100 + value: 0.786 + - type: precision_at_1000 + value: 0.109 + - type: precision_at_3 + value: 10.351 + - type: precision_at_5 + value: 7.985 + - type: recall_at_1 + value: 15.662 + - type: recall_at_10 + value: 40.066 + - type: recall_at_100 + value: 65.006 + - type: recall_at_1000 + value: 85.94000000000001 + - type: recall_at_3 + value: 27.400000000000002 + - type: recall_at_5 + value: 35.002 + - task: + type: Retrieval + dataset: + type: climate-fever + name: MTEB ClimateFEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 8.853 + - type: map_at_10 + value: 15.568000000000001 + - type: map_at_100 + value: 17.383000000000003 + - type: map_at_1000 + value: 17.584 + - type: map_at_3 + value: 12.561 + - type: map_at_5 + value: 14.056 + - type: mrr_at_1 + value: 18.958 + - type: mrr_at_10 + value: 28.288000000000004 + - type: mrr_at_100 + value: 29.432000000000002 + - type: mrr_at_1000 + value: 29.498 + - type: mrr_at_3 + value: 25.049 + - type: mrr_at_5 + value: 26.857 + - type: ndcg_at_1 + value: 18.958 + - type: ndcg_at_10 + value: 22.21 + - type: ndcg_at_100 + value: 29.596 + - type: ndcg_at_1000 + value: 33.583 + - type: ndcg_at_3 + value: 16.994999999999997 + - type: ndcg_at_5 + value: 18.95 + - type: precision_at_1 + value: 18.958 + - type: precision_at_10 + value: 7.192 + - type: precision_at_100 + value: 1.5 + - type: precision_at_1000 + value: 0.22399999999999998 + - type: precision_at_3 + value: 12.573 + - type: precision_at_5 + value: 10.202 + - type: recall_at_1 + value: 8.853 + - type: recall_at_10 + value: 28.087 + - type: recall_at_100 + value: 53.701 + - type: recall_at_1000 + value: 76.29899999999999 + - type: recall_at_3 + value: 15.913 + - type: recall_at_5 + value: 20.658 + - task: + type: Retrieval + dataset: + type: dbpedia-entity + name: MTEB DBPedia + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 9.077 + - type: map_at_10 + value: 20.788999999999998 + - type: map_at_100 + value: 30.429000000000002 + - type: map_at_1000 + value: 32.143 + - type: map_at_3 + value: 14.692 + - type: map_at_5 + value: 17.139 + - type: mrr_at_1 + value: 70.75 + - type: mrr_at_10 + value: 78.036 + - type: mrr_at_100 + value: 78.401 + - type: mrr_at_1000 + value: 78.404 + - type: mrr_at_3 + value: 76.75 + - type: mrr_at_5 + value: 77.47500000000001 + - type: ndcg_at_1 + value: 58.12500000000001 + - type: ndcg_at_10 + value: 44.015 + - type: ndcg_at_100 + value: 49.247 + - type: ndcg_at_1000 + value: 56.211999999999996 + - type: ndcg_at_3 + value: 49.151 + - type: ndcg_at_5 + value: 46.195 + - type: precision_at_1 + value: 70.75 + - type: precision_at_10 + value: 35.5 + - type: precision_at_100 + value: 11.355 + - type: precision_at_1000 + value: 2.1950000000000003 + - type: precision_at_3 + value: 53.083000000000006 + - type: precision_at_5 + value: 44.800000000000004 + - type: recall_at_1 + value: 9.077 + - type: recall_at_10 + value: 26.259 + - type: recall_at_100 + value: 56.547000000000004 + - type: recall_at_1000 + value: 78.551 + - type: recall_at_3 + value: 16.162000000000003 + - type: recall_at_5 + value: 19.753999999999998 + - task: + type: Classification + dataset: + type: mteb/emotion + name: MTEB EmotionClassification + config: default + split: test + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + metrics: + - type: accuracy + value: 49.44500000000001 + - type: f1 + value: 44.67067691783401 + - task: + type: Retrieval + dataset: + type: fever + name: MTEB FEVER + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 68.182 + - type: map_at_10 + value: 78.223 + - type: map_at_100 + value: 78.498 + - type: map_at_1000 + value: 78.512 + - type: map_at_3 + value: 76.71 + - type: map_at_5 + value: 77.725 + - type: mrr_at_1 + value: 73.177 + - type: mrr_at_10 + value: 82.513 + - type: mrr_at_100 + value: 82.633 + - type: mrr_at_1000 + value: 82.635 + - type: mrr_at_3 + value: 81.376 + - type: mrr_at_5 + value: 82.182 + - type: ndcg_at_1 + value: 73.177 + - type: ndcg_at_10 + value: 82.829 + - type: ndcg_at_100 + value: 83.84 + - type: ndcg_at_1000 + value: 84.07900000000001 + - type: ndcg_at_3 + value: 80.303 + - type: ndcg_at_5 + value: 81.846 + - type: precision_at_1 + value: 73.177 + - type: precision_at_10 + value: 10.241999999999999 + - type: precision_at_100 + value: 1.099 + - type: precision_at_1000 + value: 0.11399999999999999 + - type: precision_at_3 + value: 31.247999999999998 + - type: precision_at_5 + value: 19.697 + - type: recall_at_1 + value: 68.182 + - type: recall_at_10 + value: 92.657 + - type: recall_at_100 + value: 96.709 + - type: recall_at_1000 + value: 98.184 + - type: recall_at_3 + value: 85.9 + - type: recall_at_5 + value: 89.755 + - task: + type: Retrieval + dataset: + type: fiqa + name: MTEB FiQA2018 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 21.108 + - type: map_at_10 + value: 33.342 + - type: map_at_100 + value: 35.281 + - type: map_at_1000 + value: 35.478 + - type: map_at_3 + value: 29.067 + - type: map_at_5 + value: 31.563000000000002 + - type: mrr_at_1 + value: 41.667 + - type: mrr_at_10 + value: 49.913000000000004 + - type: mrr_at_100 + value: 50.724000000000004 + - type: mrr_at_1000 + value: 50.766 + - type: mrr_at_3 + value: 47.504999999999995 + - type: mrr_at_5 + value: 49.033 + - type: ndcg_at_1 + value: 41.667 + - type: ndcg_at_10 + value: 41.144 + - type: ndcg_at_100 + value: 48.326 + - type: ndcg_at_1000 + value: 51.486 + - type: ndcg_at_3 + value: 37.486999999999995 + - type: ndcg_at_5 + value: 38.78 + - type: precision_at_1 + value: 41.667 + - type: precision_at_10 + value: 11.358 + - type: precision_at_100 + value: 1.873 + - type: precision_at_1000 + value: 0.244 + - type: precision_at_3 + value: 25 + - type: precision_at_5 + value: 18.519 + - type: recall_at_1 + value: 21.108 + - type: recall_at_10 + value: 47.249 + - type: recall_at_100 + value: 74.52 + - type: recall_at_1000 + value: 93.31 + - type: recall_at_3 + value: 33.271 + - type: recall_at_5 + value: 39.723000000000006 + - task: + type: Retrieval + dataset: + type: hotpotqa + name: MTEB HotpotQA + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 40.317 + - type: map_at_10 + value: 64.861 + - type: map_at_100 + value: 65.697 + - type: map_at_1000 + value: 65.755 + - type: map_at_3 + value: 61.258 + - type: map_at_5 + value: 63.590999999999994 + - type: mrr_at_1 + value: 80.635 + - type: mrr_at_10 + value: 86.528 + - type: mrr_at_100 + value: 86.66199999999999 + - type: mrr_at_1000 + value: 86.666 + - type: mrr_at_3 + value: 85.744 + - type: mrr_at_5 + value: 86.24300000000001 + - type: ndcg_at_1 + value: 80.635 + - type: ndcg_at_10 + value: 73.13199999999999 + - type: ndcg_at_100 + value: 75.927 + - type: ndcg_at_1000 + value: 76.976 + - type: ndcg_at_3 + value: 68.241 + - type: ndcg_at_5 + value: 71.071 + - type: precision_at_1 + value: 80.635 + - type: precision_at_10 + value: 15.326 + - type: precision_at_100 + value: 1.7500000000000002 + - type: precision_at_1000 + value: 0.189 + - type: precision_at_3 + value: 43.961 + - type: precision_at_5 + value: 28.599999999999998 + - type: recall_at_1 + value: 40.317 + - type: recall_at_10 + value: 76.631 + - type: recall_at_100 + value: 87.495 + - type: recall_at_1000 + value: 94.362 + - type: recall_at_3 + value: 65.94200000000001 + - type: recall_at_5 + value: 71.499 + - task: + type: Classification + dataset: + type: mteb/imdb + name: MTEB ImdbClassification + config: default + split: test + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + metrics: + - type: accuracy + value: 91.686 + - type: ap + value: 87.5577120393173 + - type: f1 + value: 91.6629447355139 + - task: + type: Retrieval + dataset: + type: msmarco + name: MTEB MSMARCO + config: default + split: dev + revision: None + metrics: + - type: map_at_1 + value: 23.702 + - type: map_at_10 + value: 36.414 + - type: map_at_100 + value: 37.561 + - type: map_at_1000 + value: 37.605 + - type: map_at_3 + value: 32.456 + - type: map_at_5 + value: 34.827000000000005 + - type: mrr_at_1 + value: 24.355 + - type: mrr_at_10 + value: 37.01 + - type: mrr_at_100 + value: 38.085 + - type: mrr_at_1000 + value: 38.123000000000005 + - type: mrr_at_3 + value: 33.117999999999995 + - type: mrr_at_5 + value: 35.452 + - type: ndcg_at_1 + value: 24.384 + - type: ndcg_at_10 + value: 43.456 + - type: ndcg_at_100 + value: 48.892 + - type: ndcg_at_1000 + value: 49.964 + - type: ndcg_at_3 + value: 35.475 + - type: ndcg_at_5 + value: 39.711 + - type: precision_at_1 + value: 24.384 + - type: precision_at_10 + value: 6.7940000000000005 + - type: precision_at_100 + value: 0.951 + - type: precision_at_1000 + value: 0.104 + - type: precision_at_3 + value: 15.052999999999999 + - type: precision_at_5 + value: 11.189 + - type: recall_at_1 + value: 23.702 + - type: recall_at_10 + value: 65.057 + - type: recall_at_100 + value: 90.021 + - type: recall_at_1000 + value: 98.142 + - type: recall_at_3 + value: 43.551 + - type: recall_at_5 + value: 53.738 + - task: + type: Classification + dataset: + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (en) + config: en + split: test + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + metrics: + - type: accuracy + value: 94.62380300957591 + - type: f1 + value: 94.49871222100734 + - task: + type: Classification + dataset: + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (en) + config: en + split: test + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + metrics: + - type: accuracy + value: 77.14090287277702 + - type: f1 + value: 60.32101258220515 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (en) + config: en + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 73.84330867518494 + - type: f1 + value: 71.92248688515255 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (en) + config: en + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 78.10692669804976 + - type: f1 + value: 77.9904839122866 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-p2p + name: MTEB MedrxivClusteringP2P + config: default + split: test + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + metrics: + - type: v_measure + value: 31.822988923078444 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-s2s + name: MTEB MedrxivClusteringS2S + config: default + split: test + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + metrics: + - type: v_measure + value: 30.38394880253403 + - task: + type: Reranking + dataset: + type: mteb/mind_small + name: MTEB MindSmallReranking + config: default + split: test + revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 + metrics: + - type: map + value: 31.82504612539082 + - type: mrr + value: 32.84462298174977 + - task: + type: Retrieval + dataset: + type: nfcorpus + name: MTEB NFCorpus + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 6.029 + - type: map_at_10 + value: 14.088999999999999 + - type: map_at_100 + value: 17.601 + - type: map_at_1000 + value: 19.144 + - type: map_at_3 + value: 10.156 + - type: map_at_5 + value: 11.892 + - type: mrr_at_1 + value: 46.44 + - type: mrr_at_10 + value: 56.596999999999994 + - type: mrr_at_100 + value: 57.11000000000001 + - type: mrr_at_1000 + value: 57.14 + - type: mrr_at_3 + value: 54.334 + - type: mrr_at_5 + value: 55.774 + - type: ndcg_at_1 + value: 44.891999999999996 + - type: ndcg_at_10 + value: 37.134 + - type: ndcg_at_100 + value: 33.652 + - type: ndcg_at_1000 + value: 42.548 + - type: ndcg_at_3 + value: 41.851 + - type: ndcg_at_5 + value: 39.842 + - type: precision_at_1 + value: 46.44 + - type: precision_at_10 + value: 27.647 + - type: precision_at_100 + value: 8.309999999999999 + - type: precision_at_1000 + value: 2.146 + - type: precision_at_3 + value: 39.422000000000004 + - type: precision_at_5 + value: 34.675 + - type: recall_at_1 + value: 6.029 + - type: recall_at_10 + value: 18.907 + - type: recall_at_100 + value: 33.76 + - type: recall_at_1000 + value: 65.14999999999999 + - type: recall_at_3 + value: 11.584999999999999 + - type: recall_at_5 + value: 14.626 + - task: + type: Retrieval + dataset: + type: nq + name: MTEB NQ + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 39.373000000000005 + - type: map_at_10 + value: 55.836 + - type: map_at_100 + value: 56.611999999999995 + - type: map_at_1000 + value: 56.63 + - type: map_at_3 + value: 51.747 + - type: map_at_5 + value: 54.337999999999994 + - type: mrr_at_1 + value: 44.147999999999996 + - type: mrr_at_10 + value: 58.42699999999999 + - type: mrr_at_100 + value: 58.902 + - type: mrr_at_1000 + value: 58.914 + - type: mrr_at_3 + value: 55.156000000000006 + - type: mrr_at_5 + value: 57.291000000000004 + - type: ndcg_at_1 + value: 44.119 + - type: ndcg_at_10 + value: 63.444 + - type: ndcg_at_100 + value: 66.40599999999999 + - type: ndcg_at_1000 + value: 66.822 + - type: ndcg_at_3 + value: 55.962 + - type: ndcg_at_5 + value: 60.228 + - type: precision_at_1 + value: 44.119 + - type: precision_at_10 + value: 10.006 + - type: precision_at_100 + value: 1.17 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_3 + value: 25.135 + - type: precision_at_5 + value: 17.59 + - type: recall_at_1 + value: 39.373000000000005 + - type: recall_at_10 + value: 83.78999999999999 + - type: recall_at_100 + value: 96.246 + - type: recall_at_1000 + value: 99.324 + - type: recall_at_3 + value: 64.71900000000001 + - type: recall_at_5 + value: 74.508 + - task: + type: Retrieval + dataset: + type: quora + name: MTEB QuoraRetrieval + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 69.199 + - type: map_at_10 + value: 82.892 + - type: map_at_100 + value: 83.578 + - type: map_at_1000 + value: 83.598 + - type: map_at_3 + value: 79.948 + - type: map_at_5 + value: 81.779 + - type: mrr_at_1 + value: 79.67 + - type: mrr_at_10 + value: 86.115 + - type: mrr_at_100 + value: 86.249 + - type: mrr_at_1000 + value: 86.251 + - type: mrr_at_3 + value: 85.08200000000001 + - type: mrr_at_5 + value: 85.783 + - type: ndcg_at_1 + value: 79.67 + - type: ndcg_at_10 + value: 86.839 + - type: ndcg_at_100 + value: 88.252 + - type: ndcg_at_1000 + value: 88.401 + - type: ndcg_at_3 + value: 83.86200000000001 + - type: ndcg_at_5 + value: 85.473 + - type: precision_at_1 + value: 79.67 + - type: precision_at_10 + value: 13.19 + - type: precision_at_100 + value: 1.521 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_3 + value: 36.677 + - type: precision_at_5 + value: 24.118000000000002 + - type: recall_at_1 + value: 69.199 + - type: recall_at_10 + value: 94.321 + - type: recall_at_100 + value: 99.20400000000001 + - type: recall_at_1000 + value: 99.947 + - type: recall_at_3 + value: 85.787 + - type: recall_at_5 + value: 90.365 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering + name: MTEB RedditClustering + config: default + split: test + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + metrics: + - type: v_measure + value: 55.82810046856353 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering-p2p + name: MTEB RedditClusteringP2P + config: default + split: test + revision: 282350215ef01743dc01b456c7f5241fa8937f16 + metrics: + - type: v_measure + value: 63.38132611783628 + - task: + type: Retrieval + dataset: + type: scidocs + name: MTEB SCIDOCS + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 5.127000000000001 + - type: map_at_10 + value: 12.235 + - type: map_at_100 + value: 14.417 + - type: map_at_1000 + value: 14.75 + - type: map_at_3 + value: 8.906 + - type: map_at_5 + value: 10.591000000000001 + - type: mrr_at_1 + value: 25.2 + - type: mrr_at_10 + value: 35.879 + - type: mrr_at_100 + value: 36.935 + - type: mrr_at_1000 + value: 36.997 + - type: mrr_at_3 + value: 32.783 + - type: mrr_at_5 + value: 34.367999999999995 + - type: ndcg_at_1 + value: 25.2 + - type: ndcg_at_10 + value: 20.509 + - type: ndcg_at_100 + value: 28.67 + - type: ndcg_at_1000 + value: 34.42 + - type: ndcg_at_3 + value: 19.948 + - type: ndcg_at_5 + value: 17.166 + - type: precision_at_1 + value: 25.2 + - type: precision_at_10 + value: 10.440000000000001 + - type: precision_at_100 + value: 2.214 + - type: precision_at_1000 + value: 0.359 + - type: precision_at_3 + value: 18.533 + - type: precision_at_5 + value: 14.860000000000001 + - type: recall_at_1 + value: 5.127000000000001 + - type: recall_at_10 + value: 21.147 + - type: recall_at_100 + value: 44.946999999999996 + - type: recall_at_1000 + value: 72.89 + - type: recall_at_3 + value: 11.277 + - type: recall_at_5 + value: 15.042 + - task: + type: STS + dataset: + type: mteb/sickr-sts + name: MTEB SICK-R + config: default + split: test + revision: a6ea5a8cab320b040a23452cc28066d9beae2cee + metrics: + - type: cos_sim_pearson + value: 83.0373011786213 + - type: cos_sim_spearman + value: 79.27889560856613 + - type: euclidean_pearson + value: 80.31186315495655 + - type: euclidean_spearman + value: 79.41630415280811 + - type: manhattan_pearson + value: 80.31755140442013 + - type: manhattan_spearman + value: 79.43069870027611 + - task: + type: STS + dataset: + type: mteb/sts12-sts + name: MTEB STS12 + config: default + split: test + revision: a0d554a64d88156834ff5ae9920b964011b16384 + metrics: + - type: cos_sim_pearson + value: 84.8659751342045 + - type: cos_sim_spearman + value: 76.95377612997667 + - type: euclidean_pearson + value: 81.24552945497848 + - type: euclidean_spearman + value: 77.18236963555253 + - type: manhattan_pearson + value: 81.26477607759037 + - type: manhattan_spearman + value: 77.13821753062756 + - task: + type: STS + dataset: + type: mteb/sts13-sts + name: MTEB STS13 + config: default + split: test + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + metrics: + - type: cos_sim_pearson + value: 83.34597139044875 + - type: cos_sim_spearman + value: 84.124169425592 + - type: euclidean_pearson + value: 83.68590721511401 + - type: euclidean_spearman + value: 84.18846190846398 + - type: manhattan_pearson + value: 83.57630235061498 + - type: manhattan_spearman + value: 84.10244043726902 + - task: + type: STS + dataset: + type: mteb/sts14-sts + name: MTEB STS14 + config: default + split: test + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + metrics: + - type: cos_sim_pearson + value: 82.67641885599572 + - type: cos_sim_spearman + value: 80.46450725650428 + - type: euclidean_pearson + value: 81.61645042715865 + - type: euclidean_spearman + value: 80.61418394236874 + - type: manhattan_pearson + value: 81.55712034928871 + - type: manhattan_spearman + value: 80.57905670523951 + - task: + type: STS + dataset: + type: mteb/sts15-sts + name: MTEB STS15 + config: default + split: test + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + metrics: + - type: cos_sim_pearson + value: 88.86650310886782 + - type: cos_sim_spearman + value: 89.76081629222328 + - type: euclidean_pearson + value: 89.1530747029954 + - type: euclidean_spearman + value: 89.80990657280248 + - type: manhattan_pearson + value: 89.10640563278132 + - type: manhattan_spearman + value: 89.76282108434047 + - task: + type: STS + dataset: + type: mteb/sts16-sts + name: MTEB STS16 + config: default + split: test + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + metrics: + - type: cos_sim_pearson + value: 83.93864027911118 + - type: cos_sim_spearman + value: 85.47096193999023 + - type: euclidean_pearson + value: 85.03141840870533 + - type: euclidean_spearman + value: 85.43124029598181 + - type: manhattan_pearson + value: 84.99002664393512 + - type: manhattan_spearman + value: 85.39169195120834 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-en) + config: en-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 88.7045343749832 + - type: cos_sim_spearman + value: 89.03262221146677 + - type: euclidean_pearson + value: 89.56078218264365 + - type: euclidean_spearman + value: 89.17827006466868 + - type: manhattan_pearson + value: 89.52717595468582 + - type: manhattan_spearman + value: 89.15878115952923 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (en) + config: en + split: test + revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 + metrics: + - type: cos_sim_pearson + value: 64.20191302875551 + - type: cos_sim_spearman + value: 64.11446552557646 + - type: euclidean_pearson + value: 64.6918197393619 + - type: euclidean_spearman + value: 63.440182631197764 + - type: manhattan_pearson + value: 64.55692904121835 + - type: manhattan_spearman + value: 63.424877742756266 + - task: + type: STS + dataset: + type: mteb/stsbenchmark-sts + name: MTEB STSBenchmark + config: default + split: test + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + metrics: + - type: cos_sim_pearson + value: 86.37793104662344 + - type: cos_sim_spearman + value: 87.7357802629067 + - type: euclidean_pearson + value: 87.4286301545109 + - type: euclidean_spearman + value: 87.78452920777421 + - type: manhattan_pearson + value: 87.42445169331255 + - type: manhattan_spearman + value: 87.78537677249598 + - task: + type: Reranking + dataset: + type: mteb/scidocs-reranking + name: MTEB SciDocsRR + config: default + split: test + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + metrics: + - type: map + value: 84.31465405081792 + - type: mrr + value: 95.7173781193389 + - task: + type: Retrieval + dataset: + type: scifact + name: MTEB SciFact + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 57.760999999999996 + - type: map_at_10 + value: 67.904 + - type: map_at_100 + value: 68.539 + - type: map_at_1000 + value: 68.562 + - type: map_at_3 + value: 65.415 + - type: map_at_5 + value: 66.788 + - type: mrr_at_1 + value: 60.333000000000006 + - type: mrr_at_10 + value: 68.797 + - type: mrr_at_100 + value: 69.236 + - type: mrr_at_1000 + value: 69.257 + - type: mrr_at_3 + value: 66.667 + - type: mrr_at_5 + value: 67.967 + - type: ndcg_at_1 + value: 60.333000000000006 + - type: ndcg_at_10 + value: 72.24199999999999 + - type: ndcg_at_100 + value: 74.86 + - type: ndcg_at_1000 + value: 75.354 + - type: ndcg_at_3 + value: 67.93400000000001 + - type: ndcg_at_5 + value: 70.02199999999999 + - type: precision_at_1 + value: 60.333000000000006 + - type: precision_at_10 + value: 9.533 + - type: precision_at_100 + value: 1.09 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_3 + value: 26.778000000000002 + - type: precision_at_5 + value: 17.467 + - type: recall_at_1 + value: 57.760999999999996 + - type: recall_at_10 + value: 84.383 + - type: recall_at_100 + value: 96.267 + - type: recall_at_1000 + value: 100 + - type: recall_at_3 + value: 72.628 + - type: recall_at_5 + value: 78.094 + - task: + type: PairClassification + dataset: + type: mteb/sprintduplicatequestions-pairclassification + name: MTEB SprintDuplicateQuestions + config: default + split: test + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + metrics: + - type: cos_sim_accuracy + value: 99.8029702970297 + - type: cos_sim_ap + value: 94.9210324173411 + - type: cos_sim_f1 + value: 89.8521162672106 + - type: cos_sim_precision + value: 91.67533818938605 + - type: cos_sim_recall + value: 88.1 + - type: dot_accuracy + value: 99.69504950495049 + - type: dot_ap + value: 90.4919719146181 + - type: dot_f1 + value: 84.72289156626506 + - type: dot_precision + value: 81.76744186046511 + - type: dot_recall + value: 87.9 + - type: euclidean_accuracy + value: 99.79702970297029 + - type: euclidean_ap + value: 94.87827463795753 + - type: euclidean_f1 + value: 89.55680081507896 + - type: euclidean_precision + value: 91.27725856697819 + - type: euclidean_recall + value: 87.9 + - type: manhattan_accuracy + value: 99.7990099009901 + - type: manhattan_ap + value: 94.87587025149682 + - type: manhattan_f1 + value: 89.76298537569339 + - type: manhattan_precision + value: 90.53916581892166 + - type: manhattan_recall + value: 89 + - type: max_accuracy + value: 99.8029702970297 + - type: max_ap + value: 94.9210324173411 + - type: max_f1 + value: 89.8521162672106 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering + name: MTEB StackExchangeClustering + config: default + split: test + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + metrics: + - type: v_measure + value: 65.92385753948724 + - task: + type: Clustering + dataset: + type: mteb/stackexchange-clustering-p2p + name: MTEB StackExchangeClusteringP2P + config: default + split: test + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + metrics: + - type: v_measure + value: 33.671756975431144 + - task: + type: Reranking + dataset: + type: mteb/stackoverflowdupquestions-reranking + name: MTEB StackOverflowDupQuestions + config: default + split: test + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + metrics: + - type: map + value: 50.677928036739004 + - type: mrr + value: 51.56413133435193 + - task: + type: Summarization + dataset: + type: mteb/summeval + name: MTEB SummEval + config: default + split: test + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + metrics: + - type: cos_sim_pearson + value: 30.523589340819683 + - type: cos_sim_spearman + value: 30.187407518823235 + - type: dot_pearson + value: 29.039713969699015 + - type: dot_spearman + value: 29.114740651155508 + - task: + type: Retrieval + dataset: + type: trec-covid + name: MTEB TRECCOVID + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 0.211 + - type: map_at_10 + value: 1.6199999999999999 + - type: map_at_100 + value: 8.658000000000001 + - type: map_at_1000 + value: 21.538 + - type: map_at_3 + value: 0.575 + - type: map_at_5 + value: 0.919 + - type: mrr_at_1 + value: 78 + - type: mrr_at_10 + value: 86.18599999999999 + - type: mrr_at_100 + value: 86.18599999999999 + - type: mrr_at_1000 + value: 86.18599999999999 + - type: mrr_at_3 + value: 85 + - type: mrr_at_5 + value: 85.9 + - type: ndcg_at_1 + value: 74 + - type: ndcg_at_10 + value: 66.542 + - type: ndcg_at_100 + value: 50.163999999999994 + - type: ndcg_at_1000 + value: 45.696999999999996 + - type: ndcg_at_3 + value: 71.531 + - type: ndcg_at_5 + value: 70.45 + - type: precision_at_1 + value: 78 + - type: precision_at_10 + value: 69.39999999999999 + - type: precision_at_100 + value: 51.06 + - type: precision_at_1000 + value: 20.022000000000002 + - type: precision_at_3 + value: 76 + - type: precision_at_5 + value: 74.8 + - type: recall_at_1 + value: 0.211 + - type: recall_at_10 + value: 1.813 + - type: recall_at_100 + value: 12.098 + - type: recall_at_1000 + value: 42.618 + - type: recall_at_3 + value: 0.603 + - type: recall_at_5 + value: 0.987 + - task: + type: Retrieval + dataset: + type: webis-touche2020 + name: MTEB Touche2020 + config: default + split: test + revision: None + metrics: + - type: map_at_1 + value: 2.2079999999999997 + - type: map_at_10 + value: 7.777000000000001 + - type: map_at_100 + value: 12.825000000000001 + - type: map_at_1000 + value: 14.196 + - type: map_at_3 + value: 4.285 + - type: map_at_5 + value: 6.177 + - type: mrr_at_1 + value: 30.612000000000002 + - type: mrr_at_10 + value: 42.635 + - type: mrr_at_100 + value: 43.955 + - type: mrr_at_1000 + value: 43.955 + - type: mrr_at_3 + value: 38.435 + - type: mrr_at_5 + value: 41.088 + - type: ndcg_at_1 + value: 28.571 + - type: ndcg_at_10 + value: 20.666999999999998 + - type: ndcg_at_100 + value: 31.840000000000003 + - type: ndcg_at_1000 + value: 43.191 + - type: ndcg_at_3 + value: 23.45 + - type: ndcg_at_5 + value: 22.994 + - type: precision_at_1 + value: 30.612000000000002 + - type: precision_at_10 + value: 17.959 + - type: precision_at_100 + value: 6.755 + - type: precision_at_1000 + value: 1.4200000000000002 + - type: precision_at_3 + value: 23.810000000000002 + - type: precision_at_5 + value: 23.673 + - type: recall_at_1 + value: 2.2079999999999997 + - type: recall_at_10 + value: 13.144 + - type: recall_at_100 + value: 42.491 + - type: recall_at_1000 + value: 77.04299999999999 + - type: recall_at_3 + value: 5.3469999999999995 + - type: recall_at_5 + value: 9.139 + - task: + type: Classification + dataset: + type: mteb/toxic_conversations_50k + name: MTEB ToxicConversationsClassification + config: default + split: test + revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c + metrics: + - type: accuracy + value: 70.9044 + - type: ap + value: 14.625783489340755 + - type: f1 + value: 54.814936562590546 + - task: + type: Classification + dataset: + type: mteb/tweet_sentiment_extraction + name: MTEB TweetSentimentExtractionClassification + config: default + split: test + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + metrics: + - type: accuracy + value: 60.94227504244483 + - type: f1 + value: 61.22516038508854 + - task: + type: Clustering + dataset: + type: mteb/twentynewsgroups-clustering + name: MTEB TwentyNewsgroupsClustering + config: default + split: test + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + metrics: + - type: v_measure + value: 49.602409155145864 + - task: + type: PairClassification + dataset: + type: mteb/twittersemeval2015-pairclassification + name: MTEB TwitterSemEval2015 + config: default + split: test + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + metrics: + - type: cos_sim_accuracy + value: 86.94641473445789 + - type: cos_sim_ap + value: 76.91572747061197 + - type: cos_sim_f1 + value: 70.14348097317529 + - type: cos_sim_precision + value: 66.53254437869822 + - type: cos_sim_recall + value: 74.1688654353562 + - type: dot_accuracy + value: 84.80061989628658 + - type: dot_ap + value: 70.7952548895177 + - type: dot_f1 + value: 65.44780728844965 + - type: dot_precision + value: 61.53310104529617 + - type: dot_recall + value: 69.89445910290237 + - type: euclidean_accuracy + value: 86.94641473445789 + - type: euclidean_ap + value: 76.80774009393652 + - type: euclidean_f1 + value: 70.30522503879979 + - type: euclidean_precision + value: 68.94977168949772 + - type: euclidean_recall + value: 71.71503957783642 + - type: manhattan_accuracy + value: 86.8629671574179 + - type: manhattan_ap + value: 76.76518632600317 + - type: manhattan_f1 + value: 70.16056518946692 + - type: manhattan_precision + value: 68.360450563204 + - type: manhattan_recall + value: 72.0580474934037 + - type: max_accuracy + value: 86.94641473445789 + - type: max_ap + value: 76.91572747061197 + - type: max_f1 + value: 70.30522503879979 + - task: + type: PairClassification + dataset: + type: mteb/twitterurlcorpus-pairclassification + name: MTEB TwitterURLCorpus + config: default + split: test + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + metrics: + - type: cos_sim_accuracy + value: 89.10428066907285 + - type: cos_sim_ap + value: 86.25114759921435 + - type: cos_sim_f1 + value: 78.37857884586856 + - type: cos_sim_precision + value: 75.60818546078993 + - type: cos_sim_recall + value: 81.35971666153372 + - type: dot_accuracy + value: 87.41995575736406 + - type: dot_ap + value: 81.51838010086782 + - type: dot_f1 + value: 74.77398015435503 + - type: dot_precision + value: 71.53002390662354 + - type: dot_recall + value: 78.32614721281182 + - type: euclidean_accuracy + value: 89.12368533395428 + - type: euclidean_ap + value: 86.33456799874504 + - type: euclidean_f1 + value: 78.45496750232127 + - type: euclidean_precision + value: 75.78388462366364 + - type: euclidean_recall + value: 81.32121958731136 + - type: manhattan_accuracy + value: 89.10622113556099 + - type: manhattan_ap + value: 86.31215061745333 + - type: manhattan_f1 + value: 78.40684906011539 + - type: manhattan_precision + value: 75.89536643366722 + - type: manhattan_recall + value: 81.09023714197721 + - type: max_accuracy + value: 89.12368533395428 + - type: max_ap + value: 86.33456799874504 + - type: max_f1 + value: 78.45496750232127 +language: +- en +license: mit +--- + +# E5-large-v2 + +[Text Embeddings by Weakly-Supervised Contrastive Pre-training](https://arxiv.org/pdf/2212.03533.pdf). +Liang Wang, Nan Yang, Xiaolong Huang, Binxing Jiao, Linjun Yang, Daxin Jiang, Rangan Majumder, Furu Wei, arXiv 2022 + +This model has 24 layers and the embedding size is 1024. + +## Usage + +Below is an example to encode queries and passages from the MS-MARCO passage ranking dataset. + +```python +import torch.nn.functional as F + +from torch import Tensor +from transformers import AutoTokenizer, AutoModel + + +def average_pool(last_hidden_states: Tensor, + attention_mask: Tensor) -> Tensor: + last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0) + return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + + +# Each input text should start with "query: " or "passage: ". +# For tasks other than retrieval, you can simply use the "query: " prefix. +input_texts = ['query: how much protein should a female eat', + 'query: summit define', + "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", + "passage: Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."] + +tokenizer = AutoTokenizer.from_pretrained('intfloat/e5-large-v2') +model = AutoModel.from_pretrained('intfloat/e5-large-v2') + +# Tokenize the input texts +batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt') + +outputs = model(**batch_dict) +embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask']) + +# normalize embeddings +embeddings = F.normalize(embeddings, p=2, dim=1) +scores = (embeddings[:2] @ embeddings[2:].T) * 100 +print(scores.tolist()) +``` + +## Training Details + +Please refer to our paper at [https://arxiv.org/pdf/2212.03533.pdf](https://arxiv.org/pdf/2212.03533.pdf). + +## Benchmark Evaluation + +Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results +on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB benchmark](https://arxiv.org/abs/2210.07316). + +## Support for Sentence Transformers + +Below is an example for usage with sentence_transformers. +```python +from sentence_transformers import SentenceTransformer +model = SentenceTransformer('intfloat/e5-large-v2') +input_texts = [ + 'query: how much protein should a female eat', + 'query: summit define', + "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", + "passage: Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments." +] +embeddings = model.encode(input_texts, normalize_embeddings=True) +``` + +Package requirements + +`pip install sentence_transformers~=2.2.2` + +Contributors: [michaelfeil](https://huggingface.co/michaelfeil) + +## FAQ + +**1. Do I need to add the prefix "query: " and "passage: " to input texts?** + +Yes, this is how the model is trained, otherwise you will see a performance degradation. + +Here are some rules of thumb: +- Use "query: " and "passage: " correspondingly for asymmetric tasks such as passage retrieval in open QA, ad-hoc information retrieval. + +- Use "query: " prefix for symmetric tasks such as semantic similarity, paraphrase retrieval. + +- Use "query: " prefix if you want to use embeddings as features, such as linear probing classification, clustering. + +**2. Why are my reproduced results slightly different from reported in the model card?** + +Different versions of `transformers` and `pytorch` could cause negligible but non-zero performance differences. + +**3. Why does the cosine similarity scores distribute around 0.7 to 1.0?** + +This is a known and expected behavior as we use a low temperature 0.01 for InfoNCE contrastive loss. + +For text embedding tasks like text retrieval or semantic similarity, +what matters is the relative order of the scores instead of the absolute values, +so this should not be an issue. + +## Citation + +If you find our paper or models helpful, please consider cite as follows: + +``` +@article{wang2022text, + title={Text Embeddings by Weakly-Supervised Contrastive Pre-training}, + author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Jiao, Binxing and Yang, Linjun and Jiang, Daxin and Majumder, Rangan and Wei, Furu}, + journal={arXiv preprint arXiv:2212.03533}, + year={2022} +} +``` + +## Limitations + +This model only works for English texts. Long texts will be truncated to at most 512 tokens.