|
--- |
|
model-index: |
|
- name: XYZ-embedding-zh-v2 |
|
results: |
|
- dataset: |
|
config: default |
|
name: MTEB AFQMC |
|
revision: None |
|
split: validation |
|
type: C-MTEB/AFQMC |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 55.51799059309076 |
|
- type: cos_sim_spearman |
|
value: 58.407433584137806 |
|
- type: manhattan_pearson |
|
value: 57.17473672145622 |
|
- type: manhattan_spearman |
|
value: 58.389018054159955 |
|
- type: euclidean_pearson |
|
value: 57.19483956761451 |
|
- type: euclidean_spearman |
|
value: 58.407433584137806 |
|
- type: main_score |
|
value: 58.407433584137806 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB ATEC |
|
revision: None |
|
split: test |
|
type: C-MTEB/ATEC |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 57.31078155367183 |
|
- type: cos_sim_spearman |
|
value: 57.59782762324478 |
|
- type: manhattan_pearson |
|
value: 62.525487007985035 |
|
- type: manhattan_spearman |
|
value: 57.591139966303615 |
|
- type: euclidean_pearson |
|
value: 62.53702437760052 |
|
- type: euclidean_spearman |
|
value: 57.597828749091384 |
|
- type: main_score |
|
value: 57.59782762324478 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: zh |
|
name: MTEB AmazonReviewsClassification (zh) |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
split: test |
|
type: mteb/amazon_reviews_multi |
|
metrics: |
|
- type: accuracy |
|
value: 49.374 |
|
- type: accuracy_stderr |
|
value: 1.436636349254743 |
|
- type: f1 |
|
value: 47.115240601017774 |
|
- type: f1_stderr |
|
value: 1.5642799356594534 |
|
- type: main_score |
|
value: 49.374 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB BQ |
|
revision: None |
|
split: test |
|
type: C-MTEB/BQ |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 71.49514309404829 |
|
- type: cos_sim_spearman |
|
value: 72.66161713021279 |
|
- type: manhattan_pearson |
|
value: 71.03443640254005 |
|
- type: manhattan_spearman |
|
value: 72.63439621980275 |
|
- type: euclidean_pearson |
|
value: 71.06830370642658 |
|
- type: euclidean_spearman |
|
value: 72.66161713043078 |
|
- type: main_score |
|
value: 72.66161713021279 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB CLSClusteringP2P |
|
revision: None |
|
split: test |
|
type: C-MTEB/CLSClusteringP2P |
|
metrics: |
|
- type: v_measure |
|
value: 57.237692641281 |
|
- type: v_measure_std |
|
value: 1.2777768354339174 |
|
- type: main_score |
|
value: 57.237692641281 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB CLSClusteringS2S |
|
revision: None |
|
split: test |
|
type: C-MTEB/CLSClusteringS2S |
|
metrics: |
|
- type: v_measure |
|
value: 48.41686666939331 |
|
- type: v_measure_std |
|
value: 1.7663118461900793 |
|
- type: main_score |
|
value: 48.41686666939331 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB CMedQAv1 |
|
revision: None |
|
split: test |
|
type: C-MTEB/CMedQAv1-reranking |
|
metrics: |
|
- type: map |
|
value: 89.9766367822762 |
|
- type: mrr |
|
value: 91.88896825396824 |
|
- type: main_score |
|
value: 89.9766367822762 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB CMedQAv2 |
|
revision: None |
|
split: test |
|
type: C-MTEB/CMedQAv2-reranking |
|
metrics: |
|
- type: map |
|
value: 89.04628340075982 |
|
- type: mrr |
|
value: 91.21702380952381 |
|
- type: main_score |
|
value: 89.04628340075982 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB CmedqaRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/CmedqaRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.796 |
|
- type: map_at_10 |
|
value: 41.498000000000005 |
|
- type: map_at_100 |
|
value: 43.332 |
|
- type: map_at_1000 |
|
value: 43.429 |
|
- type: map_at_3 |
|
value: 37.172 |
|
- type: map_at_5 |
|
value: 39.617000000000004 |
|
- type: mrr_at_1 |
|
value: 42.111 |
|
- type: mrr_at_10 |
|
value: 50.726000000000006 |
|
- type: mrr_at_100 |
|
value: 51.632 |
|
- type: mrr_at_1000 |
|
value: 51.67 |
|
- type: mrr_at_3 |
|
value: 48.429 |
|
- type: mrr_at_5 |
|
value: 49.662 |
|
- type: ndcg_at_1 |
|
value: 42.111 |
|
- type: ndcg_at_10 |
|
value: 48.294 |
|
- type: ndcg_at_100 |
|
value: 55.135999999999996 |
|
- type: ndcg_at_1000 |
|
value: 56.818000000000005 |
|
- type: ndcg_at_3 |
|
value: 43.185 |
|
- type: ndcg_at_5 |
|
value: 45.266 |
|
- type: precision_at_1 |
|
value: 42.111 |
|
- type: precision_at_10 |
|
value: 10.635 |
|
- type: precision_at_100 |
|
value: 1.619 |
|
- type: precision_at_1000 |
|
value: 0.183 |
|
- type: precision_at_3 |
|
value: 24.539 |
|
- type: precision_at_5 |
|
value: 17.644000000000002 |
|
- type: recall_at_1 |
|
value: 27.796 |
|
- type: recall_at_10 |
|
value: 59.034 |
|
- type: recall_at_100 |
|
value: 86.991 |
|
- type: recall_at_1000 |
|
value: 98.304 |
|
- type: recall_at_3 |
|
value: 43.356 |
|
- type: recall_at_5 |
|
value: 49.998 |
|
- type: main_score |
|
value: 48.294 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB Cmnli |
|
revision: None |
|
split: validation |
|
type: C-MTEB/CMNLI |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 82.8983764281419 |
|
- type: cos_sim_accuracy_threshold |
|
value: 56.05731010437012 |
|
- type: cos_sim_ap |
|
value: 90.23156362696572 |
|
- type: cos_sim_f1 |
|
value: 83.83207278307574 |
|
- type: cos_sim_f1_threshold |
|
value: 52.05453634262085 |
|
- type: cos_sim_precision |
|
value: 78.91044160132068 |
|
- type: cos_sim_recall |
|
value: 89.40846387654898 |
|
- type: dot_accuracy |
|
value: 82.8983764281419 |
|
- type: dot_accuracy_threshold |
|
value: 56.05730414390564 |
|
- type: dot_ap |
|
value: 90.20952356258861 |
|
- type: dot_f1 |
|
value: 83.83207278307574 |
|
- type: dot_f1_threshold |
|
value: 52.054524421691895 |
|
- type: dot_precision |
|
value: 78.91044160132068 |
|
- type: dot_recall |
|
value: 89.40846387654898 |
|
- type: euclidean_accuracy |
|
value: 82.8983764281419 |
|
- type: euclidean_accuracy_threshold |
|
value: 93.74719858169556 |
|
- type: euclidean_ap |
|
value: 90.23156283510565 |
|
- type: euclidean_f1 |
|
value: 83.83207278307574 |
|
- type: euclidean_f1_threshold |
|
value: 97.92392253875732 |
|
- type: euclidean_precision |
|
value: 78.91044160132068 |
|
- type: euclidean_recall |
|
value: 89.40846387654898 |
|
- type: manhattan_accuracy |
|
value: 82.85027059530968 |
|
- type: manhattan_accuracy_threshold |
|
value: 3164.584159851074 |
|
- type: manhattan_ap |
|
value: 90.23178004516869 |
|
- type: manhattan_f1 |
|
value: 83.82157123834887 |
|
- type: manhattan_f1_threshold |
|
value: 3273.5992431640625 |
|
- type: manhattan_precision |
|
value: 79.76768743400211 |
|
- type: manhattan_recall |
|
value: 88.30956277764788 |
|
- type: max_accuracy |
|
value: 82.8983764281419 |
|
- type: max_ap |
|
value: 90.23178004516869 |
|
- type: max_f1 |
|
value: 83.83207278307574 |
|
task: |
|
type: PairClassification |
|
- dataset: |
|
config: default |
|
name: MTEB CovidRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/CovidRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 80.479 |
|
- type: map_at_10 |
|
value: 87.984 |
|
- type: map_at_100 |
|
value: 88.036 |
|
- type: map_at_1000 |
|
value: 88.03699999999999 |
|
- type: map_at_3 |
|
value: 87.083 |
|
- type: map_at_5 |
|
value: 87.694 |
|
- type: mrr_at_1 |
|
value: 80.927 |
|
- type: mrr_at_10 |
|
value: 88.046 |
|
- type: mrr_at_100 |
|
value: 88.099 |
|
- type: mrr_at_1000 |
|
value: 88.1 |
|
- type: mrr_at_3 |
|
value: 87.215 |
|
- type: mrr_at_5 |
|
value: 87.768 |
|
- type: ndcg_at_1 |
|
value: 80.927 |
|
- type: ndcg_at_10 |
|
value: 90.756 |
|
- type: ndcg_at_100 |
|
value: 90.96 |
|
- type: ndcg_at_1000 |
|
value: 90.975 |
|
- type: ndcg_at_3 |
|
value: 89.032 |
|
- type: ndcg_at_5 |
|
value: 90.106 |
|
- type: precision_at_1 |
|
value: 80.927 |
|
- type: precision_at_10 |
|
value: 10.011000000000001 |
|
- type: precision_at_100 |
|
value: 1.009 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 31.752999999999997 |
|
- type: precision_at_5 |
|
value: 19.6 |
|
- type: recall_at_1 |
|
value: 80.479 |
|
- type: recall_at_10 |
|
value: 99.05199999999999 |
|
- type: recall_at_100 |
|
value: 99.895 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_3 |
|
value: 94.494 |
|
- type: recall_at_5 |
|
value: 97.102 |
|
- type: main_score |
|
value: 90.756 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB DuRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/DuRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.853 |
|
- type: map_at_10 |
|
value: 85.13199999999999 |
|
- type: map_at_100 |
|
value: 87.688 |
|
- type: map_at_1000 |
|
value: 87.712 |
|
- type: map_at_3 |
|
value: 59.705 |
|
- type: map_at_5 |
|
value: 75.139 |
|
- type: mrr_at_1 |
|
value: 93.65 |
|
- type: mrr_at_10 |
|
value: 95.682 |
|
- type: mrr_at_100 |
|
value: 95.722 |
|
- type: mrr_at_1000 |
|
value: 95.724 |
|
- type: mrr_at_3 |
|
value: 95.467 |
|
- type: mrr_at_5 |
|
value: 95.612 |
|
- type: ndcg_at_1 |
|
value: 93.65 |
|
- type: ndcg_at_10 |
|
value: 91.155 |
|
- type: ndcg_at_100 |
|
value: 93.183 |
|
- type: ndcg_at_1000 |
|
value: 93.38499999999999 |
|
- type: ndcg_at_3 |
|
value: 90.648 |
|
- type: ndcg_at_5 |
|
value: 89.47699999999999 |
|
- type: precision_at_1 |
|
value: 93.65 |
|
- type: precision_at_10 |
|
value: 43.11 |
|
- type: precision_at_100 |
|
value: 4.854 |
|
- type: precision_at_1000 |
|
value: 0.49100000000000005 |
|
- type: precision_at_3 |
|
value: 81.11699999999999 |
|
- type: precision_at_5 |
|
value: 68.28999999999999 |
|
- type: recall_at_1 |
|
value: 27.853 |
|
- type: recall_at_10 |
|
value: 91.678 |
|
- type: recall_at_100 |
|
value: 98.553 |
|
- type: recall_at_1000 |
|
value: 99.553 |
|
- type: recall_at_3 |
|
value: 61.381 |
|
- type: recall_at_5 |
|
value: 78.605 |
|
- type: main_score |
|
value: 91.155 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB EcomRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/EcomRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 54.50000000000001 |
|
- type: map_at_10 |
|
value: 65.167 |
|
- type: map_at_100 |
|
value: 65.664 |
|
- type: map_at_1000 |
|
value: 65.67399999999999 |
|
- type: map_at_3 |
|
value: 62.633 |
|
- type: map_at_5 |
|
value: 64.208 |
|
- type: mrr_at_1 |
|
value: 54.50000000000001 |
|
- type: mrr_at_10 |
|
value: 65.167 |
|
- type: mrr_at_100 |
|
value: 65.664 |
|
- type: mrr_at_1000 |
|
value: 65.67399999999999 |
|
- type: mrr_at_3 |
|
value: 62.633 |
|
- type: mrr_at_5 |
|
value: 64.208 |
|
- type: ndcg_at_1 |
|
value: 54.50000000000001 |
|
- type: ndcg_at_10 |
|
value: 70.294 |
|
- type: ndcg_at_100 |
|
value: 72.564 |
|
- type: ndcg_at_1000 |
|
value: 72.841 |
|
- type: ndcg_at_3 |
|
value: 65.128 |
|
- type: ndcg_at_5 |
|
value: 67.96799999999999 |
|
- type: precision_at_1 |
|
value: 54.50000000000001 |
|
- type: precision_at_10 |
|
value: 8.64 |
|
- type: precision_at_100 |
|
value: 0.967 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 24.099999999999998 |
|
- type: precision_at_5 |
|
value: 15.840000000000002 |
|
- type: recall_at_1 |
|
value: 54.50000000000001 |
|
- type: recall_at_10 |
|
value: 86.4 |
|
- type: recall_at_100 |
|
value: 96.7 |
|
- type: recall_at_1000 |
|
value: 98.9 |
|
- type: recall_at_3 |
|
value: 72.3 |
|
- type: recall_at_5 |
|
value: 79.2 |
|
- type: main_score |
|
value: 70.294 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB IFlyTek |
|
revision: None |
|
split: validation |
|
type: C-MTEB/IFlyTek-classification |
|
metrics: |
|
- type: accuracy |
|
value: 52.743362831858406 |
|
- type: accuracy_stderr |
|
value: 0.23768288128480788 |
|
- type: f1 |
|
value: 41.1548855278405 |
|
- type: f1_stderr |
|
value: 0.4088759842813554 |
|
- type: main_score |
|
value: 52.743362831858406 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB JDReview |
|
revision: None |
|
split: test |
|
type: C-MTEB/JDReview-classification |
|
metrics: |
|
- type: accuracy |
|
value: 89.08067542213884 |
|
- type: accuracy_stderr |
|
value: 0.9559278951487445 |
|
- type: ap |
|
value: 60.875320104586564 |
|
- type: ap_stderr |
|
value: 2.137806661565934 |
|
- type: f1 |
|
value: 84.39314192399665 |
|
- type: f1_stderr |
|
value: 1.132407155321657 |
|
- type: main_score |
|
value: 89.08067542213884 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB LCQMC |
|
revision: None |
|
split: test |
|
type: C-MTEB/LCQMC |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 73.3633875566899 |
|
- type: cos_sim_spearman |
|
value: 79.27679599527615 |
|
- type: manhattan_pearson |
|
value: 79.12061667088273 |
|
- type: manhattan_spearman |
|
value: 79.26989882781706 |
|
- type: euclidean_pearson |
|
value: 79.12871362068391 |
|
- type: euclidean_spearman |
|
value: 79.27679377557219 |
|
- type: main_score |
|
value: 79.27679599527615 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB MMarcoReranking |
|
revision: None |
|
split: dev |
|
type: C-MTEB/Mmarco-reranking |
|
metrics: |
|
- type: map |
|
value: 37.68251937316638 |
|
- type: mrr |
|
value: 36.61746031746032 |
|
- type: main_score |
|
value: 37.68251937316638 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB MMarcoRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/MMarcoRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 69.401 |
|
- type: map_at_10 |
|
value: 78.8 |
|
- type: map_at_100 |
|
value: 79.077 |
|
- type: map_at_1000 |
|
value: 79.081 |
|
- type: map_at_3 |
|
value: 76.97 |
|
- type: map_at_5 |
|
value: 78.185 |
|
- type: mrr_at_1 |
|
value: 71.719 |
|
- type: mrr_at_10 |
|
value: 79.327 |
|
- type: mrr_at_100 |
|
value: 79.56400000000001 |
|
- type: mrr_at_1000 |
|
value: 79.56800000000001 |
|
- type: mrr_at_3 |
|
value: 77.736 |
|
- type: mrr_at_5 |
|
value: 78.782 |
|
- type: ndcg_at_1 |
|
value: 71.719 |
|
- type: ndcg_at_10 |
|
value: 82.505 |
|
- type: ndcg_at_100 |
|
value: 83.673 |
|
- type: ndcg_at_1000 |
|
value: 83.786 |
|
- type: ndcg_at_3 |
|
value: 79.07600000000001 |
|
- type: ndcg_at_5 |
|
value: 81.122 |
|
- type: precision_at_1 |
|
value: 71.719 |
|
- type: precision_at_10 |
|
value: 9.924 |
|
- type: precision_at_100 |
|
value: 1.049 |
|
- type: precision_at_1000 |
|
value: 0.106 |
|
- type: precision_at_3 |
|
value: 29.742 |
|
- type: precision_at_5 |
|
value: 18.937 |
|
- type: recall_at_1 |
|
value: 69.401 |
|
- type: recall_at_10 |
|
value: 93.349 |
|
- type: recall_at_100 |
|
value: 98.492 |
|
- type: recall_at_1000 |
|
value: 99.384 |
|
- type: recall_at_3 |
|
value: 84.385 |
|
- type: recall_at_5 |
|
value: 89.237 |
|
- type: main_score |
|
value: 82.505 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: zh-CN |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
split: test |
|
type: mteb/amazon_massive_intent |
|
metrics: |
|
- type: accuracy |
|
value: 77.9388029589778 |
|
- type: accuracy_stderr |
|
value: 1.416192788478398 |
|
- type: f1 |
|
value: 74.77765701086211 |
|
- type: f1_stderr |
|
value: 1.254859698486085 |
|
- type: main_score |
|
value: 77.9388029589778 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: zh-CN |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
split: test |
|
type: mteb/amazon_massive_scenario |
|
metrics: |
|
- type: accuracy |
|
value: 83.8231338264963 |
|
- type: accuracy_stderr |
|
value: 0.6973305760755886 |
|
- type: f1 |
|
value: 83.13105322628088 |
|
- type: f1_stderr |
|
value: 0.600506118139685 |
|
- type: main_score |
|
value: 83.8231338264963 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB MedicalRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/MedicalRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 57.8 |
|
- type: map_at_10 |
|
value: 64.696 |
|
- type: map_at_100 |
|
value: 65.294 |
|
- type: map_at_1000 |
|
value: 65.328 |
|
- type: map_at_3 |
|
value: 62.949999999999996 |
|
- type: map_at_5 |
|
value: 64.095 |
|
- type: mrr_at_1 |
|
value: 58.099999999999994 |
|
- type: mrr_at_10 |
|
value: 64.85 |
|
- type: mrr_at_100 |
|
value: 65.448 |
|
- type: mrr_at_1000 |
|
value: 65.482 |
|
- type: mrr_at_3 |
|
value: 63.1 |
|
- type: mrr_at_5 |
|
value: 64.23 |
|
- type: ndcg_at_1 |
|
value: 57.8 |
|
- type: ndcg_at_10 |
|
value: 68.041 |
|
- type: ndcg_at_100 |
|
value: 71.074 |
|
- type: ndcg_at_1000 |
|
value: 71.919 |
|
- type: ndcg_at_3 |
|
value: 64.584 |
|
- type: ndcg_at_5 |
|
value: 66.625 |
|
- type: precision_at_1 |
|
value: 57.8 |
|
- type: precision_at_10 |
|
value: 7.85 |
|
- type: precision_at_100 |
|
value: 0.9289999999999999 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 23.1 |
|
- type: precision_at_5 |
|
value: 14.84 |
|
- type: recall_at_1 |
|
value: 57.8 |
|
- type: recall_at_10 |
|
value: 78.5 |
|
- type: recall_at_100 |
|
value: 92.9 |
|
- type: recall_at_1000 |
|
value: 99.4 |
|
- type: recall_at_3 |
|
value: 69.3 |
|
- type: recall_at_5 |
|
value: 74.2 |
|
- type: main_score |
|
value: 68.041 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB MultilingualSentiment |
|
revision: None |
|
split: validation |
|
type: C-MTEB/MultilingualSentiment-classification |
|
metrics: |
|
- type: accuracy |
|
value: 78.60333333333334 |
|
- type: accuracy_stderr |
|
value: 0.3331499495555859 |
|
- type: f1 |
|
value: 78.4814340961856 |
|
- type: f1_stderr |
|
value: 0.45721454672060496 |
|
- type: main_score |
|
value: 78.60333333333334 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB Ocnli |
|
revision: None |
|
split: validation |
|
type: C-MTEB/OCNLI |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 80.5630752571738 |
|
- type: cos_sim_accuracy_threshold |
|
value: 53.72971296310425 |
|
- type: cos_sim_ap |
|
value: 85.61885910463258 |
|
- type: cos_sim_f1 |
|
value: 82.40469208211144 |
|
- type: cos_sim_f1_threshold |
|
value: 50.07883310317993 |
|
- type: cos_sim_precision |
|
value: 76.70609645131938 |
|
- type: cos_sim_recall |
|
value: 89.01795142555439 |
|
- type: dot_accuracy |
|
value: 80.5630752571738 |
|
- type: dot_accuracy_threshold |
|
value: 53.7297248840332 |
|
- type: dot_ap |
|
value: 85.61885910463258 |
|
- type: dot_f1 |
|
value: 82.40469208211144 |
|
- type: dot_f1_threshold |
|
value: 50.07884502410889 |
|
- type: dot_precision |
|
value: 76.70609645131938 |
|
- type: dot_recall |
|
value: 89.01795142555439 |
|
- type: euclidean_accuracy |
|
value: 80.5630752571738 |
|
- type: euclidean_accuracy_threshold |
|
value: 96.19801044464111 |
|
- type: euclidean_ap |
|
value: 85.61885910463258 |
|
- type: euclidean_f1 |
|
value: 82.40469208211144 |
|
- type: euclidean_f1_threshold |
|
value: 99.92111921310425 |
|
- type: euclidean_precision |
|
value: 76.70609645131938 |
|
- type: euclidean_recall |
|
value: 89.01795142555439 |
|
- type: manhattan_accuracy |
|
value: 80.67135896047645 |
|
- type: manhattan_accuracy_threshold |
|
value: 3323.1739044189453 |
|
- type: manhattan_ap |
|
value: 85.55348220886658 |
|
- type: manhattan_f1 |
|
value: 82.26744186046511 |
|
- type: manhattan_f1_threshold |
|
value: 3389.273452758789 |
|
- type: manhattan_precision |
|
value: 76.00716204118174 |
|
- type: manhattan_recall |
|
value: 89.65153115100317 |
|
- type: max_accuracy |
|
value: 80.67135896047645 |
|
- type: max_ap |
|
value: 85.61885910463258 |
|
- type: max_f1 |
|
value: 82.40469208211144 |
|
task: |
|
type: PairClassification |
|
- dataset: |
|
config: default |
|
name: MTEB OnlineShopping |
|
revision: None |
|
split: test |
|
type: C-MTEB/OnlineShopping-classification |
|
metrics: |
|
- type: accuracy |
|
value: 94.94 |
|
- type: accuracy_stderr |
|
value: 0.49030602688525093 |
|
- type: ap |
|
value: 93.0785841977823 |
|
- type: ap_stderr |
|
value: 0.5447383082750599 |
|
- type: f1 |
|
value: 94.92765777406245 |
|
- type: f1_stderr |
|
value: 0.4891510966106189 |
|
- type: main_score |
|
value: 94.94 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB PAWSX |
|
revision: None |
|
split: test |
|
type: C-MTEB/PAWSX |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 36.564307811370654 |
|
- type: cos_sim_spearman |
|
value: 42.44208208349051 |
|
- type: manhattan_pearson |
|
value: 42.099358471578306 |
|
- type: manhattan_spearman |
|
value: 42.50283181486304 |
|
- type: euclidean_pearson |
|
value: 42.07954956675317 |
|
- type: euclidean_spearman |
|
value: 42.453014115018554 |
|
- type: main_score |
|
value: 42.44208208349051 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB QBQTC |
|
revision: None |
|
split: test |
|
type: C-MTEB/QBQTC |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 39.19092968089104 |
|
- type: cos_sim_spearman |
|
value: 41.5174661348832 |
|
- type: manhattan_pearson |
|
value: 37.91587646684523 |
|
- type: manhattan_spearman |
|
value: 41.536668677987194 |
|
- type: euclidean_pearson |
|
value: 37.91079973901135 |
|
- type: euclidean_spearman |
|
value: 41.51833855501128 |
|
- type: main_score |
|
value: 41.5174661348832 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: zh |
|
name: MTEB STS22 (zh) |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
split: test |
|
type: mteb/sts22-crosslingual-sts |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 62.029449510721605 |
|
- type: cos_sim_spearman |
|
value: 66.31935471251364 |
|
- type: manhattan_pearson |
|
value: 63.63179975157496 |
|
- type: manhattan_spearman |
|
value: 66.3007950466125 |
|
- type: euclidean_pearson |
|
value: 63.59752734041086 |
|
- type: euclidean_spearman |
|
value: 66.31935471251364 |
|
- type: main_score |
|
value: 66.31935471251364 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB STSB |
|
revision: None |
|
split: test |
|
type: C-MTEB/STSB |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.81459862563769 |
|
- type: cos_sim_spearman |
|
value: 82.15323953301453 |
|
- type: manhattan_pearson |
|
value: 81.61904305126016 |
|
- type: manhattan_spearman |
|
value: 82.1361073852468 |
|
- type: euclidean_pearson |
|
value: 81.60799063723992 |
|
- type: euclidean_spearman |
|
value: 82.15405405083231 |
|
- type: main_score |
|
value: 82.15323953301453 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB T2Reranking |
|
revision: None |
|
split: dev |
|
type: C-MTEB/T2Reranking |
|
metrics: |
|
- type: map |
|
value: 69.13560834260383 |
|
- type: mrr |
|
value: 79.95749642669074 |
|
- type: main_score |
|
value: 69.13560834260383 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB T2Retrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/T2Retrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.041 |
|
- type: map_at_10 |
|
value: 78.509 |
|
- type: map_at_100 |
|
value: 82.083 |
|
- type: map_at_1000 |
|
value: 82.143 |
|
- type: map_at_3 |
|
value: 55.345 |
|
- type: map_at_5 |
|
value: 67.899 |
|
- type: mrr_at_1 |
|
value: 90.86 |
|
- type: mrr_at_10 |
|
value: 93.31 |
|
- type: mrr_at_100 |
|
value: 93.388 |
|
- type: mrr_at_1000 |
|
value: 93.391 |
|
- type: mrr_at_3 |
|
value: 92.92200000000001 |
|
- type: mrr_at_5 |
|
value: 93.167 |
|
- type: ndcg_at_1 |
|
value: 90.86 |
|
- type: ndcg_at_10 |
|
value: 85.875 |
|
- type: ndcg_at_100 |
|
value: 89.269 |
|
- type: ndcg_at_1000 |
|
value: 89.827 |
|
- type: ndcg_at_3 |
|
value: 87.254 |
|
- type: ndcg_at_5 |
|
value: 85.855 |
|
- type: precision_at_1 |
|
value: 90.86 |
|
- type: precision_at_10 |
|
value: 42.488 |
|
- type: precision_at_100 |
|
value: 5.029 |
|
- type: precision_at_1000 |
|
value: 0.516 |
|
- type: precision_at_3 |
|
value: 76.172 |
|
- type: precision_at_5 |
|
value: 63.759 |
|
- type: recall_at_1 |
|
value: 28.041 |
|
- type: recall_at_10 |
|
value: 84.829 |
|
- type: recall_at_100 |
|
value: 95.89999999999999 |
|
- type: recall_at_1000 |
|
value: 98.665 |
|
- type: recall_at_3 |
|
value: 57.009 |
|
- type: recall_at_5 |
|
value: 71.188 |
|
- type: main_score |
|
value: 85.875 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB TNews |
|
revision: None |
|
split: validation |
|
type: C-MTEB/TNews-classification |
|
metrics: |
|
- type: accuracy |
|
value: 54.309000000000005 |
|
- type: accuracy_stderr |
|
value: 0.4694347665011627 |
|
- type: f1 |
|
value: 52.598803987889255 |
|
- type: f1_stderr |
|
value: 0.5191189533227434 |
|
- type: main_score |
|
value: 54.309000000000005 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB ThuNewsClusteringP2P |
|
revision: None |
|
split: test |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
metrics: |
|
- type: v_measure |
|
value: 76.64191229011249 |
|
- type: v_measure_std |
|
value: 2.807206940615986 |
|
- type: main_score |
|
value: 76.64191229011249 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB ThuNewsClusteringS2S |
|
revision: None |
|
split: test |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
metrics: |
|
- type: v_measure |
|
value: 71.02529199411326 |
|
- type: v_measure_std |
|
value: 2.0547855888165945 |
|
- type: main_score |
|
value: 71.02529199411326 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB VideoRetrieval |
|
revision: None |
|
split: dev |
|
type: C-MTEB/VideoRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 67.30000000000001 |
|
- type: map_at_10 |
|
value: 76.819 |
|
- type: map_at_100 |
|
value: 77.141 |
|
- type: map_at_1000 |
|
value: 77.142 |
|
- type: map_at_3 |
|
value: 75.233 |
|
- type: map_at_5 |
|
value: 76.163 |
|
- type: mrr_at_1 |
|
value: 67.30000000000001 |
|
- type: mrr_at_10 |
|
value: 76.819 |
|
- type: mrr_at_100 |
|
value: 77.141 |
|
- type: mrr_at_1000 |
|
value: 77.142 |
|
- type: mrr_at_3 |
|
value: 75.233 |
|
- type: mrr_at_5 |
|
value: 76.163 |
|
- type: ndcg_at_1 |
|
value: 67.30000000000001 |
|
- type: ndcg_at_10 |
|
value: 80.93599999999999 |
|
- type: ndcg_at_100 |
|
value: 82.311 |
|
- type: ndcg_at_1000 |
|
value: 82.349 |
|
- type: ndcg_at_3 |
|
value: 77.724 |
|
- type: ndcg_at_5 |
|
value: 79.406 |
|
- type: precision_at_1 |
|
value: 67.30000000000001 |
|
- type: precision_at_10 |
|
value: 9.36 |
|
- type: precision_at_100 |
|
value: 0.996 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 28.299999999999997 |
|
- type: precision_at_5 |
|
value: 17.8 |
|
- type: recall_at_1 |
|
value: 67.30000000000001 |
|
- type: recall_at_10 |
|
value: 93.60000000000001 |
|
- type: recall_at_100 |
|
value: 99.6 |
|
- type: recall_at_1000 |
|
value: 99.9 |
|
- type: recall_at_3 |
|
value: 84.89999999999999 |
|
- type: recall_at_5 |
|
value: 89.0 |
|
- type: main_score |
|
value: 80.93599999999999 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB Waimai |
|
revision: None |
|
split: test |
|
type: C-MTEB/waimai-classification |
|
metrics: |
|
- type: accuracy |
|
value: 89.47 |
|
- type: accuracy_stderr |
|
value: 0.26476404589747476 |
|
- type: ap |
|
value: 75.49555223825388 |
|
- type: ap_stderr |
|
value: 0.596040511982105 |
|
- type: f1 |
|
value: 88.01797939221065 |
|
- type: f1_stderr |
|
value: 0.27168216797281214 |
|
- type: main_score |
|
value: 89.47 |
|
task: |
|
type: Classification |
|
tags: |
|
- mteb |
|
--- |
|
<h2 align="left">XYZ-embedding-zh-v2</h2> |
|
|
|
## Usage (Sentence Transformers) |
|
|
|
First install the Sentence Transformers library: |
|
|
|
```bash |
|
pip install -U sentence-transformers |
|
``` |
|
Then you can load this model and run inference. |
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
|
|
# Download from the 🤗 Hub |
|
model = SentenceTransformer("fangxq/XYZ-embedding-zh-v2") |
|
# Run inference |
|
sentences = [ |
|
'The weather is lovely today.', |
|
"It's so sunny outside!", |
|
'He drove to the stadium.', |
|
] |
|
embeddings = model.encode(sentences) |
|
print(embeddings.shape) |
|
# [3, 1792] |
|
|
|
# Get the similarity scores for the embeddings |
|
similarities = model.similarity(embeddings, embeddings) |
|
print(similarities.shape) |
|
# [3, 3] |
|
``` |
|
|
