metadata
base_model:
- Alibaba-NLP/gte-Qwen2-7B-instruct
language:
- en
- zh
model-index:
- name: infly/inf-retriever-v1
results:
- dataset:
config: default
name: MTEB CmedqaRetrieval (default)
revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301
split: dev
type: C-MTEB/CmedqaRetrieval
metrics:
- type: ndcg_at_1
value: 38.21
- type: ndcg_at_3
value: 38.438
- type: ndcg_at_5
value: 40.443
- type: ndcg_at_10
value: 43.306
- type: ndcg_at_20
value: 46.183
- type: ndcg_at_100
value: 50.646
- type: ndcg_at_1000
value: 52.82
- type: map_at_1
value: 25.153
- type: map_at_3
value: 33.038000000000004
- type: map_at_5
value: 35.228
- type: map_at_10
value: 36.924
- type: map_at_20
value: 37.96
- type: map_at_100
value: 38.792
- type: map_at_1000
value: 38.914
- type: recall_at_1
value: 25.153
- type: recall_at_3
value: 38.253
- type: recall_at_5
value: 44.507999999999996
- type: recall_at_10
value: 53.012
- type: recall_at_20
value: 62.915
- type: recall_at_100
value: 83.487
- type: recall_at_1000
value: 98.059
- type: precision_at_1
value: 38.21
- type: precision_at_3
value: 21.605
- type: precision_at_5
value: 15.664
- type: precision_at_10
value: 9.585
- type: precision_at_20
value: 5.731
- type: precision_at_100
value: 1.545
- type: precision_at_1000
value: 0.183
- type: mrr_at_1
value: 38.2096
- type: mrr_at_3
value: 43.6776
- type: mrr_at_5
value: 44.992900000000006
- type: mrr_at_10
value: 46.0602
- type: mrr_at_20
value: 46.6425
- type: mrr_at_100
value: 47.0609
- type: mrr_at_1000
value: 47.1081
- type: nauc_ndcg_at_1_max
value: 47.2941
- type: nauc_ndcg_at_1_std
value: 1.1506
- type: nauc_ndcg_at_1_diff1
value: 65.07300000000001
- type: nauc_ndcg_at_3_max
value: 40.0306
- type: nauc_ndcg_at_3_std
value: -2.1843999999999997
- type: nauc_ndcg_at_3_diff1
value: 55.63550000000001
- type: nauc_ndcg_at_5_max
value: 38.4437
- type: nauc_ndcg_at_5_std
value: -2.8594
- type: nauc_ndcg_at_5_diff1
value: 54.3432
- type: nauc_ndcg_at_10_max
value: 37.9058
- type: nauc_ndcg_at_10_std
value: -2.2464
- type: nauc_ndcg_at_10_diff1
value: 53.2127
- type: nauc_ndcg_at_20_max
value: 38.6325
- type: nauc_ndcg_at_20_std
value: -1.4661
- type: nauc_ndcg_at_20_diff1
value: 53.0613
- type: nauc_ndcg_at_100_max
value: 40.2745
- type: nauc_ndcg_at_100_std
value: 1.2013
- type: nauc_ndcg_at_100_diff1
value: 53.5367
- type: nauc_ndcg_at_1000_max
value: 40.773199999999996
- type: nauc_ndcg_at_1000_std
value: 0.4336
- type: nauc_ndcg_at_1000_diff1
value: 54.4837
- type: nauc_map_at_1_max
value: 29.7667
- type: nauc_map_at_1_std
value: -6.5036
- type: nauc_map_at_1_diff1
value: 55.916
- type: nauc_map_at_3_max
value: 34.8091
- type: nauc_map_at_3_std
value: -5.9474
- type: nauc_map_at_3_diff1
value: 54.0836
- type: nauc_map_at_5_max
value: 35.9765
- type: nauc_map_at_5_std
value: -5.1715
- type: nauc_map_at_5_diff1
value: 53.89960000000001
- type: nauc_map_at_10_max
value: 36.7189
- type: nauc_map_at_10_std
value: -4.2277000000000005
- type: nauc_map_at_10_diff1
value: 53.570499999999996
- type: nauc_map_at_20_max
value: 37.327799999999996
- type: nauc_map_at_20_std
value: -3.7319
- type: nauc_map_at_20_diff1
value: 53.662299999999995
- type: nauc_map_at_100_max
value: 37.8037
- type: nauc_map_at_100_std
value: -3.0759000000000003
- type: nauc_map_at_100_diff1
value: 53.8127
- type: nauc_map_at_1000_max
value: 37.851800000000004
- type: nauc_map_at_1000_std
value: -3.0761
- type: nauc_map_at_1000_diff1
value: 53.8489
- type: nauc_recall_at_1_max
value: 29.7667
- type: nauc_recall_at_1_std
value: -6.5036
- type: nauc_recall_at_1_diff1
value: 55.916
- type: nauc_recall_at_3_max
value: 30.2726
- type: nauc_recall_at_3_std
value: -6.4455
- type: nauc_recall_at_3_diff1
value: 47.646
- type: nauc_recall_at_5_max
value: 29.5304
- type: nauc_recall_at_5_std
value: -5.3595
- type: nauc_recall_at_5_diff1
value: 44.6824
- type: nauc_recall_at_10_max
value: 28.063399999999998
- type: nauc_recall_at_10_std
value: -2.122
- type: nauc_recall_at_10_diff1
value: 40.477999999999994
- type: nauc_recall_at_20_max
value: 27.8269
- type: nauc_recall_at_20_std
value: -0.0429
- type: nauc_recall_at_20_diff1
value: 37.2018
- type: nauc_recall_at_100_max
value: 30.131000000000004
- type: nauc_recall_at_100_std
value: 19.334899999999998
- type: nauc_recall_at_100_diff1
value: 29.274499999999996
- type: nauc_recall_at_1000_max
value: 33.964
- type: nauc_recall_at_1000_std
value: 53.2129
- type: nauc_recall_at_1000_diff1
value: 19.7514
- type: nauc_precision_at_1_max
value: 47.2941
- type: nauc_precision_at_1_std
value: 1.1506
- type: nauc_precision_at_1_diff1
value: 65.07300000000001
- type: nauc_precision_at_3_max
value: 45.4966
- type: nauc_precision_at_3_std
value: 5.432
- type: nauc_precision_at_3_diff1
value: 44.0588
- type: nauc_precision_at_5_max
value: 43.9257
- type: nauc_precision_at_5_std
value: 8.1048
- type: nauc_precision_at_5_diff1
value: 37.5711
- type: nauc_precision_at_10_max
value: 41.274499999999996
- type: nauc_precision_at_10_std
value: 12.023399999999999
- type: nauc_precision_at_10_diff1
value: 29.940299999999997
- type: nauc_precision_at_20_max
value: 39.8018
- type: nauc_precision_at_20_std
value: 15.2937
- type: nauc_precision_at_20_diff1
value: 24.4872
- type: nauc_precision_at_100_max
value: 35.012
- type: nauc_precision_at_100_std
value: 24.002499999999998
- type: nauc_precision_at_100_diff1
value: 13.2948
- type: nauc_precision_at_1000_max
value: 29.172700000000003
- type: nauc_precision_at_1000_std
value: 20.634900000000002
- type: nauc_precision_at_1000_diff1
value: 6.460000000000001
- type: nauc_mrr_at_1_max
value: 47.2941
- type: nauc_mrr_at_1_std
value: 1.1506
- type: nauc_mrr_at_1_diff1
value: 65.07300000000001
- type: nauc_mrr_at_3_max
value: 45.6503
- type: nauc_mrr_at_3_std
value: 0.5387000000000001
- type: nauc_mrr_at_3_diff1
value: 61.84889999999999
- type: nauc_mrr_at_5_max
value: 45.3089
- type: nauc_mrr_at_5_std
value: 0.7656
- type: nauc_mrr_at_5_diff1
value: 61.148
- type: nauc_mrr_at_10_max
value: 45.2298
- type: nauc_mrr_at_10_std
value: 1.15
- type: nauc_mrr_at_10_diff1
value: 60.775800000000004
- type: nauc_mrr_at_20_max
value: 45.2277
- type: nauc_mrr_at_20_std
value: 1.1927999999999999
- type: nauc_mrr_at_20_diff1
value: 60.6927
- type: nauc_mrr_at_100_max
value: 45.3238
- type: nauc_mrr_at_100_std
value: 1.3096999999999999
- type: nauc_mrr_at_100_diff1
value: 60.7616
- type: nauc_mrr_at_1000_max
value: 45.3298
- type: nauc_mrr_at_1000_std
value: 1.2893
- type: nauc_mrr_at_1000_diff1
value: 60.7901
- type: main_score
value: 43.306
task:
type: Retrieval
- dataset:
config: default
name: MTEB CovidRetrieval (default)
revision: 1271c7809071a13532e05f25fb53511ffce77117
split: dev
type: C-MTEB/CovidRetrieval
metrics:
- type: ndcg_at_1
value: 77.97699999999999
- type: ndcg_at_3
value: 84.489
- type: ndcg_at_5
value: 86.18299999999999
- type: ndcg_at_10
value: 87.15899999999999
- type: ndcg_at_20
value: 87.638
- type: ndcg_at_100
value: 87.976
- type: ndcg_at_1000
value: 88.02600000000001
- type: map_at_1
value: 77.713
- type: map_at_3
value: 82.947
- type: map_at_5
value: 83.909
- type: map_at_10
value: 84.332
- type: map_at_20
value: 84.47200000000001
- type: map_at_100
value: 84.52
- type: map_at_1000
value: 84.52199999999999
- type: recall_at_1
value: 77.713
- type: recall_at_3
value: 88.857
- type: recall_at_5
value: 92.914
- type: recall_at_10
value: 95.838
- type: recall_at_20
value: 97.682
- type: recall_at_100
value: 99.473
- type: recall_at_1000
value: 99.895
- type: precision_at_1
value: 77.97699999999999
- type: precision_at_3
value: 29.855999999999998
- type: precision_at_5
value: 18.756999999999998
- type: precision_at_10
value: 9.684
- type: precision_at_20
value: 4.936999999999999
- type: precision_at_100
value: 1.005
- type: precision_at_1000
value: 0.101
- type: mrr_at_1
value: 77.9768
- type: mrr_at_3
value: 82.947
- type: mrr_at_5
value: 83.9322
- type: mrr_at_10
value: 84.3297
- type: mrr_at_20
value: 84.4602
- type: mrr_at_100
value: 84.5085
- type: mrr_at_1000
value: 84.5098
- type: nauc_ndcg_at_1_max
value: 10.6257
- type: nauc_ndcg_at_1_std
value: -37.586999999999996
- type: nauc_ndcg_at_1_diff1
value: 87.6739
- type: nauc_ndcg_at_3_max
value: 11.4731
- type: nauc_ndcg_at_3_std
value: -39.4782
- type: nauc_ndcg_at_3_diff1
value: 85.1518
- type: nauc_ndcg_at_5_max
value: 13.3553
- type: nauc_ndcg_at_5_std
value: -38.0635
- type: nauc_ndcg_at_5_diff1
value: 84.7753
- type: nauc_ndcg_at_10_max
value: 13.5939
- type: nauc_ndcg_at_10_std
value: -35.3862
- type: nauc_ndcg_at_10_diff1
value: 85.2025
- type: nauc_ndcg_at_20_max
value: 12.3043
- type: nauc_ndcg_at_20_std
value: -34.4532
- type: nauc_ndcg_at_20_diff1
value: 85.4869
- type: nauc_ndcg_at_100_max
value: 12.359
- type: nauc_ndcg_at_100_std
value: -36.2481
- type: nauc_ndcg_at_100_diff1
value: 85.6223
- type: nauc_ndcg_at_1000_max
value: 12.554000000000002
- type: nauc_ndcg_at_1000_std
value: -36.4419
- type: nauc_ndcg_at_1000_diff1
value: 85.65469999999999
- type: nauc_map_at_1_max
value: 10.7918
- type: nauc_map_at_1_std
value: -38.0017
- type: nauc_map_at_1_diff1
value: 88.115
- type: nauc_map_at_3_max
value: 11.2948
- type: nauc_map_at_3_std
value: -38.7444
- type: nauc_map_at_3_diff1
value: 85.90090000000001
- type: nauc_map_at_5_max
value: 12.3632
- type: nauc_map_at_5_std
value: -38.1401
- type: nauc_map_at_5_diff1
value: 85.7334
- type: nauc_map_at_10_max
value: 12.567900000000002
- type: nauc_map_at_10_std
value: -37.196
- type: nauc_map_at_10_diff1
value: 85.8887
- type: nauc_map_at_20_max
value: 12.2576
- type: nauc_map_at_20_std
value: -36.9194
- type: nauc_map_at_20_diff1
value: 85.9492
- type: nauc_map_at_100_max
value: 12.313699999999999
- type: nauc_map_at_100_std
value: -37.1156
- type: nauc_map_at_100_diff1
value: 85.96589999999999
- type: nauc_map_at_1000_max
value: 12.3179
- type: nauc_map_at_1000_std
value: -37.122899999999994
- type: nauc_map_at_1000_diff1
value: 85.9666
- type: nauc_recall_at_1_max
value: 10.7918
- type: nauc_recall_at_1_std
value: -38.0017
- type: nauc_recall_at_1_diff1
value: 88.115
- type: nauc_recall_at_3_max
value: 11.5712
- type: nauc_recall_at_3_std
value: -42.0403
- type: nauc_recall_at_3_diff1
value: 82.2085
- type: nauc_recall_at_5_max
value: 19.3703
- type: nauc_recall_at_5_std
value: -36.4651
- type: nauc_recall_at_5_diff1
value: 79.02289999999999
- type: nauc_recall_at_10_max
value: 24.702099999999998
- type: nauc_recall_at_10_std
value: -13.0473
- type: nauc_recall_at_10_diff1
value: 79.0931
- type: nauc_recall_at_20_max
value: 8.377600000000001
- type: nauc_recall_at_20_std
value: 19.5544
- type: nauc_recall_at_20_diff1
value: 80.15440000000001
- type: nauc_recall_at_100_max
value: -16.4048
- type: nauc_recall_at_100_std
value: -4.217
- type: nauc_recall_at_100_diff1
value: 77.3525
- type: nauc_recall_at_1000_max
value: 35.780899999999995
- type: nauc_recall_at_1000_std
value: -17.1046
- type: nauc_recall_at_1000_diff1
value: 72.2168
- type: nauc_precision_at_1_max
value: 10.6257
- type: nauc_precision_at_1_std
value: -37.586999999999996
- type: nauc_precision_at_1_diff1
value: 87.6739
- type: nauc_precision_at_3_max
value: 12.3466
- type: nauc_precision_at_3_std
value: -35.2022
- type: nauc_precision_at_3_diff1
value: 69.3658
- type: nauc_precision_at_5_max
value: 21.8156
- type: nauc_precision_at_5_std
value: -26.7725
- type: nauc_precision_at_5_diff1
value: 55.6505
- type: nauc_precision_at_10_max
value: 28.0424
- type: nauc_precision_at_10_std
value: -3.9004999999999996
- type: nauc_precision_at_10_diff1
value: 40.0286
- type: nauc_precision_at_20_max
value: 17.8965
- type: nauc_precision_at_20_std
value: 20.0597
- type: nauc_precision_at_20_diff1
value: 19.717299999999998
- type: nauc_precision_at_100_max
value: 19.5903
- type: nauc_precision_at_100_std
value: 15.1328
- type: nauc_precision_at_100_diff1
value: -26.816699999999997
- type: nauc_precision_at_1000_max
value: 30.2629
- type: nauc_precision_at_1000_std
value: 18.6426
- type: nauc_precision_at_1000_diff1
value: -49.766
- type: nauc_mrr_at_1_max
value: 11.444799999999999
- type: nauc_mrr_at_1_std
value: -38.4039
- type: nauc_mrr_at_1_diff1
value: 87.6739
- type: nauc_mrr_at_3_max
value: 12.0501
- type: nauc_mrr_at_3_std
value: -39.6139
- type: nauc_mrr_at_3_diff1
value: 85.8329
- type: nauc_mrr_at_5_max
value: 12.549199999999999
- type: nauc_mrr_at_5_std
value: -38.6156
- type: nauc_mrr_at_5_diff1
value: 85.6892
- type: nauc_mrr_at_10_max
value: 12.647400000000001
- type: nauc_mrr_at_10_std
value: -37.7216
- type: nauc_mrr_at_10_diff1
value: 85.8803
- type: nauc_mrr_at_20_max
value: 12.3774
- type: nauc_mrr_at_20_std
value: -37.5013
- type: nauc_mrr_at_20_diff1
value: 85.95
- type: nauc_mrr_at_100_max
value: 12.4338
- type: nauc_mrr_at_100_std
value: -37.6993
- type: nauc_mrr_at_100_diff1
value: 85.9667
- type: nauc_mrr_at_1000_max
value: 12.4381
- type: nauc_mrr_at_1000_std
value: -37.7067
- type: nauc_mrr_at_1000_diff1
value: 85.9674
- type: main_score
value: 87.15899999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB DuRetrieval (default)
revision: a1a333e290fe30b10f3f56498e3a0d911a693ced
split: dev
type: C-MTEB/DuRetrieval
metrics:
- type: ndcg_at_1
value: 91.95
- type: ndcg_at_3
value: 89.564
- type: ndcg_at_5
value: 88.74
- type: ndcg_at_10
value: 90.816
- type: ndcg_at_20
value: 92.049
- type: ndcg_at_100
value: 92.944
- type: ndcg_at_1000
value: 93.101
- type: map_at_1
value: 27.468999999999998
- type: map_at_3
value: 58.833
- type: map_at_5
value: 74.146
- type: map_at_10
value: 84.49
- type: map_at_20
value: 86.583
- type: map_at_100
value: 87.15
- type: map_at_1000
value: 87.167
- type: recall_at_1
value: 27.468999999999998
- type: recall_at_3
value: 61.253
- type: recall_at_5
value: 78.605
- type: recall_at_10
value: 91.984
- type: recall_at_20
value: 96.15700000000001
- type: recall_at_100
value: 99.012
- type: recall_at_1000
value: 99.85000000000001
- type: precision_at_1
value: 91.95
- type: precision_at_3
value: 79.93299999999999
- type: precision_at_5
value: 67.52
- type: precision_at_10
value: 43.125
- type: precision_at_20
value: 23.365
- type: precision_at_100
value: 4.873
- type: precision_at_1000
value: 0.49100000000000005
- type: mrr_at_1
value: 91.95
- type: mrr_at_3
value: 94.6833
- type: mrr_at_5
value: 94.80330000000001
- type: mrr_at_10
value: 94.8327
- type: mrr_at_20
value: 94.86449999999999
- type: mrr_at_100
value: 94.8776
- type: mrr_at_1000
value: 94.87910000000001
- type: nauc_ndcg_at_1_max
value: 38.3147
- type: nauc_ndcg_at_1_std
value: 25.4415
- type: nauc_ndcg_at_1_diff1
value: 20.072400000000002
- type: nauc_ndcg_at_3_max
value: 28.4644
- type: nauc_ndcg_at_3_std
value: 16.3111
- type: nauc_ndcg_at_3_diff1
value: 4.136
- type: nauc_ndcg_at_5_max
value: 20.8656
- type: nauc_ndcg_at_5_std
value: 7.4921
- type: nauc_ndcg_at_5_diff1
value: 6.9999
- type: nauc_ndcg_at_10_max
value: 31.9021
- type: nauc_ndcg_at_10_std
value: 9.895199999999999
- type: nauc_ndcg_at_10_diff1
value: 7.4908
- type: nauc_ndcg_at_20_max
value: 35.1138
- type: nauc_ndcg_at_20_std
value: 15.992799999999999
- type: nauc_ndcg_at_20_diff1
value: 5.8599
- type: nauc_ndcg_at_100_max
value: 35.4467
- type: nauc_ndcg_at_100_std
value: 20.2691
- type: nauc_ndcg_at_100_diff1
value: 4.6742
- type: nauc_ndcg_at_1000_max
value: 34.8608
- type: nauc_ndcg_at_1000_std
value: 19.7889
- type: nauc_ndcg_at_1000_diff1
value: 4.455
- type: nauc_map_at_1_max
value: -14.544799999999999
- type: nauc_map_at_1_std
value: -36.809
- type: nauc_map_at_1_diff1
value: 48.945
- type: nauc_map_at_3_max
value: -4.3561000000000005
- type: nauc_map_at_3_std
value: -36.5114
- type: nauc_map_at_3_diff1
value: 34.5076
- type: nauc_map_at_5_max
value: 4.7926
- type: nauc_map_at_5_std
value: -29.224
- type: nauc_map_at_5_diff1
value: 23.6996
- type: nauc_map_at_10_max
value: 23.3542
- type: nauc_map_at_10_std
value: -8.1883
- type: nauc_map_at_10_diff1
value: 10.3199
- type: nauc_map_at_20_max
value: 26.1761
- type: nauc_map_at_20_std
value: 4.3185
- type: nauc_map_at_20_diff1
value: 5.7029000000000005
- type: nauc_map_at_100_max
value: 26.251
- type: nauc_map_at_100_std
value: 7.3792
- type: nauc_map_at_100_diff1
value: 4.4363
- type: nauc_map_at_1000_max
value: 26.215
- type: nauc_map_at_1000_std
value: 7.3778999999999995
- type: nauc_map_at_1000_diff1
value: 4.3981
- type: nauc_recall_at_1_max
value: -14.544799999999999
- type: nauc_recall_at_1_std
value: -36.809
- type: nauc_recall_at_1_diff1
value: 48.945
- type: nauc_recall_at_3_max
value: -8.1063
- type: nauc_recall_at_3_std
value: -39.984700000000004
- type: nauc_recall_at_3_diff1
value: 36.1148
- type: nauc_recall_at_5_max
value: -1.8721
- type: nauc_recall_at_5_std
value: -37.5321
- type: nauc_recall_at_5_diff1
value: 27.4684
- type: nauc_recall_at_10_max
value: 27.440900000000003
- type: nauc_recall_at_10_std
value: -17.471500000000002
- type: nauc_recall_at_10_diff1
value: 18.1816
- type: nauc_recall_at_20_max
value: 42.033500000000004
- type: nauc_recall_at_20_std
value: 13.2844
- type: nauc_recall_at_20_diff1
value: 11.135100000000001
- type: nauc_recall_at_100_max
value: 63.4582
- type: nauc_recall_at_100_std
value: 54.1563
- type: nauc_recall_at_100_diff1
value: 8.7789
- type: nauc_recall_at_1000_max
value: 73.73150000000001
- type: nauc_recall_at_1000_std
value: 78.3703
- type: nauc_recall_at_1000_diff1
value: 15.1756
- type: nauc_precision_at_1_max
value: 38.3147
- type: nauc_precision_at_1_std
value: 25.4415
- type: nauc_precision_at_1_diff1
value: 20.072400000000002
- type: nauc_precision_at_3_max
value: 33.7041
- type: nauc_precision_at_3_std
value: 40.7562
- type: nauc_precision_at_3_diff1
value: -41.5615
- type: nauc_precision_at_5_max
value: 26.691100000000002
- type: nauc_precision_at_5_std
value: 44.0977
- type: nauc_precision_at_5_diff1
value: -42.8939
- type: nauc_precision_at_10_max
value: 23.5844
- type: nauc_precision_at_10_std
value: 48.653400000000005
- type: nauc_precision_at_10_diff1
value: -39.3653
- type: nauc_precision_at_20_max
value: 18.6268
- type: nauc_precision_at_20_std
value: 52.2362
- type: nauc_precision_at_20_diff1
value: -38.1004
- type: nauc_precision_at_100_max
value: 15.3183
- type: nauc_precision_at_100_std
value: 52.4707
- type: nauc_precision_at_100_diff1
value: -37.5629
- type: nauc_precision_at_1000_max
value: 14.562700000000001
- type: nauc_precision_at_1000_std
value: 51.9881
- type: nauc_precision_at_1000_diff1
value: -37.6074
- type: nauc_mrr_at_1_max
value: 38.3147
- type: nauc_mrr_at_1_std
value: 25.4415
- type: nauc_mrr_at_1_diff1
value: 20.072400000000002
- type: nauc_mrr_at_3_max
value: 48.4265
- type: nauc_mrr_at_3_std
value: 32.605000000000004
- type: nauc_mrr_at_3_diff1
value: 20.5285
- type: nauc_mrr_at_5_max
value: 47.6027
- type: nauc_mrr_at_5_std
value: 32.5635
- type: nauc_mrr_at_5_diff1
value: 19.7022
- type: nauc_mrr_at_10_max
value: 47.4584
- type: nauc_mrr_at_10_std
value: 32.5057
- type: nauc_mrr_at_10_diff1
value: 19.8632
- type: nauc_mrr_at_20_max
value: 47.1959
- type: nauc_mrr_at_20_std
value: 32.3991
- type: nauc_mrr_at_20_diff1
value: 19.741
- type: nauc_mrr_at_100_max
value: 47.1371
- type: nauc_mrr_at_100_std
value: 32.263999999999996
- type: nauc_mrr_at_100_diff1
value: 19.6933
- type: nauc_mrr_at_1000_max
value: 47.1235
- type: nauc_mrr_at_1000_std
value: 32.2476
- type: nauc_mrr_at_1000_diff1
value: 19.6877
- type: main_score
value: 90.816
task:
type: Retrieval
- dataset:
config: default
name: MTEB EcomRetrieval (default)
revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9
split: dev
type: C-MTEB/EcomRetrieval
metrics:
- type: ndcg_at_1
value: 55.900000000000006
- type: ndcg_at_3
value: 66.019
- type: ndcg_at_5
value: 68.47999999999999
- type: ndcg_at_10
value: 70.678
- type: ndcg_at_20
value: 72.024
- type: ndcg_at_100
value: 72.933
- type: ndcg_at_1000
value: 73.20400000000001
- type: map_at_1
value: 55.900000000000006
- type: map_at_3
value: 63.56699999999999
- type: map_at_5
value: 64.927
- type: map_at_10
value: 65.83200000000001
- type: map_at_20
value: 66.205
- type: map_at_100
value: 66.33099999999999
- type: map_at_1000
value: 66.34
- type: recall_at_1
value: 55.900000000000006
- type: recall_at_3
value: 73.1
- type: recall_at_5
value: 79.10000000000001
- type: recall_at_10
value: 85.9
- type: recall_at_20
value: 91.2
- type: recall_at_100
value: 96.1
- type: recall_at_1000
value: 98.3
- type: precision_at_1
value: 55.900000000000006
- type: precision_at_3
value: 24.367
- type: precision_at_5
value: 15.82
- type: precision_at_10
value: 8.59
- type: precision_at_20
value: 4.5600000000000005
- type: precision_at_100
value: 0.9610000000000001
- type: precision_at_1000
value: 0.098
- type: mrr_at_1
value: 55.900000000000006
- type: mrr_at_3
value: 63.5667
- type: mrr_at_5
value: 64.92670000000001
- type: mrr_at_10
value: 65.8319
- type: mrr_at_20
value: 66.2046
- type: mrr_at_100
value: 66.3313
- type: mrr_at_1000
value: 66.3405
- type: nauc_ndcg_at_1_max
value: 7.968999999999999
- type: nauc_ndcg_at_1_std
value: -13.8408
- type: nauc_ndcg_at_1_diff1
value: 74.252
- type: nauc_ndcg_at_3_max
value: 9.5736
- type: nauc_ndcg_at_3_std
value: -11.1447
- type: nauc_ndcg_at_3_diff1
value: 69.1241
- type: nauc_ndcg_at_5_max
value: 11.5841
- type: nauc_ndcg_at_5_std
value: -10.9319
- type: nauc_ndcg_at_5_diff1
value: 68.50110000000001
- type: nauc_ndcg_at_10_max
value: 12.167
- type: nauc_ndcg_at_10_std
value: -10.105699999999999
- type: nauc_ndcg_at_10_diff1
value: 69.02199999999999
- type: nauc_ndcg_at_20_max
value: 13.3886
- type: nauc_ndcg_at_20_std
value: -8.485199999999999
- type: nauc_ndcg_at_20_diff1
value: 69.5569
- type: nauc_ndcg_at_100_max
value: 12.834100000000001
- type: nauc_ndcg_at_100_std
value: -7.8253
- type: nauc_ndcg_at_100_diff1
value: 69.7176
- type: nauc_ndcg_at_1000_max
value: 12.510499999999999
- type: nauc_ndcg_at_1000_std
value: -8.5495
- type: nauc_ndcg_at_1000_diff1
value: 69.9608
- type: nauc_map_at_1_max
value: 7.968999999999999
- type: nauc_map_at_1_std
value: -13.8408
- type: nauc_map_at_1_diff1
value: 74.252
- type: nauc_map_at_3_max
value: 9.1602
- type: nauc_map_at_3_std
value: -11.9092
- type: nauc_map_at_3_diff1
value: 70.4804
- type: nauc_map_at_5_max
value: 10.0994
- type: nauc_map_at_5_std
value: -11.874600000000001
- type: nauc_map_at_5_diff1
value: 70.2168
- type: nauc_map_at_10_max
value: 10.319799999999999
- type: nauc_map_at_10_std
value: -11.574
- type: nauc_map_at_10_diff1
value: 70.4827
- type: nauc_map_at_20_max
value: 10.5658
- type: nauc_map_at_20_std
value: -11.232000000000001
- type: nauc_map_at_20_diff1
value: 70.6199
- type: nauc_map_at_100_max
value: 10.4852
- type: nauc_map_at_100_std
value: -11.1232
- type: nauc_map_at_100_diff1
value: 70.64859999999999
- type: nauc_map_at_1000_max
value: 10.4838
- type: nauc_map_at_1000_std
value: -11.1379
- type: nauc_map_at_1000_diff1
value: 70.6609
- type: nauc_recall_at_1_max
value: 7.968999999999999
- type: nauc_recall_at_1_std
value: -13.8408
- type: nauc_recall_at_1_diff1
value: 74.252
- type: nauc_recall_at_3_max
value: 11.016399999999999
- type: nauc_recall_at_3_std
value: -8.4394
- type: nauc_recall_at_3_diff1
value: 64.3724
- type: nauc_recall_at_5_max
value: 18.305
- type: nauc_recall_at_5_std
value: -6.8603
- type: nauc_recall_at_5_diff1
value: 61.24380000000001
- type: nauc_recall_at_10_max
value: 23.8141
- type: nauc_recall_at_10_std
value: -0.9058999999999999
- type: nauc_recall_at_10_diff1
value: 60.687999999999995
- type: nauc_recall_at_20_max
value: 42.9479
- type: nauc_recall_at_20_std
value: 20.9999
- type: nauc_recall_at_20_diff1
value: 61.755700000000004
- type: nauc_recall_at_100_max
value: 63.88589999999999
- type: nauc_recall_at_100_std
value: 73.6108
- type: nauc_recall_at_100_diff1
value: 56.4067
- type: nauc_recall_at_1000_max
value: 89.56450000000001
- type: nauc_recall_at_1000_std
value: 99.23110000000001
- type: nauc_recall_at_1000_diff1
value: 59.913799999999995
- type: nauc_precision_at_1_max
value: 7.968999999999999
- type: nauc_precision_at_1_std
value: -13.8408
- type: nauc_precision_at_1_diff1
value: 74.252
- type: nauc_precision_at_3_max
value: 11.016399999999999
- type: nauc_precision_at_3_std
value: -8.4394
- type: nauc_precision_at_3_diff1
value: 64.3724
- type: nauc_precision_at_5_max
value: 18.305
- type: nauc_precision_at_5_std
value: -6.8603
- type: nauc_precision_at_5_diff1
value: 61.24380000000001
- type: nauc_precision_at_10_max
value: 23.8141
- type: nauc_precision_at_10_std
value: -0.9058999999999999
- type: nauc_precision_at_10_diff1
value: 60.687999999999995
- type: nauc_precision_at_20_max
value: 42.9479
- type: nauc_precision_at_20_std
value: 20.9999
- type: nauc_precision_at_20_diff1
value: 61.755700000000004
- type: nauc_precision_at_100_max
value: 63.88589999999999
- type: nauc_precision_at_100_std
value: 73.6108
- type: nauc_precision_at_100_diff1
value: 56.4067
- type: nauc_precision_at_1000_max
value: 89.56450000000001
- type: nauc_precision_at_1000_std
value: 99.23110000000001
- type: nauc_precision_at_1000_diff1
value: 59.913799999999995
- type: nauc_mrr_at_1_max
value: 7.968999999999999
- type: nauc_mrr_at_1_std
value: -13.8408
- type: nauc_mrr_at_1_diff1
value: 74.252
- type: nauc_mrr_at_3_max
value: 9.1602
- type: nauc_mrr_at_3_std
value: -11.9092
- type: nauc_mrr_at_3_diff1
value: 70.4804
- type: nauc_mrr_at_5_max
value: 10.0994
- type: nauc_mrr_at_5_std
value: -11.874600000000001
- type: nauc_mrr_at_5_diff1
value: 70.2168
- type: nauc_mrr_at_10_max
value: 10.319799999999999
- type: nauc_mrr_at_10_std
value: -11.574
- type: nauc_mrr_at_10_diff1
value: 70.4827
- type: nauc_mrr_at_20_max
value: 10.5658
- type: nauc_mrr_at_20_std
value: -11.232000000000001
- type: nauc_mrr_at_20_diff1
value: 70.6199
- type: nauc_mrr_at_100_max
value: 10.4852
- type: nauc_mrr_at_100_std
value: -11.1232
- type: nauc_mrr_at_100_diff1
value: 70.64859999999999
- type: nauc_mrr_at_1000_max
value: 10.4838
- type: nauc_mrr_at_1000_std
value: -11.1379
- type: nauc_mrr_at_1000_diff1
value: 70.6609
- type: main_score
value: 70.678
task:
type: Retrieval
- dataset:
config: default
name: MTEB MMarcoRetrieval (default)
revision: 539bbde593d947e2a124ba72651aafc09eb33fc2
split: dev
type: C-MTEB/MMarcoRetrieval
metrics:
- type: ndcg_at_1
value: 75.74499999999999
- type: ndcg_at_3
value: 82.188
- type: ndcg_at_5
value: 83.869
- type: ndcg_at_10
value: 85.11800000000001
- type: ndcg_at_20
value: 85.624
- type: ndcg_at_100
value: 86.05000000000001
- type: ndcg_at_1000
value: 86.177
- type: map_at_1
value: 73.33
- type: map_at_3
value: 80.333
- type: map_at_5
value: 81.349
- type: map_at_10
value: 81.906
- type: map_at_20
value: 82.059
- type: map_at_100
value: 82.127
- type: map_at_1000
value: 82.132
- type: recall_at_1
value: 73.33
- type: recall_at_3
value: 86.823
- type: recall_at_5
value: 90.814
- type: recall_at_10
value: 94.509
- type: recall_at_20
value: 96.422
- type: recall_at_100
value: 98.6
- type: recall_at_1000
value: 99.599
- type: precision_at_1
value: 75.74499999999999
- type: precision_at_3
value: 30.625999999999998
- type: precision_at_5
value: 19.281000000000002
- type: precision_at_10
value: 10.05
- type: precision_at_20
value: 5.133
- type: precision_at_100
value: 1.051
- type: precision_at_1000
value: 0.106
- type: mrr_at_1
value: 75.74499999999999
- type: mrr_at_3
value: 81.0148
- type: mrr_at_5
value: 81.8658
- type: mrr_at_10
value: 82.3666
- type: mrr_at_20
value: 82.4972
- type: mrr_at_100
value: 82.55449999999999
- type: mrr_at_1000
value: 82.5588
- type: nauc_ndcg_at_1_max
value: 13.195799999999998
- type: nauc_ndcg_at_1_std
value: -27.2173
- type: nauc_ndcg_at_1_diff1
value: 83.5531
- type: nauc_ndcg_at_3_max
value: 17.1078
- type: nauc_ndcg_at_3_std
value: -24.1926
- type: nauc_ndcg_at_3_diff1
value: 80.4983
- type: nauc_ndcg_at_5_max
value: 19.3079
- type: nauc_ndcg_at_5_std
value: -21.5239
- type: nauc_ndcg_at_5_diff1
value: 80.6484
- type: nauc_ndcg_at_10_max
value: 19.5153
- type: nauc_ndcg_at_10_std
value: -19.997799999999998
- type: nauc_ndcg_at_10_diff1
value: 80.9953
- type: nauc_ndcg_at_20_max
value: 19.5474
- type: nauc_ndcg_at_20_std
value: -19.3015
- type: nauc_ndcg_at_20_diff1
value: 81.25670000000001
- type: nauc_ndcg_at_100_max
value: 18.569399999999998
- type: nauc_ndcg_at_100_std
value: -20.4957
- type: nauc_ndcg_at_100_diff1
value: 81.3873
- type: nauc_ndcg_at_1000_max
value: 18.1522
- type: nauc_ndcg_at_1000_std
value: -21.3527
- type: nauc_ndcg_at_1000_diff1
value: 81.44590000000001
- type: nauc_map_at_1_max
value: 9.1671
- type: nauc_map_at_1_std
value: -27.6298
- type: nauc_map_at_1_diff1
value: 82.5107
- type: nauc_map_at_3_max
value: 15.1633
- type: nauc_map_at_3_std
value: -25.835200000000004
- type: nauc_map_at_3_diff1
value: 81.2211
- type: nauc_map_at_5_max
value: 16.4272
- type: nauc_map_at_5_std
value: -24.3902
- type: nauc_map_at_5_diff1
value: 81.3282
- type: nauc_map_at_10_max
value: 16.4997
- type: nauc_map_at_10_std
value: -23.860200000000003
- type: nauc_map_at_10_diff1
value: 81.46759999999999
- type: nauc_map_at_20_max
value: 16.4924
- type: nauc_map_at_20_std
value: -23.6935
- type: nauc_map_at_20_diff1
value: 81.54299999999999
- type: nauc_map_at_100_max
value: 16.381
- type: nauc_map_at_100_std
value: -23.832
- type: nauc_map_at_100_diff1
value: 81.56219999999999
- type: nauc_map_at_1000_max
value: 16.370199999999997
- type: nauc_map_at_1000_std
value: -23.8562
- type: nauc_map_at_1000_diff1
value: 81.5641
- type: nauc_recall_at_1_max
value: 9.1671
- type: nauc_recall_at_1_std
value: -27.6298
- type: nauc_recall_at_1_diff1
value: 82.5107
- type: nauc_recall_at_3_max
value: 22.362000000000002
- type: nauc_recall_at_3_std
value: -19.3576
- type: nauc_recall_at_3_diff1
value: 76.78540000000001
- type: nauc_recall_at_5_max
value: 32.5794
- type: nauc_recall_at_5_std
value: -7.4792
- type: nauc_recall_at_5_diff1
value: 75.7277
- type: nauc_recall_at_10_max
value: 42.6548
- type: nauc_recall_at_10_std
value: 12.512300000000002
- type: nauc_recall_at_10_diff1
value: 75.7877
- type: nauc_recall_at_20_max
value: 55.737
- type: nauc_recall_at_20_std
value: 39.0218
- type: nauc_recall_at_20_diff1
value: 77.1814
- type: nauc_recall_at_100_max
value: 64.1065
- type: nauc_recall_at_100_std
value: 70.1918
- type: nauc_recall_at_100_diff1
value: 78.1114
- type: nauc_recall_at_1000_max
value: 69.8721
- type: nauc_recall_at_1000_std
value: 77.2501
- type: nauc_recall_at_1000_diff1
value: 84.894
- type: nauc_precision_at_1_max
value: 13.195799999999998
- type: nauc_precision_at_1_std
value: -27.2173
- type: nauc_precision_at_1_diff1
value: 83.5531
- type: nauc_precision_at_3_max
value: 22.4513
- type: nauc_precision_at_3_std
value: -6.339
- type: nauc_precision_at_3_diff1
value: 36.5773
- type: nauc_precision_at_5_max
value: 27.0464
- type: nauc_precision_at_5_std
value: 5.2645
- type: nauc_precision_at_5_diff1
value: 22.6914
- type: nauc_precision_at_10_max
value: 28.171000000000003
- type: nauc_precision_at_10_std
value: 17.0803
- type: nauc_precision_at_10_diff1
value: 6.1321
- type: nauc_precision_at_20_max
value: 27.9335
- type: nauc_precision_at_20_std
value: 24.931800000000003
- type: nauc_precision_at_20_diff1
value: -4.4056
- type: nauc_precision_at_100_max
value: 21.8066
- type: nauc_precision_at_100_std
value: 24.659
- type: nauc_precision_at_100_diff1
value: -19.358900000000002
- type: nauc_precision_at_1000_max
value: 17.9584
- type: nauc_precision_at_1000_std
value: 20.4178
- type: nauc_precision_at_1000_diff1
value: -27.5299
- type: nauc_mrr_at_1_max
value: 13.195799999999998
- type: nauc_mrr_at_1_std
value: -27.2173
- type: nauc_mrr_at_1_diff1
value: 83.5531
- type: nauc_mrr_at_3_max
value: 16.801099999999998
- type: nauc_mrr_at_3_std
value: -24.4238
- type: nauc_mrr_at_3_diff1
value: 81.5653
- type: nauc_mrr_at_5_max
value: 17.6255
- type: nauc_mrr_at_5_std
value: -23.345399999999998
- type: nauc_mrr_at_5_diff1
value: 81.6503
- type: nauc_mrr_at_10_max
value: 17.5126
- type: nauc_mrr_at_10_std
value: -22.9439
- type: nauc_mrr_at_10_diff1
value: 81.8044
- type: nauc_mrr_at_20_max
value: 17.468700000000002
- type: nauc_mrr_at_20_std
value: -22.8673
- type: nauc_mrr_at_20_diff1
value: 81.851
- type: nauc_mrr_at_100_max
value: 17.3429
- type: nauc_mrr_at_100_std
value: -23.0229
- type: nauc_mrr_at_100_diff1
value: 81.8634
- type: nauc_mrr_at_1000_max
value: 17.3315
- type: nauc_mrr_at_1000_std
value: -23.045099999999998
- type: nauc_mrr_at_1000_diff1
value: 81.8656
- type: main_score
value: 85.11800000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB MedicalRetrieval (default)
revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6
split: dev
type: C-MTEB/MedicalRetrieval
metrics:
- type: ndcg_at_1
value: 55.00000000000001
- type: ndcg_at_3
value: 61.334
- type: ndcg_at_5
value: 62.590999999999994
- type: ndcg_at_10
value: 63.913
- type: ndcg_at_20
value: 64.748
- type: ndcg_at_100
value: 66.675
- type: ndcg_at_1000
value: 67.894
- type: map_at_1
value: 55.00000000000001
- type: map_at_3
value: 59.85
- type: map_at_5
value: 60.56
- type: map_at_10
value: 61.102999999999994
- type: map_at_20
value: 61.332
- type: map_at_100
value: 61.583
- type: map_at_1000
value: 61.626999999999995
- type: recall_at_1
value: 55.00000000000001
- type: recall_at_3
value: 65.60000000000001
- type: recall_at_5
value: 68.60000000000001
- type: recall_at_10
value: 72.7
- type: recall_at_20
value: 76
- type: recall_at_100
value: 86.6
- type: recall_at_1000
value: 96.3
- type: precision_at_1
value: 55.00000000000001
- type: precision_at_3
value: 21.867
- type: precision_at_5
value: 13.719999999999999
- type: precision_at_10
value: 7.2700000000000005
- type: precision_at_20
value: 3.8
- type: precision_at_100
value: 0.8659999999999999
- type: precision_at_1000
value: 0.096
- type: mrr_at_1
value: 55.00000000000001
- type: mrr_at_3
value: 59.85
- type: mrr_at_5
value: 60.56
- type: mrr_at_10
value: 61.1027
- type: mrr_at_20
value: 61.3323
- type: mrr_at_100
value: 61.583
- type: mrr_at_1000
value: 61.6268
- type: nauc_ndcg_at_1_max
value: 45.7614
- type: nauc_ndcg_at_1_std
value: 17.589299999999998
- type: nauc_ndcg_at_1_diff1
value: 84.6716
- type: nauc_ndcg_at_3_max
value: 43.3722
- type: nauc_ndcg_at_3_std
value: 21.6068
- type: nauc_ndcg_at_3_diff1
value: 79.70779999999999
- type: nauc_ndcg_at_5_max
value: 43.6396
- type: nauc_ndcg_at_5_std
value: 21.9309
- type: nauc_ndcg_at_5_diff1
value: 79.0538
- type: nauc_ndcg_at_10_max
value: 43.5044
- type: nauc_ndcg_at_10_std
value: 22.351
- type: nauc_ndcg_at_10_diff1
value: 78.7137
- type: nauc_ndcg_at_20_max
value: 43.616899999999994
- type: nauc_ndcg_at_20_std
value: 22.1586
- type: nauc_ndcg_at_20_diff1
value: 78.6764
- type: nauc_ndcg_at_100_max
value: 43.135400000000004
- type: nauc_ndcg_at_100_std
value: 22.4145
- type: nauc_ndcg_at_100_diff1
value: 78.35
- type: nauc_ndcg_at_1000_max
value: 43.669799999999995
- type: nauc_ndcg_at_1000_std
value: 22.17
- type: nauc_ndcg_at_1000_diff1
value: 79.0599
- type: nauc_map_at_1_max
value: 45.7614
- type: nauc_map_at_1_std
value: 17.589299999999998
- type: nauc_map_at_1_diff1
value: 84.6716
- type: nauc_map_at_3_max
value: 44.0428
- type: nauc_map_at_3_std
value: 20.6219
- type: nauc_map_at_3_diff1
value: 80.8918
- type: nauc_map_at_5_max
value: 44.2069
- type: nauc_map_at_5_std
value: 20.7613
- type: nauc_map_at_5_diff1
value: 80.545
- type: nauc_map_at_10_max
value: 44.1289
- type: nauc_map_at_10_std
value: 20.9091
- type: nauc_map_at_10_diff1
value: 80.417
- type: nauc_map_at_20_max
value: 44.1582
- type: nauc_map_at_20_std
value: 20.832700000000003
- type: nauc_map_at_20_diff1
value: 80.4097
- type: nauc_map_at_100_max
value: 44.105199999999996
- type: nauc_map_at_100_std
value: 20.8381
- type: nauc_map_at_100_diff1
value: 80.3634
- type: nauc_map_at_1000_max
value: 44.124
- type: nauc_map_at_1000_std
value: 20.846
- type: nauc_map_at_1000_diff1
value: 80.3866
- type: nauc_recall_at_1_max
value: 45.7614
- type: nauc_recall_at_1_std
value: 17.589299999999998
- type: nauc_recall_at_1_diff1
value: 84.6716
- type: nauc_recall_at_3_max
value: 41.2286
- type: nauc_recall_at_3_std
value: 24.6954
- type: nauc_recall_at_3_diff1
value: 76.0107
- type: nauc_recall_at_5_max
value: 41.7596
- type: nauc_recall_at_5_std
value: 25.866899999999998
- type: nauc_recall_at_5_diff1
value: 74.06779999999999
- type: nauc_recall_at_10_max
value: 41.261900000000004
- type: nauc_recall_at_10_std
value: 27.895999999999997
- type: nauc_recall_at_10_diff1
value: 72.3198
- type: nauc_recall_at_20_max
value: 41.6291
- type: nauc_recall_at_20_std
value: 27.720899999999997
- type: nauc_recall_at_20_diff1
value: 71.4933
- type: nauc_recall_at_100_max
value: 34.897
- type: nauc_recall_at_100_std
value: 34.8631
- type: nauc_recall_at_100_diff1
value: 63.698
- type: nauc_recall_at_1000_max
value: 43.4905
- type: nauc_recall_at_1000_std
value: 44.3006
- type: nauc_recall_at_1000_diff1
value: 66.7197
- type: nauc_precision_at_1_max
value: 45.7614
- type: nauc_precision_at_1_std
value: 17.589299999999998
- type: nauc_precision_at_1_diff1
value: 84.6716
- type: nauc_precision_at_3_max
value: 41.2286
- type: nauc_precision_at_3_std
value: 24.6954
- type: nauc_precision_at_3_diff1
value: 76.0107
- type: nauc_precision_at_5_max
value: 41.7596
- type: nauc_precision_at_5_std
value: 25.866899999999998
- type: nauc_precision_at_5_diff1
value: 74.06779999999999
- type: nauc_precision_at_10_max
value: 41.261900000000004
- type: nauc_precision_at_10_std
value: 27.895999999999997
- type: nauc_precision_at_10_diff1
value: 72.3198
- type: nauc_precision_at_20_max
value: 41.6291
- type: nauc_precision_at_20_std
value: 27.720899999999997
- type: nauc_precision_at_20_diff1
value: 71.4933
- type: nauc_precision_at_100_max
value: 34.897
- type: nauc_precision_at_100_std
value: 34.8631
- type: nauc_precision_at_100_diff1
value: 63.698
- type: nauc_precision_at_1000_max
value: 43.4905
- type: nauc_precision_at_1000_std
value: 44.3006
- type: nauc_precision_at_1000_diff1
value: 66.7197
- type: nauc_mrr_at_1_max
value: 45.7614
- type: nauc_mrr_at_1_std
value: 17.589299999999998
- type: nauc_mrr_at_1_diff1
value: 84.6716
- type: nauc_mrr_at_3_max
value: 44.0428
- type: nauc_mrr_at_3_std
value: 20.6219
- type: nauc_mrr_at_3_diff1
value: 80.8918
- type: nauc_mrr_at_5_max
value: 44.2069
- type: nauc_mrr_at_5_std
value: 20.7613
- type: nauc_mrr_at_5_diff1
value: 80.545
- type: nauc_mrr_at_10_max
value: 44.1289
- type: nauc_mrr_at_10_std
value: 20.9091
- type: nauc_mrr_at_10_diff1
value: 80.417
- type: nauc_mrr_at_20_max
value: 44.1582
- type: nauc_mrr_at_20_std
value: 20.832700000000003
- type: nauc_mrr_at_20_diff1
value: 80.4097
- type: nauc_mrr_at_100_max
value: 44.1051
- type: nauc_mrr_at_100_std
value: 20.8382
- type: nauc_mrr_at_100_diff1
value: 80.3634
- type: nauc_mrr_at_1000_max
value: 44.1239
- type: nauc_mrr_at_1000_std
value: 20.8461
- type: nauc_mrr_at_1000_diff1
value: 80.3866
- type: main_score
value: 63.913
task:
type: Retrieval
- dataset:
config: default
name: MTEB T2Retrieval (default)
revision: 8731a845f1bf500a4f111cf1070785c793d10e64
split: dev
type: C-MTEB/T2Retrieval
metrics:
- type: ndcg_at_1
value: 91.22
- type: ndcg_at_3
value: 88.018
- type: ndcg_at_5
value: 87.023
- type: ndcg_at_10
value: 87.256
- type: ndcg_at_20
value: 88.897
- type: ndcg_at_100
value: 90.386
- type: ndcg_at_1000
value: 90.81099999999999
- type: map_at_1
value: 28.203
- type: map_at_3
value: 56.16
- type: map_at_5
value: 69.28699999999999
- type: map_at_10
value: 80.361
- type: map_at_20
value: 83.152
- type: map_at_100
value: 83.894
- type: map_at_1000
value: 83.94
- type: recall_at_1
value: 28.203
- type: recall_at_3
value: 57.821
- type: recall_at_5
value: 72.616
- type: recall_at_10
value: 86.56
- type: recall_at_20
value: 92.145
- type: recall_at_100
value: 96.827
- type: recall_at_1000
value: 98.967
- type: precision_at_1
value: 91.22
- type: precision_at_3
value: 76.946
- type: precision_at_5
value: 64.822
- type: precision_at_10
value: 43.370999999999995
- type: precision_at_20
value: 23.977999999999998
- type: precision_at_100
value: 5.08
- type: precision_at_1000
value: 0.518
- type: mrr_at_1
value: 91.2195
- type: mrr_at_3
value: 93.2214
- type: mrr_at_5
value: 93.48570000000001
- type: mrr_at_10
value: 93.6156
- type: mrr_at_20
value: 93.6549
- type: mrr_at_100
value: 93.6802
- type: mrr_at_1000
value: 93.6823
- type: nauc_ndcg_at_1_max
value: 57.4777
- type: nauc_ndcg_at_1_std
value: 31.1
- type: nauc_ndcg_at_1_diff1
value: 47.8114
- type: nauc_ndcg_at_3_max
value: 47.8308
- type: nauc_ndcg_at_3_std
value: 27.538899999999998
- type: nauc_ndcg_at_3_diff1
value: 13.552900000000001
- type: nauc_ndcg_at_5_max
value: 40.672799999999995
- type: nauc_ndcg_at_5_std
value: 22.5295
- type: nauc_ndcg_at_5_diff1
value: 13.9263
- type: nauc_ndcg_at_10_max
value: 33.6021
- type: nauc_ndcg_at_10_std
value: 15.9671
- type: nauc_ndcg_at_10_diff1
value: 17.678
- type: nauc_ndcg_at_20_max
value: 39.0053
- type: nauc_ndcg_at_20_std
value: 21.218
- type: nauc_ndcg_at_20_diff1
value: 17.5961
- type: nauc_ndcg_at_100_max
value: 44.208
- type: nauc_ndcg_at_100_std
value: 27.437
- type: nauc_ndcg_at_100_diff1
value: 17.7034
- type: nauc_ndcg_at_1000_max
value: 44.8086
- type: nauc_ndcg_at_1000_std
value: 27.460600000000003
- type: nauc_ndcg_at_1000_diff1
value: 18.2625
- type: nauc_map_at_1_max
value: -24.823500000000003
- type: nauc_map_at_1_std
value: -39.5167
- type: nauc_map_at_1_diff1
value: 55.3064
- type: nauc_map_at_3_max
value: -17.5097
- type: nauc_map_at_3_std
value: -39.6402
- type: nauc_map_at_3_diff1
value: 41.255199999999995
- type: nauc_map_at_5_max
value: -5.4253
- type: nauc_map_at_5_std
value: -30.6847
- type: nauc_map_at_5_diff1
value: 33.1347
- type: nauc_map_at_10_max
value: 20.598
- type: nauc_map_at_10_std
value: -1.9429999999999998
- type: nauc_map_at_10_diff1
value: 19.081799999999998
- type: nauc_map_at_20_max
value: 30.55
- type: nauc_map_at_20_std
value: 12.5299
- type: nauc_map_at_20_diff1
value: 14.2657
- type: nauc_map_at_100_max
value: 32.8313
- type: nauc_map_at_100_std
value: 15.956000000000001
- type: nauc_map_at_100_diff1
value: 13.5621
- type: nauc_map_at_1000_max
value: 32.8957
- type: nauc_map_at_1000_std
value: 16.003
- type: nauc_map_at_1000_diff1
value: 13.5638
- type: nauc_recall_at_1_max
value: -24.823500000000003
- type: nauc_recall_at_1_std
value: -39.5167
- type: nauc_recall_at_1_diff1
value: 55.3064
- type: nauc_recall_at_3_max
value: -21.037
- type: nauc_recall_at_3_std
value: -42.2523
- type: nauc_recall_at_3_diff1
value: 39.705200000000005
- type: nauc_recall_at_5_max
value: -12.8722
- type: nauc_recall_at_5_std
value: -37.0029
- type: nauc_recall_at_5_diff1
value: 32.3874
- type: nauc_recall_at_10_max
value: 11.2835
- type: nauc_recall_at_10_std
value: -10.2755
- type: nauc_recall_at_10_diff1
value: 18.333099999999998
- type: nauc_recall_at_20_max
value: 28.602800000000002
- type: nauc_recall_at_20_std
value: 15.407499999999999
- type: nauc_recall_at_20_diff1
value: 10.1417
- type: nauc_recall_at_100_max
value: 46.5586
- type: nauc_recall_at_100_std
value: 44.6235
- type: nauc_recall_at_100_diff1
value: 5.0230999999999995
- type: nauc_recall_at_1000_max
value: 64.8937
- type: nauc_recall_at_1000_std
value: 64.5098
- type: nauc_recall_at_1000_diff1
value: 8.7287
- type: nauc_precision_at_1_max
value: 57.4777
- type: nauc_precision_at_1_std
value: 31.1
- type: nauc_precision_at_1_diff1
value: 47.8114
- type: nauc_precision_at_3_max
value: 51.668499999999995
- type: nauc_precision_at_3_std
value: 48.100500000000004
- type: nauc_precision_at_3_diff1
value: -36.261500000000005
- type: nauc_precision_at_5_max
value: 48.2878
- type: nauc_precision_at_5_std
value: 52.525
- type: nauc_precision_at_5_diff1
value: -39.636300000000006
- type: nauc_precision_at_10_max
value: 44.6478
- type: nauc_precision_at_10_std
value: 56.92549999999999
- type: nauc_precision_at_10_diff1
value: -38.2201
- type: nauc_precision_at_20_max
value: 43.105
- type: nauc_precision_at_20_std
value: 60.181700000000006
- type: nauc_precision_at_20_diff1
value: -37.841
- type: nauc_precision_at_100_max
value: 41.861
- type: nauc_precision_at_100_std
value: 61.2084
- type: nauc_precision_at_100_diff1
value: -37.7674
- type: nauc_precision_at_1000_max
value: 40.9026
- type: nauc_precision_at_1000_std
value: 60.2503
- type: nauc_precision_at_1000_diff1
value: -37.656
- type: nauc_mrr_at_1_max
value: 57.4777
- type: nauc_mrr_at_1_std
value: 31.1
- type: nauc_mrr_at_1_diff1
value: 47.8114
- type: nauc_mrr_at_3_max
value: 61.8122
- type: nauc_mrr_at_3_std
value: 35.3495
- type: nauc_mrr_at_3_diff1
value: 47.9786
- type: nauc_mrr_at_5_max
value: 62.030300000000004
- type: nauc_mrr_at_5_std
value: 35.8677
- type: nauc_mrr_at_5_diff1
value: 48.0773
- type: nauc_mrr_at_10_max
value: 61.9735
- type: nauc_mrr_at_10_std
value: 35.8413
- type: nauc_mrr_at_10_diff1
value: 48.0359
- type: nauc_mrr_at_20_max
value: 61.917100000000005
- type: nauc_mrr_at_20_std
value: 35.8176
- type: nauc_mrr_at_20_diff1
value: 48.024699999999996
- type: nauc_mrr_at_100_max
value: 61.8535
- type: nauc_mrr_at_100_std
value: 35.7606
- type: nauc_mrr_at_100_diff1
value: 48.0105
- type: nauc_mrr_at_1000_max
value: 61.8448
- type: nauc_mrr_at_1000_std
value: 35.747299999999996
- type: nauc_mrr_at_1000_diff1
value: 48.0124
- type: main_score
value: 87.256
task:
type: Retrieval
- dataset:
config: default
name: MTEB VideoRetrieval (default)
revision: 58c2597a5943a2ba48f4668c3b90d796283c5639
split: dev
type: C-MTEB/VideoRetrieval
metrics:
- type: ndcg_at_1
value: 65.9
- type: ndcg_at_3
value: 75.395
- type: ndcg_at_5
value: 76.97
- type: ndcg_at_10
value: 78.549
- type: ndcg_at_20
value: 79.489
- type: ndcg_at_100
value: 80.269
- type: ndcg_at_1000
value: 80.459
- type: map_at_1
value: 65.9
- type: map_at_3
value: 73.167
- type: map_at_5
value: 74.047
- type: map_at_10
value: 74.713
- type: map_at_20
value: 74.97399999999999
- type: map_at_100
value: 75.089
- type: map_at_1000
value: 75.09599999999999
- type: recall_at_1
value: 65.9
- type: recall_at_3
value: 81.8
- type: recall_at_5
value: 85.6
- type: recall_at_10
value: 90.4
- type: recall_at_20
value: 94.1
- type: recall_at_100
value: 98.2
- type: recall_at_1000
value: 99.7
- type: precision_at_1
value: 65.9
- type: precision_at_3
value: 27.267000000000003
- type: precision_at_5
value: 17.119999999999997
- type: precision_at_10
value: 9.04
- type: precision_at_20
value: 4.705
- type: precision_at_100
value: 0.9820000000000001
- type: precision_at_1000
value: 0.1
- type: mrr_at_1
value: 65.8
- type: mrr_at_3
value: 73.1167
- type: mrr_at_5
value: 73.9967
- type: mrr_at_10
value: 74.66329999999999
- type: mrr_at_20
value: 74.9235
- type: mrr_at_100
value: 75.0391
- type: mrr_at_1000
value: 75.0463
- type: nauc_ndcg_at_1_max
value: -7.102899999999999
- type: nauc_ndcg_at_1_std
value: -40.454
- type: nauc_ndcg_at_1_diff1
value: 73.7435
- type: nauc_ndcg_at_3_max
value: -10.2162
- type: nauc_ndcg_at_3_std
value: -51.03
- type: nauc_ndcg_at_3_diff1
value: 70.7969
- type: nauc_ndcg_at_5_max
value: -8.1372
- type: nauc_ndcg_at_5_std
value: -50.727999999999994
- type: nauc_ndcg_at_5_diff1
value: 70.4751
- type: nauc_ndcg_at_10_max
value: -8.305
- type: nauc_ndcg_at_10_std
value: -48.972500000000004
- type: nauc_ndcg_at_10_diff1
value: 70.9306
- type: nauc_ndcg_at_20_max
value: -7.61
- type: nauc_ndcg_at_20_std
value: -48.0327
- type: nauc_ndcg_at_20_diff1
value: 71.02159999999999
- type: nauc_ndcg_at_100_max
value: -7.6188
- type: nauc_ndcg_at_100_std
value: -46.3162
- type: nauc_ndcg_at_100_diff1
value: 71.1421
- type: nauc_ndcg_at_1000_max
value: -7.8627
- type: nauc_ndcg_at_1000_std
value: -46.7545
- type: nauc_ndcg_at_1000_diff1
value: 71.352
- type: nauc_map_at_1_max
value: -7.102899999999999
- type: nauc_map_at_1_std
value: -40.454
- type: nauc_map_at_1_diff1
value: 73.7435
- type: nauc_map_at_3_max
value: -9.6192
- type: nauc_map_at_3_std
value: -48.0776
- type: nauc_map_at_3_diff1
value: 71.545
- type: nauc_map_at_5_max
value: -8.5417
- type: nauc_map_at_5_std
value: -47.8473
- type: nauc_map_at_5_diff1
value: 71.42389999999999
- type: nauc_map_at_10_max
value: -8.6365
- type: nauc_map_at_10_std
value: -47.1493
- type: nauc_map_at_10_diff1
value: 71.5916
- type: nauc_map_at_20_max
value: -8.516
- type: nauc_map_at_20_std
value: -46.958800000000004
- type: nauc_map_at_20_diff1
value: 71.6177
- type: nauc_map_at_100_max
value: -8.498999999999999
- type: nauc_map_at_100_std
value: -46.7142
- type: nauc_map_at_100_diff1
value: 71.638
- type: nauc_map_at_1000_max
value: -8.4997
- type: nauc_map_at_1000_std
value: -46.7198
- type: nauc_map_at_1000_diff1
value: 71.6453
- type: nauc_recall_at_1_max
value: -7.102899999999999
- type: nauc_recall_at_1_std
value: -40.454
- type: nauc_recall_at_1_diff1
value: 73.7435
- type: nauc_recall_at_3_max
value: -12.3728
- type: nauc_recall_at_3_std
value: -62.4854
- type: nauc_recall_at_3_diff1
value: 67.9381
- type: nauc_recall_at_5_max
value: -5.5908
- type: nauc_recall_at_5_std
value: -64.0017
- type: nauc_recall_at_5_diff1
value: 66.01129999999999
- type: nauc_recall_at_10_max
value: -5.2142
- type: nauc_recall_at_10_std
value: -59.1241
- type: nauc_recall_at_10_diff1
value: 67.0484
- type: nauc_recall_at_20_max
value: 6.0224
- type: nauc_recall_at_20_std
value: -52.987899999999996
- type: nauc_recall_at_20_diff1
value: 65.76859999999999
- type: nauc_recall_at_100_max
value: 33.408500000000004
- type: nauc_recall_at_100_std
value: 15.325800000000001
- type: nauc_recall_at_100_diff1
value: 59.588699999999996
- type: nauc_recall_at_1000_max
value: 100
- type: nauc_recall_at_1000_std
value: 95.6427
- type: nauc_recall_at_1000_diff1
value: 91.28540000000001
- type: nauc_precision_at_1_max
value: -7.102899999999999
- type: nauc_precision_at_1_std
value: -40.454
- type: nauc_precision_at_1_diff1
value: 73.7435
- type: nauc_precision_at_3_max
value: -12.3728
- type: nauc_precision_at_3_std
value: -62.4854
- type: nauc_precision_at_3_diff1
value: 67.9381
- type: nauc_precision_at_5_max
value: -5.5908
- type: nauc_precision_at_5_std
value: -64.0017
- type: nauc_precision_at_5_diff1
value: 66.01129999999999
- type: nauc_precision_at_10_max
value: -5.2142
- type: nauc_precision_at_10_std
value: -59.1241
- type: nauc_precision_at_10_diff1
value: 67.0484
- type: nauc_precision_at_20_max
value: 6.0224
- type: nauc_precision_at_20_std
value: -52.987899999999996
- type: nauc_precision_at_20_diff1
value: 65.76859999999999
- type: nauc_precision_at_100_max
value: 33.408500000000004
- type: nauc_precision_at_100_std
value: 15.325800000000001
- type: nauc_precision_at_100_diff1
value: 59.588699999999996
- type: nauc_precision_at_1000_max
value: 100
- type: nauc_precision_at_1000_std
value: 95.6427
- type: nauc_precision_at_1000_diff1
value: 91.28540000000001
- type: nauc_mrr_at_1_max
value: -7.704999999999999
- type: nauc_mrr_at_1_std
value: -40.435300000000005
- type: nauc_mrr_at_1_diff1
value: 73.9529
- type: nauc_mrr_at_3_max
value: -9.9824
- type: nauc_mrr_at_3_std
value: -48.0572
- type: nauc_mrr_at_3_diff1
value: 71.6749
- type: nauc_mrr_at_5_max
value: -8.9185
- type: nauc_mrr_at_5_std
value: -47.8265
- type: nauc_mrr_at_5_diff1
value: 71.5584
- type: nauc_mrr_at_10_max
value: -9.023299999999999
- type: nauc_mrr_at_10_std
value: -47.1288
- type: nauc_mrr_at_10_diff1
value: 71.7295
- type: nauc_mrr_at_20_max
value: -8.9076
- type: nauc_mrr_at_20_std
value: -46.9382
- type: nauc_mrr_at_20_diff1
value: 71.7573
- type: nauc_mrr_at_100_max
value: -8.892700000000001
- type: nauc_mrr_at_100_std
value: -46.6939
- type: nauc_mrr_at_100_diff1
value: 71.7782
- type: nauc_mrr_at_1000_max
value: -8.8935
- type: nauc_mrr_at_1000_std
value: -46.699400000000004
- type: nauc_mrr_at_1000_diff1
value: 71.7855
- type: main_score
value: 78.549
task:
type: Retrieval
tags:
- sentence-transformers
- transformers
- sentence-similarity
- mteb
license: apache-2.0
INF-Retriever-v1
Model Overview
INF-Retriever-v1 is an LLM-based dense retrieval model developed by INF TECH. It is built upon the gte-Qwen2-7B-instruct model and specifically fine-tuned to excel in retrieval tasks, particularly for Chinese and English data.
As of January 23, 2025, INF-Retriever-v1 ranks both No.1 on the Automated Heterogeneous Information Retrieval Benchmark of version 24.04 & 24.05(AIR-Bench), showcasing its cutting-edge performance in heterogeneous information retrieval tasks.
Key Features
Optimized for Chinese and English retrieval: The model has been specifically fine-tuned with retrieval-focused datasets in both languages, significantly improving its accuracy and efficiency for a variety of retrieval scenarios.
Top-tier performance: INF-Retriever-v1 has achieved outstanding results on the AIR-Bench leaderboard, making it a top choice for heterogeneous information retrieval tasks across various domains.
Usage
Sentence Transformers
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("infly/inf-retriever-v1", trust_remote_code=True)
# In case you want to reduce the maximum length:
model.max_seq_length = 8192
queries = [
"how much protein should a female eat",
"summit define",
]
documents = [
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.",
]
query_embeddings = model.encode(queries, prompt_name="query")
document_embeddings = model.encode(documents)
scores = (query_embeddings @ document_embeddings.T) * 100
print(scores.tolist())
# [[86.8702392578125, 67.82364654541016], [59.51014709472656, 82.33668518066406]]
Transformers
import torch
import torch.nn.functional as F
from torch import Tensor
from transformers import AutoTokenizer, AutoModel
def last_token_pool(last_hidden_states: Tensor,
attention_mask: Tensor) -> Tensor:
left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])
if left_padding:
return last_hidden_states[:, -1]
else:
sequence_lengths = attention_mask.sum(dim=1) - 1
batch_size = last_hidden_states.shape[0]
return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths]
def get_detailed_instruct(task_description: str, query: str) -> str:
return f'Instruct: {task_description}\nQuery: {query}'
# Each query must come with a one-sentence instruction that describes the task
task = 'Given a web search query, retrieve relevant passages that answer the query'
queries = [
get_detailed_instruct(task, 'how much protein should a female eat'),
get_detailed_instruct(task, 'summit define')
]
# No need to add instruction for retrieval documents
documents = [
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."
]
input_texts = queries + documents
tokenizer = AutoTokenizer.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True)
model = AutoModel.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True)
max_length = 8192
# Tokenize the input texts
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
outputs = model(**batch_dict)
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
# normalize embeddings
embeddings = F.normalize(embeddings, p=2, dim=1)
scores = (embeddings[:2] @ embeddings[2:].T) * 100
print(scores.tolist())
# [[86.87025451660156, 67.82366180419922], [59.510135650634766, 82.33667755126953]]
Evaluation
AIR-Bench
INF-Retriever-v1 has demonstrated superior retrieval capabilities across multiple domains and languages. The results from the Automated Heterogeneous Information Retrieval Benchmark (AIR-Bench) as of January 23, 2025, are as follows:
AIR-Bench_24.04 (Bilingual, EN & ZH)
| Model Name | Average⬆️ | wiki_en | wiki_zh | web_en | web_zh | healthcare_en | healthcare_zh | law_en | arxiv_en | news_en | news_zh | finance_en | finance_zh | msmarco_en |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| E5-mistral-7b-instruct | 45.26 | 61.67 | 55.97 | 44.41 | 45.96 | 56.32 | 35.79 | 19.32 | 44.78 | 48.18 | 35.99 | 54.79 | 26.11 | 59.03 |
| BGE-M3 | 46.65 | 60.49 | 62.36 | 47.35 | 50.38 | 49.1 | 42.38 | 26.68 | 40.76 | 48.04 | 40.75 | 51.52 | 32.18 | 54.4 |
| BGE-Multilingual-Gemma2 | 46.83 | 63.71 | 67.3 | 50.38 | 53.24 | 47.24 | 42.13 | 22.58 | 23.28 | 50.91 | 44.02 | 49.3 | 31.6 | 63.14 |
| GTE-Qwen2-7B-instruct | 48.38 | 63.46 | 66.44 | 51.2 | 51.98 | 54.2 | 38.82 | 22.31 | 40.27 | 54.07 | 43.03 | 58.2 | 26.63 | 58.39 |
| INF-Retriever-v1 | 52.56 | 65.25 | 68.44 | 52.13 | 56.6 | 56.96 | 42.03 | 34.51 | 50.62 | 53.32 | 50.02 | 58.34 | 35.42 | 59.64 |
AIR-Bench_24.05 (Multilingual, 13 languages)
Although INF-Retriever-v1 has been fine-tuned exclusively on English and Chinese, it continues to perform exceptionally well across other languages, securing the No. 1 position on this multilingual benchmark.
| Model Name | Average⬆️ | wiki_en | wiki_zh | wiki_ar | wiki_bn | wiki_de | wiki_es | wiki_fa | wiki_fr | wiki_hi | wiki_id | wiki_ja | wiki_ko | wiki_ru | web_en | web_zh | web_ar | web_bn | web_de | web_es | web_fa | web_fr | web_hi | web_id | web_ja | web_ko | web_ru | healthcare_en | healthcare_zh | healthcare_de | healthcare_es | healthcare_fr | law_en | law_de | law_fr | arxiv_en | science_ru | news_en | news_zh | news_ar | news_bn | news_de | news_es | news_fa | news_fr | news_hi | news_id | news_ja | news_ko | news_ru | finance_en | finance_zh | finance_ar | finance_fr |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| GTE-Qwen2-7B-instruct | 50.05 | 73.59 | 67.5 | 59.44 | 58.17 | 63.96 | 67.62 | 57.05 | 70.32 | 60.54 | 61.81 | 62.88 | 59.17 | 62.95 | 58.99 | 51.66 | 55.56 | 51.45 | 48.62 | 54.11 | 49.54 | 55.16 | 53.06 | 55.51 | 57.27 | 57.54 | 55.88 | 54.46 | 38.66 | 53.92 | 53.78 | 30.29 | 22.75 | 13.18 | 13.15 | 41.32 | 45.21 | 52.74 | 43.17 | 37.63 | 61.31 | 44.89 | 45.21 | 30.1 | 49.76 | 30.28 | 46.44 | 44.13 | 47.19 | 46.55 | 59.23 | 34.61 | 43.56 | 39.57 |
| Multilingual-E5-large-instruct | 51.11 | 68.62 | 62.82 | 63.21 | 64.45 | 65.81 | 68.1 | 64.2 | 69.72 | 71.81 | 66.36 | 64.12 | 64.79 | 62.57 | 41.58 | 47.06 | 56.4 | 56.17 | 50.87 | 52.24 | 58.68 | 50.2 | 56.32 | 54.49 | 54.89 | 55.81 | 54.97 | 54.02 | 39.76 | 52.06 | 51.74 | 36.64 | 16.9 | 15.59 | 15.12 | 39.52 | 56.86 | 44.28 | 35.46 | 48.2 | 49.31 | 47.84 | 45.99 | 45.59 | 50.58 | 39.66 | 48.59 | 47.6 | 50.52 | 48.81 | 52.79 | 37.72 | 48.95 | 42.74 |
| BGE-M3 | 51.31 | 69.7 | 63.52 | 59.65 | 64.33 | 64.68 | 65.4 | 61.14 | 66.04 | 69.02 | 66.3 | 60.86 | 62.36 | 60.18 | 53.88 | 50.2 | 52.53 | 55.53 | 51.89 | 51.78 | 55.81 | 51.46 | 57.06 | 53.14 | 54.75 | 55.28 | 54.53 | 49.05 | 42.31 | 49 | 53.05 | 39.29 | 26.95 | 20.11 | 20.2 | 41.64 | 55.18 | 47.34 | 41 | 44.93 | 59.03 | 47.87 | 44.7 | 43.81 | 49.52 | 42.12 | 47.45 | 47.09 | 48.14 | 48.31 | 52.92 | 40.23 | 45.76 | 41.44 |
| BGE-Multilingual-Gemma2 | 54.46 | 72.8 | 68.64 | 63.42 | 69.48 | 67.91 | 71.79 | 67.57 | 71.28 | 75.39 | 68.91 | 68.29 | 66.78 | 64.15 | 56.48 | 53.04 | 59.97 | 59.68 | 57.72 | 58.2 | 62.43 | 59.54 | 64.5 | 60 | 60.26 | 59.64 | 60.12 | 47.48 | 42.35 | 55.4 | 63.13 | 45.13 | 22.6 | 15.75 | 14.29 | 24 | 44.13 | 50.29 | 43.42 | 48.41 | 58.77 | 52.05 | 49.9 | 43.4 | 56.8 | 44.89 | 50.65 | 51.51 | 51.64 | 51.48 | 50.08 | 39.23 | 50.25 | 51.1 |
| INF-Retriever-v1 | 54.47 | 73.52 | 69.45 | 63.13 | 61.58 | 66.8 | 69.29 | 63.03 | 69.74 | 69.02 | 68.63 | 63.45 | 64.44 | 62.74 | 57.6 | 56.46 | 58.48 | 53.7 | 55.2 | 57.08 | 53.27 | 57.35 | 55.64 | 58.85 | 59.52 | 60.01 | 58.79 | 57.03 | 41.82 | 55.46 | 57.6 | 43.25 | 34.76 | 21.75 | 21.87 | 51.38 | 59.72 | 52.7 | 49.78 | 49.11 | 43.62 | 51.47 | 49.52 | 40.43 | 54.54 | 38.57 | 51.06 | 51.12 | 53.15 | 51.88 | 59.44 | 44.13 | 50.71 | 44.2 |
Contributors
Supervisors
Wei Chu • Yinghui Xu • Yuan Qi
INF memory team
Junhan Yang ([email protected]) • Jiahe Wan • Yichen Yao ([email protected])