q275343119 commited on
Commit
5560186
·
1 Parent(s): 7699424

init - 初始化

Browse files
.gitignore ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.pyc
2
+ model_infos.json
3
+ space
4
+ .venv
5
+ results
6
+ mteb
7
+ **/.DS_Store
8
+
9
+ # FastAPI
10
+ **/__pycache__/
11
+ .pytest_cache/
12
+ .coverage
13
+ htmlcov/
14
+
15
+ # Gradio
16
+ flagged/
17
+ gradio_cached_examples/
18
+ tmp/
19
+ logs/
20
+
21
+ # IDE
22
+ .idea/
23
+ .vscode/
24
+
25
+ # 环境相关
26
+ .env
27
+ .env.*
DESCRIPTION.md ADDED
@@ -0,0 +1 @@
 
 
1
+ Massive Text Embedding Benchmark (MTEB) Leaderboard.
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2025 笑尿伊人
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
app.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 16:59
3
+ # @Author : q275343119
4
+ # @File : app.py
5
+ # @Description: Main entry point for the application
6
+ from app.ui.home import init_home
7
+ from app.ui.about import init_about
8
+
9
+ import gradio as gr
10
+
11
+
12
+ def main():
13
+ home = init_home()
14
+ about = init_about()
15
+
16
+ demo = gr.TabbedInterface(
17
+ [home, about],
18
+ ["Home", "About"],
19
+ css="footer {display: none}"
20
+ )
21
+ demo.launch()
22
+
23
+
24
+ if __name__ == '__main__':
25
+ main()
app/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:13
3
+ # @Author : q275343119
4
+ # @File : __init__.py.py
5
+ # @Description:
app/backend/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:16
3
+ # @Author : q275343119
4
+ # @File : __init__.py.py
5
+ # @Description: Backend module initialization
app/backend/constant.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from enum import Enum
2
+
3
+
4
+ class Navigation(Enum):
5
+ TEXT_LEADERBOARD = "Text Leaderboard"
6
+ MULTIMODAL_LEADERBOARD = "Multimodal Leaderboard"
7
+
8
+
9
+ class TaskCategory(Enum):
10
+ LAW = "Law"
11
+ CODE = "Code"
12
+ CONVERSATIONAL = "Conversational"
13
+ TECH = "Tech"
14
+ LONG_CONTEXT = "Long-context"
15
+ MULTILINGUAL = "Multilingual"
16
+
17
+
18
+ class ModelProvider(Enum):
19
+ OPENAI = "OpenAI"
20
+ VOYAGEAI = "VoyageAI"
21
+ COHERE = "Cohere"
22
+ OTHERS = "Others"
23
+
24
+
25
+ class EvaluationMetric(Enum):
26
+ NDCG_1 = "NDCG@1"
27
+ NDCG_3 = "NDCG@3"
28
+ NDCG_5 = "NDCG@5"
29
+ NDCG_10 = "NDCG@10"
30
+ NDCG_20 = "NDCG@20"
31
+ NDCG_50 = "NDCG@50"
32
+ NDCG_100 = "NDCG@100"
33
+ RECALL_1 = "RECALL@1"
34
+ RECALL_3 = "RECALL@3"
35
+ RECALL_5 = "RECALL@5"
36
+ RECALL_10 = "RECALL@10"
37
+ RECALL_20 = "RECALL@20"
38
+ RECALL_50 = "RECALL@50"
39
+ RECALL_100 = "RECALL@100"
40
+ PRECISION_1 = "PRECISION@1"
41
+ PRECISION_3 = "PRECISION@3"
42
+ PRECISION_5 = "PRECISION@5"
43
+ PRECISION_10 = "PRECISION@10"
44
+ PRECISION_20 = "PRECISION@20"
45
+ PRECISION_50 = "PRECISION@50"
46
+ PRECISION_100 = "PRECISION@100"
app/backend/data_engine.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Data service provider
3
+ """
4
+ import json
5
+ from typing import List
6
+
7
+ import pandas as pd
8
+
9
+ from app.backend.constant import ModelProvider
10
+
11
+
12
+ class DataEngine:
13
+
14
+ def __init__(self):
15
+ self.df = self.init_dataframe()
16
+
17
+ @property
18
+ def leaderboards(self):
19
+ """
20
+ Get leaderboard data
21
+ """
22
+ with open('./mock_data/leaderboard.json', 'r') as f:
23
+ return json.load(f)
24
+
25
+ @property
26
+ def models(self):
27
+ """
28
+ Get models data
29
+ """
30
+ with open('./mock_data/models.json', 'r') as f:
31
+ return json.load(f)
32
+
33
+ @property
34
+ def tasks(self):
35
+ """
36
+ Get tasks data
37
+ """
38
+ with open('./mock_data/tasks.json', 'r') as f:
39
+ return json.load(f)
40
+
41
+ @property
42
+ def results(self):
43
+ """
44
+ Get results data
45
+ """
46
+ with open('./mock_data/results.json', 'r') as f:
47
+ return json.load(f)
48
+
49
+ def init_dataframe(self):
50
+ """
51
+ Initialize DataFrame
52
+ """
53
+ d = {"hello": [123], "world": [456]}
54
+ return pd.DataFrame(d)
55
+
56
+ def _check_providers(self, organization: str, providers: List):
57
+ if not providers:
58
+ return True
59
+ if "Others" in providers:
60
+ if organization not in (
61
+ ModelProvider.OPENAI.value, ModelProvider.COHERE.value, ModelProvider.VOYAGEAI.value):
62
+ return True
63
+ return organization in providers
64
+
65
+ def filter_df(self, leaderboard: str, task: str, providers: List, sort_key: str):
66
+
67
+ tasks = []
68
+ for lb in self.leaderboards:
69
+ if lb["name"] == leaderboard:
70
+ tasks = lb["tasks"]
71
+ break
72
+ df_list = []
73
+ for t in (filter(lambda x: x.upper() == task.upper(), tasks)):
74
+ datasets = []
75
+ for ta in self.tasks:
76
+ if ta["slug"].upper() == t.upper():
77
+ datasets = ta["datasets"]
78
+ break
79
+ for model in self.models:
80
+ if t in model["tasks"] and self._check_providers(model["organization"], providers):
81
+
82
+ for dataset in datasets:
83
+ results = self.results[dataset]
84
+ for result in results:
85
+ if result['model_name'] == model["model_name"]:
86
+ d = result["results"]
87
+ d["class"] = result["class"]
88
+ d["organization"] = result["organization"]
89
+ d["model_name"] = result["model_name"]
90
+ df = pd.DataFrame([d])
91
+ df = df[["class", "organization", "model_name", "ndcg_at_1", "ndcg_at_3", "ndcg_at_5",
92
+ "ndcg_at_10",
93
+ "ndcg_at_20", "ndcg_at_50", "ndcg_at_100", "recall_at_1", "recall_at_3",
94
+ "recall_at_5", "recall_at_10",
95
+ "recall_at_20", "recall_at_50", "recall_at_100", "precision_at_1",
96
+ "precision_at_3", "precision_at_5",
97
+ "precision_at_10", "precision_at_20", "precision_at_50", "precision_at_100"]]
98
+ df_list.append(df)
99
+ if df_list:
100
+ return pd.concat(df_list).sort_values(by=sort_key.replace("@", '_at_').lower())
101
+ return pd.DataFrame(columns=["class", "organization", "model_name", "ndcg_at_1", "ndcg_at_3", "ndcg_at_5",
102
+ "ndcg_at_10",
103
+ "ndcg_at_20", "ndcg_at_50", "ndcg_at_100", "recall_at_1", "recall_at_3",
104
+ "recall_at_5", "recall_at_10",
105
+ "recall_at_20", "recall_at_50", "recall_at_100", "precision_at_1",
106
+ "precision_at_3", "precision_at_5",
107
+ "precision_at_10", "precision_at_20", "precision_at_50", "precision_at_100"])
108
+
109
+ def get_model_result(self, model: dict, task_datasets_map: dict, results: dict):
110
+ """
111
+ get_model_result
112
+ """
113
+ model_class = model["class"]
114
+ model_organization = model["organization"]
115
+ model_model_name = model["model_name"]
116
+ for leaderboard in model["leaderboards"]:
117
+ for task in model["tasks"]:
118
+ for dateset in task_datasets_map.get(task, []):
119
+ for result in results[dateset]:
120
+ if result["model_name"] == model_model_name:
121
+ d_result = result["results"]
122
+ d_result["class"] = model_class
123
+ d_result["organization"] = model_organization
124
+ d_result["model_name"] = model_model_name
125
+ d_result["leaderboard"] = leaderboard
126
+ d_result["dateset"] = dateset
127
+ d_result["task"] = task
128
+ yield d_result
129
+
130
+ def jsons_to_df(self):
131
+
132
+ # change leaderboards to task_leaderboard_map
133
+ task_leaderboard_map = {}
134
+ leaderboards = self.leaderboards
135
+ for leaderboard in leaderboards:
136
+ for task in leaderboard["tasks"]:
137
+ task_leaderboard_map[task] = leaderboard["name"]
138
+
139
+ # change tasks to task_datasets_map
140
+ task_datasets_map = {}
141
+ for task in self.tasks:
142
+ task_datasets_map[task["slug"]] = task["datasets"]
143
+
144
+ df_results_list = []
145
+ results = self.results
146
+ for model in self.models:
147
+ for d_result in self.get_model_result(model, task_datasets_map, results):
148
+ if d_result:
149
+ df_results_list.append(pd.DataFrame([d_result]))
150
+
151
+ if df_results_list:
152
+ df_result = pd.concat(df_results_list)
153
+ return df_result[
154
+ ["leaderboard", "task", "class", "organization", "model_name", "dateset", "ndcg_at_1", "ndcg_at_3",
155
+ "ndcg_at_5",
156
+ "ndcg_at_10",
157
+ "ndcg_at_20", "ndcg_at_50", "ndcg_at_100", "recall_at_1", "recall_at_3",
158
+ "recall_at_5", "recall_at_10",
159
+ "recall_at_20", "recall_at_50", "recall_at_100", "precision_at_1",
160
+ "precision_at_3", "precision_at_5",
161
+ "precision_at_10", "precision_at_20", "precision_at_50", "precision_at_100"]], leaderboards
162
+ return pd.DataFrame(
163
+ columns=["leaderboard", "task", "class", "organization", "model_name", "dateset", "ndcg_at_1", "ndcg_at_3",
164
+ "ndcg_at_5",
165
+ "ndcg_at_10",
166
+ "ndcg_at_20", "ndcg_at_50", "ndcg_at_100", "recall_at_1", "recall_at_3",
167
+ "recall_at_5", "recall_at_10",
168
+ "recall_at_20", "recall_at_50", "recall_at_100", "precision_at_1",
169
+ "precision_at_3", "precision_at_5",
170
+ "precision_at_10", "precision_at_20", "precision_at_50", "precision_at_100"]), leaderboards
171
+
172
+ def filter_by_providers(self, df_result: pd.DataFrame, providers: List):
173
+ """
174
+ filter_by_providers
175
+ """
176
+ return df_result[df_result['organization'].apply(lambda x: self._check_providers(x, providers))]
177
+
178
+ def summarize_dataframe(self):
179
+ """
180
+ Summarize data statistics
181
+ """
app/ui/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:16
3
+ # @Author : q275343119
4
+ # @File : __init__.py.py
5
+ # @Description:
app/ui/about.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:20
3
+ # @Author : q275343119
4
+ # @File : about.py
5
+ # @Description: About page implementation
6
+
7
+ import gradio as gr
8
+
9
+
10
+ def init_about():
11
+ """Initialize the about page"""
12
+ with gr.Blocks() as block:
13
+ with gr.Column():
14
+ # Motivation section
15
+ gr.Markdown("## Motivation")
16
+ gr.Markdown("""
17
+ [Placeholder] This section explains why we created this embedding benchmark:
18
+ - To standardize embedding evaluation
19
+ - To promote transparency in model performance
20
+ - To accelerate progress in embedding research
21
+ """)
22
+
23
+ # Organizers section
24
+ gr.Markdown("## Organizers")
25
+ gr.Markdown("""
26
+ [Placeholder] The MTEB benchmark is organized by:
27
+ - Person A - Organization X
28
+ - Person B - Organization Y
29
+ - Person C - Organization Z
30
+ """)
31
+
32
+ # FAQ section
33
+ gr.Markdown("## Frequently Asked Questions")
34
+ gr.Markdown("""
35
+ [Placeholder] Common questions about MTEB:
36
+
37
+ **Q: How can I submit my model?**
38
+ A: Follow the submission guidelines in our GitHub repository.
39
+
40
+ **Q: What metrics are used?**
41
+ A: We use various metrics depending on the task type.
42
+
43
+ **Q: How often is the leaderboard updated?**
44
+ A: The leaderboard is updated daily.
45
+ """)
46
+
47
+ block.queue(max_size=10)
48
+ return block
app/ui/component/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:25
3
+ # @Author : q275343119
4
+ # @File : __init__.py.py
5
+ # @Description:
app/ui/component/df_component.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import gradio as gr
3
+
4
+ from app.backend.data_engine import DataEngine
5
+
6
+
7
+ class DataFrameComponent:
8
+
9
+ def __init__(self, data_engine: DataEngine):
10
+ self.data_engine = data_engine
11
+
12
+ def show(self, df=None):
13
+ with gr.Row():
14
+ # Create DataFrame display area
15
+ if df is None:
16
+ df = self.data_engine.init_dataframe()
17
+ print(df)
18
+ df_display = gr.DataFrame(df, col_count=(len(df.columns), 'fixed'))
19
+
20
+ return df_display
app/ui/component/filter_component.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:26
3
+ # @Author : q275343119
4
+ # @File : filter_component.py
5
+ # @Description: Filter component implementation
6
+ """
7
+ Filter component for data filtering
8
+ """
9
+ from typing import List
10
+ import gradio as gr
11
+
12
+ from app.backend.data_engine import DataEngine
13
+
14
+
15
+ class FilterComponent:
16
+
17
+ def __init__(self, data_engine: DataEngine, navigations: List, model_provides: List,
18
+ evaluation_metrics: List):
19
+ self.data_engine = data_engine
20
+ self.navigations = navigations
21
+ self.task_categories = []
22
+ self.model_provides = model_provides
23
+ self.evaluation_metrics = evaluation_metrics
24
+
25
+ def show(self):
26
+ with gr.Row():
27
+ navigations = gr.Radio(
28
+ label="Navigation",
29
+ choices=self.navigations,
30
+ value=self.navigations[0],
31
+ interactive=True,
32
+ elem_classes=["filter-checkbox-group"],
33
+ scale=2,
34
+ )
35
+
36
+ model_provides = gr.CheckboxGroup(
37
+ label="Model Provides",
38
+ choices=self.model_provides,
39
+ value=self.model_provides,
40
+ interactive=True,
41
+ elem_classes=["filter-checkbox-group"],
42
+ scale=2,
43
+ )
44
+ evaluation_metrics = gr.Dropdown(
45
+ label="Evaluation Metrics",
46
+ choices=self.evaluation_metrics,
47
+ value="NDCG@10",
48
+ interactive=True,
49
+ elem_classes=["filter-checkbox-group"],
50
+ scale=2,
51
+ )
52
+
53
+ return navigations, model_provides, evaluation_metrics
app/ui/component/subtabs_component.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List
2
+ import gradio as gr
3
+ import pandas as pd
4
+
5
+ from app.backend.data_engine import DataEngine
6
+
7
+
8
+ class SubtabsComponent:
9
+
10
+ def __init__(self, data_engine: DataEngine):
11
+ self.data_engine = data_engine
12
+
13
+ def show(self, navigations: str = None, model_provides: List = None, evaluation_metrics: str = None):
14
+ df_result, leaderboards = self.data_engine.jsons_to_df()
15
+
16
+ navigations = "Text" if navigations is None else navigations.split(" ", maxsplit=1)[0]
17
+ model_provides = [] if model_provides is None else model_provides
18
+ evaluation_metrics = "NDCG@10" if evaluation_metrics is None else evaluation_metrics
19
+
20
+ df_result = self.data_engine.filter_by_providers(df_result, model_provides).sort_values(
21
+ by=evaluation_metrics.replace("@", '_at_').lower())
22
+
23
+ items = []
24
+ for leaderboard in leaderboards:
25
+
26
+ with gr.Column(visible=leaderboard["name"] == navigations) as column:
27
+ with gr.Tabs():
28
+ with gr.TabItem("overall"):
29
+ df_leaderboard = df_result[df_result["leaderboard"] == leaderboard["slug"]]
30
+
31
+ gr_df = gr.Dataframe(df_leaderboard)
32
+ items.append(gr_df)
33
+
34
+ for task in leaderboard["tasks"]:
35
+ with gr.TabItem(task):
36
+ df = df_leaderboard[df_leaderboard["task"] == task]
37
+
38
+ gr_df = gr.Dataframe(df)
39
+ items.append(gr_df)
40
+ items.append(column)
41
+ return items
app/ui/home.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:20
3
+ # @Author : q275343119
4
+ # @File : home.py
5
+ # @Description: Home page implementation and UI components
6
+ import pandas as pd
7
+
8
+ from app.backend.constant import Navigation, TaskCategory, ModelProvider, EvaluationMetric
9
+ from app.backend.data_engine import DataEngine
10
+ from app.ui.component.df_component import DataFrameComponent
11
+ from app.ui.component.filter_component import FilterComponent
12
+ from app.ui.component.subtabs_component import SubtabsComponent
13
+ from app.ui.static import HOME_CSS
14
+
15
+ import gradio as gr
16
+
17
+ NUM_DATASETS = 1
18
+ NUM_SCORES = 2
19
+ NUM_MODELS = 3
20
+
21
+
22
+ def init_home():
23
+ """
24
+ Initialize the home page
25
+ """
26
+ data_engine = DataEngine()
27
+
28
+ with gr.Blocks(css=HOME_CSS) as block:
29
+ gr.Markdown(f"""
30
+ [Voyageai] Massive Text Embedding Benchmark (MTEB) Leaderboard. To submit, refer to the <a href="https://github.com/embeddings-benchmark/mteb/blob/main/docs/adding_a_model.md" target="_blank" style="text-decoration: underline">MTEB GitHub repository</a> 🤗 Refer to the [MTEB paper](https://arxiv.org/abs/2210.07316) for details on metrics, tasks and models. Also check out [MTEB Arena](https://huggingface.co/spaces/mteb/arena) ⚔️
31
+ """)
32
+
33
+ filter_area = FilterComponent(
34
+ data_engine,
35
+ [element.value for element in Navigation],
36
+ [element.value for element in ModelProvider],
37
+ [element.value for element in EvaluationMetric],
38
+ )
39
+
40
+ navigations, model_provides, evaluation_metrics = filter_area.show()
41
+
42
+ sub_tabs = SubtabsComponent(data_engine)
43
+ columns = sub_tabs.show()
44
+
45
+ # df_area = DataFrameComponent(data_engine)
46
+ # df_display = df_area.show(pd.DataFrame(columns=[element.value for element in Navigation]))
47
+
48
+
49
+ gr.Markdown(f"""
50
+ - **Total Datasets**: {NUM_DATASETS}
51
+ - **Total Languages**: 113
52
+ - **Total Scores**: {NUM_SCORES}
53
+ - **Total Models**: {NUM_MODELS}
54
+ """ + r"""
55
+ Made with ❤️ for NLP. If this work is useful to you, please consider citing:
56
+
57
+ ```bibtex
58
+ @article{muennighoff2022mteb,
59
+ doi = {10.48550/ARXIV.2210.07316},
60
+ url = {https://arxiv.org/abs/2210.07316},
61
+ author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Lo{\"\i}c and Reimers, Nils},
62
+ title = {MTEB: Massive Text Embedding Benchmark},
63
+ publisher = {arXiv},
64
+ journal={arXiv preprint arXiv:2210.07316},
65
+ year = {2022}
66
+ }
67
+ ```
68
+ """)
69
+ # all_outputs = list[sub_tab_map.values()]
70
+ block.load(sub_tabs.show,
71
+ inputs=[navigations, model_provides, evaluation_metrics], outputs=columns)
72
+
73
+ gr.on(triggers=[navigations.change,
74
+ model_provides.change,
75
+ evaluation_metrics.change],
76
+ inputs=[navigations, model_provides, evaluation_metrics],
77
+ fn=sub_tabs.show,
78
+ outputs=columns
79
+ )
80
+
81
+
82
+ block.queue(max_size=10)
83
+ return block
app/ui/static.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ # @Date : 2024/12/24 17:16
3
+ # @Author : q275343119
4
+ # @File : static.py
5
+ # @Description:
6
+
7
+ HOME_CSS = """
8
+ table > thead {
9
+ white-space: normal
10
+ }
11
+
12
+ table {
13
+ --cell-width-1: 250px
14
+ }
15
+
16
+ table > tbody > tr > td:nth-child(2) > div {
17
+ overflow-x: auto
18
+ }
19
+
20
+ .filter-checkbox-group {
21
+ max-width: max-content;
22
+ }
23
+ """
mock_data/leaderboard.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "slug": "text",
4
+ "name": "Text",
5
+ "tasks": [
6
+ "law",
7
+ "long-context",
8
+ "finance",
9
+ "conversational",
10
+ "tech",
11
+ "multilingual",
12
+ "code",
13
+ "healthcare"
14
+ ]
15
+ },
16
+ {
17
+ "slug": "multimodal",
18
+ "name": "Multimodal",
19
+ "tasks": [
20
+ "text-to-photo",
21
+ "document-screenshot",
22
+ "figures-and-tables",
23
+ "text-to-text"
24
+ ]
25
+ }
26
+ ]
mock_data/models.json ADDED
@@ -0,0 +1,400 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+
3
+ {
4
+ "class": "sentence-transformers",
5
+ "organization": "BAAI",
6
+ "model_name": "bge-law-transformer",
7
+ "leaderboards": [
8
+ "text"
9
+ ],
10
+ "tasks": [
11
+ "law"
12
+ ]
13
+ },
14
+ {
15
+ "class": "custom",
16
+ "organization": "OpenAI",
17
+ "model_name": "openai-law-embed-v2",
18
+ "leaderboards": [
19
+ "text"
20
+ ],
21
+ "tasks": [
22
+ "law"
23
+ ]
24
+ },
25
+ {
26
+ "class": "proprietary",
27
+ "organization": "Google",
28
+ "model_name": "google-law-model-v1",
29
+ "leaderboards": [
30
+ "text"
31
+ ],
32
+ "tasks": [
33
+ "law"
34
+ ]
35
+ },
36
+ {
37
+ "class": "sentence-transformers",
38
+ "organization": "HuggingFace",
39
+ "model_name": "hf-long-context-transformer",
40
+ "leaderboards": [
41
+ "text"
42
+ ],
43
+ "tasks": [
44
+ "long-context"
45
+ ]
46
+ },
47
+ {
48
+ "class": "custom",
49
+ "organization": "Apple",
50
+ "model_name": "apple-long-context-embed",
51
+ "leaderboards": [
52
+ "text"
53
+ ],
54
+ "tasks": [
55
+ "long-context"
56
+ ]
57
+ },
58
+ {
59
+ "class": "proprietary",
60
+ "organization": "Microsoft",
61
+ "model_name": "microsoft-long-context-v1",
62
+ "leaderboards": [
63
+ "text"
64
+ ],
65
+ "tasks": [
66
+ "long-context"
67
+ ]
68
+ },
69
+ {
70
+ "class": "sentence-transformers",
71
+ "organization": "HuggingFace",
72
+ "model_name": "hf-finance-transformer",
73
+ "leaderboards": [
74
+ "text"
75
+ ],
76
+ "tasks": [
77
+ "finance"
78
+ ]
79
+ },
80
+ {
81
+ "class": "custom",
82
+ "organization": "OpenAI",
83
+ "model_name": "openai-finance-embed-v1",
84
+ "leaderboards": [
85
+ "text"
86
+ ],
87
+ "tasks": [
88
+ "finance"
89
+ ]
90
+ },
91
+ {
92
+ "class": "proprietary",
93
+ "organization": "Google",
94
+ "model_name": "google-finance-embed-v1",
95
+ "leaderboards": [
96
+ "text"
97
+ ],
98
+ "tasks": [
99
+ "finance"
100
+ ]
101
+ },
102
+ {
103
+ "class": "custom",
104
+ "organization": "Microsoft",
105
+ "model_name": "microsoft-conversational-v2",
106
+ "leaderboards": [
107
+ "text"
108
+ ],
109
+ "tasks": [
110
+ "conversational"
111
+ ]
112
+ },
113
+ {
114
+ "class": "sentence-transformers",
115
+ "organization": "Facebook",
116
+ "model_name": "fb-conversational-embed",
117
+ "leaderboards": [
118
+ "text"
119
+ ],
120
+ "tasks": [
121
+ "conversational"
122
+ ]
123
+ },
124
+ {
125
+ "class": "proprietary",
126
+ "organization": "Amazon",
127
+ "model_name": "amazon-conversational-transformer",
128
+ "leaderboards": [
129
+ "text"
130
+ ],
131
+ "tasks": [
132
+ "conversational"
133
+ ]
134
+ },
135
+ {
136
+ "class": "sentence-transformers",
137
+ "organization": "Tesla",
138
+ "model_name": "tesla-tech-model",
139
+ "leaderboards": [
140
+ "text"
141
+ ],
142
+ "tasks": [
143
+ "tech"
144
+ ]
145
+ },
146
+ {
147
+ "class": "custom",
148
+ "organization": "Snowflake",
149
+ "model_name": "snowflake-tech-embed",
150
+ "leaderboards": [
151
+ "text"
152
+ ],
153
+ "tasks": [
154
+ "tech"
155
+ ]
156
+ },
157
+ {
158
+ "class": "proprietary",
159
+ "organization": "DeepMind",
160
+ "model_name": "dm-tech-v1",
161
+ "leaderboards": [
162
+ "text"
163
+ ],
164
+ "tasks": [
165
+ "tech"
166
+ ]
167
+ },
168
+ {
169
+ "class": "sentence-transformers",
170
+ "organization": "Tesla",
171
+ "model_name": "tesla-multilingual",
172
+ "leaderboards": [
173
+ "text"
174
+ ],
175
+ "tasks": [
176
+ "multilingual"
177
+ ]
178
+ },
179
+ {
180
+ "class": "custom",
181
+ "organization": "Apple",
182
+ "model_name": "apple-multilingual-embed",
183
+ "leaderboards": [
184
+ "text"
185
+ ],
186
+ "tasks": [
187
+ "multilingual"
188
+ ]
189
+ },
190
+ {
191
+ "class": "proprietary",
192
+ "organization": "Google",
193
+ "model_name": "google-multilingual-model",
194
+ "leaderboards": [
195
+ "text"
196
+ ],
197
+ "tasks": [
198
+ "multilingual"
199
+ ]
200
+ },
201
+ {
202
+ "class": "custom",
203
+ "organization": "Microsoft",
204
+ "model_name": "microsoft-code-embed",
205
+ "leaderboards": [
206
+ "text"
207
+ ],
208
+ "tasks": [
209
+ "code"
210
+ ]
211
+ },
212
+ {
213
+ "class": "sentence-transformers",
214
+ "organization": "HuggingFace",
215
+ "model_name": "hf-code-transformer",
216
+ "leaderboards": [
217
+ "text"
218
+ ],
219
+ "tasks": [
220
+ "code"
221
+ ]
222
+ },
223
+ {
224
+ "class": "proprietary",
225
+ "organization": "Apple",
226
+ "model_name": "apple-code-model",
227
+ "leaderboards": [
228
+ "text"
229
+ ],
230
+ "tasks": [
231
+ "code"
232
+ ]
233
+ },
234
+ {
235
+ "class": "custom",
236
+ "organization": "Microsoft",
237
+ "model_name": "microsoft-healthcare-v2",
238
+ "leaderboards": [
239
+ "text"
240
+ ],
241
+ "tasks": [
242
+ "healthcare"
243
+ ]
244
+ },
245
+ {
246
+ "class": "sentence-transformers",
247
+ "organization": "Facebook",
248
+ "model_name": "fb-healthcare-embed",
249
+ "leaderboards": [
250
+ "text"
251
+ ],
252
+ "tasks": [
253
+ "healthcare"
254
+ ]
255
+ },
256
+ {
257
+ "class": "proprietary",
258
+ "organization": "Google",
259
+ "model_name": "google-healthcare-model",
260
+ "leaderboards": [
261
+ "text"
262
+ ],
263
+ "tasks": [
264
+ "healthcare"
265
+ ]
266
+ },
267
+
268
+ {
269
+ "class": "sentence-transformers",
270
+ "organization": "Facebook",
271
+ "model_name": "fb-text-to-photo",
272
+ "leaderboards": [
273
+ "multimodal"
274
+ ],
275
+ "tasks": [
276
+ "text-to-photo"
277
+ ]
278
+ },
279
+ {
280
+ "class": "proprietary",
281
+ "organization": "Apple",
282
+ "model_name": "apple-text-to-photo-v2",
283
+ "leaderboards": [
284
+ "multimodal"
285
+ ],
286
+ "tasks": [
287
+ "text-to-photo"
288
+ ]
289
+ },
290
+ {
291
+ "class": "custom",
292
+ "organization": "Microsoft",
293
+ "model_name": "microsoft-text-to-photo",
294
+ "leaderboards": [
295
+ "multimodal"
296
+ ],
297
+ "tasks": [
298
+ "text-to-photo"
299
+ ]
300
+ },
301
+ {
302
+ "class": "custom",
303
+ "organization": "Microsoft",
304
+ "model_name": "microsoft-document-screenshot",
305
+ "leaderboards": [
306
+ "multimodal"
307
+ ],
308
+ "tasks": [
309
+ "document-screenshot"
310
+ ]
311
+ },
312
+ {
313
+ "class": "sentence-transformers",
314
+ "organization": "Google",
315
+ "model_name": "google-document-screenshot",
316
+ "leaderboards": [
317
+ "multimodal"
318
+ ],
319
+ "tasks": [
320
+ "document-screenshot"
321
+ ]
322
+ },
323
+ {
324
+ "class": "proprietary",
325
+ "organization": "Facebook",
326
+ "model_name": "fb-document-screenshot",
327
+ "leaderboards": [
328
+ "multimodal"
329
+ ],
330
+ "tasks": [
331
+ "document-screenshot"
332
+ ]
333
+ },
334
+ {
335
+ "class": "sentence-transformers",
336
+ "organization": "Google",
337
+ "model_name": "google-figures-tables",
338
+ "leaderboards": [
339
+ "multimodal"
340
+ ],
341
+ "tasks": [
342
+ "figures-and-tables"
343
+ ]
344
+ },
345
+ {
346
+ "class": "custom",
347
+ "organization": "Microsoft",
348
+ "model_name": "microsoft-figures-tables",
349
+ "leaderboards": [
350
+ "multimodal"
351
+ ],
352
+ "tasks": [
353
+ "figures-and-tables"
354
+ ]
355
+ },
356
+ {
357
+ "class": "proprietary",
358
+ "organization": "Apple",
359
+ "model_name": "apple-figures-tables",
360
+ "leaderboards": [
361
+ "multimodal"
362
+ ],
363
+ "tasks": [
364
+ "figures-and-tables"
365
+ ]
366
+ },
367
+ {
368
+ "class": "sentence-transformers",
369
+ "organization": "OpenAI",
370
+ "model_name": "openai-text-to-text",
371
+ "leaderboards": [
372
+ "multimodal"
373
+ ],
374
+ "tasks": [
375
+ "text-to-text"
376
+ ]
377
+ },
378
+ {
379
+ "class": "proprietary",
380
+ "organization": "Facebook",
381
+ "model_name": "fb-text-to-text",
382
+ "leaderboards": [
383
+ "multimodal"
384
+ ],
385
+ "tasks": [
386
+ "text-to-text"
387
+ ]
388
+ },
389
+ {
390
+ "class": "custom",
391
+ "organization": "Apple",
392
+ "model_name": "apple-text-to-text",
393
+ "leaderboards": [
394
+ "multimodal"
395
+ ],
396
+ "tasks": [
397
+ "text-to-text"
398
+ ]
399
+ }
400
+ ]
mock_data/results.json ADDED
@@ -0,0 +1,3098 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "ALIA_casedocs": [
3
+ {
4
+ "class": "sentence-transformers",
5
+ "organization": "BAAI",
6
+ "model_name": "bge-law-transformer",
7
+ "results": {
8
+ "ndcg_at_1": 0.36478,
9
+ "ndcg_at_3": 0.37725,
10
+ "ndcg_at_5": 0.44341,
11
+ "ndcg_at_10": 0.42686,
12
+ "ndcg_at_20": 0.54972,
13
+ "ndcg_at_50": 0.51785,
14
+ "ndcg_at_100": 0.54966,
15
+ "recall_at_1": 0.12306,
16
+ "recall_at_3": 0.27949,
17
+ "recall_at_5": 0.35275,
18
+ "recall_at_10": 0.44979,
19
+ "recall_at_20": 0.62146,
20
+ "recall_at_50": 0.76205,
21
+ "recall_at_100": 0.93419,
22
+ "precision_at_1": 0.43235,
23
+ "precision_at_3": 0.32557,
24
+ "precision_at_5": 0.25043,
25
+ "precision_at_10": 0.17796,
26
+ "precision_at_20": 0.11168,
27
+ "precision_at_50": 0.05925,
28
+ "precision_at_100": 0.03773
29
+ }
30
+ },
31
+ {
32
+ "class": "custom",
33
+ "organization": "OpenAI",
34
+ "model_name": "openai-law-embed-v2",
35
+ "results": {
36
+ "ndcg_at_1": 0.32971,
37
+ "ndcg_at_3": 0.38459,
38
+ "ndcg_at_5": 0.41729,
39
+ "ndcg_at_10": 0.43828,
40
+ "ndcg_at_20": 0.53335,
41
+ "ndcg_at_50": 0.53531,
42
+ "ndcg_at_100": 0.53924,
43
+ "recall_at_1": 0.12671,
44
+ "recall_at_3": 0.25399,
45
+ "recall_at_5": 0.35906,
46
+ "recall_at_10": 0.44872,
47
+ "recall_at_20": 0.60041,
48
+ "recall_at_50": 0.77739,
49
+ "recall_at_100": 0.9348,
50
+ "precision_at_1": 0.44211,
51
+ "precision_at_3": 0.32576,
52
+ "precision_at_5": 0.25242,
53
+ "precision_at_10": 0.17539,
54
+ "precision_at_20": 0.11747,
55
+ "precision_at_50": 0.0544,
56
+ "precision_at_100": 0.03865
57
+ }
58
+ },
59
+ {
60
+ "class": "proprietary",
61
+ "organization": "Google",
62
+ "model_name": "google-law-model-v1",
63
+ "results": {
64
+ "ndcg_at_1": 0.4045,
65
+ "ndcg_at_3": 0.3939,
66
+ "ndcg_at_5": 0.42908,
67
+ "ndcg_at_10": 0.43616,
68
+ "ndcg_at_20": 0.52403,
69
+ "ndcg_at_50": 0.52347,
70
+ "ndcg_at_100": 0.53751,
71
+ "recall_at_1": 0.14181,
72
+ "recall_at_3": 0.2506,
73
+ "recall_at_5": 0.36251,
74
+ "recall_at_10": 0.4489,
75
+ "recall_at_20": 0.62968,
76
+ "recall_at_50": 0.78569,
77
+ "recall_at_100": 0.93525,
78
+ "precision_at_1": 0.40471,
79
+ "precision_at_3": 0.31683,
80
+ "precision_at_5": 0.26878,
81
+ "precision_at_10": 0.17019,
82
+ "precision_at_20": 0.11306,
83
+ "precision_at_50": 0.05615,
84
+ "precision_at_100": 0.03734
85
+ }
86
+ }
87
+ ],
88
+ "LegalTexts": [
89
+ {
90
+ "class": "sentence-transformers",
91
+ "organization": "BAAI",
92
+ "model_name": "bge-law-transformer",
93
+ "results": {
94
+ "ndcg_at_1": 0.36192,
95
+ "ndcg_at_3": 0.44016,
96
+ "ndcg_at_5": 0.40487,
97
+ "ndcg_at_10": 0.40586,
98
+ "ndcg_at_20": 0.54346,
99
+ "ndcg_at_50": 0.50676,
100
+ "ndcg_at_100": 0.53148,
101
+ "recall_at_1": 0.14222,
102
+ "recall_at_3": 0.27691,
103
+ "recall_at_5": 0.36361,
104
+ "recall_at_10": 0.44561,
105
+ "recall_at_20": 0.61007,
106
+ "recall_at_50": 0.77295,
107
+ "recall_at_100": 0.94334,
108
+ "precision_at_1": 0.35762,
109
+ "precision_at_3": 0.31123,
110
+ "precision_at_5": 0.25394,
111
+ "precision_at_10": 0.17507,
112
+ "precision_at_20": 0.11435,
113
+ "precision_at_50": 0.05183,
114
+ "precision_at_100": 0.03373
115
+ }
116
+ },
117
+ {
118
+ "class": "custom",
119
+ "organization": "OpenAI",
120
+ "model_name": "openai-law-embed-v2",
121
+ "results": {
122
+ "ndcg_at_1": 0.46065,
123
+ "ndcg_at_3": 0.38512,
124
+ "ndcg_at_5": 0.36361,
125
+ "ndcg_at_10": 0.43889,
126
+ "ndcg_at_20": 0.51997,
127
+ "ndcg_at_50": 0.50135,
128
+ "ndcg_at_100": 0.53673,
129
+ "recall_at_1": 0.13369,
130
+ "recall_at_3": 0.27221,
131
+ "recall_at_5": 0.35675,
132
+ "recall_at_10": 0.44555,
133
+ "recall_at_20": 0.62188,
134
+ "recall_at_50": 0.76927,
135
+ "recall_at_100": 0.94952,
136
+ "precision_at_1": 0.44624,
137
+ "precision_at_3": 0.32536,
138
+ "precision_at_5": 0.25935,
139
+ "precision_at_10": 0.17961,
140
+ "precision_at_20": 0.1146,
141
+ "precision_at_50": 0.0558,
142
+ "precision_at_100": 0.03893
143
+ }
144
+ },
145
+ {
146
+ "class": "proprietary",
147
+ "organization": "Google",
148
+ "model_name": "google-law-model-v1",
149
+ "results": {
150
+ "ndcg_at_1": 0.30446,
151
+ "ndcg_at_3": 0.43084,
152
+ "ndcg_at_5": 0.40613,
153
+ "ndcg_at_10": 0.44502,
154
+ "ndcg_at_20": 0.52514,
155
+ "ndcg_at_50": 0.53524,
156
+ "ndcg_at_100": 0.54568,
157
+ "recall_at_1": 0.14275,
158
+ "recall_at_3": 0.25138,
159
+ "recall_at_5": 0.36204,
160
+ "recall_at_10": 0.44248,
161
+ "recall_at_20": 0.60406,
162
+ "recall_at_50": 0.75297,
163
+ "recall_at_100": 0.94421,
164
+ "precision_at_1": 0.41081,
165
+ "precision_at_3": 0.30649,
166
+ "precision_at_5": 0.26292,
167
+ "precision_at_10": 0.17944,
168
+ "precision_at_20": 0.11021,
169
+ "precision_at_50": 0.0529,
170
+ "precision_at_100": 0.03297
171
+ }
172
+ }
173
+ ],
174
+ "CaseLaw": [
175
+ {
176
+ "class": "sentence-transformers",
177
+ "organization": "BAAI",
178
+ "model_name": "bge-law-transformer",
179
+ "results": {
180
+ "ndcg_at_1": 0.36211,
181
+ "ndcg_at_3": 0.40056,
182
+ "ndcg_at_5": 0.41713,
183
+ "ndcg_at_10": 0.40765,
184
+ "ndcg_at_20": 0.4876,
185
+ "ndcg_at_50": 0.53407,
186
+ "ndcg_at_100": 0.53122,
187
+ "recall_at_1": 0.12847,
188
+ "recall_at_3": 0.25477,
189
+ "recall_at_5": 0.36699,
190
+ "recall_at_10": 0.4467,
191
+ "recall_at_20": 0.60769,
192
+ "recall_at_50": 0.76661,
193
+ "recall_at_100": 0.94573,
194
+ "precision_at_1": 0.44695,
195
+ "precision_at_3": 0.32835,
196
+ "precision_at_5": 0.26066,
197
+ "precision_at_10": 0.17599,
198
+ "precision_at_20": 0.11326,
199
+ "precision_at_50": 0.05851,
200
+ "precision_at_100": 0.03426
201
+ }
202
+ },
203
+ {
204
+ "class": "custom",
205
+ "organization": "OpenAI",
206
+ "model_name": "openai-law-embed-v2",
207
+ "results": {
208
+ "ndcg_at_1": 0.42982,
209
+ "ndcg_at_3": 0.38709,
210
+ "ndcg_at_5": 0.37816,
211
+ "ndcg_at_10": 0.41592,
212
+ "ndcg_at_20": 0.52352,
213
+ "ndcg_at_50": 0.52101,
214
+ "ndcg_at_100": 0.53155,
215
+ "recall_at_1": 0.14278,
216
+ "recall_at_3": 0.26978,
217
+ "recall_at_5": 0.35648,
218
+ "recall_at_10": 0.44579,
219
+ "recall_at_20": 0.61447,
220
+ "recall_at_50": 0.76217,
221
+ "recall_at_100": 0.94399,
222
+ "precision_at_1": 0.40751,
223
+ "precision_at_3": 0.30081,
224
+ "precision_at_5": 0.2541,
225
+ "precision_at_10": 0.17927,
226
+ "precision_at_20": 0.11016,
227
+ "precision_at_50": 0.05374,
228
+ "precision_at_100": 0.03766
229
+ }
230
+ },
231
+ {
232
+ "class": "proprietary",
233
+ "organization": "Google",
234
+ "model_name": "google-law-model-v1",
235
+ "results": {
236
+ "ndcg_at_1": 0.44943,
237
+ "ndcg_at_3": 0.43604,
238
+ "ndcg_at_5": 0.42766,
239
+ "ndcg_at_10": 0.44154,
240
+ "ndcg_at_20": 0.52754,
241
+ "ndcg_at_50": 0.54695,
242
+ "ndcg_at_100": 0.54296,
243
+ "recall_at_1": 0.12455,
244
+ "recall_at_3": 0.25695,
245
+ "recall_at_5": 0.35183,
246
+ "recall_at_10": 0.44062,
247
+ "recall_at_20": 0.60774,
248
+ "recall_at_50": 0.75593,
249
+ "recall_at_100": 0.94127,
250
+ "precision_at_1": 0.42763,
251
+ "precision_at_3": 0.313,
252
+ "precision_at_5": 0.26384,
253
+ "precision_at_10": 0.17497,
254
+ "precision_at_20": 0.11667,
255
+ "precision_at_50": 0.05425,
256
+ "precision_at_100": 0.03931
257
+ }
258
+ }
259
+ ],
260
+ "ResearchPapers": [
261
+ {
262
+ "class": "sentence-transformers",
263
+ "organization": "HuggingFace",
264
+ "model_name": "hf-long-context-transformer",
265
+ "results": {
266
+ "ndcg_at_1": 0.31451,
267
+ "ndcg_at_3": 0.44086,
268
+ "ndcg_at_5": 0.40646,
269
+ "ndcg_at_10": 0.4224,
270
+ "ndcg_at_20": 0.49329,
271
+ "ndcg_at_50": 0.51133,
272
+ "ndcg_at_100": 0.54613,
273
+ "recall_at_1": 0.14023,
274
+ "recall_at_3": 0.25622,
275
+ "recall_at_5": 0.36902,
276
+ "recall_at_10": 0.4443,
277
+ "recall_at_20": 0.60141,
278
+ "recall_at_50": 0.76068,
279
+ "recall_at_100": 0.93481,
280
+ "precision_at_1": 0.44999,
281
+ "precision_at_3": 0.30118,
282
+ "precision_at_5": 0.26018,
283
+ "precision_at_10": 0.17529,
284
+ "precision_at_20": 0.11333,
285
+ "precision_at_50": 0.0599,
286
+ "precision_at_100": 0.03698
287
+ }
288
+ },
289
+ {
290
+ "class": "custom",
291
+ "organization": "Apple",
292
+ "model_name": "apple-long-context-embed",
293
+ "results": {
294
+ "ndcg_at_1": 0.33481,
295
+ "ndcg_at_3": 0.40097,
296
+ "ndcg_at_5": 0.35182,
297
+ "ndcg_at_10": 0.42715,
298
+ "ndcg_at_20": 0.48506,
299
+ "ndcg_at_50": 0.52616,
300
+ "ndcg_at_100": 0.53372,
301
+ "recall_at_1": 0.12212,
302
+ "recall_at_3": 0.25664,
303
+ "recall_at_5": 0.35762,
304
+ "recall_at_10": 0.44112,
305
+ "recall_at_20": 0.629,
306
+ "recall_at_50": 0.76181,
307
+ "recall_at_100": 0.93975,
308
+ "precision_at_1": 0.3747,
309
+ "precision_at_3": 0.30879,
310
+ "precision_at_5": 0.26735,
311
+ "precision_at_10": 0.17013,
312
+ "precision_at_20": 0.11509,
313
+ "precision_at_50": 0.05378,
314
+ "precision_at_100": 0.03815
315
+ }
316
+ },
317
+ {
318
+ "class": "proprietary",
319
+ "organization": "Microsoft",
320
+ "model_name": "microsoft-long-context-v1",
321
+ "results": {
322
+ "ndcg_at_1": 0.30163,
323
+ "ndcg_at_3": 0.42854,
324
+ "ndcg_at_5": 0.35301,
325
+ "ndcg_at_10": 0.41328,
326
+ "ndcg_at_20": 0.46942,
327
+ "ndcg_at_50": 0.54489,
328
+ "ndcg_at_100": 0.54623,
329
+ "recall_at_1": 0.14336,
330
+ "recall_at_3": 0.27145,
331
+ "recall_at_5": 0.3677,
332
+ "recall_at_10": 0.44922,
333
+ "recall_at_20": 0.60745,
334
+ "recall_at_50": 0.75395,
335
+ "recall_at_100": 0.93033,
336
+ "precision_at_1": 0.37077,
337
+ "precision_at_3": 0.3186,
338
+ "precision_at_5": 0.25815,
339
+ "precision_at_10": 0.17424,
340
+ "precision_at_20": 0.11122,
341
+ "precision_at_50": 0.05416,
342
+ "precision_at_100": 0.03459
343
+ }
344
+ }
345
+ ],
346
+ "Books": [
347
+ {
348
+ "class": "sentence-transformers",
349
+ "organization": "HuggingFace",
350
+ "model_name": "hf-long-context-transformer",
351
+ "results": {
352
+ "ndcg_at_1": 0.36261,
353
+ "ndcg_at_3": 0.43934,
354
+ "ndcg_at_5": 0.38147,
355
+ "ndcg_at_10": 0.44412,
356
+ "ndcg_at_20": 0.51783,
357
+ "ndcg_at_50": 0.51233,
358
+ "ndcg_at_100": 0.54471,
359
+ "recall_at_1": 0.14233,
360
+ "recall_at_3": 0.2751,
361
+ "recall_at_5": 0.36276,
362
+ "recall_at_10": 0.44174,
363
+ "recall_at_20": 0.61989,
364
+ "recall_at_50": 0.75488,
365
+ "recall_at_100": 0.93724,
366
+ "precision_at_1": 0.37642,
367
+ "precision_at_3": 0.30001,
368
+ "precision_at_5": 0.26099,
369
+ "precision_at_10": 0.17765,
370
+ "precision_at_20": 0.1169,
371
+ "precision_at_50": 0.05827,
372
+ "precision_at_100": 0.03998
373
+ }
374
+ },
375
+ {
376
+ "class": "custom",
377
+ "organization": "Apple",
378
+ "model_name": "apple-long-context-embed",
379
+ "results": {
380
+ "ndcg_at_1": 0.36391,
381
+ "ndcg_at_3": 0.41878,
382
+ "ndcg_at_5": 0.35613,
383
+ "ndcg_at_10": 0.42244,
384
+ "ndcg_at_20": 0.47992,
385
+ "ndcg_at_50": 0.52095,
386
+ "ndcg_at_100": 0.54423,
387
+ "recall_at_1": 0.13303,
388
+ "recall_at_3": 0.25958,
389
+ "recall_at_5": 0.36674,
390
+ "recall_at_10": 0.44978,
391
+ "recall_at_20": 0.60272,
392
+ "recall_at_50": 0.75622,
393
+ "recall_at_100": 0.94982,
394
+ "precision_at_1": 0.36653,
395
+ "precision_at_3": 0.32442,
396
+ "precision_at_5": 0.2673,
397
+ "precision_at_10": 0.17172,
398
+ "precision_at_20": 0.11823,
399
+ "precision_at_50": 0.05541,
400
+ "precision_at_100": 0.03613
401
+ }
402
+ },
403
+ {
404
+ "class": "proprietary",
405
+ "organization": "Microsoft",
406
+ "model_name": "microsoft-long-context-v1",
407
+ "results": {
408
+ "ndcg_at_1": 0.40574,
409
+ "ndcg_at_3": 0.40208,
410
+ "ndcg_at_5": 0.43823,
411
+ "ndcg_at_10": 0.40922,
412
+ "ndcg_at_20": 0.47995,
413
+ "ndcg_at_50": 0.50552,
414
+ "ndcg_at_100": 0.53766,
415
+ "recall_at_1": 0.12922,
416
+ "recall_at_3": 0.27522,
417
+ "recall_at_5": 0.35574,
418
+ "recall_at_10": 0.44328,
419
+ "recall_at_20": 0.60411,
420
+ "recall_at_50": 0.76089,
421
+ "recall_at_100": 0.94238,
422
+ "precision_at_1": 0.35448,
423
+ "precision_at_3": 0.30718,
424
+ "precision_at_5": 0.25895,
425
+ "precision_at_10": 0.17633,
426
+ "precision_at_20": 0.11984,
427
+ "precision_at_50": 0.05793,
428
+ "precision_at_100": 0.03789
429
+ }
430
+ }
431
+ ],
432
+ "LegalDocuments": [
433
+ {
434
+ "class": "sentence-transformers",
435
+ "organization": "HuggingFace",
436
+ "model_name": "hf-long-context-transformer",
437
+ "results": {
438
+ "ndcg_at_1": 0.36806,
439
+ "ndcg_at_3": 0.36677,
440
+ "ndcg_at_5": 0.39546,
441
+ "ndcg_at_10": 0.42128,
442
+ "ndcg_at_20": 0.5002,
443
+ "ndcg_at_50": 0.53357,
444
+ "ndcg_at_100": 0.53802,
445
+ "recall_at_1": 0.12385,
446
+ "recall_at_3": 0.27164,
447
+ "recall_at_5": 0.36313,
448
+ "recall_at_10": 0.4499,
449
+ "recall_at_20": 0.60007,
450
+ "recall_at_50": 0.76891,
451
+ "recall_at_100": 0.93358,
452
+ "precision_at_1": 0.3996,
453
+ "precision_at_3": 0.30551,
454
+ "precision_at_5": 0.25785,
455
+ "precision_at_10": 0.17764,
456
+ "precision_at_20": 0.11254,
457
+ "precision_at_50": 0.05461,
458
+ "precision_at_100": 0.0374
459
+ }
460
+ },
461
+ {
462
+ "class": "custom",
463
+ "organization": "Apple",
464
+ "model_name": "apple-long-context-embed",
465
+ "results": {
466
+ "ndcg_at_1": 0.42178,
467
+ "ndcg_at_3": 0.36932,
468
+ "ndcg_at_5": 0.4149,
469
+ "ndcg_at_10": 0.42505,
470
+ "ndcg_at_20": 0.51548,
471
+ "ndcg_at_50": 0.52734,
472
+ "ndcg_at_100": 0.53818,
473
+ "recall_at_1": 0.13936,
474
+ "recall_at_3": 0.26506,
475
+ "recall_at_5": 0.35152,
476
+ "recall_at_10": 0.44632,
477
+ "recall_at_20": 0.62205,
478
+ "recall_at_50": 0.78145,
479
+ "recall_at_100": 0.93817,
480
+ "precision_at_1": 0.41881,
481
+ "precision_at_3": 0.3196,
482
+ "precision_at_5": 0.26914,
483
+ "precision_at_10": 0.17815,
484
+ "precision_at_20": 0.11278,
485
+ "precision_at_50": 0.05694,
486
+ "precision_at_100": 0.03641
487
+ }
488
+ },
489
+ {
490
+ "class": "proprietary",
491
+ "organization": "Microsoft",
492
+ "model_name": "microsoft-long-context-v1",
493
+ "results": {
494
+ "ndcg_at_1": 0.33571,
495
+ "ndcg_at_3": 0.3897,
496
+ "ndcg_at_5": 0.43525,
497
+ "ndcg_at_10": 0.4356,
498
+ "ndcg_at_20": 0.50017,
499
+ "ndcg_at_50": 0.51182,
500
+ "ndcg_at_100": 0.53435,
501
+ "recall_at_1": 0.12901,
502
+ "recall_at_3": 0.27841,
503
+ "recall_at_5": 0.36943,
504
+ "recall_at_10": 0.44663,
505
+ "recall_at_20": 0.62974,
506
+ "recall_at_50": 0.75753,
507
+ "recall_at_100": 0.94961,
508
+ "precision_at_1": 0.39253,
509
+ "precision_at_3": 0.3123,
510
+ "precision_at_5": 0.26144,
511
+ "precision_at_10": 0.17133,
512
+ "precision_at_20": 0.11174,
513
+ "precision_at_50": 0.05662,
514
+ "precision_at_100": 0.03452
515
+ }
516
+ }
517
+ ],
518
+ "FinancialStatements": [
519
+ {
520
+ "class": "sentence-transformers",
521
+ "organization": "HuggingFace",
522
+ "model_name": "hf-finance-transformer",
523
+ "results": {
524
+ "ndcg_at_1": 0.48434,
525
+ "ndcg_at_3": 0.35799,
526
+ "ndcg_at_5": 0.41527,
527
+ "ndcg_at_10": 0.44231,
528
+ "ndcg_at_20": 0.54103,
529
+ "ndcg_at_50": 0.52956,
530
+ "ndcg_at_100": 0.5308,
531
+ "recall_at_1": 0.13159,
532
+ "recall_at_3": 0.25513,
533
+ "recall_at_5": 0.36668,
534
+ "recall_at_10": 0.44413,
535
+ "recall_at_20": 0.60097,
536
+ "recall_at_50": 0.785,
537
+ "recall_at_100": 0.93302,
538
+ "precision_at_1": 0.39292,
539
+ "precision_at_3": 0.32611,
540
+ "precision_at_5": 0.25737,
541
+ "precision_at_10": 0.17719,
542
+ "precision_at_20": 0.11676,
543
+ "precision_at_50": 0.0543,
544
+ "precision_at_100": 0.03738
545
+ }
546
+ },
547
+ {
548
+ "class": "custom",
549
+ "organization": "OpenAI",
550
+ "model_name": "openai-finance-embed-v1",
551
+ "results": {
552
+ "ndcg_at_1": 0.45535,
553
+ "ndcg_at_3": 0.42909,
554
+ "ndcg_at_5": 0.3542,
555
+ "ndcg_at_10": 0.44902,
556
+ "ndcg_at_20": 0.53363,
557
+ "ndcg_at_50": 0.50212,
558
+ "ndcg_at_100": 0.53157,
559
+ "recall_at_1": 0.14736,
560
+ "recall_at_3": 0.26212,
561
+ "recall_at_5": 0.35148,
562
+ "recall_at_10": 0.44468,
563
+ "recall_at_20": 0.62848,
564
+ "recall_at_50": 0.78889,
565
+ "recall_at_100": 0.94323,
566
+ "precision_at_1": 0.39045,
567
+ "precision_at_3": 0.30464,
568
+ "precision_at_5": 0.26797,
569
+ "precision_at_10": 0.17381,
570
+ "precision_at_20": 0.11574,
571
+ "precision_at_50": 0.05665,
572
+ "precision_at_100": 0.03701
573
+ }
574
+ },
575
+ {
576
+ "class": "proprietary",
577
+ "organization": "Google",
578
+ "model_name": "google-finance-embed-v1",
579
+ "results": {
580
+ "ndcg_at_1": 0.4488,
581
+ "ndcg_at_3": 0.40741,
582
+ "ndcg_at_5": 0.44834,
583
+ "ndcg_at_10": 0.43025,
584
+ "ndcg_at_20": 0.50183,
585
+ "ndcg_at_50": 0.52363,
586
+ "ndcg_at_100": 0.53737,
587
+ "recall_at_1": 0.1222,
588
+ "recall_at_3": 0.2625,
589
+ "recall_at_5": 0.35385,
590
+ "recall_at_10": 0.44159,
591
+ "recall_at_20": 0.60495,
592
+ "recall_at_50": 0.77579,
593
+ "recall_at_100": 0.93091,
594
+ "precision_at_1": 0.43508,
595
+ "precision_at_3": 0.30428,
596
+ "precision_at_5": 0.25365,
597
+ "precision_at_10": 0.17445,
598
+ "precision_at_20": 0.11813,
599
+ "precision_at_50": 0.05169,
600
+ "precision_at_100": 0.03702
601
+ }
602
+ }
603
+ ],
604
+ "StockMarketData": [
605
+ {
606
+ "class": "sentence-transformers",
607
+ "organization": "HuggingFace",
608
+ "model_name": "hf-finance-transformer",
609
+ "results": {
610
+ "ndcg_at_1": 0.34875,
611
+ "ndcg_at_3": 0.39934,
612
+ "ndcg_at_5": 0.40177,
613
+ "ndcg_at_10": 0.41756,
614
+ "ndcg_at_20": 0.53874,
615
+ "ndcg_at_50": 0.54897,
616
+ "ndcg_at_100": 0.53181,
617
+ "recall_at_1": 0.12345,
618
+ "recall_at_3": 0.27656,
619
+ "recall_at_5": 0.36947,
620
+ "recall_at_10": 0.44182,
621
+ "recall_at_20": 0.626,
622
+ "recall_at_50": 0.75834,
623
+ "recall_at_100": 0.93638,
624
+ "precision_at_1": 0.37242,
625
+ "precision_at_3": 0.31243,
626
+ "precision_at_5": 0.25467,
627
+ "precision_at_10": 0.1711,
628
+ "precision_at_20": 0.11973,
629
+ "precision_at_50": 0.05367,
630
+ "precision_at_100": 0.03905
631
+ }
632
+ },
633
+ {
634
+ "class": "custom",
635
+ "organization": "OpenAI",
636
+ "model_name": "openai-finance-embed-v1",
637
+ "results": {
638
+ "ndcg_at_1": 0.38596,
639
+ "ndcg_at_3": 0.39188,
640
+ "ndcg_at_5": 0.35061,
641
+ "ndcg_at_10": 0.44788,
642
+ "ndcg_at_20": 0.47752,
643
+ "ndcg_at_50": 0.54408,
644
+ "ndcg_at_100": 0.5449,
645
+ "recall_at_1": 0.13509,
646
+ "recall_at_3": 0.25732,
647
+ "recall_at_5": 0.35538,
648
+ "recall_at_10": 0.44672,
649
+ "recall_at_20": 0.6046,
650
+ "recall_at_50": 0.77502,
651
+ "recall_at_100": 0.93839,
652
+ "precision_at_1": 0.43411,
653
+ "precision_at_3": 0.31135,
654
+ "precision_at_5": 0.26273,
655
+ "precision_at_10": 0.1768,
656
+ "precision_at_20": 0.11835,
657
+ "precision_at_50": 0.05292,
658
+ "precision_at_100": 0.03624
659
+ }
660
+ },
661
+ {
662
+ "class": "proprietary",
663
+ "organization": "Google",
664
+ "model_name": "google-finance-embed-v1",
665
+ "results": {
666
+ "ndcg_at_1": 0.48918,
667
+ "ndcg_at_3": 0.36458,
668
+ "ndcg_at_5": 0.36347,
669
+ "ndcg_at_10": 0.40554,
670
+ "ndcg_at_20": 0.50052,
671
+ "ndcg_at_50": 0.50264,
672
+ "ndcg_at_100": 0.53588,
673
+ "recall_at_1": 0.13486,
674
+ "recall_at_3": 0.27385,
675
+ "recall_at_5": 0.36561,
676
+ "recall_at_10": 0.4445,
677
+ "recall_at_20": 0.61532,
678
+ "recall_at_50": 0.76438,
679
+ "recall_at_100": 0.94084,
680
+ "precision_at_1": 0.4365,
681
+ "precision_at_3": 0.31413,
682
+ "precision_at_5": 0.25735,
683
+ "precision_at_10": 0.17959,
684
+ "precision_at_20": 0.11096,
685
+ "precision_at_50": 0.05511,
686
+ "precision_at_100": 0.03121
687
+ }
688
+ }
689
+ ],
690
+ "BankingRecords": [
691
+ {
692
+ "class": "sentence-transformers",
693
+ "organization": "HuggingFace",
694
+ "model_name": "hf-finance-transformer",
695
+ "results": {
696
+ "ndcg_at_1": 0.38203,
697
+ "ndcg_at_3": 0.37418,
698
+ "ndcg_at_5": 0.4351,
699
+ "ndcg_at_10": 0.41743,
700
+ "ndcg_at_20": 0.49473,
701
+ "ndcg_at_50": 0.52136,
702
+ "ndcg_at_100": 0.53756,
703
+ "recall_at_1": 0.14753,
704
+ "recall_at_3": 0.25793,
705
+ "recall_at_5": 0.35851,
706
+ "recall_at_10": 0.44339,
707
+ "recall_at_20": 0.6243,
708
+ "recall_at_50": 0.75142,
709
+ "recall_at_100": 0.94824,
710
+ "precision_at_1": 0.35872,
711
+ "precision_at_3": 0.32266,
712
+ "precision_at_5": 0.2636,
713
+ "precision_at_10": 0.17931,
714
+ "precision_at_20": 0.11076,
715
+ "precision_at_50": 0.05252,
716
+ "precision_at_100": 0.03767
717
+ }
718
+ },
719
+ {
720
+ "class": "custom",
721
+ "organization": "OpenAI",
722
+ "model_name": "openai-finance-embed-v1",
723
+ "results": {
724
+ "ndcg_at_1": 0.40739,
725
+ "ndcg_at_3": 0.43647,
726
+ "ndcg_at_5": 0.39726,
727
+ "ndcg_at_10": 0.42447,
728
+ "ndcg_at_20": 0.54815,
729
+ "ndcg_at_50": 0.50476,
730
+ "ndcg_at_100": 0.54706,
731
+ "recall_at_1": 0.12288,
732
+ "recall_at_3": 0.26529,
733
+ "recall_at_5": 0.35948,
734
+ "recall_at_10": 0.44849,
735
+ "recall_at_20": 0.60992,
736
+ "recall_at_50": 0.77764,
737
+ "recall_at_100": 0.93974,
738
+ "precision_at_1": 0.36072,
739
+ "precision_at_3": 0.32013,
740
+ "precision_at_5": 0.26196,
741
+ "precision_at_10": 0.17291,
742
+ "precision_at_20": 0.11549,
743
+ "precision_at_50": 0.05603,
744
+ "precision_at_100": 0.03867
745
+ }
746
+ },
747
+ {
748
+ "class": "proprietary",
749
+ "organization": "Google",
750
+ "model_name": "google-finance-embed-v1",
751
+ "results": {
752
+ "ndcg_at_1": 0.31985,
753
+ "ndcg_at_3": 0.44625,
754
+ "ndcg_at_5": 0.42884,
755
+ "ndcg_at_10": 0.40019,
756
+ "ndcg_at_20": 0.5205,
757
+ "ndcg_at_50": 0.50303,
758
+ "ndcg_at_100": 0.53178,
759
+ "recall_at_1": 0.12598,
760
+ "recall_at_3": 0.25343,
761
+ "recall_at_5": 0.35641,
762
+ "recall_at_10": 0.44316,
763
+ "recall_at_20": 0.60534,
764
+ "recall_at_50": 0.76815,
765
+ "recall_at_100": 0.93912,
766
+ "precision_at_1": 0.37354,
767
+ "precision_at_3": 0.32065,
768
+ "precision_at_5": 0.25275,
769
+ "precision_at_10": 0.17665,
770
+ "precision_at_20": 0.11192,
771
+ "precision_at_50": 0.05703,
772
+ "precision_at_100": 0.03476
773
+ }
774
+ }
775
+ ],
776
+ "ChatLogs": [
777
+ {
778
+ "class": "custom",
779
+ "organization": "Microsoft",
780
+ "model_name": "microsoft-conversational-v2",
781
+ "results": {
782
+ "ndcg_at_1": 0.31086,
783
+ "ndcg_at_3": 0.41518,
784
+ "ndcg_at_5": 0.42979,
785
+ "ndcg_at_10": 0.41864,
786
+ "ndcg_at_20": 0.47152,
787
+ "ndcg_at_50": 0.54462,
788
+ "ndcg_at_100": 0.53909,
789
+ "recall_at_1": 0.13135,
790
+ "recall_at_3": 0.27782,
791
+ "recall_at_5": 0.36798,
792
+ "recall_at_10": 0.44585,
793
+ "recall_at_20": 0.61007,
794
+ "recall_at_50": 0.76913,
795
+ "recall_at_100": 0.94458,
796
+ "precision_at_1": 0.44751,
797
+ "precision_at_3": 0.31363,
798
+ "precision_at_5": 0.25568,
799
+ "precision_at_10": 0.17145,
800
+ "precision_at_20": 0.11113,
801
+ "precision_at_50": 0.05038,
802
+ "precision_at_100": 0.03531
803
+ }
804
+ },
805
+ {
806
+ "class": "sentence-transformers",
807
+ "organization": "Facebook",
808
+ "model_name": "fb-conversational-embed",
809
+ "results": {
810
+ "ndcg_at_1": 0.35364,
811
+ "ndcg_at_3": 0.37922,
812
+ "ndcg_at_5": 0.36366,
813
+ "ndcg_at_10": 0.40685,
814
+ "ndcg_at_20": 0.46173,
815
+ "ndcg_at_50": 0.5461,
816
+ "ndcg_at_100": 0.53954,
817
+ "recall_at_1": 0.13445,
818
+ "recall_at_3": 0.27737,
819
+ "recall_at_5": 0.369,
820
+ "recall_at_10": 0.44018,
821
+ "recall_at_20": 0.61156,
822
+ "recall_at_50": 0.7562,
823
+ "recall_at_100": 0.94805,
824
+ "precision_at_1": 0.43949,
825
+ "precision_at_3": 0.32901,
826
+ "precision_at_5": 0.26666,
827
+ "precision_at_10": 0.17573,
828
+ "precision_at_20": 0.11971,
829
+ "precision_at_50": 0.0564,
830
+ "precision_at_100": 0.03335
831
+ }
832
+ },
833
+ {
834
+ "class": "proprietary",
835
+ "organization": "Amazon",
836
+ "model_name": "amazon-conversational-transformer",
837
+ "results": {
838
+ "ndcg_at_1": 0.38162,
839
+ "ndcg_at_3": 0.35782,
840
+ "ndcg_at_5": 0.37348,
841
+ "ndcg_at_10": 0.42507,
842
+ "ndcg_at_20": 0.47226,
843
+ "ndcg_at_50": 0.54381,
844
+ "ndcg_at_100": 0.54003,
845
+ "recall_at_1": 0.12125,
846
+ "recall_at_3": 0.27809,
847
+ "recall_at_5": 0.36768,
848
+ "recall_at_10": 0.44121,
849
+ "recall_at_20": 0.61645,
850
+ "recall_at_50": 0.7761,
851
+ "recall_at_100": 0.93316,
852
+ "precision_at_1": 0.38107,
853
+ "precision_at_3": 0.30445,
854
+ "precision_at_5": 0.26095,
855
+ "precision_at_10": 0.17977,
856
+ "precision_at_20": 0.11521,
857
+ "precision_at_50": 0.05176,
858
+ "precision_at_100": 0.03837
859
+ }
860
+ }
861
+ ],
862
+ "CustomerServiceInteractions": [
863
+ {
864
+ "class": "custom",
865
+ "organization": "Microsoft",
866
+ "model_name": "microsoft-conversational-v2",
867
+ "results": {
868
+ "ndcg_at_1": 0.43075,
869
+ "ndcg_at_3": 0.3791,
870
+ "ndcg_at_5": 0.44601,
871
+ "ndcg_at_10": 0.41317,
872
+ "ndcg_at_20": 0.46408,
873
+ "ndcg_at_50": 0.52089,
874
+ "ndcg_at_100": 0.53379,
875
+ "recall_at_1": 0.13676,
876
+ "recall_at_3": 0.26395,
877
+ "recall_at_5": 0.3681,
878
+ "recall_at_10": 0.44254,
879
+ "recall_at_20": 0.62697,
880
+ "recall_at_50": 0.75481,
881
+ "recall_at_100": 0.93131,
882
+ "precision_at_1": 0.42118,
883
+ "precision_at_3": 0.3138,
884
+ "precision_at_5": 0.25566,
885
+ "precision_at_10": 0.17184,
886
+ "precision_at_20": 0.11613,
887
+ "precision_at_50": 0.05884,
888
+ "precision_at_100": 0.03492
889
+ }
890
+ },
891
+ {
892
+ "class": "sentence-transformers",
893
+ "organization": "Facebook",
894
+ "model_name": "fb-conversational-embed",
895
+ "results": {
896
+ "ndcg_at_1": 0.45517,
897
+ "ndcg_at_3": 0.38531,
898
+ "ndcg_at_5": 0.39457,
899
+ "ndcg_at_10": 0.41228,
900
+ "ndcg_at_20": 0.49646,
901
+ "ndcg_at_50": 0.54303,
902
+ "ndcg_at_100": 0.54205,
903
+ "recall_at_1": 0.13851,
904
+ "recall_at_3": 0.27383,
905
+ "recall_at_5": 0.36674,
906
+ "recall_at_10": 0.44846,
907
+ "recall_at_20": 0.60926,
908
+ "recall_at_50": 0.77397,
909
+ "recall_at_100": 0.94151,
910
+ "precision_at_1": 0.38343,
911
+ "precision_at_3": 0.30666,
912
+ "precision_at_5": 0.25943,
913
+ "precision_at_10": 0.17165,
914
+ "precision_at_20": 0.11814,
915
+ "precision_at_50": 0.05299,
916
+ "precision_at_100": 0.03727
917
+ }
918
+ },
919
+ {
920
+ "class": "proprietary",
921
+ "organization": "Amazon",
922
+ "model_name": "amazon-conversational-transformer",
923
+ "results": {
924
+ "ndcg_at_1": 0.40956,
925
+ "ndcg_at_3": 0.35892,
926
+ "ndcg_at_5": 0.37056,
927
+ "ndcg_at_10": 0.40777,
928
+ "ndcg_at_20": 0.54674,
929
+ "ndcg_at_50": 0.51362,
930
+ "ndcg_at_100": 0.53597,
931
+ "recall_at_1": 0.13679,
932
+ "recall_at_3": 0.26475,
933
+ "recall_at_5": 0.35469,
934
+ "recall_at_10": 0.44882,
935
+ "recall_at_20": 0.601,
936
+ "recall_at_50": 0.75559,
937
+ "recall_at_100": 0.94204,
938
+ "precision_at_1": 0.43718,
939
+ "precision_at_3": 0.31947,
940
+ "precision_at_5": 0.25611,
941
+ "precision_at_10": 0.17802,
942
+ "precision_at_20": 0.11883,
943
+ "precision_at_50": 0.0566,
944
+ "precision_at_100": 0.03416
945
+ }
946
+ }
947
+ ],
948
+ "ForumPosts": [
949
+ {
950
+ "class": "custom",
951
+ "organization": "Microsoft",
952
+ "model_name": "microsoft-conversational-v2",
953
+ "results": {
954
+ "ndcg_at_1": 0.45408,
955
+ "ndcg_at_3": 0.37915,
956
+ "ndcg_at_5": 0.43796,
957
+ "ndcg_at_10": 0.42478,
958
+ "ndcg_at_20": 0.46097,
959
+ "ndcg_at_50": 0.53337,
960
+ "ndcg_at_100": 0.54571,
961
+ "recall_at_1": 0.12766,
962
+ "recall_at_3": 0.26067,
963
+ "recall_at_5": 0.36289,
964
+ "recall_at_10": 0.44937,
965
+ "recall_at_20": 0.6009,
966
+ "recall_at_50": 0.75516,
967
+ "recall_at_100": 0.93326,
968
+ "precision_at_1": 0.36415,
969
+ "precision_at_3": 0.30269,
970
+ "precision_at_5": 0.25359,
971
+ "precision_at_10": 0.17167,
972
+ "precision_at_20": 0.1171,
973
+ "precision_at_50": 0.05523,
974
+ "precision_at_100": 0.03043
975
+ }
976
+ },
977
+ {
978
+ "class": "sentence-transformers",
979
+ "organization": "Facebook",
980
+ "model_name": "fb-conversational-embed",
981
+ "results": {
982
+ "ndcg_at_1": 0.32926,
983
+ "ndcg_at_3": 0.42901,
984
+ "ndcg_at_5": 0.40443,
985
+ "ndcg_at_10": 0.42423,
986
+ "ndcg_at_20": 0.49838,
987
+ "ndcg_at_50": 0.50405,
988
+ "ndcg_at_100": 0.54763,
989
+ "recall_at_1": 0.14542,
990
+ "recall_at_3": 0.25511,
991
+ "recall_at_5": 0.36165,
992
+ "recall_at_10": 0.44936,
993
+ "recall_at_20": 0.60242,
994
+ "recall_at_50": 0.78978,
995
+ "recall_at_100": 0.94908,
996
+ "precision_at_1": 0.44124,
997
+ "precision_at_3": 0.30716,
998
+ "precision_at_5": 0.25761,
999
+ "precision_at_10": 0.17781,
1000
+ "precision_at_20": 0.11188,
1001
+ "precision_at_50": 0.05638,
1002
+ "precision_at_100": 0.03341
1003
+ }
1004
+ },
1005
+ {
1006
+ "class": "proprietary",
1007
+ "organization": "Amazon",
1008
+ "model_name": "amazon-conversational-transformer",
1009
+ "results": {
1010
+ "ndcg_at_1": 0.4498,
1011
+ "ndcg_at_3": 0.37517,
1012
+ "ndcg_at_5": 0.37576,
1013
+ "ndcg_at_10": 0.43931,
1014
+ "ndcg_at_20": 0.51543,
1015
+ "ndcg_at_50": 0.51242,
1016
+ "ndcg_at_100": 0.53351,
1017
+ "recall_at_1": 0.13577,
1018
+ "recall_at_3": 0.25436,
1019
+ "recall_at_5": 0.35619,
1020
+ "recall_at_10": 0.44014,
1021
+ "recall_at_20": 0.60306,
1022
+ "recall_at_50": 0.78889,
1023
+ "recall_at_100": 0.93044,
1024
+ "precision_at_1": 0.36037,
1025
+ "precision_at_3": 0.32616,
1026
+ "precision_at_5": 0.26172,
1027
+ "precision_at_10": 0.17806,
1028
+ "precision_at_20": 0.11806,
1029
+ "precision_at_50": 0.05701,
1030
+ "precision_at_100": 0.03974
1031
+ }
1032
+ }
1033
+ ],
1034
+ "TechBlogs": [
1035
+ {
1036
+ "class": "sentence-transformers",
1037
+ "organization": "Tesla",
1038
+ "model_name": "tesla-tech-model",
1039
+ "results": {
1040
+ "ndcg_at_1": 0.30643,
1041
+ "ndcg_at_3": 0.4002,
1042
+ "ndcg_at_5": 0.41746,
1043
+ "ndcg_at_10": 0.42285,
1044
+ "ndcg_at_20": 0.47855,
1045
+ "ndcg_at_50": 0.50593,
1046
+ "ndcg_at_100": 0.53188,
1047
+ "recall_at_1": 0.12854,
1048
+ "recall_at_3": 0.25888,
1049
+ "recall_at_5": 0.35743,
1050
+ "recall_at_10": 0.44339,
1051
+ "recall_at_20": 0.61108,
1052
+ "recall_at_50": 0.7883,
1053
+ "recall_at_100": 0.93209,
1054
+ "precision_at_1": 0.37368,
1055
+ "precision_at_3": 0.31051,
1056
+ "precision_at_5": 0.25881,
1057
+ "precision_at_10": 0.17447,
1058
+ "precision_at_20": 0.11475,
1059
+ "precision_at_50": 0.0534,
1060
+ "precision_at_100": 0.03568
1061
+ }
1062
+ },
1063
+ {
1064
+ "class": "custom",
1065
+ "organization": "Snowflake",
1066
+ "model_name": "snowflake-tech-embed",
1067
+ "results": {
1068
+ "ndcg_at_1": 0.41676,
1069
+ "ndcg_at_3": 0.38764,
1070
+ "ndcg_at_5": 0.42896,
1071
+ "ndcg_at_10": 0.41236,
1072
+ "ndcg_at_20": 0.50488,
1073
+ "ndcg_at_50": 0.52271,
1074
+ "ndcg_at_100": 0.54134,
1075
+ "recall_at_1": 0.12853,
1076
+ "recall_at_3": 0.27275,
1077
+ "recall_at_5": 0.35719,
1078
+ "recall_at_10": 0.44164,
1079
+ "recall_at_20": 0.62852,
1080
+ "recall_at_50": 0.75658,
1081
+ "recall_at_100": 0.93782,
1082
+ "precision_at_1": 0.42004,
1083
+ "precision_at_3": 0.32972,
1084
+ "precision_at_5": 0.26613,
1085
+ "precision_at_10": 0.1704,
1086
+ "precision_at_20": 0.11991,
1087
+ "precision_at_50": 0.0538,
1088
+ "precision_at_100": 0.03601
1089
+ }
1090
+ },
1091
+ {
1092
+ "class": "proprietary",
1093
+ "organization": "DeepMind",
1094
+ "model_name": "dm-tech-v1",
1095
+ "results": {
1096
+ "ndcg_at_1": 0.46949,
1097
+ "ndcg_at_3": 0.37344,
1098
+ "ndcg_at_5": 0.42941,
1099
+ "ndcg_at_10": 0.40618,
1100
+ "ndcg_at_20": 0.46027,
1101
+ "ndcg_at_50": 0.52979,
1102
+ "ndcg_at_100": 0.53935,
1103
+ "recall_at_1": 0.13937,
1104
+ "recall_at_3": 0.27528,
1105
+ "recall_at_5": 0.36444,
1106
+ "recall_at_10": 0.44732,
1107
+ "recall_at_20": 0.60973,
1108
+ "recall_at_50": 0.7619,
1109
+ "recall_at_100": 0.9477,
1110
+ "precision_at_1": 0.42538,
1111
+ "precision_at_3": 0.31344,
1112
+ "precision_at_5": 0.25846,
1113
+ "precision_at_10": 0.17736,
1114
+ "precision_at_20": 0.11519,
1115
+ "precision_at_50": 0.05737,
1116
+ "precision_at_100": 0.03307
1117
+ }
1118
+ }
1119
+ ],
1120
+ "DeviceSpecifications": [
1121
+ {
1122
+ "class": "sentence-transformers",
1123
+ "organization": "Tesla",
1124
+ "model_name": "tesla-tech-model",
1125
+ "results": {
1126
+ "ndcg_at_1": 0.38662,
1127
+ "ndcg_at_3": 0.35902,
1128
+ "ndcg_at_5": 0.42961,
1129
+ "ndcg_at_10": 0.41038,
1130
+ "ndcg_at_20": 0.5141,
1131
+ "ndcg_at_50": 0.51426,
1132
+ "ndcg_at_100": 0.54536,
1133
+ "recall_at_1": 0.13181,
1134
+ "recall_at_3": 0.26616,
1135
+ "recall_at_5": 0.35425,
1136
+ "recall_at_10": 0.44259,
1137
+ "recall_at_20": 0.60538,
1138
+ "recall_at_50": 0.76624,
1139
+ "recall_at_100": 0.93642,
1140
+ "precision_at_1": 0.35288,
1141
+ "precision_at_3": 0.32686,
1142
+ "precision_at_5": 0.25708,
1143
+ "precision_at_10": 0.17507,
1144
+ "precision_at_20": 0.11839,
1145
+ "precision_at_50": 0.05949,
1146
+ "precision_at_100": 0.03646
1147
+ }
1148
+ },
1149
+ {
1150
+ "class": "custom",
1151
+ "organization": "Snowflake",
1152
+ "model_name": "snowflake-tech-embed",
1153
+ "results": {
1154
+ "ndcg_at_1": 0.36366,
1155
+ "ndcg_at_3": 0.44325,
1156
+ "ndcg_at_5": 0.43545,
1157
+ "ndcg_at_10": 0.44984,
1158
+ "ndcg_at_20": 0.47496,
1159
+ "ndcg_at_50": 0.53153,
1160
+ "ndcg_at_100": 0.54062,
1161
+ "recall_at_1": 0.13236,
1162
+ "recall_at_3": 0.26072,
1163
+ "recall_at_5": 0.36499,
1164
+ "recall_at_10": 0.4421,
1165
+ "recall_at_20": 0.6297,
1166
+ "recall_at_50": 0.75303,
1167
+ "recall_at_100": 0.93599,
1168
+ "precision_at_1": 0.43327,
1169
+ "precision_at_3": 0.31096,
1170
+ "precision_at_5": 0.26165,
1171
+ "precision_at_10": 0.17379,
1172
+ "precision_at_20": 0.11198,
1173
+ "precision_at_50": 0.05142,
1174
+ "precision_at_100": 0.03999
1175
+ }
1176
+ },
1177
+ {
1178
+ "class": "proprietary",
1179
+ "organization": "DeepMind",
1180
+ "model_name": "dm-tech-v1",
1181
+ "results": {
1182
+ "ndcg_at_1": 0.47907,
1183
+ "ndcg_at_3": 0.35582,
1184
+ "ndcg_at_5": 0.38363,
1185
+ "ndcg_at_10": 0.44638,
1186
+ "ndcg_at_20": 0.53994,
1187
+ "ndcg_at_50": 0.52256,
1188
+ "ndcg_at_100": 0.53509,
1189
+ "recall_at_1": 0.14258,
1190
+ "recall_at_3": 0.25902,
1191
+ "recall_at_5": 0.35659,
1192
+ "recall_at_10": 0.44741,
1193
+ "recall_at_20": 0.60291,
1194
+ "recall_at_50": 0.76419,
1195
+ "recall_at_100": 0.94188,
1196
+ "precision_at_1": 0.44829,
1197
+ "precision_at_3": 0.32927,
1198
+ "precision_at_5": 0.25151,
1199
+ "precision_at_10": 0.17668,
1200
+ "precision_at_20": 0.11602,
1201
+ "precision_at_50": 0.0587,
1202
+ "precision_at_100": 0.03387
1203
+ }
1204
+ }
1205
+ ],
1206
+ "SoftwareDocumentation": [
1207
+ {
1208
+ "class": "sentence-transformers",
1209
+ "organization": "Tesla",
1210
+ "model_name": "tesla-tech-model",
1211
+ "results": {
1212
+ "ndcg_at_1": 0.33891,
1213
+ "ndcg_at_3": 0.3776,
1214
+ "ndcg_at_5": 0.40428,
1215
+ "ndcg_at_10": 0.41962,
1216
+ "ndcg_at_20": 0.47764,
1217
+ "ndcg_at_50": 0.52146,
1218
+ "ndcg_at_100": 0.54563,
1219
+ "recall_at_1": 0.14107,
1220
+ "recall_at_3": 0.26422,
1221
+ "recall_at_5": 0.36519,
1222
+ "recall_at_10": 0.44727,
1223
+ "recall_at_20": 0.61643,
1224
+ "recall_at_50": 0.78874,
1225
+ "recall_at_100": 0.94459,
1226
+ "precision_at_1": 0.44509,
1227
+ "precision_at_3": 0.30578,
1228
+ "precision_at_5": 0.25257,
1229
+ "precision_at_10": 0.17249,
1230
+ "precision_at_20": 0.11159,
1231
+ "precision_at_50": 0.05556,
1232
+ "precision_at_100": 0.03451
1233
+ }
1234
+ },
1235
+ {
1236
+ "class": "custom",
1237
+ "organization": "Snowflake",
1238
+ "model_name": "snowflake-tech-embed",
1239
+ "results": {
1240
+ "ndcg_at_1": 0.43238,
1241
+ "ndcg_at_3": 0.37988,
1242
+ "ndcg_at_5": 0.39416,
1243
+ "ndcg_at_10": 0.41702,
1244
+ "ndcg_at_20": 0.54051,
1245
+ "ndcg_at_50": 0.53707,
1246
+ "ndcg_at_100": 0.54506,
1247
+ "recall_at_1": 0.14298,
1248
+ "recall_at_3": 0.25608,
1249
+ "recall_at_5": 0.35831,
1250
+ "recall_at_10": 0.44458,
1251
+ "recall_at_20": 0.61525,
1252
+ "recall_at_50": 0.77235,
1253
+ "recall_at_100": 0.94504,
1254
+ "precision_at_1": 0.35217,
1255
+ "precision_at_3": 0.32198,
1256
+ "precision_at_5": 0.26531,
1257
+ "precision_at_10": 0.17925,
1258
+ "precision_at_20": 0.11595,
1259
+ "precision_at_50": 0.05453,
1260
+ "precision_at_100": 0.03694
1261
+ }
1262
+ },
1263
+ {
1264
+ "class": "proprietary",
1265
+ "organization": "DeepMind",
1266
+ "model_name": "dm-tech-v1",
1267
+ "results": {
1268
+ "ndcg_at_1": 0.39185,
1269
+ "ndcg_at_3": 0.39625,
1270
+ "ndcg_at_5": 0.40315,
1271
+ "ndcg_at_10": 0.40738,
1272
+ "ndcg_at_20": 0.48067,
1273
+ "ndcg_at_50": 0.51536,
1274
+ "ndcg_at_100": 0.54435,
1275
+ "recall_at_1": 0.12579,
1276
+ "recall_at_3": 0.25129,
1277
+ "recall_at_5": 0.35922,
1278
+ "recall_at_10": 0.44186,
1279
+ "recall_at_20": 0.62642,
1280
+ "recall_at_50": 0.7736,
1281
+ "recall_at_100": 0.94205,
1282
+ "precision_at_1": 0.40707,
1283
+ "precision_at_3": 0.32124,
1284
+ "precision_at_5": 0.26377,
1285
+ "precision_at_10": 0.17724,
1286
+ "precision_at_20": 0.11045,
1287
+ "precision_at_50": 0.05565,
1288
+ "precision_at_100": 0.0339
1289
+ }
1290
+ }
1291
+ ],
1292
+ "ParallelCorpora": [
1293
+ {
1294
+ "class": "sentence-transformers",
1295
+ "organization": "Tesla",
1296
+ "model_name": "tesla-multilingual",
1297
+ "results": {
1298
+ "ndcg_at_1": 0.4117,
1299
+ "ndcg_at_3": 0.36836,
1300
+ "ndcg_at_5": 0.38378,
1301
+ "ndcg_at_10": 0.44263,
1302
+ "ndcg_at_20": 0.51812,
1303
+ "ndcg_at_50": 0.53216,
1304
+ "ndcg_at_100": 0.54057,
1305
+ "recall_at_1": 0.14566,
1306
+ "recall_at_3": 0.2682,
1307
+ "recall_at_5": 0.35595,
1308
+ "recall_at_10": 0.44596,
1309
+ "recall_at_20": 0.61659,
1310
+ "recall_at_50": 0.78716,
1311
+ "recall_at_100": 0.94259,
1312
+ "precision_at_1": 0.38429,
1313
+ "precision_at_3": 0.3212,
1314
+ "precision_at_5": 0.25935,
1315
+ "precision_at_10": 0.17743,
1316
+ "precision_at_20": 0.11238,
1317
+ "precision_at_50": 0.05918,
1318
+ "precision_at_100": 0.03416
1319
+ }
1320
+ },
1321
+ {
1322
+ "class": "custom",
1323
+ "organization": "Apple",
1324
+ "model_name": "apple-multilingual-embed",
1325
+ "results": {
1326
+ "ndcg_at_1": 0.4331,
1327
+ "ndcg_at_3": 0.39469,
1328
+ "ndcg_at_5": 0.36942,
1329
+ "ndcg_at_10": 0.44221,
1330
+ "ndcg_at_20": 0.50165,
1331
+ "ndcg_at_50": 0.50989,
1332
+ "ndcg_at_100": 0.54776,
1333
+ "recall_at_1": 0.13383,
1334
+ "recall_at_3": 0.26805,
1335
+ "recall_at_5": 0.35224,
1336
+ "recall_at_10": 0.44017,
1337
+ "recall_at_20": 0.61241,
1338
+ "recall_at_50": 0.78675,
1339
+ "recall_at_100": 0.9482,
1340
+ "precision_at_1": 0.40393,
1341
+ "precision_at_3": 0.32532,
1342
+ "precision_at_5": 0.26905,
1343
+ "precision_at_10": 0.1713,
1344
+ "precision_at_20": 0.11994,
1345
+ "precision_at_50": 0.058,
1346
+ "precision_at_100": 0.03086
1347
+ }
1348
+ },
1349
+ {
1350
+ "class": "proprietary",
1351
+ "organization": "Google",
1352
+ "model_name": "google-multilingual-model",
1353
+ "results": {
1354
+ "ndcg_at_1": 0.48879,
1355
+ "ndcg_at_3": 0.38319,
1356
+ "ndcg_at_5": 0.40288,
1357
+ "ndcg_at_10": 0.44327,
1358
+ "ndcg_at_20": 0.54182,
1359
+ "ndcg_at_50": 0.52196,
1360
+ "ndcg_at_100": 0.53643,
1361
+ "recall_at_1": 0.14996,
1362
+ "recall_at_3": 0.26212,
1363
+ "recall_at_5": 0.36817,
1364
+ "recall_at_10": 0.44209,
1365
+ "recall_at_20": 0.61969,
1366
+ "recall_at_50": 0.78785,
1367
+ "recall_at_100": 0.94373,
1368
+ "precision_at_1": 0.3636,
1369
+ "precision_at_3": 0.31523,
1370
+ "precision_at_5": 0.26818,
1371
+ "precision_at_10": 0.17667,
1372
+ "precision_at_20": 0.11862,
1373
+ "precision_at_50": 0.05379,
1374
+ "precision_at_100": 0.03766
1375
+ }
1376
+ }
1377
+ ],
1378
+ "TranslationDatasets": [
1379
+ {
1380
+ "class": "sentence-transformers",
1381
+ "organization": "Tesla",
1382
+ "model_name": "tesla-multilingual",
1383
+ "results": {
1384
+ "ndcg_at_1": 0.39585,
1385
+ "ndcg_at_3": 0.37619,
1386
+ "ndcg_at_5": 0.39101,
1387
+ "ndcg_at_10": 0.42069,
1388
+ "ndcg_at_20": 0.51587,
1389
+ "ndcg_at_50": 0.50951,
1390
+ "ndcg_at_100": 0.53074,
1391
+ "recall_at_1": 0.14944,
1392
+ "recall_at_3": 0.2717,
1393
+ "recall_at_5": 0.35949,
1394
+ "recall_at_10": 0.44643,
1395
+ "recall_at_20": 0.61472,
1396
+ "recall_at_50": 0.78585,
1397
+ "recall_at_100": 0.94071,
1398
+ "precision_at_1": 0.35347,
1399
+ "precision_at_3": 0.30395,
1400
+ "precision_at_5": 0.25844,
1401
+ "precision_at_10": 0.17669,
1402
+ "precision_at_20": 0.11242,
1403
+ "precision_at_50": 0.05902,
1404
+ "precision_at_100": 0.03638
1405
+ }
1406
+ },
1407
+ {
1408
+ "class": "custom",
1409
+ "organization": "Apple",
1410
+ "model_name": "apple-multilingual-embed",
1411
+ "results": {
1412
+ "ndcg_at_1": 0.45491,
1413
+ "ndcg_at_3": 0.35047,
1414
+ "ndcg_at_5": 0.37814,
1415
+ "ndcg_at_10": 0.44019,
1416
+ "ndcg_at_20": 0.50439,
1417
+ "ndcg_at_50": 0.54898,
1418
+ "ndcg_at_100": 0.53765,
1419
+ "recall_at_1": 0.12732,
1420
+ "recall_at_3": 0.26139,
1421
+ "recall_at_5": 0.36501,
1422
+ "recall_at_10": 0.44899,
1423
+ "recall_at_20": 0.61531,
1424
+ "recall_at_50": 0.77864,
1425
+ "recall_at_100": 0.94728,
1426
+ "precision_at_1": 0.40739,
1427
+ "precision_at_3": 0.32296,
1428
+ "precision_at_5": 0.26689,
1429
+ "precision_at_10": 0.17109,
1430
+ "precision_at_20": 0.11667,
1431
+ "precision_at_50": 0.05416,
1432
+ "precision_at_100": 0.03433
1433
+ }
1434
+ },
1435
+ {
1436
+ "class": "proprietary",
1437
+ "organization": "Google",
1438
+ "model_name": "google-multilingual-model",
1439
+ "results": {
1440
+ "ndcg_at_1": 0.40725,
1441
+ "ndcg_at_3": 0.35109,
1442
+ "ndcg_at_5": 0.43066,
1443
+ "ndcg_at_10": 0.44471,
1444
+ "ndcg_at_20": 0.54054,
1445
+ "ndcg_at_50": 0.52224,
1446
+ "ndcg_at_100": 0.53616,
1447
+ "recall_at_1": 0.1283,
1448
+ "recall_at_3": 0.25027,
1449
+ "recall_at_5": 0.36327,
1450
+ "recall_at_10": 0.44976,
1451
+ "recall_at_20": 0.62801,
1452
+ "recall_at_50": 0.75493,
1453
+ "recall_at_100": 0.9364,
1454
+ "precision_at_1": 0.41833,
1455
+ "precision_at_3": 0.30363,
1456
+ "precision_at_5": 0.26561,
1457
+ "precision_at_10": 0.17904,
1458
+ "precision_at_20": 0.11677,
1459
+ "precision_at_50": 0.05544,
1460
+ "precision_at_100": 0.03877
1461
+ }
1462
+ }
1463
+ ],
1464
+ "MultilingualText": [
1465
+ {
1466
+ "class": "sentence-transformers",
1467
+ "organization": "Tesla",
1468
+ "model_name": "tesla-multilingual",
1469
+ "results": {
1470
+ "ndcg_at_1": 0.31978,
1471
+ "ndcg_at_3": 0.36664,
1472
+ "ndcg_at_5": 0.44156,
1473
+ "ndcg_at_10": 0.43269,
1474
+ "ndcg_at_20": 0.51214,
1475
+ "ndcg_at_50": 0.53334,
1476
+ "ndcg_at_100": 0.54545,
1477
+ "recall_at_1": 0.14908,
1478
+ "recall_at_3": 0.25033,
1479
+ "recall_at_5": 0.36675,
1480
+ "recall_at_10": 0.44204,
1481
+ "recall_at_20": 0.60777,
1482
+ "recall_at_50": 0.75273,
1483
+ "recall_at_100": 0.93035,
1484
+ "precision_at_1": 0.36493,
1485
+ "precision_at_3": 0.31317,
1486
+ "precision_at_5": 0.2637,
1487
+ "precision_at_10": 0.17319,
1488
+ "precision_at_20": 0.11788,
1489
+ "precision_at_50": 0.05131,
1490
+ "precision_at_100": 0.03879
1491
+ }
1492
+ },
1493
+ {
1494
+ "class": "custom",
1495
+ "organization": "Apple",
1496
+ "model_name": "apple-multilingual-embed",
1497
+ "results": {
1498
+ "ndcg_at_1": 0.32276,
1499
+ "ndcg_at_3": 0.38335,
1500
+ "ndcg_at_5": 0.37129,
1501
+ "ndcg_at_10": 0.43166,
1502
+ "ndcg_at_20": 0.477,
1503
+ "ndcg_at_50": 0.52339,
1504
+ "ndcg_at_100": 0.53949,
1505
+ "recall_at_1": 0.14319,
1506
+ "recall_at_3": 0.25435,
1507
+ "recall_at_5": 0.36554,
1508
+ "recall_at_10": 0.44812,
1509
+ "recall_at_20": 0.61978,
1510
+ "recall_at_50": 0.78079,
1511
+ "recall_at_100": 0.93749,
1512
+ "precision_at_1": 0.36464,
1513
+ "precision_at_3": 0.30309,
1514
+ "precision_at_5": 0.25552,
1515
+ "precision_at_10": 0.17779,
1516
+ "precision_at_20": 0.11349,
1517
+ "precision_at_50": 0.05506,
1518
+ "precision_at_100": 0.03023
1519
+ }
1520
+ },
1521
+ {
1522
+ "class": "proprietary",
1523
+ "organization": "Google",
1524
+ "model_name": "google-multilingual-model",
1525
+ "results": {
1526
+ "ndcg_at_1": 0.45851,
1527
+ "ndcg_at_3": 0.37408,
1528
+ "ndcg_at_5": 0.38938,
1529
+ "ndcg_at_10": 0.42358,
1530
+ "ndcg_at_20": 0.53014,
1531
+ "ndcg_at_50": 0.53765,
1532
+ "ndcg_at_100": 0.53464,
1533
+ "recall_at_1": 0.13339,
1534
+ "recall_at_3": 0.27829,
1535
+ "recall_at_5": 0.36404,
1536
+ "recall_at_10": 0.44777,
1537
+ "recall_at_20": 0.62085,
1538
+ "recall_at_50": 0.759,
1539
+ "recall_at_100": 0.93079,
1540
+ "precision_at_1": 0.3659,
1541
+ "precision_at_3": 0.31028,
1542
+ "precision_at_5": 0.25392,
1543
+ "precision_at_10": 0.17075,
1544
+ "precision_at_20": 0.11919,
1545
+ "precision_at_50": 0.05181,
1546
+ "precision_at_100": 0.03628
1547
+ }
1548
+ }
1549
+ ],
1550
+ "GitHubRepositories": [
1551
+ {
1552
+ "class": "custom",
1553
+ "organization": "Microsoft",
1554
+ "model_name": "microsoft-code-embed",
1555
+ "results": {
1556
+ "ndcg_at_1": 0.34252,
1557
+ "ndcg_at_3": 0.39355,
1558
+ "ndcg_at_5": 0.37716,
1559
+ "ndcg_at_10": 0.43307,
1560
+ "ndcg_at_20": 0.48186,
1561
+ "ndcg_at_50": 0.51359,
1562
+ "ndcg_at_100": 0.53475,
1563
+ "recall_at_1": 0.14452,
1564
+ "recall_at_3": 0.25338,
1565
+ "recall_at_5": 0.36496,
1566
+ "recall_at_10": 0.44909,
1567
+ "recall_at_20": 0.61582,
1568
+ "recall_at_50": 0.78303,
1569
+ "recall_at_100": 0.94025,
1570
+ "precision_at_1": 0.36587,
1571
+ "precision_at_3": 0.31529,
1572
+ "precision_at_5": 0.25364,
1573
+ "precision_at_10": 0.17285,
1574
+ "precision_at_20": 0.11465,
1575
+ "precision_at_50": 0.05142,
1576
+ "precision_at_100": 0.03968
1577
+ }
1578
+ },
1579
+ {
1580
+ "class": "sentence-transformers",
1581
+ "organization": "HuggingFace",
1582
+ "model_name": "hf-code-transformer",
1583
+ "results": {
1584
+ "ndcg_at_1": 0.37997,
1585
+ "ndcg_at_3": 0.36795,
1586
+ "ndcg_at_5": 0.35415,
1587
+ "ndcg_at_10": 0.42952,
1588
+ "ndcg_at_20": 0.49271,
1589
+ "ndcg_at_50": 0.54416,
1590
+ "ndcg_at_100": 0.53553,
1591
+ "recall_at_1": 0.13285,
1592
+ "recall_at_3": 0.25972,
1593
+ "recall_at_5": 0.3653,
1594
+ "recall_at_10": 0.44467,
1595
+ "recall_at_20": 0.62214,
1596
+ "recall_at_50": 0.77127,
1597
+ "recall_at_100": 0.93559,
1598
+ "precision_at_1": 0.35963,
1599
+ "precision_at_3": 0.31517,
1600
+ "precision_at_5": 0.26978,
1601
+ "precision_at_10": 0.17536,
1602
+ "precision_at_20": 0.11722,
1603
+ "precision_at_50": 0.0576,
1604
+ "precision_at_100": 0.03423
1605
+ }
1606
+ },
1607
+ {
1608
+ "class": "proprietary",
1609
+ "organization": "Apple",
1610
+ "model_name": "apple-code-model",
1611
+ "results": {
1612
+ "ndcg_at_1": 0.428,
1613
+ "ndcg_at_3": 0.37358,
1614
+ "ndcg_at_5": 0.37884,
1615
+ "ndcg_at_10": 0.44992,
1616
+ "ndcg_at_20": 0.46798,
1617
+ "ndcg_at_50": 0.51038,
1618
+ "ndcg_at_100": 0.54978,
1619
+ "recall_at_1": 0.14302,
1620
+ "recall_at_3": 0.27512,
1621
+ "recall_at_5": 0.36887,
1622
+ "recall_at_10": 0.44374,
1623
+ "recall_at_20": 0.61517,
1624
+ "recall_at_50": 0.76992,
1625
+ "recall_at_100": 0.94901,
1626
+ "precision_at_1": 0.44804,
1627
+ "precision_at_3": 0.31162,
1628
+ "precision_at_5": 0.25807,
1629
+ "precision_at_10": 0.1792,
1630
+ "precision_at_20": 0.11099,
1631
+ "precision_at_50": 0.05133,
1632
+ "precision_at_100": 0.03809
1633
+ }
1634
+ }
1635
+ ],
1636
+ "CodeSnippets": [
1637
+ {
1638
+ "class": "custom",
1639
+ "organization": "Microsoft",
1640
+ "model_name": "microsoft-code-embed",
1641
+ "results": {
1642
+ "ndcg_at_1": 0.48199,
1643
+ "ndcg_at_3": 0.35464,
1644
+ "ndcg_at_5": 0.37767,
1645
+ "ndcg_at_10": 0.42106,
1646
+ "ndcg_at_20": 0.47295,
1647
+ "ndcg_at_50": 0.53758,
1648
+ "ndcg_at_100": 0.53898,
1649
+ "recall_at_1": 0.14917,
1650
+ "recall_at_3": 0.25646,
1651
+ "recall_at_5": 0.3548,
1652
+ "recall_at_10": 0.44751,
1653
+ "recall_at_20": 0.60868,
1654
+ "recall_at_50": 0.77432,
1655
+ "recall_at_100": 0.93076,
1656
+ "precision_at_1": 0.42371,
1657
+ "precision_at_3": 0.32897,
1658
+ "precision_at_5": 0.26665,
1659
+ "precision_at_10": 0.17022,
1660
+ "precision_at_20": 0.11437,
1661
+ "precision_at_50": 0.05524,
1662
+ "precision_at_100": 0.03127
1663
+ }
1664
+ },
1665
+ {
1666
+ "class": "sentence-transformers",
1667
+ "organization": "HuggingFace",
1668
+ "model_name": "hf-code-transformer",
1669
+ "results": {
1670
+ "ndcg_at_1": 0.31713,
1671
+ "ndcg_at_3": 0.39786,
1672
+ "ndcg_at_5": 0.35876,
1673
+ "ndcg_at_10": 0.40049,
1674
+ "ndcg_at_20": 0.49391,
1675
+ "ndcg_at_50": 0.51027,
1676
+ "ndcg_at_100": 0.53921,
1677
+ "recall_at_1": 0.13616,
1678
+ "recall_at_3": 0.26097,
1679
+ "recall_at_5": 0.36709,
1680
+ "recall_at_10": 0.44995,
1681
+ "recall_at_20": 0.60652,
1682
+ "recall_at_50": 0.7856,
1683
+ "recall_at_100": 0.94528,
1684
+ "precision_at_1": 0.37037,
1685
+ "precision_at_3": 0.32462,
1686
+ "precision_at_5": 0.26304,
1687
+ "precision_at_10": 0.17964,
1688
+ "precision_at_20": 0.11173,
1689
+ "precision_at_50": 0.05597,
1690
+ "precision_at_100": 0.03243
1691
+ }
1692
+ },
1693
+ {
1694
+ "class": "proprietary",
1695
+ "organization": "Apple",
1696
+ "model_name": "apple-code-model",
1697
+ "results": {
1698
+ "ndcg_at_1": 0.38266,
1699
+ "ndcg_at_3": 0.39202,
1700
+ "ndcg_at_5": 0.35493,
1701
+ "ndcg_at_10": 0.43881,
1702
+ "ndcg_at_20": 0.48872,
1703
+ "ndcg_at_50": 0.53869,
1704
+ "ndcg_at_100": 0.53683,
1705
+ "recall_at_1": 0.1461,
1706
+ "recall_at_3": 0.27657,
1707
+ "recall_at_5": 0.35192,
1708
+ "recall_at_10": 0.44538,
1709
+ "recall_at_20": 0.60702,
1710
+ "recall_at_50": 0.78463,
1711
+ "recall_at_100": 0.9486,
1712
+ "precision_at_1": 0.42423,
1713
+ "precision_at_3": 0.31381,
1714
+ "precision_at_5": 0.26868,
1715
+ "precision_at_10": 0.175,
1716
+ "precision_at_20": 0.11001,
1717
+ "precision_at_50": 0.05263,
1718
+ "precision_at_100": 0.03728
1719
+ }
1720
+ }
1721
+ ],
1722
+ "ProgrammingQuestions": [
1723
+ {
1724
+ "class": "custom",
1725
+ "organization": "Microsoft",
1726
+ "model_name": "microsoft-code-embed",
1727
+ "results": {
1728
+ "ndcg_at_1": 0.47235,
1729
+ "ndcg_at_3": 0.42062,
1730
+ "ndcg_at_5": 0.44524,
1731
+ "ndcg_at_10": 0.40797,
1732
+ "ndcg_at_20": 0.48262,
1733
+ "ndcg_at_50": 0.50422,
1734
+ "ndcg_at_100": 0.54219,
1735
+ "recall_at_1": 0.12357,
1736
+ "recall_at_3": 0.26331,
1737
+ "recall_at_5": 0.36803,
1738
+ "recall_at_10": 0.4472,
1739
+ "recall_at_20": 0.61063,
1740
+ "recall_at_50": 0.75564,
1741
+ "recall_at_100": 0.93419,
1742
+ "precision_at_1": 0.42289,
1743
+ "precision_at_3": 0.30065,
1744
+ "precision_at_5": 0.26393,
1745
+ "precision_at_10": 0.1705,
1746
+ "precision_at_20": 0.11886,
1747
+ "precision_at_50": 0.0574,
1748
+ "precision_at_100": 0.03028
1749
+ }
1750
+ },
1751
+ {
1752
+ "class": "sentence-transformers",
1753
+ "organization": "HuggingFace",
1754
+ "model_name": "hf-code-transformer",
1755
+ "results": {
1756
+ "ndcg_at_1": 0.47084,
1757
+ "ndcg_at_3": 0.36371,
1758
+ "ndcg_at_5": 0.36771,
1759
+ "ndcg_at_10": 0.4319,
1760
+ "ndcg_at_20": 0.5426,
1761
+ "ndcg_at_50": 0.54877,
1762
+ "ndcg_at_100": 0.53845,
1763
+ "recall_at_1": 0.13776,
1764
+ "recall_at_3": 0.27296,
1765
+ "recall_at_5": 0.35521,
1766
+ "recall_at_10": 0.44614,
1767
+ "recall_at_20": 0.60422,
1768
+ "recall_at_50": 0.7653,
1769
+ "recall_at_100": 0.93355,
1770
+ "precision_at_1": 0.37316,
1771
+ "precision_at_3": 0.30778,
1772
+ "precision_at_5": 0.26899,
1773
+ "precision_at_10": 0.17148,
1774
+ "precision_at_20": 0.11726,
1775
+ "precision_at_50": 0.05475,
1776
+ "precision_at_100": 0.03405
1777
+ }
1778
+ },
1779
+ {
1780
+ "class": "proprietary",
1781
+ "organization": "Apple",
1782
+ "model_name": "apple-code-model",
1783
+ "results": {
1784
+ "ndcg_at_1": 0.42288,
1785
+ "ndcg_at_3": 0.35297,
1786
+ "ndcg_at_5": 0.36994,
1787
+ "ndcg_at_10": 0.42116,
1788
+ "ndcg_at_20": 0.51656,
1789
+ "ndcg_at_50": 0.50791,
1790
+ "ndcg_at_100": 0.53042,
1791
+ "recall_at_1": 0.12078,
1792
+ "recall_at_3": 0.26329,
1793
+ "recall_at_5": 0.36007,
1794
+ "recall_at_10": 0.44965,
1795
+ "recall_at_20": 0.61431,
1796
+ "recall_at_50": 0.7613,
1797
+ "recall_at_100": 0.94383,
1798
+ "precision_at_1": 0.43091,
1799
+ "precision_at_3": 0.32639,
1800
+ "precision_at_5": 0.25782,
1801
+ "precision_at_10": 0.17673,
1802
+ "precision_at_20": 0.11918,
1803
+ "precision_at_50": 0.05437,
1804
+ "precision_at_100": 0.03343
1805
+ }
1806
+ }
1807
+ ],
1808
+ "PatientRecords": [
1809
+ {
1810
+ "class": "custom",
1811
+ "organization": "Microsoft",
1812
+ "model_name": "microsoft-healthcare-v2",
1813
+ "results": {
1814
+ "ndcg_at_1": 0.36495,
1815
+ "ndcg_at_3": 0.43287,
1816
+ "ndcg_at_5": 0.36448,
1817
+ "ndcg_at_10": 0.43512,
1818
+ "ndcg_at_20": 0.53344,
1819
+ "ndcg_at_50": 0.52122,
1820
+ "ndcg_at_100": 0.53324,
1821
+ "recall_at_1": 0.13818,
1822
+ "recall_at_3": 0.26486,
1823
+ "recall_at_5": 0.3598,
1824
+ "recall_at_10": 0.44427,
1825
+ "recall_at_20": 0.62507,
1826
+ "recall_at_50": 0.76953,
1827
+ "recall_at_100": 0.94294,
1828
+ "precision_at_1": 0.44488,
1829
+ "precision_at_3": 0.32785,
1830
+ "precision_at_5": 0.26495,
1831
+ "precision_at_10": 0.17904,
1832
+ "precision_at_20": 0.11725,
1833
+ "precision_at_50": 0.05159,
1834
+ "precision_at_100": 0.03069
1835
+ }
1836
+ },
1837
+ {
1838
+ "class": "sentence-transformers",
1839
+ "organization": "Facebook",
1840
+ "model_name": "fb-healthcare-embed",
1841
+ "results": {
1842
+ "ndcg_at_1": 0.34502,
1843
+ "ndcg_at_3": 0.41964,
1844
+ "ndcg_at_5": 0.42443,
1845
+ "ndcg_at_10": 0.41718,
1846
+ "ndcg_at_20": 0.54506,
1847
+ "ndcg_at_50": 0.52925,
1848
+ "ndcg_at_100": 0.54439,
1849
+ "recall_at_1": 0.13915,
1850
+ "recall_at_3": 0.26605,
1851
+ "recall_at_5": 0.36128,
1852
+ "recall_at_10": 0.44337,
1853
+ "recall_at_20": 0.62886,
1854
+ "recall_at_50": 0.78315,
1855
+ "recall_at_100": 0.94994,
1856
+ "precision_at_1": 0.3723,
1857
+ "precision_at_3": 0.30295,
1858
+ "precision_at_5": 0.26884,
1859
+ "precision_at_10": 0.17781,
1860
+ "precision_at_20": 0.1105,
1861
+ "precision_at_50": 0.0583,
1862
+ "precision_at_100": 0.03843
1863
+ }
1864
+ },
1865
+ {
1866
+ "class": "proprietary",
1867
+ "organization": "Google",
1868
+ "model_name": "google-healthcare-model",
1869
+ "results": {
1870
+ "ndcg_at_1": 0.37816,
1871
+ "ndcg_at_3": 0.35029,
1872
+ "ndcg_at_5": 0.41213,
1873
+ "ndcg_at_10": 0.40133,
1874
+ "ndcg_at_20": 0.47368,
1875
+ "ndcg_at_50": 0.52187,
1876
+ "ndcg_at_100": 0.53866,
1877
+ "recall_at_1": 0.13869,
1878
+ "recall_at_3": 0.27732,
1879
+ "recall_at_5": 0.36667,
1880
+ "recall_at_10": 0.4432,
1881
+ "recall_at_20": 0.60019,
1882
+ "recall_at_50": 0.78793,
1883
+ "recall_at_100": 0.93892,
1884
+ "precision_at_1": 0.38429,
1885
+ "precision_at_3": 0.31032,
1886
+ "precision_at_5": 0.25127,
1887
+ "precision_at_10": 0.17171,
1888
+ "precision_at_20": 0.11083,
1889
+ "precision_at_50": 0.05943,
1890
+ "precision_at_100": 0.03355
1891
+ }
1892
+ }
1893
+ ],
1894
+ "MedicalImages": [
1895
+ {
1896
+ "class": "custom",
1897
+ "organization": "Microsoft",
1898
+ "model_name": "microsoft-healthcare-v2",
1899
+ "results": {
1900
+ "ndcg_at_1": 0.315,
1901
+ "ndcg_at_3": 0.3643,
1902
+ "ndcg_at_5": 0.41396,
1903
+ "ndcg_at_10": 0.43675,
1904
+ "ndcg_at_20": 0.4832,
1905
+ "ndcg_at_50": 0.54346,
1906
+ "ndcg_at_100": 0.53251,
1907
+ "recall_at_1": 0.12382,
1908
+ "recall_at_3": 0.25542,
1909
+ "recall_at_5": 0.35478,
1910
+ "recall_at_10": 0.44497,
1911
+ "recall_at_20": 0.62098,
1912
+ "recall_at_50": 0.76209,
1913
+ "recall_at_100": 0.94931,
1914
+ "precision_at_1": 0.39217,
1915
+ "precision_at_3": 0.30446,
1916
+ "precision_at_5": 0.26112,
1917
+ "precision_at_10": 0.17512,
1918
+ "precision_at_20": 0.11011,
1919
+ "precision_at_50": 0.05949,
1920
+ "precision_at_100": 0.0364
1921
+ }
1922
+ },
1923
+ {
1924
+ "class": "sentence-transformers",
1925
+ "organization": "Facebook",
1926
+ "model_name": "fb-healthcare-embed",
1927
+ "results": {
1928
+ "ndcg_at_1": 0.37332,
1929
+ "ndcg_at_3": 0.44424,
1930
+ "ndcg_at_5": 0.40001,
1931
+ "ndcg_at_10": 0.42532,
1932
+ "ndcg_at_20": 0.48689,
1933
+ "ndcg_at_50": 0.53648,
1934
+ "ndcg_at_100": 0.5487,
1935
+ "recall_at_1": 0.12879,
1936
+ "recall_at_3": 0.26795,
1937
+ "recall_at_5": 0.35338,
1938
+ "recall_at_10": 0.44899,
1939
+ "recall_at_20": 0.61746,
1940
+ "recall_at_50": 0.78418,
1941
+ "recall_at_100": 0.93593,
1942
+ "precision_at_1": 0.40559,
1943
+ "precision_at_3": 0.3142,
1944
+ "precision_at_5": 0.26203,
1945
+ "precision_at_10": 0.17335,
1946
+ "precision_at_20": 0.1108,
1947
+ "precision_at_50": 0.0533,
1948
+ "precision_at_100": 0.03044
1949
+ }
1950
+ },
1951
+ {
1952
+ "class": "proprietary",
1953
+ "organization": "Google",
1954
+ "model_name": "google-healthcare-model",
1955
+ "results": {
1956
+ "ndcg_at_1": 0.33651,
1957
+ "ndcg_at_3": 0.43117,
1958
+ "ndcg_at_5": 0.35613,
1959
+ "ndcg_at_10": 0.44266,
1960
+ "ndcg_at_20": 0.49934,
1961
+ "ndcg_at_50": 0.52851,
1962
+ "ndcg_at_100": 0.54564,
1963
+ "recall_at_1": 0.12355,
1964
+ "recall_at_3": 0.26755,
1965
+ "recall_at_5": 0.36563,
1966
+ "recall_at_10": 0.44859,
1967
+ "recall_at_20": 0.61649,
1968
+ "recall_at_50": 0.78753,
1969
+ "recall_at_100": 0.9462,
1970
+ "precision_at_1": 0.39591,
1971
+ "precision_at_3": 0.32248,
1972
+ "precision_at_5": 0.26823,
1973
+ "precision_at_10": 0.1737,
1974
+ "precision_at_20": 0.11591,
1975
+ "precision_at_50": 0.05707,
1976
+ "precision_at_100": 0.03386
1977
+ }
1978
+ }
1979
+ ],
1980
+ "ClinicalTrials": [
1981
+ {
1982
+ "class": "custom",
1983
+ "organization": "Microsoft",
1984
+ "model_name": "microsoft-healthcare-v2",
1985
+ "results": {
1986
+ "ndcg_at_1": 0.35301,
1987
+ "ndcg_at_3": 0.43665,
1988
+ "ndcg_at_5": 0.42374,
1989
+ "ndcg_at_10": 0.41009,
1990
+ "ndcg_at_20": 0.54703,
1991
+ "ndcg_at_50": 0.54576,
1992
+ "ndcg_at_100": 0.53171,
1993
+ "recall_at_1": 0.13686,
1994
+ "recall_at_3": 0.25579,
1995
+ "recall_at_5": 0.36714,
1996
+ "recall_at_10": 0.44398,
1997
+ "recall_at_20": 0.61054,
1998
+ "recall_at_50": 0.77278,
1999
+ "recall_at_100": 0.93894,
2000
+ "precision_at_1": 0.44804,
2001
+ "precision_at_3": 0.3038,
2002
+ "precision_at_5": 0.25241,
2003
+ "precision_at_10": 0.17617,
2004
+ "precision_at_20": 0.11375,
2005
+ "precision_at_50": 0.05908,
2006
+ "precision_at_100": 0.03765
2007
+ }
2008
+ },
2009
+ {
2010
+ "class": "sentence-transformers",
2011
+ "organization": "Facebook",
2012
+ "model_name": "fb-healthcare-embed",
2013
+ "results": {
2014
+ "ndcg_at_1": 0.39992,
2015
+ "ndcg_at_3": 0.35186,
2016
+ "ndcg_at_5": 0.39482,
2017
+ "ndcg_at_10": 0.43634,
2018
+ "ndcg_at_20": 0.46919,
2019
+ "ndcg_at_50": 0.52712,
2020
+ "ndcg_at_100": 0.54092,
2021
+ "recall_at_1": 0.13142,
2022
+ "recall_at_3": 0.27946,
2023
+ "recall_at_5": 0.36885,
2024
+ "recall_at_10": 0.4464,
2025
+ "recall_at_20": 0.60732,
2026
+ "recall_at_50": 0.78932,
2027
+ "recall_at_100": 0.93876,
2028
+ "precision_at_1": 0.42916,
2029
+ "precision_at_3": 0.30007,
2030
+ "precision_at_5": 0.25223,
2031
+ "precision_at_10": 0.17696,
2032
+ "precision_at_20": 0.1132,
2033
+ "precision_at_50": 0.05023,
2034
+ "precision_at_100": 0.03638
2035
+ }
2036
+ },
2037
+ {
2038
+ "class": "proprietary",
2039
+ "organization": "Google",
2040
+ "model_name": "google-healthcare-model",
2041
+ "results": {
2042
+ "ndcg_at_1": 0.33725,
2043
+ "ndcg_at_3": 0.40388,
2044
+ "ndcg_at_5": 0.39817,
2045
+ "ndcg_at_10": 0.43164,
2046
+ "ndcg_at_20": 0.46068,
2047
+ "ndcg_at_50": 0.51296,
2048
+ "ndcg_at_100": 0.54346,
2049
+ "recall_at_1": 0.13486,
2050
+ "recall_at_3": 0.26501,
2051
+ "recall_at_5": 0.35261,
2052
+ "recall_at_10": 0.44446,
2053
+ "recall_at_20": 0.6295,
2054
+ "recall_at_50": 0.78965,
2055
+ "recall_at_100": 0.94848,
2056
+ "precision_at_1": 0.44756,
2057
+ "precision_at_3": 0.30211,
2058
+ "precision_at_5": 0.25199,
2059
+ "precision_at_10": 0.17035,
2060
+ "precision_at_20": 0.11085,
2061
+ "precision_at_50": 0.05663,
2062
+ "precision_at_100": 0.03558
2063
+ }
2064
+ }
2065
+ ],
2066
+ "TextPhotoPairs": [
2067
+ {
2068
+ "class": "sentence-transformers",
2069
+ "organization": "Facebook",
2070
+ "model_name": "fb-text-to-photo",
2071
+ "results": {
2072
+ "ndcg_at_1": 0.38969,
2073
+ "ndcg_at_3": 0.37443,
2074
+ "ndcg_at_5": 0.38385,
2075
+ "ndcg_at_10": 0.43381,
2076
+ "ndcg_at_20": 0.47417,
2077
+ "ndcg_at_50": 0.53955,
2078
+ "ndcg_at_100": 0.53257,
2079
+ "recall_at_1": 0.1295,
2080
+ "recall_at_3": 0.27905,
2081
+ "recall_at_5": 0.35864,
2082
+ "recall_at_10": 0.44506,
2083
+ "recall_at_20": 0.60178,
2084
+ "recall_at_50": 0.76573,
2085
+ "recall_at_100": 0.94895,
2086
+ "precision_at_1": 0.3582,
2087
+ "precision_at_3": 0.32281,
2088
+ "precision_at_5": 0.26238,
2089
+ "precision_at_10": 0.17916,
2090
+ "precision_at_20": 0.11831,
2091
+ "precision_at_50": 0.05255,
2092
+ "precision_at_100": 0.03192
2093
+ }
2094
+ },
2095
+ {
2096
+ "class": "proprietary",
2097
+ "organization": "Apple",
2098
+ "model_name": "apple-text-to-photo-v2",
2099
+ "results": {
2100
+ "ndcg_at_1": 0.37115,
2101
+ "ndcg_at_3": 0.35804,
2102
+ "ndcg_at_5": 0.41873,
2103
+ "ndcg_at_10": 0.41871,
2104
+ "ndcg_at_20": 0.49689,
2105
+ "ndcg_at_50": 0.52029,
2106
+ "ndcg_at_100": 0.54502,
2107
+ "recall_at_1": 0.13004,
2108
+ "recall_at_3": 0.26266,
2109
+ "recall_at_5": 0.3649,
2110
+ "recall_at_10": 0.44746,
2111
+ "recall_at_20": 0.60394,
2112
+ "recall_at_50": 0.75227,
2113
+ "recall_at_100": 0.94193,
2114
+ "precision_at_1": 0.42163,
2115
+ "precision_at_3": 0.31858,
2116
+ "precision_at_5": 0.26983,
2117
+ "precision_at_10": 0.17879,
2118
+ "precision_at_20": 0.11332,
2119
+ "precision_at_50": 0.05277,
2120
+ "precision_at_100": 0.03069
2121
+ }
2122
+ },
2123
+ {
2124
+ "class": "custom",
2125
+ "organization": "Microsoft",
2126
+ "model_name": "microsoft-text-to-photo",
2127
+ "results": {
2128
+ "ndcg_at_1": 0.38312,
2129
+ "ndcg_at_3": 0.36662,
2130
+ "ndcg_at_5": 0.36532,
2131
+ "ndcg_at_10": 0.43171,
2132
+ "ndcg_at_20": 0.54142,
2133
+ "ndcg_at_50": 0.53123,
2134
+ "ndcg_at_100": 0.53362,
2135
+ "recall_at_1": 0.13918,
2136
+ "recall_at_3": 0.25559,
2137
+ "recall_at_5": 0.36491,
2138
+ "recall_at_10": 0.44649,
2139
+ "recall_at_20": 0.60659,
2140
+ "recall_at_50": 0.77238,
2141
+ "recall_at_100": 0.93495,
2142
+ "precision_at_1": 0.37461,
2143
+ "precision_at_3": 0.32237,
2144
+ "precision_at_5": 0.26687,
2145
+ "precision_at_10": 0.17813,
2146
+ "precision_at_20": 0.11664,
2147
+ "precision_at_50": 0.05836,
2148
+ "precision_at_100": 0.03771
2149
+ }
2150
+ }
2151
+ ],
2152
+ "ImageCaptions": [
2153
+ {
2154
+ "class": "sentence-transformers",
2155
+ "organization": "Facebook",
2156
+ "model_name": "fb-text-to-photo",
2157
+ "results": {
2158
+ "ndcg_at_1": 0.39061,
2159
+ "ndcg_at_3": 0.4024,
2160
+ "ndcg_at_5": 0.39395,
2161
+ "ndcg_at_10": 0.43832,
2162
+ "ndcg_at_20": 0.54166,
2163
+ "ndcg_at_50": 0.52527,
2164
+ "ndcg_at_100": 0.54258,
2165
+ "recall_at_1": 0.14862,
2166
+ "recall_at_3": 0.27141,
2167
+ "recall_at_5": 0.36146,
2168
+ "recall_at_10": 0.44509,
2169
+ "recall_at_20": 0.60311,
2170
+ "recall_at_50": 0.7862,
2171
+ "recall_at_100": 0.94634,
2172
+ "precision_at_1": 0.42609,
2173
+ "precision_at_3": 0.30639,
2174
+ "precision_at_5": 0.25778,
2175
+ "precision_at_10": 0.1767,
2176
+ "precision_at_20": 0.11035,
2177
+ "precision_at_50": 0.05098,
2178
+ "precision_at_100": 0.03691
2179
+ }
2180
+ },
2181
+ {
2182
+ "class": "proprietary",
2183
+ "organization": "Apple",
2184
+ "model_name": "apple-text-to-photo-v2",
2185
+ "results": {
2186
+ "ndcg_at_1": 0.38782,
2187
+ "ndcg_at_3": 0.39907,
2188
+ "ndcg_at_5": 0.44192,
2189
+ "ndcg_at_10": 0.41861,
2190
+ "ndcg_at_20": 0.50653,
2191
+ "ndcg_at_50": 0.50371,
2192
+ "ndcg_at_100": 0.53094,
2193
+ "recall_at_1": 0.14718,
2194
+ "recall_at_3": 0.26991,
2195
+ "recall_at_5": 0.35298,
2196
+ "recall_at_10": 0.44161,
2197
+ "recall_at_20": 0.61503,
2198
+ "recall_at_50": 0.76317,
2199
+ "recall_at_100": 0.93096,
2200
+ "precision_at_1": 0.36238,
2201
+ "precision_at_3": 0.32202,
2202
+ "precision_at_5": 0.26725,
2203
+ "precision_at_10": 0.17919,
2204
+ "precision_at_20": 0.11515,
2205
+ "precision_at_50": 0.05398,
2206
+ "precision_at_100": 0.03531
2207
+ }
2208
+ },
2209
+ {
2210
+ "class": "custom",
2211
+ "organization": "Microsoft",
2212
+ "model_name": "microsoft-text-to-photo",
2213
+ "results": {
2214
+ "ndcg_at_1": 0.34286,
2215
+ "ndcg_at_3": 0.35004,
2216
+ "ndcg_at_5": 0.36618,
2217
+ "ndcg_at_10": 0.43906,
2218
+ "ndcg_at_20": 0.48724,
2219
+ "ndcg_at_50": 0.53748,
2220
+ "ndcg_at_100": 0.53192,
2221
+ "recall_at_1": 0.12018,
2222
+ "recall_at_3": 0.26834,
2223
+ "recall_at_5": 0.36894,
2224
+ "recall_at_10": 0.44752,
2225
+ "recall_at_20": 0.61956,
2226
+ "recall_at_50": 0.77971,
2227
+ "recall_at_100": 0.94013,
2228
+ "precision_at_1": 0.37471,
2229
+ "precision_at_3": 0.30041,
2230
+ "precision_at_5": 0.26722,
2231
+ "precision_at_10": 0.17754,
2232
+ "precision_at_20": 0.11076,
2233
+ "precision_at_50": 0.05329,
2234
+ "precision_at_100": 0.03729
2235
+ }
2236
+ }
2237
+ ],
2238
+ "VisualDescriptions": [
2239
+ {
2240
+ "class": "sentence-transformers",
2241
+ "organization": "Facebook",
2242
+ "model_name": "fb-text-to-photo",
2243
+ "results": {
2244
+ "ndcg_at_1": 0.3731,
2245
+ "ndcg_at_3": 0.35184,
2246
+ "ndcg_at_5": 0.38967,
2247
+ "ndcg_at_10": 0.44264,
2248
+ "ndcg_at_20": 0.49059,
2249
+ "ndcg_at_50": 0.54856,
2250
+ "ndcg_at_100": 0.54512,
2251
+ "recall_at_1": 0.1389,
2252
+ "recall_at_3": 0.27555,
2253
+ "recall_at_5": 0.36946,
2254
+ "recall_at_10": 0.44591,
2255
+ "recall_at_20": 0.60349,
2256
+ "recall_at_50": 0.78323,
2257
+ "recall_at_100": 0.93264,
2258
+ "precision_at_1": 0.37308,
2259
+ "precision_at_3": 0.31917,
2260
+ "precision_at_5": 0.25591,
2261
+ "precision_at_10": 0.17212,
2262
+ "precision_at_20": 0.11848,
2263
+ "precision_at_50": 0.05681,
2264
+ "precision_at_100": 0.03208
2265
+ }
2266
+ },
2267
+ {
2268
+ "class": "proprietary",
2269
+ "organization": "Apple",
2270
+ "model_name": "apple-text-to-photo-v2",
2271
+ "results": {
2272
+ "ndcg_at_1": 0.32523,
2273
+ "ndcg_at_3": 0.40374,
2274
+ "ndcg_at_5": 0.35299,
2275
+ "ndcg_at_10": 0.40289,
2276
+ "ndcg_at_20": 0.48648,
2277
+ "ndcg_at_50": 0.51942,
2278
+ "ndcg_at_100": 0.5367,
2279
+ "recall_at_1": 0.13546,
2280
+ "recall_at_3": 0.25256,
2281
+ "recall_at_5": 0.36284,
2282
+ "recall_at_10": 0.44043,
2283
+ "recall_at_20": 0.61199,
2284
+ "recall_at_50": 0.78481,
2285
+ "recall_at_100": 0.93342,
2286
+ "precision_at_1": 0.4099,
2287
+ "precision_at_3": 0.31956,
2288
+ "precision_at_5": 0.25034,
2289
+ "precision_at_10": 0.1741,
2290
+ "precision_at_20": 0.11492,
2291
+ "precision_at_50": 0.05379,
2292
+ "precision_at_100": 0.03808
2293
+ }
2294
+ },
2295
+ {
2296
+ "class": "custom",
2297
+ "organization": "Microsoft",
2298
+ "model_name": "microsoft-text-to-photo",
2299
+ "results": {
2300
+ "ndcg_at_1": 0.35668,
2301
+ "ndcg_at_3": 0.37626,
2302
+ "ndcg_at_5": 0.43703,
2303
+ "ndcg_at_10": 0.44126,
2304
+ "ndcg_at_20": 0.52407,
2305
+ "ndcg_at_50": 0.52237,
2306
+ "ndcg_at_100": 0.53844,
2307
+ "recall_at_1": 0.14218,
2308
+ "recall_at_3": 0.26515,
2309
+ "recall_at_5": 0.35722,
2310
+ "recall_at_10": 0.44893,
2311
+ "recall_at_20": 0.60871,
2312
+ "recall_at_50": 0.76138,
2313
+ "recall_at_100": 0.94685,
2314
+ "precision_at_1": 0.44046,
2315
+ "precision_at_3": 0.32624,
2316
+ "precision_at_5": 0.25043,
2317
+ "precision_at_10": 0.17745,
2318
+ "precision_at_20": 0.11075,
2319
+ "precision_at_50": 0.05735,
2320
+ "precision_at_100": 0.03549
2321
+ }
2322
+ }
2323
+ ],
2324
+ "ScannedDocuments": [
2325
+ {
2326
+ "class": "custom",
2327
+ "organization": "Microsoft",
2328
+ "model_name": "microsoft-document-screenshot",
2329
+ "results": {
2330
+ "ndcg_at_1": 0.46084,
2331
+ "ndcg_at_3": 0.40767,
2332
+ "ndcg_at_5": 0.41619,
2333
+ "ndcg_at_10": 0.43795,
2334
+ "ndcg_at_20": 0.52227,
2335
+ "ndcg_at_50": 0.50727,
2336
+ "ndcg_at_100": 0.53102,
2337
+ "recall_at_1": 0.13957,
2338
+ "recall_at_3": 0.26535,
2339
+ "recall_at_5": 0.35164,
2340
+ "recall_at_10": 0.44968,
2341
+ "recall_at_20": 0.60755,
2342
+ "recall_at_50": 0.75067,
2343
+ "recall_at_100": 0.93596,
2344
+ "precision_at_1": 0.36538,
2345
+ "precision_at_3": 0.30673,
2346
+ "precision_at_5": 0.26419,
2347
+ "precision_at_10": 0.17926,
2348
+ "precision_at_20": 0.11508,
2349
+ "precision_at_50": 0.05761,
2350
+ "precision_at_100": 0.03545
2351
+ }
2352
+ },
2353
+ {
2354
+ "class": "sentence-transformers",
2355
+ "organization": "Google",
2356
+ "model_name": "google-document-screenshot",
2357
+ "results": {
2358
+ "ndcg_at_1": 0.42588,
2359
+ "ndcg_at_3": 0.38053,
2360
+ "ndcg_at_5": 0.38053,
2361
+ "ndcg_at_10": 0.43037,
2362
+ "ndcg_at_20": 0.48407,
2363
+ "ndcg_at_50": 0.53389,
2364
+ "ndcg_at_100": 0.54636,
2365
+ "recall_at_1": 0.13767,
2366
+ "recall_at_3": 0.2562,
2367
+ "recall_at_5": 0.36555,
2368
+ "recall_at_10": 0.44749,
2369
+ "recall_at_20": 0.60477,
2370
+ "recall_at_50": 0.77073,
2371
+ "recall_at_100": 0.94816,
2372
+ "precision_at_1": 0.35243,
2373
+ "precision_at_3": 0.32045,
2374
+ "precision_at_5": 0.26921,
2375
+ "precision_at_10": 0.176,
2376
+ "precision_at_20": 0.11158,
2377
+ "precision_at_50": 0.0586,
2378
+ "precision_at_100": 0.03261
2379
+ }
2380
+ },
2381
+ {
2382
+ "class": "proprietary",
2383
+ "organization": "Facebook",
2384
+ "model_name": "fb-document-screenshot",
2385
+ "results": {
2386
+ "ndcg_at_1": 0.43047,
2387
+ "ndcg_at_3": 0.4036,
2388
+ "ndcg_at_5": 0.39222,
2389
+ "ndcg_at_10": 0.41055,
2390
+ "ndcg_at_20": 0.51586,
2391
+ "ndcg_at_50": 0.52629,
2392
+ "ndcg_at_100": 0.53482,
2393
+ "recall_at_1": 0.14323,
2394
+ "recall_at_3": 0.27662,
2395
+ "recall_at_5": 0.36309,
2396
+ "recall_at_10": 0.44137,
2397
+ "recall_at_20": 0.62843,
2398
+ "recall_at_50": 0.76847,
2399
+ "recall_at_100": 0.93851,
2400
+ "precision_at_1": 0.38161,
2401
+ "precision_at_3": 0.3267,
2402
+ "precision_at_5": 0.2601,
2403
+ "precision_at_10": 0.17009,
2404
+ "precision_at_20": 0.11318,
2405
+ "precision_at_50": 0.05571,
2406
+ "precision_at_100": 0.03147
2407
+ }
2408
+ }
2409
+ ],
2410
+ "PDFScreenshots": [
2411
+ {
2412
+ "class": "custom",
2413
+ "organization": "Microsoft",
2414
+ "model_name": "microsoft-document-screenshot",
2415
+ "results": {
2416
+ "ndcg_at_1": 0.36798,
2417
+ "ndcg_at_3": 0.44422,
2418
+ "ndcg_at_5": 0.42524,
2419
+ "ndcg_at_10": 0.40728,
2420
+ "ndcg_at_20": 0.45382,
2421
+ "ndcg_at_50": 0.54888,
2422
+ "ndcg_at_100": 0.54772,
2423
+ "recall_at_1": 0.12838,
2424
+ "recall_at_3": 0.26112,
2425
+ "recall_at_5": 0.36557,
2426
+ "recall_at_10": 0.44539,
2427
+ "recall_at_20": 0.61758,
2428
+ "recall_at_50": 0.76291,
2429
+ "recall_at_100": 0.93611,
2430
+ "precision_at_1": 0.3649,
2431
+ "precision_at_3": 0.30367,
2432
+ "precision_at_5": 0.25259,
2433
+ "precision_at_10": 0.17048,
2434
+ "precision_at_20": 0.1125,
2435
+ "precision_at_50": 0.05955,
2436
+ "precision_at_100": 0.03235
2437
+ }
2438
+ },
2439
+ {
2440
+ "class": "sentence-transformers",
2441
+ "organization": "Google",
2442
+ "model_name": "google-document-screenshot",
2443
+ "results": {
2444
+ "ndcg_at_1": 0.37616,
2445
+ "ndcg_at_3": 0.39252,
2446
+ "ndcg_at_5": 0.42646,
2447
+ "ndcg_at_10": 0.40989,
2448
+ "ndcg_at_20": 0.50146,
2449
+ "ndcg_at_50": 0.54703,
2450
+ "ndcg_at_100": 0.53725,
2451
+ "recall_at_1": 0.14408,
2452
+ "recall_at_3": 0.26808,
2453
+ "recall_at_5": 0.35366,
2454
+ "recall_at_10": 0.44319,
2455
+ "recall_at_20": 0.6165,
2456
+ "recall_at_50": 0.7821,
2457
+ "recall_at_100": 0.93025,
2458
+ "precision_at_1": 0.39636,
2459
+ "precision_at_3": 0.31408,
2460
+ "precision_at_5": 0.26203,
2461
+ "precision_at_10": 0.17071,
2462
+ "precision_at_20": 0.11895,
2463
+ "precision_at_50": 0.05926,
2464
+ "precision_at_100": 0.03287
2465
+ }
2466
+ },
2467
+ {
2468
+ "class": "proprietary",
2469
+ "organization": "Facebook",
2470
+ "model_name": "fb-document-screenshot",
2471
+ "results": {
2472
+ "ndcg_at_1": 0.31996,
2473
+ "ndcg_at_3": 0.41517,
2474
+ "ndcg_at_5": 0.44132,
2475
+ "ndcg_at_10": 0.43229,
2476
+ "ndcg_at_20": 0.49242,
2477
+ "ndcg_at_50": 0.51934,
2478
+ "ndcg_at_100": 0.54795,
2479
+ "recall_at_1": 0.14381,
2480
+ "recall_at_3": 0.27718,
2481
+ "recall_at_5": 0.35044,
2482
+ "recall_at_10": 0.44854,
2483
+ "recall_at_20": 0.61955,
2484
+ "recall_at_50": 0.75965,
2485
+ "recall_at_100": 0.93734,
2486
+ "precision_at_1": 0.40568,
2487
+ "precision_at_3": 0.31345,
2488
+ "precision_at_5": 0.2586,
2489
+ "precision_at_10": 0.17238,
2490
+ "precision_at_20": 0.11896,
2491
+ "precision_at_50": 0.05155,
2492
+ "precision_at_100": 0.03821
2493
+ }
2494
+ }
2495
+ ],
2496
+ "DigitalForms": [
2497
+ {
2498
+ "class": "custom",
2499
+ "organization": "Microsoft",
2500
+ "model_name": "microsoft-document-screenshot",
2501
+ "results": {
2502
+ "ndcg_at_1": 0.46682,
2503
+ "ndcg_at_3": 0.39785,
2504
+ "ndcg_at_5": 0.43051,
2505
+ "ndcg_at_10": 0.43718,
2506
+ "ndcg_at_20": 0.54244,
2507
+ "ndcg_at_50": 0.50392,
2508
+ "ndcg_at_100": 0.53782,
2509
+ "recall_at_1": 0.13334,
2510
+ "recall_at_3": 0.27244,
2511
+ "recall_at_5": 0.3552,
2512
+ "recall_at_10": 0.44848,
2513
+ "recall_at_20": 0.62882,
2514
+ "recall_at_50": 0.757,
2515
+ "recall_at_100": 0.93483,
2516
+ "precision_at_1": 0.39889,
2517
+ "precision_at_3": 0.30788,
2518
+ "precision_at_5": 0.26953,
2519
+ "precision_at_10": 0.17837,
2520
+ "precision_at_20": 0.11811,
2521
+ "precision_at_50": 0.05076,
2522
+ "precision_at_100": 0.03752
2523
+ }
2524
+ },
2525
+ {
2526
+ "class": "sentence-transformers",
2527
+ "organization": "Google",
2528
+ "model_name": "google-document-screenshot",
2529
+ "results": {
2530
+ "ndcg_at_1": 0.33865,
2531
+ "ndcg_at_3": 0.39382,
2532
+ "ndcg_at_5": 0.43972,
2533
+ "ndcg_at_10": 0.43263,
2534
+ "ndcg_at_20": 0.54872,
2535
+ "ndcg_at_50": 0.50251,
2536
+ "ndcg_at_100": 0.53444,
2537
+ "recall_at_1": 0.12182,
2538
+ "recall_at_3": 0.2718,
2539
+ "recall_at_5": 0.36731,
2540
+ "recall_at_10": 0.44774,
2541
+ "recall_at_20": 0.62798,
2542
+ "recall_at_50": 0.78709,
2543
+ "recall_at_100": 0.93895,
2544
+ "precision_at_1": 0.3716,
2545
+ "precision_at_3": 0.32182,
2546
+ "precision_at_5": 0.25292,
2547
+ "precision_at_10": 0.17558,
2548
+ "precision_at_20": 0.11234,
2549
+ "precision_at_50": 0.05107,
2550
+ "precision_at_100": 0.03798
2551
+ }
2552
+ },
2553
+ {
2554
+ "class": "proprietary",
2555
+ "organization": "Facebook",
2556
+ "model_name": "fb-document-screenshot",
2557
+ "results": {
2558
+ "ndcg_at_1": 0.4152,
2559
+ "ndcg_at_3": 0.39801,
2560
+ "ndcg_at_5": 0.42631,
2561
+ "ndcg_at_10": 0.42054,
2562
+ "ndcg_at_20": 0.51262,
2563
+ "ndcg_at_50": 0.52299,
2564
+ "ndcg_at_100": 0.53012,
2565
+ "recall_at_1": 0.12061,
2566
+ "recall_at_3": 0.27809,
2567
+ "recall_at_5": 0.36153,
2568
+ "recall_at_10": 0.44364,
2569
+ "recall_at_20": 0.62267,
2570
+ "recall_at_50": 0.75502,
2571
+ "recall_at_100": 0.93745,
2572
+ "precision_at_1": 0.38713,
2573
+ "precision_at_3": 0.32182,
2574
+ "precision_at_5": 0.25983,
2575
+ "precision_at_10": 0.17831,
2576
+ "precision_at_20": 0.11659,
2577
+ "precision_at_50": 0.05705,
2578
+ "precision_at_100": 0.03146
2579
+ }
2580
+ }
2581
+ ],
2582
+ "ScientificFigures": [
2583
+ {
2584
+ "class": "sentence-transformers",
2585
+ "organization": "Google",
2586
+ "model_name": "google-figures-tables",
2587
+ "results": {
2588
+ "ndcg_at_1": 0.45908,
2589
+ "ndcg_at_3": 0.38422,
2590
+ "ndcg_at_5": 0.40931,
2591
+ "ndcg_at_10": 0.42148,
2592
+ "ndcg_at_20": 0.48164,
2593
+ "ndcg_at_50": 0.52527,
2594
+ "ndcg_at_100": 0.54208,
2595
+ "recall_at_1": 0.14275,
2596
+ "recall_at_3": 0.26978,
2597
+ "recall_at_5": 0.36195,
2598
+ "recall_at_10": 0.44262,
2599
+ "recall_at_20": 0.60387,
2600
+ "recall_at_50": 0.76902,
2601
+ "recall_at_100": 0.93051,
2602
+ "precision_at_1": 0.3634,
2603
+ "precision_at_3": 0.31565,
2604
+ "precision_at_5": 0.26485,
2605
+ "precision_at_10": 0.17592,
2606
+ "precision_at_20": 0.11899,
2607
+ "precision_at_50": 0.05322,
2608
+ "precision_at_100": 0.03504
2609
+ }
2610
+ },
2611
+ {
2612
+ "class": "custom",
2613
+ "organization": "Microsoft",
2614
+ "model_name": "microsoft-figures-tables",
2615
+ "results": {
2616
+ "ndcg_at_1": 0.35309,
2617
+ "ndcg_at_3": 0.39324,
2618
+ "ndcg_at_5": 0.39668,
2619
+ "ndcg_at_10": 0.43974,
2620
+ "ndcg_at_20": 0.53656,
2621
+ "ndcg_at_50": 0.5255,
2622
+ "ndcg_at_100": 0.53754,
2623
+ "recall_at_1": 0.1289,
2624
+ "recall_at_3": 0.25626,
2625
+ "recall_at_5": 0.36932,
2626
+ "recall_at_10": 0.44247,
2627
+ "recall_at_20": 0.62367,
2628
+ "recall_at_50": 0.77186,
2629
+ "recall_at_100": 0.93111,
2630
+ "precision_at_1": 0.40936,
2631
+ "precision_at_3": 0.32993,
2632
+ "precision_at_5": 0.26435,
2633
+ "precision_at_10": 0.17186,
2634
+ "precision_at_20": 0.11841,
2635
+ "precision_at_50": 0.05078,
2636
+ "precision_at_100": 0.03341
2637
+ }
2638
+ },
2639
+ {
2640
+ "class": "proprietary",
2641
+ "organization": "Apple",
2642
+ "model_name": "apple-figures-tables",
2643
+ "results": {
2644
+ "ndcg_at_1": 0.47641,
2645
+ "ndcg_at_3": 0.35375,
2646
+ "ndcg_at_5": 0.41473,
2647
+ "ndcg_at_10": 0.40328,
2648
+ "ndcg_at_20": 0.47554,
2649
+ "ndcg_at_50": 0.54443,
2650
+ "ndcg_at_100": 0.54503,
2651
+ "recall_at_1": 0.12211,
2652
+ "recall_at_3": 0.25477,
2653
+ "recall_at_5": 0.35309,
2654
+ "recall_at_10": 0.44582,
2655
+ "recall_at_20": 0.61233,
2656
+ "recall_at_50": 0.76764,
2657
+ "recall_at_100": 0.9335,
2658
+ "precision_at_1": 0.39097,
2659
+ "precision_at_3": 0.32707,
2660
+ "precision_at_5": 0.25097,
2661
+ "precision_at_10": 0.17326,
2662
+ "precision_at_20": 0.11361,
2663
+ "precision_at_50": 0.05631,
2664
+ "precision_at_100": 0.0309
2665
+ }
2666
+ }
2667
+ ],
2668
+ "DataTables": [
2669
+ {
2670
+ "class": "sentence-transformers",
2671
+ "organization": "Google",
2672
+ "model_name": "google-figures-tables",
2673
+ "results": {
2674
+ "ndcg_at_1": 0.33694,
2675
+ "ndcg_at_3": 0.38421,
2676
+ "ndcg_at_5": 0.43533,
2677
+ "ndcg_at_10": 0.43528,
2678
+ "ndcg_at_20": 0.46134,
2679
+ "ndcg_at_50": 0.50154,
2680
+ "ndcg_at_100": 0.54112,
2681
+ "recall_at_1": 0.12832,
2682
+ "recall_at_3": 0.27122,
2683
+ "recall_at_5": 0.36142,
2684
+ "recall_at_10": 0.44989,
2685
+ "recall_at_20": 0.60414,
2686
+ "recall_at_50": 0.76647,
2687
+ "recall_at_100": 0.94696,
2688
+ "precision_at_1": 0.37434,
2689
+ "precision_at_3": 0.3226,
2690
+ "precision_at_5": 0.26582,
2691
+ "precision_at_10": 0.17201,
2692
+ "precision_at_20": 0.11972,
2693
+ "precision_at_50": 0.05518,
2694
+ "precision_at_100": 0.03778
2695
+ }
2696
+ },
2697
+ {
2698
+ "class": "custom",
2699
+ "organization": "Microsoft",
2700
+ "model_name": "microsoft-figures-tables",
2701
+ "results": {
2702
+ "ndcg_at_1": 0.38555,
2703
+ "ndcg_at_3": 0.35724,
2704
+ "ndcg_at_5": 0.36338,
2705
+ "ndcg_at_10": 0.44112,
2706
+ "ndcg_at_20": 0.49923,
2707
+ "ndcg_at_50": 0.50662,
2708
+ "ndcg_at_100": 0.5358,
2709
+ "recall_at_1": 0.13923,
2710
+ "recall_at_3": 0.25662,
2711
+ "recall_at_5": 0.35427,
2712
+ "recall_at_10": 0.44734,
2713
+ "recall_at_20": 0.61585,
2714
+ "recall_at_50": 0.77176,
2715
+ "recall_at_100": 0.93153,
2716
+ "precision_at_1": 0.3562,
2717
+ "precision_at_3": 0.30284,
2718
+ "precision_at_5": 0.25418,
2719
+ "precision_at_10": 0.17348,
2720
+ "precision_at_20": 0.11927,
2721
+ "precision_at_50": 0.05035,
2722
+ "precision_at_100": 0.0311
2723
+ }
2724
+ },
2725
+ {
2726
+ "class": "proprietary",
2727
+ "organization": "Apple",
2728
+ "model_name": "apple-figures-tables",
2729
+ "results": {
2730
+ "ndcg_at_1": 0.32631,
2731
+ "ndcg_at_3": 0.41485,
2732
+ "ndcg_at_5": 0.40319,
2733
+ "ndcg_at_10": 0.43302,
2734
+ "ndcg_at_20": 0.48586,
2735
+ "ndcg_at_50": 0.53117,
2736
+ "ndcg_at_100": 0.53798,
2737
+ "recall_at_1": 0.12914,
2738
+ "recall_at_3": 0.27601,
2739
+ "recall_at_5": 0.35433,
2740
+ "recall_at_10": 0.44641,
2741
+ "recall_at_20": 0.62933,
2742
+ "recall_at_50": 0.7689,
2743
+ "recall_at_100": 0.94162,
2744
+ "precision_at_1": 0.44586,
2745
+ "precision_at_3": 0.32414,
2746
+ "precision_at_5": 0.26125,
2747
+ "precision_at_10": 0.17699,
2748
+ "precision_at_20": 0.11306,
2749
+ "precision_at_50": 0.05331,
2750
+ "precision_at_100": 0.03628
2751
+ }
2752
+ }
2753
+ ],
2754
+ "ChartsAndGraphs": [
2755
+ {
2756
+ "class": "sentence-transformers",
2757
+ "organization": "Google",
2758
+ "model_name": "google-figures-tables",
2759
+ "results": {
2760
+ "ndcg_at_1": 0.49583,
2761
+ "ndcg_at_3": 0.40585,
2762
+ "ndcg_at_5": 0.44957,
2763
+ "ndcg_at_10": 0.43504,
2764
+ "ndcg_at_20": 0.52163,
2765
+ "ndcg_at_50": 0.50501,
2766
+ "ndcg_at_100": 0.54562,
2767
+ "recall_at_1": 0.13405,
2768
+ "recall_at_3": 0.27466,
2769
+ "recall_at_5": 0.3673,
2770
+ "recall_at_10": 0.44928,
2771
+ "recall_at_20": 0.60835,
2772
+ "recall_at_50": 0.75721,
2773
+ "recall_at_100": 0.93124,
2774
+ "precision_at_1": 0.40517,
2775
+ "precision_at_3": 0.31256,
2776
+ "precision_at_5": 0.26884,
2777
+ "precision_at_10": 0.17107,
2778
+ "precision_at_20": 0.11214,
2779
+ "precision_at_50": 0.05087,
2780
+ "precision_at_100": 0.03532
2781
+ }
2782
+ },
2783
+ {
2784
+ "class": "custom",
2785
+ "organization": "Microsoft",
2786
+ "model_name": "microsoft-figures-tables",
2787
+ "results": {
2788
+ "ndcg_at_1": 0.32672,
2789
+ "ndcg_at_3": 0.42643,
2790
+ "ndcg_at_5": 0.44754,
2791
+ "ndcg_at_10": 0.42451,
2792
+ "ndcg_at_20": 0.54396,
2793
+ "ndcg_at_50": 0.53566,
2794
+ "ndcg_at_100": 0.5423,
2795
+ "recall_at_1": 0.14287,
2796
+ "recall_at_3": 0.27625,
2797
+ "recall_at_5": 0.36987,
2798
+ "recall_at_10": 0.44625,
2799
+ "recall_at_20": 0.62609,
2800
+ "recall_at_50": 0.75191,
2801
+ "recall_at_100": 0.93102,
2802
+ "precision_at_1": 0.41913,
2803
+ "precision_at_3": 0.30844,
2804
+ "precision_at_5": 0.25488,
2805
+ "precision_at_10": 0.17645,
2806
+ "precision_at_20": 0.11864,
2807
+ "precision_at_50": 0.05695,
2808
+ "precision_at_100": 0.03949
2809
+ }
2810
+ },
2811
+ {
2812
+ "class": "proprietary",
2813
+ "organization": "Apple",
2814
+ "model_name": "apple-figures-tables",
2815
+ "results": {
2816
+ "ndcg_at_1": 0.39147,
2817
+ "ndcg_at_3": 0.42671,
2818
+ "ndcg_at_5": 0.42096,
2819
+ "ndcg_at_10": 0.43618,
2820
+ "ndcg_at_20": 0.48445,
2821
+ "ndcg_at_50": 0.51566,
2822
+ "ndcg_at_100": 0.5336,
2823
+ "recall_at_1": 0.13291,
2824
+ "recall_at_3": 0.26486,
2825
+ "recall_at_5": 0.35456,
2826
+ "recall_at_10": 0.4442,
2827
+ "recall_at_20": 0.61189,
2828
+ "recall_at_50": 0.78357,
2829
+ "recall_at_100": 0.94492,
2830
+ "precision_at_1": 0.44567,
2831
+ "precision_at_3": 0.3215,
2832
+ "precision_at_5": 0.25692,
2833
+ "precision_at_10": 0.17791,
2834
+ "precision_at_20": 0.11699,
2835
+ "precision_at_50": 0.05242,
2836
+ "precision_at_100": 0.03666
2837
+ }
2838
+ }
2839
+ ],
2840
+ "ParaphraseDatasets": [
2841
+ {
2842
+ "class": "sentence-transformers",
2843
+ "organization": "OpenAI",
2844
+ "model_name": "openai-text-to-text",
2845
+ "results": {
2846
+ "ndcg_at_1": 0.37017,
2847
+ "ndcg_at_3": 0.41477,
2848
+ "ndcg_at_5": 0.40753,
2849
+ "ndcg_at_10": 0.43993,
2850
+ "ndcg_at_20": 0.54746,
2851
+ "ndcg_at_50": 0.50086,
2852
+ "ndcg_at_100": 0.54416,
2853
+ "recall_at_1": 0.14535,
2854
+ "recall_at_3": 0.25829,
2855
+ "recall_at_5": 0.36108,
2856
+ "recall_at_10": 0.44594,
2857
+ "recall_at_20": 0.61139,
2858
+ "recall_at_50": 0.75527,
2859
+ "recall_at_100": 0.93483,
2860
+ "precision_at_1": 0.37863,
2861
+ "precision_at_3": 0.30376,
2862
+ "precision_at_5": 0.26006,
2863
+ "precision_at_10": 0.17798,
2864
+ "precision_at_20": 0.11749,
2865
+ "precision_at_50": 0.05015,
2866
+ "precision_at_100": 0.03747
2867
+ }
2868
+ },
2869
+ {
2870
+ "class": "proprietary",
2871
+ "organization": "Facebook",
2872
+ "model_name": "fb-text-to-text",
2873
+ "results": {
2874
+ "ndcg_at_1": 0.34583,
2875
+ "ndcg_at_3": 0.38838,
2876
+ "ndcg_at_5": 0.41596,
2877
+ "ndcg_at_10": 0.40132,
2878
+ "ndcg_at_20": 0.46592,
2879
+ "ndcg_at_50": 0.52785,
2880
+ "ndcg_at_100": 0.54444,
2881
+ "recall_at_1": 0.12692,
2882
+ "recall_at_3": 0.27724,
2883
+ "recall_at_5": 0.35507,
2884
+ "recall_at_10": 0.44902,
2885
+ "recall_at_20": 0.60721,
2886
+ "recall_at_50": 0.78016,
2887
+ "recall_at_100": 0.94368,
2888
+ "precision_at_1": 0.44247,
2889
+ "precision_at_3": 0.31464,
2890
+ "precision_at_5": 0.26538,
2891
+ "precision_at_10": 0.17087,
2892
+ "precision_at_20": 0.1152,
2893
+ "precision_at_50": 0.0552,
2894
+ "precision_at_100": 0.03908
2895
+ }
2896
+ },
2897
+ {
2898
+ "class": "custom",
2899
+ "organization": "Apple",
2900
+ "model_name": "apple-text-to-text",
2901
+ "results": {
2902
+ "ndcg_at_1": 0.43867,
2903
+ "ndcg_at_3": 0.42089,
2904
+ "ndcg_at_5": 0.43117,
2905
+ "ndcg_at_10": 0.42119,
2906
+ "ndcg_at_20": 0.48891,
2907
+ "ndcg_at_50": 0.51436,
2908
+ "ndcg_at_100": 0.53202,
2909
+ "recall_at_1": 0.13818,
2910
+ "recall_at_3": 0.26497,
2911
+ "recall_at_5": 0.35916,
2912
+ "recall_at_10": 0.44884,
2913
+ "recall_at_20": 0.61392,
2914
+ "recall_at_50": 0.77141,
2915
+ "recall_at_100": 0.93491,
2916
+ "precision_at_1": 0.43396,
2917
+ "precision_at_3": 0.32882,
2918
+ "precision_at_5": 0.2606,
2919
+ "precision_at_10": 0.17913,
2920
+ "precision_at_20": 0.11763,
2921
+ "precision_at_50": 0.05776,
2922
+ "precision_at_100": 0.03729
2923
+ }
2924
+ }
2925
+ ],
2926
+ "SummarizationDatasets": [
2927
+ {
2928
+ "class": "sentence-transformers",
2929
+ "organization": "OpenAI",
2930
+ "model_name": "openai-text-to-text",
2931
+ "results": {
2932
+ "ndcg_at_1": 0.44164,
2933
+ "ndcg_at_3": 0.39176,
2934
+ "ndcg_at_5": 0.42126,
2935
+ "ndcg_at_10": 0.40666,
2936
+ "ndcg_at_20": 0.47097,
2937
+ "ndcg_at_50": 0.51879,
2938
+ "ndcg_at_100": 0.53449,
2939
+ "recall_at_1": 0.13735,
2940
+ "recall_at_3": 0.26976,
2941
+ "recall_at_5": 0.35961,
2942
+ "recall_at_10": 0.44875,
2943
+ "recall_at_20": 0.60002,
2944
+ "recall_at_50": 0.77108,
2945
+ "recall_at_100": 0.94887,
2946
+ "precision_at_1": 0.36923,
2947
+ "precision_at_3": 0.31749,
2948
+ "precision_at_5": 0.26518,
2949
+ "precision_at_10": 0.1757,
2950
+ "precision_at_20": 0.11329,
2951
+ "precision_at_50": 0.05302,
2952
+ "precision_at_100": 0.03504
2953
+ }
2954
+ },
2955
+ {
2956
+ "class": "proprietary",
2957
+ "organization": "Facebook",
2958
+ "model_name": "fb-text-to-text",
2959
+ "results": {
2960
+ "ndcg_at_1": 0.39128,
2961
+ "ndcg_at_3": 0.37037,
2962
+ "ndcg_at_5": 0.41629,
2963
+ "ndcg_at_10": 0.42365,
2964
+ "ndcg_at_20": 0.48047,
2965
+ "ndcg_at_50": 0.50028,
2966
+ "ndcg_at_100": 0.54065,
2967
+ "recall_at_1": 0.12546,
2968
+ "recall_at_3": 0.27352,
2969
+ "recall_at_5": 0.3577,
2970
+ "recall_at_10": 0.44456,
2971
+ "recall_at_20": 0.61389,
2972
+ "recall_at_50": 0.78284,
2973
+ "recall_at_100": 0.94875,
2974
+ "precision_at_1": 0.36158,
2975
+ "precision_at_3": 0.30161,
2976
+ "precision_at_5": 0.25081,
2977
+ "precision_at_10": 0.17885,
2978
+ "precision_at_20": 0.11355,
2979
+ "precision_at_50": 0.05582,
2980
+ "precision_at_100": 0.03625
2981
+ }
2982
+ },
2983
+ {
2984
+ "class": "custom",
2985
+ "organization": "Apple",
2986
+ "model_name": "apple-text-to-text",
2987
+ "results": {
2988
+ "ndcg_at_1": 0.41353,
2989
+ "ndcg_at_3": 0.38203,
2990
+ "ndcg_at_5": 0.40762,
2991
+ "ndcg_at_10": 0.42419,
2992
+ "ndcg_at_20": 0.54232,
2993
+ "ndcg_at_50": 0.50979,
2994
+ "ndcg_at_100": 0.53747,
2995
+ "recall_at_1": 0.13057,
2996
+ "recall_at_3": 0.27323,
2997
+ "recall_at_5": 0.35986,
2998
+ "recall_at_10": 0.44274,
2999
+ "recall_at_20": 0.60114,
3000
+ "recall_at_50": 0.78097,
3001
+ "recall_at_100": 0.93288,
3002
+ "precision_at_1": 0.36877,
3003
+ "precision_at_3": 0.30569,
3004
+ "precision_at_5": 0.26832,
3005
+ "precision_at_10": 0.17365,
3006
+ "precision_at_20": 0.11018,
3007
+ "precision_at_50": 0.05929,
3008
+ "precision_at_100": 0.03584
3009
+ }
3010
+ }
3011
+ ],
3012
+ "TextRewriting": [
3013
+ {
3014
+ "class": "sentence-transformers",
3015
+ "organization": "OpenAI",
3016
+ "model_name": "openai-text-to-text",
3017
+ "results": {
3018
+ "ndcg_at_1": 0.48474,
3019
+ "ndcg_at_3": 0.38171,
3020
+ "ndcg_at_5": 0.35397,
3021
+ "ndcg_at_10": 0.4255,
3022
+ "ndcg_at_20": 0.48694,
3023
+ "ndcg_at_50": 0.50416,
3024
+ "ndcg_at_100": 0.53381,
3025
+ "recall_at_1": 0.12291,
3026
+ "recall_at_3": 0.26987,
3027
+ "recall_at_5": 0.35696,
3028
+ "recall_at_10": 0.44125,
3029
+ "recall_at_20": 0.62412,
3030
+ "recall_at_50": 0.76108,
3031
+ "recall_at_100": 0.94192,
3032
+ "precision_at_1": 0.38738,
3033
+ "precision_at_3": 0.32594,
3034
+ "precision_at_5": 0.25406,
3035
+ "precision_at_10": 0.17381,
3036
+ "precision_at_20": 0.11778,
3037
+ "precision_at_50": 0.05507,
3038
+ "precision_at_100": 0.03618
3039
+ }
3040
+ },
3041
+ {
3042
+ "class": "proprietary",
3043
+ "organization": "Facebook",
3044
+ "model_name": "fb-text-to-text",
3045
+ "results": {
3046
+ "ndcg_at_1": 0.36275,
3047
+ "ndcg_at_3": 0.44189,
3048
+ "ndcg_at_5": 0.36213,
3049
+ "ndcg_at_10": 0.40005,
3050
+ "ndcg_at_20": 0.46742,
3051
+ "ndcg_at_50": 0.53436,
3052
+ "ndcg_at_100": 0.53368,
3053
+ "recall_at_1": 0.13381,
3054
+ "recall_at_3": 0.26192,
3055
+ "recall_at_5": 0.36402,
3056
+ "recall_at_10": 0.44473,
3057
+ "recall_at_20": 0.62827,
3058
+ "recall_at_50": 0.78524,
3059
+ "recall_at_100": 0.94364,
3060
+ "precision_at_1": 0.37658,
3061
+ "precision_at_3": 0.31831,
3062
+ "precision_at_5": 0.25089,
3063
+ "precision_at_10": 0.17347,
3064
+ "precision_at_20": 0.11446,
3065
+ "precision_at_50": 0.05213,
3066
+ "precision_at_100": 0.037
3067
+ }
3068
+ },
3069
+ {
3070
+ "class": "custom",
3071
+ "organization": "Apple",
3072
+ "model_name": "apple-text-to-text",
3073
+ "results": {
3074
+ "ndcg_at_1": 0.4582,
3075
+ "ndcg_at_3": 0.38506,
3076
+ "ndcg_at_5": 0.41886,
3077
+ "ndcg_at_10": 0.40925,
3078
+ "ndcg_at_20": 0.52724,
3079
+ "ndcg_at_50": 0.52855,
3080
+ "ndcg_at_100": 0.54976,
3081
+ "recall_at_1": 0.13668,
3082
+ "recall_at_3": 0.26086,
3083
+ "recall_at_5": 0.35085,
3084
+ "recall_at_10": 0.44351,
3085
+ "recall_at_20": 0.62795,
3086
+ "recall_at_50": 0.78031,
3087
+ "recall_at_100": 0.93716,
3088
+ "precision_at_1": 0.35769,
3089
+ "precision_at_3": 0.32803,
3090
+ "precision_at_5": 0.26491,
3091
+ "precision_at_10": 0.17186,
3092
+ "precision_at_20": 0.11379,
3093
+ "precision_at_50": 0.05455,
3094
+ "precision_at_100": 0.03843
3095
+ }
3096
+ }
3097
+ ]
3098
+ }
mock_data/tasks.json ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "slug": "law",
4
+ "name": "Law",
5
+ "datasets": [
6
+ "ALIA_casedocs",
7
+ "LegalTexts",
8
+ "CaseLaw"
9
+ ]
10
+ },
11
+ {
12
+ "slug": "finance",
13
+ "name": "Finance",
14
+ "datasets": [
15
+ "FinancialStatements",
16
+ "StockMarketData",
17
+ "BankingRecords"
18
+ ]
19
+ },
20
+ {
21
+ "slug": "healthcare",
22
+ "name": "Healthcare",
23
+ "datasets": [
24
+ "PatientRecords",
25
+ "MedicalImages",
26
+ "ClinicalTrials"
27
+ ]
28
+ },
29
+ {
30
+ "slug": "tech",
31
+ "name": "Technology",
32
+ "datasets": [
33
+ "TechBlogs",
34
+ "DeviceSpecifications",
35
+ "SoftwareDocumentation"
36
+ ]
37
+ },
38
+ {
39
+ "slug": "conversational",
40
+ "name": "Conversational",
41
+ "datasets": [
42
+ "ChatLogs",
43
+ "CustomerServiceInteractions",
44
+ "ForumPosts"
45
+ ]
46
+ },
47
+ {
48
+ "slug": "code",
49
+ "name": "Code",
50
+ "datasets": [
51
+ "GitHubRepositories",
52
+ "CodeSnippets",
53
+ "ProgrammingQuestions"
54
+ ]
55
+ },
56
+ {
57
+ "slug": "multilingual",
58
+ "name": "Multilingual",
59
+ "datasets": [
60
+ "ParallelCorpora",
61
+ "TranslationDatasets",
62
+ "MultilingualText"
63
+ ]
64
+ },
65
+ {
66
+ "slug": "long-context",
67
+ "name": "Long Context",
68
+ "datasets": [
69
+ "ResearchPapers",
70
+ "Books",
71
+ "LegalDocuments"
72
+ ]
73
+ },
74
+ {
75
+ "slug": "text-to-photo",
76
+ "name": "Text to Photo",
77
+ "datasets": [
78
+ "TextPhotoPairs",
79
+ "ImageCaptions",
80
+ "VisualDescriptions"
81
+ ]
82
+ },
83
+ {
84
+ "slug": "document-screenshot",
85
+ "name": "Document Screenshot",
86
+ "datasets": [
87
+ "ScannedDocuments",
88
+ "PDFScreenshots",
89
+ "DigitalForms"
90
+ ]
91
+ },
92
+ {
93
+ "slug": "figures-and-tables",
94
+ "name": "Figures and Tables",
95
+ "datasets": [
96
+ "ScientificFigures",
97
+ "DataTables",
98
+ "ChartsAndGraphs"
99
+ ]
100
+ },
101
+ {
102
+ "slug": "text-to-text",
103
+ "name": "Text to Text",
104
+ "datasets": [
105
+ "ParaphraseDatasets",
106
+ "SummarizationDatasets",
107
+ "TextRewriting"
108
+ ]
109
+ }
110
+ ]
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ gradio==4.20.0
2
+ uvicorn>=0.14.0
3
+ fastapi<0.113.0
utils/__init__.py ADDED
File without changes