2024-02-05 23:52:05 Publish script update

app.py

@@ -5,28 +5,46 @@ import pandas as pd
 import plotly
 from pandas.api.types import is_numeric_dtype
 
-from pipeline.config import LLMBoardConfig, QueriesConfig
+from pipeline.config import LLMBoardConfig
 
 README = """
-<br/><h2>About this project</h2>
-<p>
-This project analyses different models and providers from the perspective of an application developer.
-<br/>
-Models are asked to summarize a text in different languages and using different output formats with following prompt:
-</p>
-<pre>
-<code>{}</code>
-</pre>
+This projects compares different large language models and their providers for real time applications and mass data processing. 
+While other boards compare LLMs on different human intelligence tasks we focus on features related to business and engineering aspects such as response times, pricing and data streaming capabilities. 
+
+We chose a task of newspaper articles summarization as it represents a very standard type of task where model has to understand unstructured natural language text, process it and output text in a specified format.
+For this version we chose English, Polish and Japanese languages, with Japanese representing languages using logographic alphabets. This will verify the effectiveness of the LLM for different language groups.
+
+We used the following prompt:
+
+```
+Summarize me this text, the summary should be in {language}
+```
+
+Where language variable is original language of the text as we wanted to avoid the model translating the text to English during summarization.
+
+The model was asked to return the output in three formats: markdown, json and function call. Note that currently function calls are only supported by Open AI API.
+To do that we added following text to the query:
+
+```
+...
+```
+
+When  measuring execution time we used `time.time()` result saved to variable  before making the call to API and compared it to `time.time()` result  after receiving the results. We used litellm python library for all of  the models which naturally adds some overhead compared to pure curl  calls.
+
+In order to count tokens we split the output string by  whitespace \w regex character. For data which was impossible to obtain through the API, such as model sizes we only used official sources such  as developers' release blogs and their documentation.
+
+When it comes to pricing most providers charge per token count, while HuggingFace Endpoints allow the user to choose machine type and host the model repository on it. The user is then charged by the running time of the machine. In this project we attempted to use HF Endpoints as much as possible due to their popularity and transparency of how the model is executed.
 """
 
-summary_df: pd.DataFrame = pd.read_csv("data/2024-02-03 22:42:23.322179_summary.csv")
+summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
 time_of_day_comparison_df = pd.read_csv("data/2024-02-03 22:33:36.957992_time_of_day_comparison.csv")
-general_plots = pd.read_csv("data/2024-02-03 22:42:30.259126_general_plot.csv")
-model_costs_df = pd.read_csv("data/2024-01-26 16:32:54.481637_model_costs.csv")
+general_plots = pd.read_csv("data/2024-02-05 12:03:42.452218_general_plot.csv")
+model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv")
 
 
 with open("data/time_of_day_plot.json", "r") as f:
     time_of_day_plot = plotly.io.from_json(f.read())
+    time_of_day_plot.update_layout(autosize=True)
 
 
 searched_model_name = ""
@@ -113,15 +131,21 @@ collapse_output_method_button = gr.Button("Collapse output method")
 last_textbox = 0
 
 with gr.Blocks() as demo:
-    gr.HTML("<h1>LLM Board</h1>" + README.format(QueriesConfig().base_query_template))
+    gr.HTML("<h1>LLM Board</h1>")
 
     with gr.Row():
         filter_textbox.render()
         filter_button.render()
-    with gr.Tab("Basic information"):
+    
+    with gr.Tab("About this project"):
+        gr.Markdown(README)
+    with gr.Tab("General plots"):
         for index, row in general_plots.iterrows():
-            gr.Plot(plotly.io.from_json(row["plot_json"]), label=row["description"], scale=1)
-            gr.Markdown(str(row["comment"]))
+            plot = plotly.io.from_json(row["plot_json"])
+            plot.update_layout(autosize=True)
+            gr.Plot(plot, label=row["description"], scale=1)
+            if pd.notna(row["comment"]):
+                gr.Markdown(str(row["comment"]))
     with gr.Tab("Output characteristics"):
         with gr.Row():
             collapse_languages_button.render()
@@ -130,8 +154,14 @@ with gr.Blocks() as demo:
 
     with gr.Tab("Preformance by time of the day"):
         time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day")
-        time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot", scale=1, width="100%")
-        gr.Markdown("Hourly measurements were made in Central European Time.")
+        time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot", scale=1)
+        gr.Markdown("""
+These measurements were made by testing the models using the same dataset as in the other comparisons every hour for 24 hours.
+
+Hours and times of day in the table and in the plot are based on Central European Time.
+                    
+Measurements were made during a normal work week.
+""")
 
     with gr.Tab("Costs comparison"):
         models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")

data/2024-02-05 12:03:42.452218_general_plot.csv

@@ -0,0 +1,442 @@
+path,plot_object,description,comment,plot_json
+./html/plots/response_times.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.27102681]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.69058203]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.79273835]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison',
+              'offsetgroup': 'chat-bison',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.89024363]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.66830386]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k',
+              'offsetgroup': 'chat-bison-32k',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.12817777]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.73886213]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.03678889]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.29307391]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([14.32373439]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([15.5354884]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.6742315]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.44499114]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model endpoints response times'},
+               'xaxis': {'anchor': 'y',
+                         'categoryarray': [Mistral-7B-Instruct-v0.2,
+                                           Mixtral-8x7B-Instruct-v0.1,
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison, chat-bison (PaLM 2), chat-
+                                           bison-32k, chat-bison-32k (PaLM 2 32K),
+                                           gemini-pro, gpt-3.5-turbo, gpt-4,
+                                           gpt-4-turbo, llama-2-70b-chat,
+                                           zephyr-7b-beta],
+                         'categoryorder': 'array',
+                         'domain': [0.0, 1.0],
+                         'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
+})",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[4.690582027777991],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.7927383464436197],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.668303858522159],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.738862129823485],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.036788886637026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.293073908173212],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[14.323734385105828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.535488395149827],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.6742314998615178],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.4449911407752936],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
+./html/plots/execution_costs.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.041]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.5018]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.2624]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.062568]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.072612]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05180257]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05200691]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.01634787]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.01052]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.01309]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.01304]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Costs of execution of 800 test queries per model'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
+})",Costs of execution of 800 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.041],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[0.5018],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[1.2624],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.062568],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.07261200000000001],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.05180256798532274],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.052006909343931415],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.016347871700922646],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.01052],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.013090000000000001],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.01304],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 800 test queries per model""},""barmode"":""relative""}}"
+./html/plots/model_sizes.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([70.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([56.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.1]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model sizes in billions of parameters'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
+})",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[70.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"

data/2024-02-05 12:03:45.281624_model_costs.csv

@@ -0,0 +1,12 @@
+model,provider,cost_per_token,provider_pricing
+gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
+gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
+gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
+llama-2-70b-chat,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+zephyr-7b-beta,Hugging Face Inference Endpoint,$0.15 / 1M tokens,$1.3 / hour
+Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$0.14 / 1M tokens,$1.3 / hour
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.068 / 1M tokens,$0.6 / hour
+gemini-pro,Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)"
+chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
+chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"

data/2024-02-05 23:33:22.947120_summary.csv

@@ -0,0 +1,55 @@
+model,language,template_name,execution_time,characters_count,words_count,chunk_sizes_mean,chunk_sizes_min,chunk_sizes_max,chunk_generation_times_mean,chunk_generation_times_min,chunk_generation_times_max
+Mistral-7B-Instruct-v0.2,english,json,4.278648148264204,342.8,55.15,4.1311870534561415,1.0,15.0,0.040544769239987996,0.00024437904357910156,60.7178213596344
+Mistral-7B-Instruct-v0.2,english,markdown,3.7188014924526214,366.85,57.1375,4.239202657807309,1.0,16.0,0.03371991898375372,0.020945072174072266,0.25538039207458496
+Mistral-7B-Instruct-v0.2,japanese,json,2.413569121701377,52.07857142857143,3.414285714285714,1.2991803278688525,1.0,11.0,0.039267738759305254,0.00047779083251953125,3.370748996734619
+Mistral-7B-Instruct-v0.2,japanese,markdown,2.323612022399902,42.25,1.5833333333333333,1.1069868995633187,1.0,6.0,0.03958142719935121,0.0004932880401611328,0.2205650806427002
+Mixtral-8x7B-Instruct-v0.1,english,json,3.643989532061343,404.10849056603774,62.575471698113205,4.137546333747872,1.0,15.0,0.014243678199270911,0.00020456314086914062,3.4127349853515625
+Mixtral-8x7B-Instruct-v0.1,english,markdown,5.995457515120506,533.9270833333334,85.15625,4.360071452875127,1.0,15.0,0.016480832612607788,0.00021076202392578125,1.7103569507598877
+Mixtral-8x7B-Instruct-v0.1,japanese,json,7.938035423738243,565.6496350364963,64.76642335766424,2.248940739450926,1.0,109.0,0.023964353726961752,0.00020885467529296875,76.45081758499146
+Mixtral-8x7B-Instruct-v0.1,japanese,markdown,9.979603266716003,431.71666666666664,5.983333333333333,1.0810483702683527,1.0,12.0,0.017558807731893653,0.0002148151397705078,2.994176149368286
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,json,1.9458302193674548,366.25,61.58965517241379,4.169774654522613,1.0,13.0,0.012296600931852907,0.00024318695068359375,0.17363858222961426
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,markdown,1.6830505162477494,282.55,47.1125,3.9572829131652663,1.0,14.0,0.012217252319600401,0.0003170967102050781,0.15227961540222168
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,json,1.4167885013989039,72.81428571428572,8.8,2.0274463007159906,1.0,13.0,0.014790566363975749,0.00031280517578125,0.3310587406158447
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,markdown,1.336316986878713,89.78333333333333,12.716666666666667,2.432054176072235,1.0,13.0,0.013910464400900675,0.0005300045013427734,0.176621675491333
+chat-bison,english,json,3.446989095211029,550.05,86.0,89.4390243902439,1.0,138.0,0.4258628240445765,0.0006544589996337891,2.2015340328216553
+chat-bison,japanese,json,4.333498167991638,246.1,12.0,33.48299319727891,3.0,124.0,0.47770044917152044,0.0005426406860351562,3.5425846576690674
+chat-bison (PaLM 2),english,json,2.5406186148524283,381.7267857142857,60.88125,78.31727422604872,1.0,142.0,0.3495202942369709,0.0003688335418701172,7.334931373596191
+chat-bison (PaLM 2),english,markdown,2.9635989278554917,490.4875,78.5375,84.20386266094421,1.0,138.0,0.3641034907025841,0.0004639625549316406,4.600815534591675
+chat-bison (PaLM 2),japanese,json,3.0616454005241396,232.68,10.48,34.21764705882353,3.0,124.0,0.326771635167739,0.0005857944488525391,2.7724690437316895
+chat-bison (PaLM 2),japanese,markdown,4.669546687602997,431.725,13.85,31.74448529411765,2.0,124.0,0.27883222348549785,0.0005550384521484375,7.114030122756958
+chat-bison-32k,english,json,3.74513863325119,538.4,84.25,88.26229508196721,1.0,134.0,0.47832588094179745,0.0006556510925292969,3.0114917755126953
+chat-bison-32k,japanese,json,4.5112169027328495,241.45,11.8,35.77037037037037,6.0,124.0,0.5467217816246881,0.0005869865417480469,4.387756824493408
+chat-bison-32k (PaLM 2 32K),english,json,7.142214598400252,335.47232142857143,53.799107142857146,77.18344289235826,1.0,133.0,1.4512595853652734,0.00041985511779785156,101.5493803024292
+chat-bison-32k (PaLM 2 32K),english,markdown,3.44630788564682,429.95,69.35,82.6826923076923,1.0,133.0,0.5048719226167753,0.0005412101745605469,18.035786390304565
+chat-bison-32k (PaLM 2 32K),japanese,json,5.072022783756256,272.85,10.35,34.320754716981135,4.0,124.0,0.5336985453119818,0.0005052089691162109,52.47814655303955
+chat-bison-32k (PaLM 2 32K),japanese,markdown,6.1971998631954195,446.45,21.4,32.118705035971225,5.0,124.0,0.3856057263106751,0.0005400180816650391,7.165225505828857
+gemini-pro,english,json,2.7415290858475547,388.5206321334504,62.01755926251097,204.58853444290338,1.0,988.0,1.4202116817663073,0.0004703998565673828,13.73231840133667
+gemini-pro,english,markdown,3.0089986324310303,487.1458333333333,80.67708333333333,183.39607843137256,12.0,472.0,1.1130499073103362,0.0006177425384521484,4.875006198883057
+gemini-pro,japanese,json,4.5300196591070145,276.6694915254237,6.983050847457627,91.1927374301676,4.0,333.0,1.4752027082709627,0.0006062984466552734,8.291066646575928
+gemini-pro,japanese,markdown,6.166934008781727,422.7307692307692,9.096153846153847,90.83471074380165,9.0,266.0,1.3123543331445742,0.0006639957427978516,13.754565954208374
+gpt-3.5-turbo,english,call,2.908525016600803,456.85911016949154,72.26059322033899,,,,,,
+gpt-3.5-turbo,english,json,2.566863525494622,398.56960556844547,65.32366589327147,4.806746320443226,1.0,16.0,0.015585779219764062,0.0005395412445068359,1.4500916004180908
+gpt-3.5-turbo,english,markdown,2.3388922008184285,506.90384615384613,81.99038461538461,5.050584403142365,1.0,16.0,0.01715806382989308,0.0005617141723632812,1.127073049545288
+gpt-3.5-turbo,japanese,call,7.255881716940138,382.95,6.351111111111111,,,,,,
+gpt-3.5-turbo,japanese,json,5.226112197780009,237.1069182389937,3.559748427672956,1.1291481969569905,1.0,7.0,0.02129129201220198,0.0005452632904052734,13.75982117652893
+gpt-3.5-turbo,japanese,markdown,7.29901300883684,358.72131147540983,6.065573770491803,1.1894977168949772,1.0,15.0,0.021750167313542772,0.0005612373352050781,13.71114993095398
+gpt-4,english,call,9.15708539162022,374.16815144766144,59.661469933184854,,,,,,
+gpt-4,english,json,7.963626848197565,383.3707317073171,61.69512195121951,4.7834568389659005,1.0,16.0,0.08687007305138472,0.000537872314453125,3.531284809112549
+gpt-4,english,markdown,10.02310108244419,428.35,67.825,4.990243192078054,1.0,16.0,0.10377185442257747,0.0005447864532470703,2.087376117706299
+gpt-4,japanese,call,24.259297592319243,214.9076023391813,1.1695906432748537,,,,,,
+gpt-4,japanese,json,22.703774025460252,211.63025210084032,4.168067226890757,1.1685226429101707,1.0,7.0,0.11835955503744466,0.0005507469177246094,3.5608017444610596
+gpt-4,japanese,markdown,31.99519391655922,275.1,2.925,1.1327980234712787,1.0,9.0,0.12606545905407157,0.0005619525909423828,2.9794347286224365
+gpt-4-turbo,english,call,7.319288367695279,372.86,58.532222222222224,,,,,,
+gpt-4-turbo,english,json,6.457074524425879,407.0243902439024,64.18414634146342,4.735998183701559,1.0,17.0,0.06474667943090798,0.0006806850433349609,5.431884765625
+gpt-4-turbo,english,markdown,8.325546002388,462.1125,71.925,4.984360253471754,1.0,16.0,0.07950969425127868,0.0006999969482421875,2.0926766395568848
+gpt-4-turbo,japanese,call,32.67405146055443,372.7860465116279,24.886046511627907,,,,,,
+gpt-4-turbo,japanese,json,19.359180968999862,250.44166666666666,6.183333333333334,1.184261339007763,1.0,8.0,0.08582140138847597,0.0006830692291259766,13.252992153167725
+gpt-4-turbo,japanese,markdown,20.977174520492554,283.55,4.45,1.1460038395473375,1.0,10.0,0.07991695286011809,0.0007085800170898438,2.3845231533050537
+llama-2-70b-chat,english,json,2.917410132025375,355.7463556851312,54.861030126336246,4.02267032967033,1.0,15.0,0.019103390617685005,0.0001990795135498047,3.8762006759643555
+llama-2-70b-chat,english,markdown,4.99397261045417,827.5714285714286,125.95918367346938,4.177931176591799,1.0,16.0,0.018161741699259625,0.0002124309539794922,0.7439091205596924
+llama-2-70b-chat,japanese,json,5.265992468681889,369.3768115942029,42.56521739130435,2.3362207250561435,1.0,15.0,0.023359512199899433,0.00020837783813476562,0.9314718246459961
+llama-2-70b-chat,japanese,markdown,11.084085723449444,1379.6896551724137,158.01724137931035,2.2644745033674796,1.0,16.0,0.01566780037868879,0.0002117156982421875,76.44498753547668
+zephyr-7b-beta,english,json,3.94231437737087,367.57232704402514,60.257861635220124,4.262252042007002,1.0,14.0,0.03629181767914927,0.0003070831298828125,0.4896988868713379
+zephyr-7b-beta,english,markdown,4.959704548120499,358.9375,58.3625,4.390672782874618,1.0,15.0,0.05067269182351022,0.00040459632873535156,75.15552616119385
+zephyr-7b-beta,japanese,json,2.339889786640803,41.583333333333336,3.6333333333333333,1.1566991191469633,1.0,7.0,0.042134473762636,0.018090486526489258,0.4599947929382324
+zephyr-7b-beta,japanese,markdown,2.3176693956057233,38.1,2.3833333333333333,1.0343891402714933,1.0,4.0,0.041092377011053165,0.02583909034729004,0.4606599807739258

pipeline/models.py

@@ -76,7 +76,8 @@ MODELS = [
         "together_ai/togethercomputer/llama-2-70b-chat",
         None,
         "Together AI",
-        cost_per_million_tokens=0.2,
+        cost_per_million_tokens=0.9,
+        size_billion_parameters=70,
         # https://github.com/facebookresearch/llama/issues/148
         input_size=2048,
     ),
@@ -144,3 +145,7 @@ MODELS = [
         cost_per_million_output_tokens=0.5,
     ),
 ]
+
+MODELS = [model for model in MODELS 
+          if model.model_name=="together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1"
+          or model.model_name=="huggingface/HuggingFaceH4/zephyr-7b-beta"]