2024-02-03 22:50:03 Publish script update

app.py

@@ -19,9 +19,9 @@ Models are asked to summarize a text in different languages and using different
 </pre>
 """
 
-summary_df: pd.DataFrame = pd.read_csv("data/2024-01-25 13:30:17.207984_summary.csv")
-time_of_day_comparison_df = pd.read_csv("data/2024-01-25 13:30:20.959750_time_of_day_comparison.csv")
-general_plots = pd.read_csv("data/2024-01-26 16:44:14.180186_general_plot.csv")
+summary_df: pd.DataFrame = pd.read_csv("data/2024-02-03 22:42:23.322179_summary.csv")
+time_of_day_comparison_df = pd.read_csv("data/2024-02-03 22:33:36.957992_time_of_day_comparison.csv")
+general_plots = pd.read_csv("data/2024-02-03 22:42:30.259126_general_plot.csv")
 model_costs_df = pd.read_csv("data/2024-01-26 16:32:54.481637_model_costs.csv")
 
 
@@ -81,7 +81,7 @@ def dataframes():
         dataframe_style(
             time_of_day_comparison_df[time_of_day_comparison_df.model.str.lower().str.contains(searched_model_name)]
         ),
-        dataframe_style(model_costs_df[model_costs_df.Model.str.lower().str.contains(searched_model_name)]),
+        dataframe_style(model_costs_df[model_costs_df.model.str.lower().str.contains(searched_model_name)]),
     )
 
 
@@ -129,8 +129,9 @@ with gr.Blocks() as demo:
         summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Statistics")
 
     with gr.Tab("Preformance by time of the day"):
-        time_of_day_comparison_ui = gr.DataFrame(time_of_day_comparison_df, label="Time of day")
+        time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day")
         time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot")
+        gr.Markdown("Hourly measurements were made in Central European Time.")
 
     with gr.Tab("Costs comparison"):
         models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")

data/2024-01-30 13:08:12.199559_general_plot.csv

@@ -0,0 +1,398 @@
+path,plot_object,description,comment,plot_json
+./html/plots/response_times.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.17454003]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.14219763]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.29026454]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison',
+              'offsetgroup': 'chat-bison',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.89024363]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k',
+              'offsetgroup': 'chat-bison-32k',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.12817777]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.72138064]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([5.28237349]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([15.52184566]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([14.00852903]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.0658799]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.23879932]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model endpoints response times'},
+               'xaxis': {'anchor': 'y',
+                         'categoryarray': [Mistral-7B-Instruct-v0.2,
+                                           Mixtral-8x7B-Instruct-v0.1,
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison, chat-bison-32k, gemini-pro,
+                                           gpt-3.5-turbo, gpt-4, gpt-4-turbo,
+                                           llama-2-70b-chat, zephyr-7b-beta],
+                         'categoryorder': 'array',
+                         'domain': [0.0, 1.0],
+                         'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
+})",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.174540030956268],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[7.142197625471814],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.2902645373344421],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[4.72138064004937],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[5.282373493889353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[15.5218456586202],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[14.008529031276703],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[6.06587989590749],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.2387993240356447],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison-32k"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
+./html/plots/execution_costs.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.471009]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.49821]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([11.69823]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.095657]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.3244509]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.1279793]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05851047]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.01705275]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.10229025]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Costs of execution of 780 test queries per model'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
+})",Costs of execution of 780 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.47100899999999996],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[4.49821],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[11.698229999999999],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.095657],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.3244509],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.1279792968829473],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.05851047111882104],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.017052749236424764],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.10229025],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 780 test queries per model""},""barmode"":""relative""}}"
+./html/plots/model_sizes.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([56.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.1]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model sizes in billions of parameters'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
+})",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"

data/2024-01-30 18:39:54.448407_time_of_day_comparison.csv

@@ -0,0 +1,27 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon,3.023258109887441,172.18333333333334,22.95
+Mistral-7B-Instruct-v0.2,evening,,,
+Mistral-7B-Instruct-v0.2,morning,3.174540030956268,113.0,16.27
+Mixtral-8x7B-Instruct-v0.1,afternoon,7.426922065019608,450.4875,51.85
+Mixtral-8x7B-Instruct-v0.1,morning,7.142197625471814,494.9583333333333,57.74166666666667
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.538642168045044,169.46666666666667,26.083333333333332
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.2902645373344421,106.18,16.74
+chat-bison,morning,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon,3.866238150000572,381.7875,40.3875
+chat-bison-32k,morning,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon,4.844334444403648,371.25,38.7375
+gemini-pro,afternoon,3.6972034962972007,352.2,33.9375
+gemini-pro,morning,4.72138064004937,425.09166666666664,49.891666666666666
+gpt-3.5-turbo,afternoon,4.228875203738137,364.92857142857144,36.42857142857143
+gpt-3.5-turbo,evening,4.351089119911194,305.25,28.5
+gpt-3.5-turbo,morning,5.336839800789242,479.2785714285714,53.92142857142857
+gpt-4,afternoon,20.52127287968868,307.175,31.975
+gpt-4,morning,15.5218456586202,389.1875,44.125
+gpt-4-turbo,afternoon,23.51892908215523,396.40833333333336,41.858333333333334
+gpt-4-turbo,morning,14.008529031276703,433.075,49.975
+llama-2-70b-chat,afternoon,4.851686128755895,671.2317073170732,83.5609756097561
+llama-2-70b-chat,evening,5.552149415016174,967.5,141.25
+llama-2-70b-chat,morning,6.175082007679371,662.7416666666667,94.30833333333334
+zephyr-7b-beta,afternoon,3.0400254487991334,163.45,23.8
+zephyr-7b-beta,evening,,,
+zephyr-7b-beta,morning,3.2387993240356447,112.57,17.26

data/2024-01-30 18:40:08.467683_summary.csv

@@ -0,0 +1,75 @@
+model,language,template_name,execution_time,characters_count,words_count,chunk_sizes_mean,chunk_sizes_min,chunk_sizes_max,chunk_generation_times_mean,chunk_generation_times_min,chunk_generation_times_max
+Mistral-7B-Instruct-v0.2,english,call,,,,,,,,,
+Mistral-7B-Instruct-v0.2,english,json,7.318650853633881,222.2,35.95,4.157156220767072,1.0,14.0,0.10865272596910582,0.00024437904357910156,60.7178213596344
+Mistral-7B-Instruct-v0.2,english,markdown,3.4006769716739655,324.0,50.9,4.217377155873739,1.0,16.0,0.03379235289362949,0.020945072174072266,0.25538039207458496
+Mistral-7B-Instruct-v0.2,japanese,call,,,,,,,,,
+Mistral-7B-Instruct-v0.2,japanese,json,2.129413692156474,44.516666666666666,3.1666666666666665,1.338176352705411,1.0,11.0,0.038890062568182934,0.020466089248657227,0.23994874954223633
+Mistral-7B-Instruct-v0.2,japanese,markdown,2.217114305496216,38.9,1.475,1.1035460992907802,1.0,6.0,0.03959447390644263,0.0004932880401611328,0.2205650806427002
+Mixtral-8x7B-Instruct-v0.1,english,call,,,,,,,,,
+Mixtral-8x7B-Instruct-v0.1,english,json,3.2277008444070816,397.18333333333334,61.68333333333333,4.278456014362657,1.0,15.0,0.016790826658800218,0.0002071857452392578,1.011472463607788
+Mixtral-8x7B-Instruct-v0.1,english,markdown,7.234642684459686,565.725,90.475,4.366846777306059,1.0,15.0,0.018897794563767535,0.00021576881408691406,1.7103569507598877
+Mixtral-8x7B-Instruct-v0.1,japanese,call,,,,,,,,,
+Mixtral-8x7B-Instruct-v0.1,japanese,json,8.271932434617428,536.4833333333333,58.78333333333333,2.1412226435175947,1.0,15.0,0.026507280039896726,0.00021338462829589844,76.45081758499146
+Mixtral-8x7B-Instruct-v0.1,japanese,markdown,10.715969985723495,419.625,5.75,1.0792824074074074,1.0,10.0,0.018218461148164892,0.00021529197692871094,2.994176149368286
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,call,,,,,,,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,json,1.3998499751091003,203.15,34.6,4.120689655172414,1.0,13.0,0.01180358711652775,0.0003421306610107422,0.13397955894470215
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,markdown,1.5995279610157014,248.2,41.625,3.9303246239113223,1.0,14.0,0.012339331286348055,0.0003273487091064453,0.15227961540222168
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,call,,,,,,,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,json,1.2796377182006835,59.766666666666666,7.2,1.9966592427616927,1.0,13.0,0.01475177509482029,0.00041413307189941406,0.18952107429504395
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,markdown,1.3147150695323944,80.225,11.25,2.3595588235294116,1.0,13.0,0.013936110629754908,0.0005300045013427734,0.176621675491333
+chat-bison,english,call,,,,,,,,,
+chat-bison,english,json,3.446989095211029,550.05,86.0,89.4390243902439,1.0,138.0,0.4258628240445765,0.0006544589996337891,2.2015340328216553
+chat-bison,japanese,call,,,,,,,,,
+chat-bison,japanese,json,4.333498167991638,246.1,12.0,33.48299319727891,3.0,124.0,0.47770044917152044,0.0005426406860351562,3.5425846576690674
+chat-bison (PaLM 2),english,call,,,,,,,,,
+chat-bison (PaLM 2),english,json,2.9914507985115053,380.9,60.65,81.91397849462365,1.0,135.0,0.46110378285889986,0.0008225440979003906,2.025282144546509
+chat-bison (PaLM 2),english,markdown,3.1855568289756775,480.45,76.6,86.56756756756756,12.0,129.0,0.4221067342672262,0.00087738037109375,2.063969373703003
+chat-bison (PaLM 2),japanese,call,,,,,,,,,
+chat-bison (PaLM 2),japanese,json,3.69934686422348,230.85,10.5,34.714285714285715,6.0,124.0,0.4294193310845167,0.0006532669067382812,2.7724690437316895
+chat-bison (PaLM 2),japanese,markdown,5.588598108291626,434.95,13.8,32.09963099630996,10.0,124.0,0.3500617219073306,0.0005550384521484375,7.114030122756958
+chat-bison-32k,english,call,,,,,,,,,
+chat-bison-32k,english,json,3.74513863325119,538.4,84.25,88.26229508196721,1.0,134.0,0.47832588094179745,0.0006556510925292969,3.0114917755126953
+chat-bison-32k,japanese,call,,,,,,,,,
+chat-bison-32k,japanese,json,4.5112169027328495,241.45,11.8,35.77037037037037,6.0,124.0,0.5467217816246881,0.0005869865417480469,4.387756824493408
+chat-bison-32k (PaLM 2 32K),english,call,,,,,,,,,
+chat-bison-32k (PaLM 2 32K),english,json,2.968719494342804,335.75,53.85,77.183908045977,1.0,133.0,0.4880794711496638,0.0005464553833007812,1.8793001174926758
+chat-bison-32k (PaLM 2 32K),english,markdown,3.229217529296875,429.95,69.35,82.6826923076923,1.0,133.0,0.4584116041660309,0.0005412101745605469,2.0599677562713623
+chat-bison-32k (PaLM 2 32K),japanese,call,,,,,,,,,
+chat-bison-32k (PaLM 2 32K),japanese,json,6.968563377857208,272.85,10.35,34.320754716981135,4.0,124.0,0.7706441234492656,0.0005052089691162109,52.47814655303955
+chat-bison-32k (PaLM 2 32K),japanese,markdown,6.2108373761177065,446.45,21.4,32.118705035971225,5.0,124.0,0.38614349571063367,0.0007460117340087891,7.091885089874268
+gemini-pro,english,call,,,,,,,,,
+gemini-pro,english,json,3.3069549907337534,478.03333333333336,75.23333333333333,178.14906832298138,1.0,988.0,1.107387858147947,0.0006723403930664062,8.669384241104126
+gemini-pro,english,markdown,3.3248310733485864,547.225,90.025,159.77372262773721,12.0,472.0,0.879381437371247,0.0007510185241699219,2.820554733276367
+gemini-pro,japanese,call,,,,,,,,,
+gemini-pro,japanese,json,4.618957793458979,220.95,4.666666666666667,77.52631578947368,4.0,237.0,1.2511649340914006,0.0006062984466552734,5.698127031326294
+gemini-pro,japanese,markdown,6.4115668955971215,383.975,7.675,83.02162162162162,14.0,266.0,1.1660070625511376,0.0006639957427978516,13.754565954208374
+gpt-3.5-turbo,english,call,3.124613248166584,602.9285714285714,97.85714285714286,,,,,,
+gpt-3.5-turbo,english,json,2.4552148609626583,520.0853658536586,83.79268292682927,4.997304898054839,1.0,16.0,0.017422531230516167,0.0005590915679931641,0.4925534725189209
+gpt-3.5-turbo,english,markdown,2.666209004142068,564.75,91.06818181818181,5.092008196721311,1.0,16.0,0.017853882908821107,0.0005643367767333984,1.127073049545288
+gpt-3.5-turbo,japanese,call,8.609365148308836,372.23809523809524,4.392857142857143,,,,,,
+gpt-3.5-turbo,japanese,json,5.703425769564472,236.84146341463415,3.341463414634146,1.1251375934186896,1.0,7.0,0.022491555757866794,0.0005545616149902344,13.75982117652893
+gpt-3.5-turbo,japanese,markdown,7.63344073877102,357.4047619047619,6.880952380952381,1.2151704039504574,1.0,15.0,0.02293831178999754,0.0005612373352050781,13.71114993095398
+gpt-4,english,call,10.829727350137173,441.225,69.05,,,,,,
+gpt-4,english,json,9.5267409324646,470.65,74.625,4.912839248434238,1.0,16.0,0.08728140598796852,0.0005440711975097656,1.9534873962402344
+gpt-4,english,markdown,10.24732003211975,427.8,67.45,5.041838538597525,1.0,15.0,0.10614711590633719,0.0005447864532470703,2.087376117706299
+gpt-4,japanese,call,26.076972967539078,223.9,1.15,,,,,,
+gpt-4,japanese,json,23.192965501394028,208.9,4.05,1.1566998892580287,1.0,7.0,0.11818944638881705,0.0005507469177246094,2.4770302772521973
+gpt-4,japanese,markdown,36.14162145853042,282.65,3.15,1.1283433133732534,1.0,9.0,0.13859879746884404,0.0005800724029541016,2.9794347286224365
+gpt-4-turbo,english,call,8.286203730106354,462.8,72.1,,,,,,
+gpt-4-turbo,english,json,7.81642444729805,544.875,84.45,4.914317925591883,1.0,17.0,0.06131222159376005,0.0007143020629882812,0.7760622501373291
+gpt-4-turbo,english,markdown,8.539259791374207,472.6,73.15,5.014323607427055,1.0,15.0,0.0789825123248113,0.0007131099700927734,2.0926766395568848
+gpt-4-turbo,japanese,call,47.0068474650383,402.2,24.35,,,,,,
+gpt-4-turbo,japanese,json,20.065124291181565,270.525,6.075,1.1806873977086743,1.0,8.0,0.08205758946486919,0.0006937980651855469,13.252992153167725
+gpt-4-turbo,japanese,markdown,22.259230959415437,277.35,3.95,1.1460743801652893,1.0,8.0,0.0860337075123117,0.0007085800170898438,2.3845231533050537
+llama-2-70b-chat,english,call,,,,,,,,,
+llama-2-70b-chat,english,json,3.174426339921497,405.09375,62.359375,4.130975143403441,1.0,15.0,0.016708370984322405,0.00021314620971679688,0.5100884437561035
+llama-2-70b-chat,english,markdown,4.972272306680679,951.7857142857143,147.47619047619048,4.128369307032944,1.0,16.0,0.01363343600582685,0.0002124309539794922,0.5661592483520508
+llama-2-70b-chat,japanese,call,,,,,,,,,
+llama-2-70b-chat,japanese,json,4.583555459976196,356.2741935483871,42.41935483870968,2.4138345536006995,1.0,15.0,0.017530749553071386,0.00020837783813476562,0.35228896141052246
+llama-2-70b-chat,japanese,markdown,11.858248114585876,1293.3333333333333,154.38095238095238,2.3937951701040014,1.0,16.0,0.017392761688814075,0.0002117156982421875,76.44498753547668
+zephyr-7b-beta,english,call,,,,,,,,,
+zephyr-7b-beta,english,json,2.845248854160309,238.1,39.2,4.274685816876122,1.0,14.0,0.03622233246986485,0.0003070831298828125,0.4896988868713379
+zephyr-7b-beta,english,markdown,5.826004445552826,318.825,51.825,4.382474226804124,1.0,15.0,0.06875277114488006,0.00040459632873535156,75.15552616119385
+zephyr-7b-beta,japanese,call,,,,,,,,,
+zephyr-7b-beta,japanese,json,2.1363336205482484,35.666666666666664,3.466666666666667,1.1674849972722312,1.0,7.0,0.042095991411432825,0.018090486526489258,0.4599947929382324
+zephyr-7b-beta,japanese,markdown,2.2039071798324583,35.225,2.225,1.0345080763582966,1.0,4.0,0.041054743327058184,0.02583909034729004,0.4606599807739258

data/2024-02-01 08:08:06.822774_time_of_day_comparison.csv

@@ -0,0 +1,34 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon,3.023258109887441,172.18333333333334,22.95
+Mistral-7B-Instruct-v0.2,evening,3.2679351409276327,216.63333333333333,31.5
+Mistral-7B-Instruct-v0.2,morning,3.174540030956268,113.0,16.27
+Mixtral-8x7B-Instruct-v0.1,afternoon,7.426922065019608,450.4875,51.85
+Mixtral-8x7B-Instruct-v0.1,evening,4.979682341865871,457.8,53.708333333333336
+Mixtral-8x7B-Instruct-v0.1,morning,7.142197625471814,494.9583333333333,57.74166666666667
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.538642168045044,169.46666666666667,26.083333333333332
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.7001954774061838,222.51666666666668,35.575
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.2902645373344421,106.18,16.74
+chat-bison,morning,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon,3.866238150000572,381.7875,40.3875
+chat-bison (PaLM 2),evening,2.8466253956158956,358.525,39.25833333333333
+chat-bison-32k,morning,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon,4.844334444403648,371.25,38.7375
+chat-bison-32k (PaLM 2 32K),evening,3.981116022666295,348.93333333333334,36.525
+gemini-pro,afternoon,3.6972034962972007,352.2,33.9375
+gemini-pro,evening,3.810917937329837,335.68333333333334,36.21666666666667
+gemini-pro,morning,4.72138064004937,425.09166666666664,49.891666666666666
+gpt-3.5-turbo,afternoon,4.228875203738137,364.92857142857144,36.42857142857143
+gpt-3.5-turbo,evening,3.9572247514184915,375.75,37.660377358490564
+gpt-3.5-turbo,morning,5.336839800789242,479.2785714285714,53.92142857142857
+gpt-4,afternoon,20.52127287968868,307.175,31.975
+gpt-4,evening,16.028902428833085,306.075,31.925
+gpt-4,morning,15.5218456586202,389.1875,44.125
+gpt-4-turbo,afternoon,23.51892908215523,396.40833333333336,41.858333333333334
+gpt-4-turbo,evening,15.113789739608764,455.105,39.97
+gpt-4-turbo,morning,14.008529031276703,433.075,49.975
+llama-2-70b-chat,afternoon,4.851686128755895,671.2317073170732,83.5609756097561
+llama-2-70b-chat,evening,4.701648034746685,582.578125,73.9921875
+llama-2-70b-chat,morning,6.175082007679371,662.7416666666667,94.30833333333334
+zephyr-7b-beta,afternoon,3.0400254487991334,163.45,23.8
+zephyr-7b-beta,evening,3.3303639868895214,218.425,33.65
+zephyr-7b-beta,morning,3.2387993240356447,112.57,17.26

data/2024-02-02 10:02:32.863904_time_of_day_comparison.csv

@@ -0,0 +1,58 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon,3.2297915895779927,224.50833333333333,32.375
+Mistral-7B-Instruct-v0.2,evening,3.335640662057059,237.52857142857144,35.33571428571429
+Mistral-7B-Instruct-v0.2,midnight,,,
+Mistral-7B-Instruct-v0.2,morning,3.267787692944209,154.65,23.283333333333335
+Mistral-7B-Instruct-v0.2,noon,3.1783012986183166,210.325,30.975
+Mixtral-8x7B-Instruct-v0.1,afternoon,6.686205209493637,451.71,60.95
+Mixtral-8x7B-Instruct-v0.1,evening,4.822802252936781,414.6791666666667,55.68333333333333
+Mixtral-8x7B-Instruct-v0.1,midnight,5.600324046611786,399.5,61.7
+Mixtral-8x7B-Instruct-v0.1,morning,4.805075550901479,350.13125,47.046875
+Mixtral-8x7B-Instruct-v0.1,noon,6.976712214946747,492.05,66.15
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.6058823029200235,213.325,33.90833333333333
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.722607149396624,243.87857142857143,39.43571428571428
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,2.0674875736236573,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.6125914414723714,224.34444444444443,37.12222222222222
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,noon,1.6206423223018647,227.325,36.3
+chat-bison,morning,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon,3.2259692871570587,389.43,51.34
+chat-bison (PaLM 2),evening,2.756077417731285,370.17083333333335,50.0625
+chat-bison (PaLM 2),midnight,2.501569759845734,381.95,60.9
+chat-bison (PaLM 2),morning,2.4648332941532134,381.865,60.92
+chat-bison (PaLM 2),noon,2.706343245506287,307.6,35.7
+chat-bison-32k,morning,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon,4.288204146623611,362.5,46.555
+chat-bison-32k (PaLM 2 32K),evening,8.945300988356273,342.34166666666664,45.1875
+chat-bison-32k (PaLM 2 32K),midnight,7.170754861831665,335.75,53.85
+chat-bison-32k (PaLM 2 32K),morning,5.129021260738373,335.75,53.85
+chat-bison-32k (PaLM 2 32K),noon,4.41904046535492,304.3,32.1
+gemini-pro,afternoon,3.268519560079924,360.825,45.92
+gemini-pro,evening,3.265133913461264,355.7375,48.05416666666667
+gemini-pro,midnight,2.8754740890703703,364.45,57.0
+gemini-pro,morning,3.592642272646363,396.7375,56.6
+gemini-pro,noon,3.1745591791052568,309.125,33.675
+gpt-3.5-turbo,afternoon,4.410703988353233,390.6659192825112,42.2914798206278
+gpt-3.5-turbo,evening,4.809755622192356,392.99825174825173,43.07867132867133
+gpt-3.5-turbo,midnight,3.440265198548635,380.8333333333333,43.05
+gpt-3.5-turbo,morning,4.75027750830197,426.46590909090907,50.22386363636364
+gpt-3.5-turbo,noon,4.429612889289856,323.81,27.68
+gpt-4,afternoon,18.126467501112433,315.16136363636366,36.856818181818184
+gpt-4,evening,15.545672736714817,313.95,37.3
+gpt-4,midnight,12.410712122917175,330.6166666666667,41.416666666666664
+gpt-4,morning,14.07501753171285,335.77647058823527,42.84117647058824
+gpt-4,noon,19.315037021636964,273.52,25.12
+gpt-4-turbo,afternoon,22.23356882767244,366.7227272727273,45.41590909090909
+gpt-4-turbo,evening,16.361396548577716,400.25892857142856,45.276785714285715
+gpt-4-turbo,midnight,9.732182610034943,387.1,51.4
+gpt-4-turbo,morning,13.437257664694506,361.0573529411765,47.733823529411765
+gpt-4-turbo,noon,16.244312160015106,346.79,37.4
+llama-2-70b-chat,afternoon,4.869195998604618,553.0742574257425,75.0990099009901
+llama-2-70b-chat,evening,3.9572451173042764,470.4717741935484,64.35887096774194
+llama-2-70b-chat,midnight,2.7049551407496133,312.85,48.55
+llama-2-70b-chat,morning,4.015529702738376,456.196875,67.375
+llama-2-70b-chat,noon,5.845867207175807,337.275,44.625
+zephyr-7b-beta,afternoon,3.5596186633871385,273.65833333333336,43.233333333333334
+zephyr-7b-beta,evening,3.4374013015202114,242.40714285714284,37.892857142857146
+zephyr-7b-beta,midnight,,,
+zephyr-7b-beta,morning,3.3750870446364085,158.19166666666666,24.941666666666666
+zephyr-7b-beta,noon,3.3402509868144987,216.9,33.575

data/2024-02-02 10:13:06.642849_general_plot.csv

@@ -0,0 +1,428 @@
+path,plot_object,description,comment,plot_json
+./html/plots/response_times.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.27102681]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([5.47596174]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.66062556]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison',
+              'offsetgroup': 'chat-bison',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.89024363]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.79700603]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k',
+              'offsetgroup': 'chat-bison-32k',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.12817777]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.21499148]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.37706134]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.63898239]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([15.72262285]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([16.46241833]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.2692996]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.44499114]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model endpoints response times'},
+               'xaxis': {'anchor': 'y',
+                         'categoryarray': [Mistral-7B-Instruct-v0.2,
+                                           Mixtral-8x7B-Instruct-v0.1,
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison, chat-bison (PaLM 2), chat-
+                                           bison-32k, chat-bison-32k (PaLM 2 32K),
+                                           gemini-pro, gpt-3.5-turbo, gpt-4,
+                                           gpt-4-turbo, llama-2-70b-chat,
+                                           zephyr-7b-beta],
+                         'categoryorder': 'array',
+                         'domain': [0.0, 1.0],
+                         'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
+})",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[5.475961740149392],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.6606255621910095],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.7970060307638986],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.214991477898189],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.3770613408857777],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.63898239252044],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[15.72262285346881],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[16.462418334769165],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[4.269299596734112],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.4449911407752936],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
+./html/plots/execution_costs.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.23604]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.5988]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([5.8344]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.040992]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.143622]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.15650749]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.08633046]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.02132704]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05781]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05291]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.04954]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Costs of execution of 800 test queries per model'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
+})",Costs of execution of 800 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.23604],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[2.5987999999999998],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[5.8344000000000005],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.040992],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.14362200000000003],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.15650748757521313],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.08633045819070603],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.02132703948020935],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.05781],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.05291],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.04954],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 800 test queries per model""},""barmode"":""relative""}}"
+./html/plots/model_sizes.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([56.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.1]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model sizes in billions of parameters'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
+})",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"

data/2024-02-02 11:10:33.807657_general_plot.csv

@@ -0,0 +1,293 @@
+path,plot_object,description,comment,plot_json
+./html/plots/response_times.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.27102681]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([5.47596174]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.66062556]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison',
+              'offsetgroup': 'chat-bison',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.89024363]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.79700603]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k',
+              'offsetgroup': 'chat-bison-32k',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.12817777]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.21499148]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.37706134]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.63898239]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([15.72262285]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([16.46241833]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.2692996]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.44499114]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model endpoints response times'},
+               'xaxis': {'anchor': 'y',
+                         'categoryarray': [Mistral-7B-Instruct-v0.2,
+                                           Mixtral-8x7B-Instruct-v0.1,
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison, chat-bison (PaLM 2), chat-
+                                           bison-32k, chat-bison-32k (PaLM 2 32K),
+                                           gemini-pro, gpt-3.5-turbo, gpt-4,
+                                           gpt-4-turbo, llama-2-70b-chat,
+                                           zephyr-7b-beta],
+                         'categoryorder': 'array',
+                         'domain': [0.0, 1.0],
+                         'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
+})",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[5.475961740149392],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.6606255621910095],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.7970060307638986],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.214991477898189],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.3770613408857777],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.63898239252044],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[15.72262285346881],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[16.462418334769165],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[4.269299596734112],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.4449911407752936],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
+./html/plots/execution_costs.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': '',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': '',
+              'offsetgroup': '',
+              'orientation': 'v',
+              'showlegend': False,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo', 'gpt-4-turbo', 'gpt-4', 'llama-2-70b-chat',
+                          'Mixtral-8x7B-Instruct-v0.1', 'zephyr-7b-beta',
+                          'Mistral-7B-Instruct-v0.2', 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+                          'gemini-pro', 'chat-bison (PaLM 2)', 'chat-bison-32k (PaLM 2 32K)'],
+                         dtype=object),
+              'xaxis': 'x',
+              'y': array([0.22948   , 2.9468    , 6.342     , 0.02828   , 0.12708   , 0.04109804,
+                          0.10571385, 0.00933233, 0.05789   , 0.04731   , 0.04454   ]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Costs of execution of 800 test queries per model'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
+})",Costs of execution of 800 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":"""",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":"""",""offsetgroup"":"""",""orientation"":""v"",""showlegend"":false,""textposition"":""auto"",""x"":[""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4"",""llama-2-70b-chat"",""Mixtral-8x7B-Instruct-v0.1"",""zephyr-7b-beta"",""Mistral-7B-Instruct-v0.2"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""gemini-pro"",""chat-bison (PaLM 2)"",""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.22948,2.9467999999999996,6.3420000000000005,0.02828,0.12708,0.0410980390045378,0.1057138456636005,0.009332333167394003,0.05789,0.04731,0.044539999999999996],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 800 test queries per model""},""barmode"":""relative""}}"
+./html/plots/model_sizes.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([56.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.1]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model sizes in billions of parameters'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
+})",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"

data/2024-02-03 21:01:06.672013_time_of_day_comparison.csv

@@ -0,0 +1,58 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon,3.2297915895779927,224.50833333333333,32.375
+Mistral-7B-Instruct-v0.2,evening,3.335640662057059,237.52857142857144,35.33571428571429
+Mistral-7B-Instruct-v0.2,midnight,,,
+Mistral-7B-Instruct-v0.2,morning,3.267787692944209,154.65,23.283333333333335
+Mistral-7B-Instruct-v0.2,noon,3.1783012986183166,210.325,30.975
+Mixtral-8x7B-Instruct-v0.1,afternoon,5.647441667449138,361.92105263157896,51.305263157894736
+Mixtral-8x7B-Instruct-v0.1,evening,4.120577350668951,346.51,48.5125
+Mixtral-8x7B-Instruct-v0.1,midnight,5.600324046611786,399.5,61.7
+Mixtral-8x7B-Instruct-v0.1,morning,4.245225775510745,318.15714285714284,45.17857142857143
+Mixtral-8x7B-Instruct-v0.1,noon,5.2255242466926575,350.46,50.36
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.7891986429691316,292.65833333333336,48.225
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.8029413998126984,282.245,46.325
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,2.0674875736236573,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.7550641377766927,283.4533333333333,47.29
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,noon,1.8389003086090088,314.46,52.08
+chat-bison,morning,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon,2.8929106787631387,385.43684210526317,55.810526315789474
+chat-bison (PaLM 2),evening,2.7066626918315886,374.8975,54.42
+chat-bison (PaLM 2),midnight,2.501569759845734,381.95,60.9
+chat-bison (PaLM 2),morning,2.4689753359014337,381.8840909090909,60.902272727272724
+chat-bison (PaLM 2),noon,2.571754927635193,352.17,50.81
+chat-bison-32k,morning,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon,4.44931383258418,349.0105263157895,49.86052631578947
+chat-bison-32k (PaLM 2 32K),evening,7.76503027677536,339.705,48.6525
+chat-bison-32k (PaLM 2 32K),midnight,7.170754861831665,335.75,53.85
+chat-bison-32k (PaLM 2 32K),morning,7.92237914909016,335.75,53.85
+chat-bison-32k (PaLM 2 32K),noon,6.040619640350342,323.17,45.15
+gemini-pro,afternoon,2.8898058522299785,366.08684210526314,52.444736842105264
+gemini-pro,evening,3.060831026541881,364.98,52.845
+gemini-pro,midnight,2.8754740890703703,364.45,57.0
+gemini-pro,morning,3.172877737902161,386.9125,58.09464285714286
+gemini-pro,noon,2.7814492097834953,348.55,49.4
+gpt-3.5-turbo,afternoon,4.1847387138678105,398.0991253644315,45.470845481049565
+gpt-3.5-turbo,evening,4.304991676779234,399.01502145922746,44.74034334763949
+gpt-3.5-turbo,midnight,3.5683146858215333,404.84,43.88
+gpt-3.5-turbo,morning,4.437713119534958,423.19741379310346,49.02155172413793
+gpt-3.5-turbo,noon,4.098882734775543,346.4125,34.7375
+gpt-4,afternoon,16.17388726863186,320.5529411764706,39.75588235294118
+gpt-4,evening,14.089318896690486,316.045652173913,38.46413043478261
+gpt-4,midnight,12.091028991371694,316.73,37.44
+gpt-4,morning,12.958622653407113,330.034375,41.71145833333333
+gpt-4,noon,17.375231413543226,295.78125,31.23125
+gpt-4-turbo,afternoon,18.663996428251266,364.4514705882353,47.06470588235294
+gpt-4-turbo,evening,15.48614443048187,399.3239130434783,46.46630434782609
+gpt-4-turbo,midnight,11.938190605640411,384.78,44.55
+gpt-4-turbo,morning,13.66163076336185,369.496875,48.915625
+gpt-4-turbo,noon,16.01451396048069,374.3375,45.11875
+llama-2-70b-chat,afternoon,3.9614109361273613,429.84293193717275,60.924083769633505
+llama-2-70b-chat,evening,3.514792161901998,407.02941176470586,57.77205882352941
+llama-2-70b-chat,midnight,2.7049551407496133,312.85,48.55
+llama-2-70b-chat,morning,3.5679600898884543,378.98214285714283,56.725
+llama-2-70b-chat,noon,4.026921252409617,294.06,42.54
+zephyr-7b-beta,afternoon,3.5596186633871385,273.65833333333336,43.233333333333334
+zephyr-7b-beta,evening,3.4374013015202114,242.40714285714284,37.892857142857146
+zephyr-7b-beta,midnight,,,
+zephyr-7b-beta,morning,3.3750870446364085,158.19166666666666,24.941666666666666
+zephyr-7b-beta,noon,3.3402509868144987,216.9,33.575

data/2024-02-03 22:21:49.754184_time_of_day_comparison.csv

@@ -0,0 +1,267 @@
+model,hour,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,0.0,,,
+Mistral-7B-Instruct-v0.2,1.0,1.9149879813194275,37.8,2.95
+Mistral-7B-Instruct-v0.2,2.0,,,
+Mistral-7B-Instruct-v0.2,3.0,,,
+Mistral-7B-Instruct-v0.2,4.0,,,
+Mistral-7B-Instruct-v0.2,5.0,,,
+Mistral-7B-Instruct-v0.2,6.0,,,
+Mistral-7B-Instruct-v0.2,7.0,,,
+Mistral-7B-Instruct-v0.2,8.0,,,
+Mistral-7B-Instruct-v0.2,9.0,,,
+Mistral-7B-Instruct-v0.2,10.0,,,
+Mistral-7B-Instruct-v0.2,11.0,3.734026002883911,362.9,58.35
+Mistral-7B-Instruct-v0.2,12.0,3.1783012986183166,210.325,30.975
+Mistral-7B-Instruct-v0.2,13.0,,,
+Mistral-7B-Instruct-v0.2,14.0,3.2796461462974547,233.8,33.525
+Mistral-7B-Instruct-v0.2,15.0,3.749682915210724,362.9,58.35
+Mistral-7B-Instruct-v0.2,16.0,3.023258109887441,172.18333333333334,22.95
+Mistral-7B-Instruct-v0.2,17.0,,,
+Mistral-7B-Instruct-v0.2,18.0,3.741873788833618,362.9,58.35
+Mistral-7B-Instruct-v0.2,19.0,,,
+Mistral-7B-Instruct-v0.2,20.0,3.225523215532303,210.325,30.975
+Mistral-7B-Instruct-v0.2,21.0,3.2891411036252975,219.7875,31.7625
+Mistral-7B-Instruct-v0.2,22.0,,,
+Mistral-7B-Instruct-v0.2,23.0,,,
+Mixtral-8x7B-Instruct-v0.1,0.0,5.600324046611786,399.5,61.7
+Mixtral-8x7B-Instruct-v0.1,1.0,3.643988568608354,389.36,55.27
+Mixtral-8x7B-Instruct-v0.1,2.0,3.1196579193246774,275.875,42.4
+Mixtral-8x7B-Instruct-v0.1,3.0,3.017132020765735,316.8,49.025
+Mixtral-8x7B-Instruct-v0.1,4.0,3.0949330102829706,200.375,30.9
+Mixtral-8x7B-Instruct-v0.1,5.0,2.8691824844905307,279.65,42.95
+Mixtral-8x7B-Instruct-v0.1,6.0,2.9245848854382834,251.4,38.7
+Mixtral-8x7B-Instruct-v0.1,7.0,3.039147071838379,244.925,37.8
+Mixtral-8x7B-Instruct-v0.1,8.0,3.6833377838134767,236.73333333333332,36.68333333333333
+Mixtral-8x7B-Instruct-v0.1,9.0,2.35319835799081,286.25,44.75
+Mixtral-8x7B-Instruct-v0.1,10.0,4.525930786132813,311.3,48.55
+Mixtral-8x7B-Instruct-v0.1,11.0,4.267878273700146,308.85,47.7
+Mixtral-8x7B-Instruct-v0.1,12.0,5.2255242466926575,350.46,50.36
+Mixtral-8x7B-Instruct-v0.1,13.0,4.550832962644273,371.5375,57.975
+Mixtral-8x7B-Instruct-v0.1,14.0,6.3550496492812885,390.675,55.725
+Mixtral-8x7B-Instruct-v0.1,15.0,5.536225740114848,298.05,46.2
+Mixtral-8x7B-Instruct-v0.1,16.0,6.244929383286332,410.9916666666667,51.69166666666667
+Mixtral-8x7B-Instruct-v0.1,17.0,4.749357461929321,255.2,39.15
+Mixtral-8x7B-Instruct-v0.1,18.0,3.441127454080889,312.875,48.525
+Mixtral-8x7B-Instruct-v0.1,19.0,3.921023598888464,279.5625,43.225
+Mixtral-8x7B-Instruct-v0.1,20.0,3.5051895811937857,380.725,53.875
+Mixtral-8x7B-Instruct-v0.1,21.0,4.853644103915603,408.34166666666664,49.56666666666667
+Mixtral-8x7B-Instruct-v0.1,22.0,3.1714347918828327,284.9,43.825
+Mixtral-8x7B-Instruct-v0.1,23.0,4.876900292932987,321.725,49.875
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,0.0,2.0674875736236573,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,1.0,1.2007620811462403,48.35,5.8
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,2.0,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,3.0,2.0712238788604735,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,4.0,1.9543781518936156,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,5.0,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,6.0,2.110265576839447,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,7.0,2.0211721837520598,372.25,62.425
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,8.0,1.99203217625618,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,9.0,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,10.0,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,11.0,1.9041210174560548,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,12.0,1.8389003086090088,314.46,52.08
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,13.0,1.9219599564870198,353.6666666666667,59.266666666666666
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,14.0,1.7107620418071747,227.325,36.3
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,15.0,1.8755218625068664,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,16.0,1.538642168045044,169.46666666666667,26.083333333333332
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,17.0,1.9580047667026519,371.875,62.425
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,18.0,1.857077181339264,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,19.0,1.938528937101364,371.625,62.3
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,20.0,1.628979456424713,227.325,36.3
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,21.0,1.7358034878969193,220.1125,35.2125
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,22.0,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,23.0,2.094106078147888,372.05,62.6
+chat-bison,1.0,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),0.0,2.501569759845734,381.95,60.9
+chat-bison (PaLM 2),1.0,2.476984361807505,381.95,60.9
+chat-bison (PaLM 2),2.0,2.438890391588211,381.95,60.9
+chat-bison (PaLM 2),3.0,2.440149003267288,381.95,60.9
+chat-bison (PaLM 2),4.0,2.4795879781246186,381.95,60.9
+chat-bison (PaLM 2),5.0,2.461125838756561,381.8,61.1
+chat-bison (PaLM 2),6.0,2.469940322637558,381.95,60.9
+chat-bison (PaLM 2),7.0,2.4428869783878326,381.95,60.9
+chat-bison (PaLM 2),8.0,2.4800265431404114,381.8833333333333,60.88333333333333
+chat-bison (PaLM 2),9.0,2.465018570423126,381.95,60.9
+chat-bison (PaLM 2),10.0,2.4786566257476808,381.75,60.85
+chat-bison (PaLM 2),11.0,2.499571430683136,381.65,60.88333333333333
+chat-bison (PaLM 2),12.0,2.571754927635193,352.17,50.81
+chat-bison (PaLM 2),13.0,2.522790166735649,409.325,65.3875
+chat-bison (PaLM 2),14.0,2.8620632350444795,373.375,52.9875
+chat-bison (PaLM 2),15.0,2.650241893529892,377.8,60.425
+chat-bison (PaLM 2),16.0,3.4333535691102344,381.84166666666664,47.225
+chat-bison (PaLM 2),17.0,2.5084280292193095,381.95,60.9
+chat-bison (PaLM 2),18.0,2.615047258138657,381.45,60.85
+chat-bison (PaLM 2),19.0,2.821215844154358,382.225,61.025
+chat-bison (PaLM 2),20.0,2.591941976547241,344.725,48.275
+chat-bison (PaLM 2),21.0,2.7827586233615875,383.275,47.65833333333333
+chat-bison (PaLM 2),22.0,2.813016355037689,381.95,60.9
+chat-bison (PaLM 2),23.0,2.463971793651581,381.85,60.875
+chat-bison-32k,1.0,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),0.0,7.170754861831665,335.75,53.85
+chat-bison-32k (PaLM 2 32K),1.0,9.889513119061787,335.75,53.85
+chat-bison-32k (PaLM 2 32K),2.0,4.82983283996582,335.75,53.85
+chat-bison-32k (PaLM 2 32K),3.0,6.070689761638642,335.75,53.85
+chat-bison-32k (PaLM 2 32K),4.0,8.697703075408935,335.75,53.85
+chat-bison-32k (PaLM 2 32K),5.0,2.9196789026260377,335.75,53.85
+chat-bison-32k (PaLM 2 32K),6.0,10.046520245075225,335.75,53.85
+chat-bison-32k (PaLM 2 32K),7.0,8.814902746677399,335.75,53.85
+chat-bison-32k (PaLM 2 32K),8.0,8.755691254138947,335.75,53.85
+chat-bison-32k (PaLM 2 32K),9.0,2.943085324764252,335.75,53.85
+chat-bison-32k (PaLM 2 32K),10.0,13.628323638439179,335.75,53.85
+chat-bison-32k (PaLM 2 32K),11.0,7.315447652339936,335.75,53.85
+chat-bison-32k (PaLM 2 32K),12.0,6.040619640350342,323.17,45.15
+chat-bison-32k (PaLM 2 32K),13.0,3.8712039202451707,359.3,57.725
+chat-bison-32k (PaLM 2 32K),14.0,4.875012156367302,339.6875,46.1375
+chat-bison-32k (PaLM 2 32K),15.0,2.951279658079147,335.75,53.85
+chat-bison-32k (PaLM 2 32K),16.0,4.577468846241633,359.4166666666667,43.775
+chat-bison-32k (PaLM 2 32K),17.0,5.394908706347148,335.75,53.85
+chat-bison-32k (PaLM 2 32K),18.0,7.554238790273667,335.75,53.85
+chat-bison-32k (PaLM 2 32K),19.0,10.186308375000953,335.75,53.85
+chat-bison-32k (PaLM 2 32K),20.0,5.36962095797062,320.025,42.975
+chat-bison-32k (PaLM 2 32K),21.0,5.4035370687643685,359.4166666666667,43.775
+chat-bison-32k (PaLM 2 32K),22.0,12.272967928647995,335.75,53.85
+chat-bison-32k (PaLM 2 32K),23.0,10.500626176595688,335.75,53.85
+gemini-pro,0.0,2.8754740890703703,364.45,57.0
+gemini-pro,1.0,3.731358753455864,375.3,55.09
+gemini-pro,2.0,2.664601039886475,358.825,57.425
+gemini-pro,3.0,2.8943973687978892,387.0,61.725
+gemini-pro,4.0,2.8736069202423096,371.375,60.025
+gemini-pro,5.0,2.656540846824646,357.25,56.7
+gemini-pro,6.0,2.8120841443538667,390.4,62.45
+gemini-pro,7.0,2.70631699073009,378.2,60.35
+gemini-pro,8.0,2.7259290743682345,377.1166666666667,60.38333333333333
+gemini-pro,9.0,2.6013306021690368,396.15,63.35
+gemini-pro,10.0,2.4627303073280737,381.7,59.75
+gemini-pro,11.0,2.4739445872226002,360.6,57.95
+gemini-pro,12.0,2.7814492097834953,348.55,49.4
+gemini-pro,13.0,2.559203814856614,392.975,63.875
+gemini-pro,14.0,3.1300881316787317,343.3625,47.4625
+gemini-pro,15.0,2.4760417878627776,361.275,58.075
+gemini-pro,16.0,3.3130789541565213,353.7,41.59166666666667
+gemini-pro,17.0,2.492802628016068,388.51666666666665,61.8
+gemini-pro,18.0,2.7202366315401516,362.925,57.825
+gemini-pro,19.0,2.6147810145269466,392.6875,62.0125
+gemini-pro,20.0,3.2052203896757843,341.85,46.6875
+gemini-pro,21.0,3.6886135495227315,358.10833333333335,44.858333333333334
+gemini-pro,22.0,2.6609258592128753,376.325,60.05
+gemini-pro,23.0,2.590940225124359,367.15,58.6
+gpt-3.5-turbo,0.0,3.5683146858215333,404.84,43.88
+gpt-3.5-turbo,1.0,4.272891883608661,444.9625,48.91875
+gpt-3.5-turbo,2.0,2.6514626717567444,390.22,54.64
+gpt-3.5-turbo,3.0,8.802967548370361,438.15833333333336,48.75
+gpt-3.5-turbo,4.0,3.9857387453317643,409.6,38.875
+gpt-3.5-turbo,5.0,3.3526177048683166,391.56666666666666,45.35
+gpt-3.5-turbo,6.0,3.9461031516393024,413.46666666666664,46.666666666666664
+gpt-3.5-turbo,7.0,3.563987676302592,396.7,48.483333333333334
+gpt-3.5-turbo,8.0,3.8539037982622784,401.06666666666666,46.88333333333333
+gpt-3.5-turbo,9.0,3.774983811378479,373.15,41.81666666666667
+gpt-3.5-turbo,10.0,,,
+gpt-3.5-turbo,11.0,2.8019149283568066,398.40833333333336,55.5
+gpt-3.5-turbo,12.0,4.098882734775543,346.4125,34.7375
+gpt-3.5-turbo,13.0,3.8091073359761918,414.22857142857146,50.82142857142857
+gpt-3.5-turbo,14.0,3.901234769821167,389.9,47.55833333333333
+gpt-3.5-turbo,15.0,4.8184381556510925,411.03,42.87
+gpt-3.5-turbo,16.0,3.899510098587383,376.54545454545456,43.67727272727273
+gpt-3.5-turbo,17.0,5.015711693763733,425.6,42.83
+gpt-3.5-turbo,18.0,3.721581067641576,378.26666666666665,44.46666666666667
+gpt-3.5-turbo,19.0,3.851715175310771,399.625,46.0
+gpt-3.5-turbo,20.0,3.153710008992089,380.1611111111111,47.05555555555556
+gpt-3.5-turbo,21.0,4.117136875174681,407.26153846153846,40.63076923076923
+gpt-3.5-turbo,22.0,3.208261313041051,396.3,46.266666666666666
+gpt-3.5-turbo,23.0,8.566175188620885,441.65833333333336,49.28333333333333
+gpt-4,0.0,12.091028991371694,316.73,37.44
+gpt-4,1.0,12.780311059551078,338.25416666666666,40.09166666666667
+gpt-4,2.0,11.616398258209228,341.39,48.28
+gpt-4,3.0,12.220598830779393,324.725,40.541666666666664
+gpt-4,4.0,13.938926541805268,303.1625,31.9875
+gpt-4,5.0,12.497149809201558,319.71666666666664,40.63333333333333
+gpt-4,6.0,13.388921443621317,330.0833333333333,41.46666666666667
+gpt-4,7.0,14.678142400582631,323.53333333333336,40.733333333333334
+gpt-4,8.0,14.978816239039103,317.0,39.43333333333333
+gpt-4,9.0,15.373240264256795,324.98333333333335,40.5
+gpt-4,10.0,,,
+gpt-4,11.0,11.441057385516768,344.78333333333336,50.025
+gpt-4,12.0,17.375231413543226,295.78125,31.23125
+gpt-4,13.0,15.355684958185469,337.90714285714284,44.85
+gpt-4,14.0,14.926335791746775,325.48333333333335,41.733333333333334
+gpt-4,15.0,17.480767397880555,301.15,34.69
+gpt-4,16.0,17.60613609231226,325.15,39.6
+gpt-4,17.0,14.354735706791732,299.63,35.66
+gpt-4,18.0,13.452181231975555,317.68333333333334,39.675
+gpt-4,19.0,14.330684673786163,317.45,39.791666666666664
+gpt-4,20.0,13.774037790298461,318.76666666666665,40.80555555555556
+gpt-4,21.0,15.565142148249858,308.5769230769231,33.707692307692305
+gpt-4,22.0,15.137496527036031,323.8666666666667,41.00833333333333
+gpt-4,23.0,10.696240643493267,317.28333333333336,40.175
+gpt-4-turbo,0.0,11.938190605640411,384.78,44.55
+gpt-4-turbo,1.0,14.554479810595513,382.94166666666666,48.0625
+gpt-4-turbo,2.0,13.72197850227356,380.31,49.62
+gpt-4-turbo,3.0,11.976015224059422,370.85833333333335,49.108333333333334
+gpt-4-turbo,4.0,17.072669982910156,366.9875,44.7875
+gpt-4-turbo,5.0,15.055192097028096,296.6666666666667,43.03333333333333
+gpt-4-turbo,6.0,15.722527865568797,378.6166666666667,51.416666666666664
+gpt-4-turbo,7.0,7.6716701745986935,302.23333333333335,40.96666666666667
+gpt-4-turbo,8.0,11.272664443651836,392.1166666666667,49.266666666666666
+gpt-4-turbo,9.0,20.45977386633555,332.1166666666667,43.5
+gpt-4-turbo,10.0,,,
+gpt-4-turbo,11.0,10.300394964218139,406.775,60.791666666666664
+gpt-4-turbo,12.0,16.01451396048069,374.3375,45.11875
+gpt-4-turbo,13.0,13.703989851474763,327.0571428571429,46.714285714285715
+gpt-4-turbo,14.0,11.651468181610108,347.85833333333335,46.458333333333336
+gpt-4-turbo,15.0,37.896924748420716,367.4,49.16
+gpt-4-turbo,16.0,18.2455517671325,395.83181818181816,48.56818181818182
+gpt-4-turbo,17.0,15.710689465999604,364.73,42.88
+gpt-4-turbo,18.0,16.597523101170857,379.60833333333335,47.983333333333334
+gpt-4-turbo,19.0,19.196569097042083,355.68333333333334,46.958333333333336
+gpt-4-turbo,20.0,14.754687264230515,401.8777777777778,49.75555555555555
+gpt-4-turbo,21.0,14.39423566047962,439.87692307692305,42.72692307692308
+gpt-4-turbo,22.0,17.963759408394495,437.6,50.358333333333334
+gpt-4-turbo,23.0,11.649714199701945,332.7083333333333,43.733333333333334
+llama-2-70b-chat,0.0,2.7049551407496133,312.85,48.55
+llama-2-70b-chat,1.0,3.5678966999053956,355.78,52.5
+llama-2-70b-chat,2.0,2.7264933708386545,341.675,52.75
+llama-2-70b-chat,3.0,3.13531751036644,355.5,54.75
+llama-2-70b-chat,4.0,2.6207807064056396,262.85,40.7
+llama-2-70b-chat,5.0,2.4901255933861983,331.85,51.2
+llama-2-70b-chat,6.0,3.237797245834813,298.0,46.025
+llama-2-70b-chat,7.0,2.8074560369764057,309.8,47.225
+llama-2-70b-chat,8.0,2.643855281498121,270.68333333333334,41.93333333333333
+llama-2-70b-chat,9.0,2.270249330997467,355.5,54.75
+llama-2-70b-chat,10.0,2.6760833263397217,165.55,25.2
+llama-2-70b-chat,11.0,3.1485369634628295,298.56666666666666,45.766666666666666
+llama-2-70b-chat,12.0,4.026921252409617,294.06,42.54
+llama-2-70b-chat,13.0,4.110724348770945,428.7375,65.5375
+llama-2-70b-chat,14.0,4.592028452555339,423.8875,60.8125
+llama-2-70b-chat,15.0,2.76978215304288,285.925,44.15
+llama-2-70b-chat,16.0,4.2957686424255375,572.3083333333333,74.625
+llama-2-70b-chat,17.0,2.720098500781589,254.86666666666667,39.28333333333333
+llama-2-70b-chat,18.0,2.9921934604644775,326.375,50.025
+llama-2-70b-chat,19.0,2.7523183100151294,283.95,43.8375
+llama-2-70b-chat,20.0,3.1451149598146095,345.975,49.65
+llama-2-70b-chat,21.0,4.558388976727501,556.1583333333333,71.875
+llama-2-70b-chat,22.0,3.0530899854806752,348.0,53.65
+llama-2-70b-chat,23.0,2.941225928068161,355.5,54.75
+zephyr-7b-beta,0.0,,,
+zephyr-7b-beta,1.0,1.9264068841934203,29.75,3.3
+zephyr-7b-beta,2.0,,,
+zephyr-7b-beta,3.0,,,
+zephyr-7b-beta,4.0,,,
+zephyr-7b-beta,5.0,,,
+zephyr-7b-beta,6.0,,,
+zephyr-7b-beta,7.0,,,
+zephyr-7b-beta,8.0,,,
+zephyr-7b-beta,9.0,,,
+zephyr-7b-beta,10.0,,,
+zephyr-7b-beta,11.0,4.056525647640228,386.3,63.35
+zephyr-7b-beta,12.0,3.3402509868144987,216.9,33.575
+zephyr-7b-beta,13.0,,,
+zephyr-7b-beta,14.0,4.056431210041046,399.0,64.9
+zephyr-7b-beta,15.0,4.06974081993103,386.3,63.35
+zephyr-7b-beta,16.0,3.304698485362379,214.1625,32.7875
+zephyr-7b-beta,17.0,,,
+zephyr-7b-beta,18.0,4.079625189304352,386.3,63.35
+zephyr-7b-beta,19.0,,,
+zephyr-7b-beta,20.0,3.3111458659172057,216.9,33.575
+zephyr-7b-beta,21.0,3.339973047375679,219.1875,33.6875
+zephyr-7b-beta,22.0,,,
+zephyr-7b-beta,23.0,,,

data/2024-02-03 22:30:15.856748_time_of_day_comparison.csv

@@ -0,0 +1,91 @@
+model,time,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon 12-14,3.228973722457886,222.0625,32.25
+Mistral-7B-Instruct-v0.2,early morning 6-8,,,
+Mistral-7B-Instruct-v0.2,evening 18-20,3.397640073299408,261.18333333333334,40.1
+Mistral-7B-Instruct-v0.2,late afternoon 15-17,3.2048643112182615,219.8625,31.8
+Mistral-7B-Instruct-v0.2,morning 9-11,3.734026002883911,362.9,58.35
+Mistral-7B-Instruct-v0.2,night 0-2,1.9149879813194275,37.8,2.95
+Mistral-7B-Instruct-v0.2,night 21-23,3.389284573495388,175.79375,25.68125
+Mistral-7B-Instruct-v0.2,night 3-5,,,
+Mixtral-8x7B-Instruct-v0.1,afternoon 12-14,5.362903979589355,369.3192307692308,54.353846153846156
+Mixtral-8x7B-Instruct-v0.1,early morning 6-8,3.2748278152374994,243.2642857142857,37.57857142857143
+Mixtral-8x7B-Instruct-v0.1,evening 18-20,3.6435119574237023,326.69,48.545
+Mixtral-8x7B-Instruct-v0.1,late afternoon 15-17,5.80184749175942,347.9681818181818,47.27272727272727
+Mixtral-8x7B-Instruct-v0.1,morning 9-11,3.9661054956285575,304.82,47.28
+Mixtral-8x7B-Instruct-v0.1,night 0-2,3.8265925436529495,362.25625,52.85625
+Mixtral-8x7B-Instruct-v0.1,night 21-23,5.62397656769588,395.15714285714284,49.02857142857143
+Mixtral-8x7B-Instruct-v0.1,night 3-5,3.010503646099206,262.8,40.56
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon 12-14,1.8381905496120452,308.795,51.08
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning 6-8,2.0273348593711855,372.13,62.53
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening 18-20,1.7984187936782836,313.99,51.96
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon 15-17,1.7547113946505954,285.17857142857144,46.9
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning 9-11,1.9041210174560548,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 0-2,1.6341248273849487,210.2,34.2
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 21-23,1.5875422928068372,192.78333333333333,31.261111111111113
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 3-5,2.0128010153770446,372.05,62.6
+chat-bison,night 0-2,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon 12-14,2.6460144804074215,376.28076923076924,55.965384615384615
+chat-bison (PaLM 2),early morning 6-8,2.4665334616388592,381.9214285714286,60.892857142857146
+chat-bison (PaLM 2),evening 18-20,2.688272579908371,367.07,55.89
+chat-bison (PaLM 2),late afternoon 15-17,3.0387172081253744,381.1363636363636,53.35454545454545
+chat-bison (PaLM 2),morning 9-11,2.488477897644043,381.73,60.88
+chat-bison (PaLM 2),night 0-2,2.468383938074112,381.95,60.9
+chat-bison (PaLM 2),night 21-23,2.7250528037548065,382.725,52.95
+chat-bison (PaLM 2),night 3-5,2.460119960308075,381.92,60.94
+chat-bison-32k,night 0-2,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon 12-14,5.014458654477046,339.3692307692308,49.323076923076925
+chat-bison-32k (PaLM 2 32K),early morning 6-8,9.141417106560299,335.75,53.85
+chat-bison-32k (PaLM 2 32K),evening 18-20,7.7332194912433625,329.46,49.5
+chat-bison-32k (PaLM 2 32K),late afternoon 15-17,4.5047362284226855,348.65909090909093,48.35454545454545
+chat-bison-32k (PaLM 2 32K),morning 9-11,7.7035503840446475,335.75,53.85
+chat-bison-32k (PaLM 2 32K),night 0-2,7.7498266498247785,335.75,53.85
+chat-bison-32k (PaLM 2 32K),night 21-23,7.796841062307358,349.95,47.805
+chat-bison-32k (PaLM 2 32K),night 3-5,6.491292915344238,335.75,53.85
+gemini-pro,afternoon 12-14,2.816922114008949,360.62307692307695,53.25769230769231
+gemini-pro,early morning 6-8,2.7453590130460435,381.2214285714286,60.964285714285715
+gemini-pro,evening 18-20,2.86901999375759,366.4,55.045
+gemini-pro,late afternoon 15-17,2.9268629640903114,364.57272727272726,50.1
+gemini-pro,morning 9-11,2.497767536007628,371.93,59.39
+gemini-pro,night 0-2,3.338477972348531,369.825,55.9125
+gemini-pro,night 21-23,3.7189874940246117,390.9142857142857,51.35
+gemini-pro,night 3-5,2.8375814715210272,374.8,60.04
+gpt-3.5-turbo,afternoon 12-14,3.9458200880459375,381.4428571428571,43.76190476190476
+gpt-3.5-turbo,early morning 6-8,3.787998208734724,403.74444444444447,47.34444444444444
+gpt-3.5-turbo,evening 18-20,3.5153889304115657,385.18095238095236,46.01428571428571
+gpt-3.5-turbo,late afternoon 15-17,4.384064777692159,396.4357142857143,43.28333333333333
+gpt-3.5-turbo,morning 9-11,3.126271222697364,389.9888888888889,50.93888888888889
+gpt-3.5-turbo,night 0-2,3.822115447632102,426.71923076923076,49.05
+gpt-3.5-turbo,night 21-23,5.110168156187617,422.73510971786834,45.358934169278996
+gpt-3.5-turbo,night 3-5,6.062970260473398,418.6192307692308,44.926923076923075
+gpt-4,afternoon 12-14,16.002364798386893,318.3095238095238,38.77142857142857
+gpt-4,early morning 6-8,14.348626694414351,323.5388888888889,40.544444444444444
+gpt-4,evening 18-20,13.841120740345547,318.0809523809524,40.19285714285714
+gpt-4,late afternoon 15-17,16.80607506932254,313.3595238095238,37.49285714285714
+gpt-4,morning 9-11,12.759107512468733,338.18333333333334,46.85
+gpt-4,night 0-2,12.3578163115329,334.075,41.35
+gpt-4,night 21-23,14.298642643005493,314.336,37.012
+gpt-4,night 3-5,12.813134506115546,316.93461538461537,37.93076923076923
+gpt-4-turbo,afternoon 12-14,13.997754749229976,351.01190476190476,46.03333333333333
+gpt-4-turbo,early morning 6-8,11.555620827939775,357.65555555555557,47.21666666666667
+gpt-4-turbo,evening 18-20,16.550320884159632,382.31666666666666,48.45
+gpt-4-turbo,late afternoon 15-17,22.320911452883767,381.65714285714284,47.35476190476191
+gpt-4-turbo,morning 9-11,13.686854598257277,381.8888888888889,55.02777777777778
+gpt-4-turbo,night 0-2,13.770663784850727,382.7613636363636,47.61818181818182
+gpt-4-turbo,night 21-23,14.592236209392548,413.61,44.8
+gpt-4-turbo,night 3-5,14.254795966698573,352.54615384615386,46.37692307692308
+llama-2-70b-chat,afternoon 12-14,4.234376892130426,375.44615384615383,55.238461538461536
+llama-2-70b-chat,early morning 6-8,2.8660141522424265,289.6642857142857,44.614285714285714
+llama-2-70b-chat,evening 18-20,2.9706250462084185,317.245,47.4
+llama-2-70b-chat,late afternoon 15-17,3.6833307104881365,433.6636363636364,59.445454545454545
+llama-2-70b-chat,morning 9-11,2.872361832027194,283.35,43.45
+llama-2-70b-chat,night 0-2,3.249819871626402,346.8875,52.06875
+llama-2-70b-chat,night 21-23,4.719581684340602,572.1689655172414,79.83103448275862
+llama-2-70b-chat,night 3-5,2.8264514451677147,313.71,48.42
+zephyr-7b-beta,afternoon 12-14,3.5789777278900146,277.6,44.016666666666666
+zephyr-7b-beta,early morning 6-8,,,
+zephyr-7b-beta,evening 18-20,3.5673056403795878,273.3666666666667,43.5
+zephyr-7b-beta,late afternoon 15-17,3.4592524923459447,248.59,38.9
+zephyr-7b-beta,morning 9-11,4.056525647640228,386.3,63.35
+zephyr-7b-beta,night 0-2,1.9264068841934203,29.75,3.3
+zephyr-7b-beta,night 21-23,3.45343524068594,176.23125,27.21875
+zephyr-7b-beta,night 3-5,,,

data/2024-02-03 22:31:14.132394_time_of_day_comparison.csv

@@ -0,0 +1,91 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,afternoon 12-14,3.228973722457886,222.0625,32.25
+Mistral-7B-Instruct-v0.2,early morning 6-8,,,
+Mistral-7B-Instruct-v0.2,evening 18-20,3.397640073299408,261.18333333333334,40.1
+Mistral-7B-Instruct-v0.2,late afternoon 15-17,3.2048643112182615,219.8625,31.8
+Mistral-7B-Instruct-v0.2,morning 9-11,3.734026002883911,362.9,58.35
+Mistral-7B-Instruct-v0.2,night 0-2,1.9149879813194275,37.8,2.95
+Mistral-7B-Instruct-v0.2,night 21-23,3.389284573495388,175.79375,25.68125
+Mistral-7B-Instruct-v0.2,night 3-5,,,
+Mixtral-8x7B-Instruct-v0.1,afternoon 12-14,5.362903979589355,369.3192307692308,54.353846153846156
+Mixtral-8x7B-Instruct-v0.1,early morning 6-8,3.2748278152374994,243.2642857142857,37.57857142857143
+Mixtral-8x7B-Instruct-v0.1,evening 18-20,3.6435119574237023,326.69,48.545
+Mixtral-8x7B-Instruct-v0.1,late afternoon 15-17,5.80184749175942,347.9681818181818,47.27272727272727
+Mixtral-8x7B-Instruct-v0.1,morning 9-11,3.9661054956285575,304.82,47.28
+Mixtral-8x7B-Instruct-v0.1,night 0-2,3.8265925436529495,362.25625,52.85625
+Mixtral-8x7B-Instruct-v0.1,night 21-23,5.62397656769588,395.15714285714284,49.02857142857143
+Mixtral-8x7B-Instruct-v0.1,night 3-5,3.010503646099206,262.8,40.56
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon 12-14,1.8381905496120452,308.795,51.08
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning 6-8,2.0273348593711855,372.13,62.53
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening 18-20,1.7984187936782836,313.99,51.96
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon 15-17,1.7547113946505954,285.17857142857144,46.9
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning 9-11,1.9041210174560548,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 0-2,1.6341248273849487,210.2,34.2
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 21-23,1.5875422928068372,192.78333333333333,31.261111111111113
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night 3-5,2.0128010153770446,372.05,62.6
+chat-bison,night 0-2,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),afternoon 12-14,2.6460144804074215,376.28076923076924,55.965384615384615
+chat-bison (PaLM 2),early morning 6-8,2.4665334616388592,381.9214285714286,60.892857142857146
+chat-bison (PaLM 2),evening 18-20,2.688272579908371,367.07,55.89
+chat-bison (PaLM 2),late afternoon 15-17,3.0387172081253744,381.1363636363636,53.35454545454545
+chat-bison (PaLM 2),morning 9-11,2.488477897644043,381.73,60.88
+chat-bison (PaLM 2),night 0-2,2.468383938074112,381.95,60.9
+chat-bison (PaLM 2),night 21-23,2.7250528037548065,382.725,52.95
+chat-bison (PaLM 2),night 3-5,2.460119960308075,381.92,60.94
+chat-bison-32k,night 0-2,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),afternoon 12-14,5.014458654477046,339.3692307692308,49.323076923076925
+chat-bison-32k (PaLM 2 32K),early morning 6-8,9.141417106560299,335.75,53.85
+chat-bison-32k (PaLM 2 32K),evening 18-20,7.7332194912433625,329.46,49.5
+chat-bison-32k (PaLM 2 32K),late afternoon 15-17,4.5047362284226855,348.65909090909093,48.35454545454545
+chat-bison-32k (PaLM 2 32K),morning 9-11,7.7035503840446475,335.75,53.85
+chat-bison-32k (PaLM 2 32K),night 0-2,7.7498266498247785,335.75,53.85
+chat-bison-32k (PaLM 2 32K),night 21-23,7.796841062307358,349.95,47.805
+chat-bison-32k (PaLM 2 32K),night 3-5,6.491292915344238,335.75,53.85
+gemini-pro,afternoon 12-14,2.816922114008949,360.62307692307695,53.25769230769231
+gemini-pro,early morning 6-8,2.7453590130460435,381.2214285714286,60.964285714285715
+gemini-pro,evening 18-20,2.86901999375759,366.4,55.045
+gemini-pro,late afternoon 15-17,2.9268629640903114,364.57272727272726,50.1
+gemini-pro,morning 9-11,2.497767536007628,371.93,59.39
+gemini-pro,night 0-2,3.338477972348531,369.825,55.9125
+gemini-pro,night 21-23,3.7189874940246117,390.9142857142857,51.35
+gemini-pro,night 3-5,2.8375814715210272,374.8,60.04
+gpt-3.5-turbo,afternoon 12-14,3.9458200880459375,381.4428571428571,43.76190476190476
+gpt-3.5-turbo,early morning 6-8,3.787998208734724,403.74444444444447,47.34444444444444
+gpt-3.5-turbo,evening 18-20,3.5153889304115657,385.18095238095236,46.01428571428571
+gpt-3.5-turbo,late afternoon 15-17,4.384064777692159,396.4357142857143,43.28333333333333
+gpt-3.5-turbo,morning 9-11,3.126271222697364,389.9888888888889,50.93888888888889
+gpt-3.5-turbo,night 0-2,3.822115447632102,426.71923076923076,49.05
+gpt-3.5-turbo,night 21-23,5.110168156187617,422.73510971786834,45.358934169278996
+gpt-3.5-turbo,night 3-5,6.062970260473398,418.6192307692308,44.926923076923075
+gpt-4,afternoon 12-14,16.002364798386893,318.3095238095238,38.77142857142857
+gpt-4,early morning 6-8,14.348626694414351,323.5388888888889,40.544444444444444
+gpt-4,evening 18-20,13.841120740345547,318.0809523809524,40.19285714285714
+gpt-4,late afternoon 15-17,16.80607506932254,313.3595238095238,37.49285714285714
+gpt-4,morning 9-11,12.759107512468733,338.18333333333334,46.85
+gpt-4,night 0-2,12.3578163115329,334.075,41.35
+gpt-4,night 21-23,14.298642643005493,314.336,37.012
+gpt-4,night 3-5,12.813134506115546,316.93461538461537,37.93076923076923
+gpt-4-turbo,afternoon 12-14,13.997754749229976,351.01190476190476,46.03333333333333
+gpt-4-turbo,early morning 6-8,11.555620827939775,357.65555555555557,47.21666666666667
+gpt-4-turbo,evening 18-20,16.550320884159632,382.31666666666666,48.45
+gpt-4-turbo,late afternoon 15-17,22.320911452883767,381.65714285714284,47.35476190476191
+gpt-4-turbo,morning 9-11,13.686854598257277,381.8888888888889,55.02777777777778
+gpt-4-turbo,night 0-2,13.770663784850727,382.7613636363636,47.61818181818182
+gpt-4-turbo,night 21-23,14.592236209392548,413.61,44.8
+gpt-4-turbo,night 3-5,14.254795966698573,352.54615384615386,46.37692307692308
+llama-2-70b-chat,afternoon 12-14,4.234376892130426,375.44615384615383,55.238461538461536
+llama-2-70b-chat,early morning 6-8,2.8660141522424265,289.6642857142857,44.614285714285714
+llama-2-70b-chat,evening 18-20,2.9706250462084185,317.245,47.4
+llama-2-70b-chat,late afternoon 15-17,3.6833307104881365,433.6636363636364,59.445454545454545
+llama-2-70b-chat,morning 9-11,2.872361832027194,283.35,43.45
+llama-2-70b-chat,night 0-2,3.249819871626402,346.8875,52.06875
+llama-2-70b-chat,night 21-23,4.719581684340602,572.1689655172414,79.83103448275862
+llama-2-70b-chat,night 3-5,2.8264514451677147,313.71,48.42
+zephyr-7b-beta,afternoon 12-14,3.5789777278900146,277.6,44.016666666666666
+zephyr-7b-beta,early morning 6-8,,,
+zephyr-7b-beta,evening 18-20,3.5673056403795878,273.3666666666667,43.5
+zephyr-7b-beta,late afternoon 15-17,3.4592524923459447,248.59,38.9
+zephyr-7b-beta,morning 9-11,4.056525647640228,386.3,63.35
+zephyr-7b-beta,night 0-2,1.9264068841934203,29.75,3.3
+zephyr-7b-beta,night 21-23,3.45343524068594,176.23125,27.21875
+zephyr-7b-beta,night 3-5,,,

data/2024-02-03 22:33:36.957992_time_of_day_comparison.csv

@@ -0,0 +1,91 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,0-2 midnight,1.9149879813194275,37.8,2.95
+Mistral-7B-Instruct-v0.2,12-14 afternoon,3.228973722457886,222.0625,32.25
+Mistral-7B-Instruct-v0.2,15-17 late afternoon,3.2048643112182615,219.8625,31.8
+Mistral-7B-Instruct-v0.2,18-20 evening,3.397640073299408,261.18333333333334,40.1
+Mistral-7B-Instruct-v0.2,21-23 late evening,3.389284573495388,175.79375,25.68125
+Mistral-7B-Instruct-v0.2,3-5 night,,,
+Mistral-7B-Instruct-v0.2,6-8 early morning,,,
+Mistral-7B-Instruct-v0.2,9-11 morning,3.734026002883911,362.9,58.35
+Mixtral-8x7B-Instruct-v0.1,0-2 midnight,3.8265925436529495,362.25625,52.85625
+Mixtral-8x7B-Instruct-v0.1,12-14 afternoon,5.362903979589355,369.3192307692308,54.353846153846156
+Mixtral-8x7B-Instruct-v0.1,15-17 late afternoon,5.80184749175942,347.9681818181818,47.27272727272727
+Mixtral-8x7B-Instruct-v0.1,18-20 evening,3.6435119574237023,326.69,48.545
+Mixtral-8x7B-Instruct-v0.1,21-23 late evening,5.62397656769588,395.15714285714284,49.02857142857143
+Mixtral-8x7B-Instruct-v0.1,3-5 night,3.010503646099206,262.8,40.56
+Mixtral-8x7B-Instruct-v0.1,6-8 early morning,3.2748278152374994,243.2642857142857,37.57857142857143
+Mixtral-8x7B-Instruct-v0.1,9-11 morning,3.9661054956285575,304.82,47.28
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,0-2 midnight,1.6341248273849487,210.2,34.2
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,12-14 afternoon,1.8381905496120452,308.795,51.08
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,15-17 late afternoon,1.7547113946505954,285.17857142857144,46.9
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,18-20 evening,1.7984187936782836,313.99,51.96
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,21-23 late evening,1.5875422928068372,192.78333333333333,31.261111111111113
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,3-5 night,2.0128010153770446,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,6-8 early morning,2.0273348593711855,372.13,62.53
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,9-11 morning,1.9041210174560548,372.05,62.6
+chat-bison,0-2 midnight,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),0-2 midnight,2.468383938074112,381.95,60.9
+chat-bison (PaLM 2),12-14 afternoon,2.6460144804074215,376.28076923076924,55.965384615384615
+chat-bison (PaLM 2),15-17 late afternoon,3.0387172081253744,381.1363636363636,53.35454545454545
+chat-bison (PaLM 2),18-20 evening,2.688272579908371,367.07,55.89
+chat-bison (PaLM 2),21-23 late evening,2.7250528037548065,382.725,52.95
+chat-bison (PaLM 2),3-5 night,2.460119960308075,381.92,60.94
+chat-bison (PaLM 2),6-8 early morning,2.4665334616388592,381.9214285714286,60.892857142857146
+chat-bison (PaLM 2),9-11 morning,2.488477897644043,381.73,60.88
+chat-bison-32k,0-2 midnight,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),0-2 midnight,7.7498266498247785,335.75,53.85
+chat-bison-32k (PaLM 2 32K),12-14 afternoon,5.014458654477046,339.3692307692308,49.323076923076925
+chat-bison-32k (PaLM 2 32K),15-17 late afternoon,4.5047362284226855,348.65909090909093,48.35454545454545
+chat-bison-32k (PaLM 2 32K),18-20 evening,7.7332194912433625,329.46,49.5
+chat-bison-32k (PaLM 2 32K),21-23 late evening,7.796841062307358,349.95,47.805
+chat-bison-32k (PaLM 2 32K),3-5 night,6.491292915344238,335.75,53.85
+chat-bison-32k (PaLM 2 32K),6-8 early morning,9.141417106560299,335.75,53.85
+chat-bison-32k (PaLM 2 32K),9-11 morning,7.7035503840446475,335.75,53.85
+gemini-pro,0-2 midnight,3.338477972348531,369.825,55.9125
+gemini-pro,12-14 afternoon,2.816922114008949,360.62307692307695,53.25769230769231
+gemini-pro,15-17 late afternoon,2.9268629640903114,364.57272727272726,50.1
+gemini-pro,18-20 evening,2.86901999375759,366.4,55.045
+gemini-pro,21-23 late evening,3.7189874940246117,390.9142857142857,51.35
+gemini-pro,3-5 night,2.8375814715210272,374.8,60.04
+gemini-pro,6-8 early morning,2.7453590130460435,381.2214285714286,60.964285714285715
+gemini-pro,9-11 morning,2.497767536007628,371.93,59.39
+gpt-3.5-turbo,0-2 midnight,3.822115447632102,426.71923076923076,49.05
+gpt-3.5-turbo,12-14 afternoon,3.9458200880459375,381.4428571428571,43.76190476190476
+gpt-3.5-turbo,15-17 late afternoon,4.384064777692159,396.4357142857143,43.28333333333333
+gpt-3.5-turbo,18-20 evening,3.5153889304115657,385.18095238095236,46.01428571428571
+gpt-3.5-turbo,21-23 late evening,5.110168156187617,422.73510971786834,45.358934169278996
+gpt-3.5-turbo,3-5 night,6.062970260473398,418.6192307692308,44.926923076923075
+gpt-3.5-turbo,6-8 early morning,3.787998208734724,403.74444444444447,47.34444444444444
+gpt-3.5-turbo,9-11 morning,3.126271222697364,389.9888888888889,50.93888888888889
+gpt-4,0-2 midnight,12.3578163115329,334.075,41.35
+gpt-4,12-14 afternoon,16.002364798386893,318.3095238095238,38.77142857142857
+gpt-4,15-17 late afternoon,16.80607506932254,313.3595238095238,37.49285714285714
+gpt-4,18-20 evening,13.841120740345547,318.0809523809524,40.19285714285714
+gpt-4,21-23 late evening,14.298642643005493,314.336,37.012
+gpt-4,3-5 night,12.813134506115546,316.93461538461537,37.93076923076923
+gpt-4,6-8 early morning,14.348626694414351,323.5388888888889,40.544444444444444
+gpt-4,9-11 morning,12.759107512468733,338.18333333333334,46.85
+gpt-4-turbo,0-2 midnight,13.770663784850727,382.7613636363636,47.61818181818182
+gpt-4-turbo,12-14 afternoon,13.997754749229976,351.01190476190476,46.03333333333333
+gpt-4-turbo,15-17 late afternoon,22.320911452883767,381.65714285714284,47.35476190476191
+gpt-4-turbo,18-20 evening,16.550320884159632,382.31666666666666,48.45
+gpt-4-turbo,21-23 late evening,14.592236209392548,413.61,44.8
+gpt-4-turbo,3-5 night,14.254795966698573,352.54615384615386,46.37692307692308
+gpt-4-turbo,6-8 early morning,11.555620827939775,357.65555555555557,47.21666666666667
+gpt-4-turbo,9-11 morning,13.686854598257277,381.8888888888889,55.02777777777778
+llama-2-70b-chat,0-2 midnight,3.249819871626402,346.8875,52.06875
+llama-2-70b-chat,12-14 afternoon,4.234376892130426,375.44615384615383,55.238461538461536
+llama-2-70b-chat,15-17 late afternoon,3.6833307104881365,433.6636363636364,59.445454545454545
+llama-2-70b-chat,18-20 evening,2.9706250462084185,317.245,47.4
+llama-2-70b-chat,21-23 late evening,4.719581684340602,572.1689655172414,79.83103448275862
+llama-2-70b-chat,3-5 night,2.8264514451677147,313.71,48.42
+llama-2-70b-chat,6-8 early morning,2.8660141522424265,289.6642857142857,44.614285714285714
+llama-2-70b-chat,9-11 morning,2.872361832027194,283.35,43.45
+zephyr-7b-beta,0-2 midnight,1.9264068841934203,29.75,3.3
+zephyr-7b-beta,12-14 afternoon,3.5789777278900146,277.6,44.016666666666666
+zephyr-7b-beta,15-17 late afternoon,3.4592524923459447,248.59,38.9
+zephyr-7b-beta,18-20 evening,3.5673056403795878,273.3666666666667,43.5
+zephyr-7b-beta,21-23 late evening,3.45343524068594,176.23125,27.21875
+zephyr-7b-beta,3-5 night,,,
+zephyr-7b-beta,6-8 early morning,,,
+zephyr-7b-beta,9-11 morning,4.056525647640228,386.3,63.35

data/2024-02-03 22:42:23.322179_summary.csv

@@ -0,0 +1,75 @@
+model,language,template_name,execution_time,characters_count,words_count,chunk_sizes_mean,chunk_sizes_min,chunk_sizes_max,chunk_generation_times_mean,chunk_generation_times_min,chunk_generation_times_max
+Mistral-7B-Instruct-v0.2,english,call,,,,,,,,,
+Mistral-7B-Instruct-v0.2,english,json,4.278648148264204,342.8,55.15,4.1311870534561415,1.0,15.0,0.040544769239987996,0.00024437904357910156,60.7178213596344
+Mistral-7B-Instruct-v0.2,english,markdown,3.7188014924526214,366.85,57.1375,4.239202657807309,1.0,16.0,0.03371991898375372,0.020945072174072266,0.25538039207458496
+Mistral-7B-Instruct-v0.2,japanese,call,,,,,,,,,
+Mistral-7B-Instruct-v0.2,japanese,json,2.413569121701377,52.07857142857143,3.414285714285714,1.2991803278688525,1.0,11.0,0.039267738759305254,0.00047779083251953125,3.370748996734619
+Mistral-7B-Instruct-v0.2,japanese,markdown,2.323612022399902,42.25,1.5833333333333333,1.1069868995633187,1.0,6.0,0.03958142719935121,0.0004932880401611328,0.2205650806427002
+Mixtral-8x7B-Instruct-v0.1,english,call,,,,,,,,,
+Mixtral-8x7B-Instruct-v0.1,english,json,3.643989532061343,295.4172413793103,45.744827586206895,4.137546333747872,1.0,15.0,0.014243678199270911,0.00020456314086914062,3.4127349853515625
+Mixtral-8x7B-Instruct-v0.1,english,markdown,5.995457515120506,512.57,81.75,4.360071452875127,1.0,15.0,0.016480832612607788,0.00021076202392578125,1.7103569507598877
+Mixtral-8x7B-Instruct-v0.1,japanese,call,,,,,,,,,
+Mixtral-8x7B-Instruct-v0.1,japanese,json,7.938035423738243,553.5285714285715,63.378571428571426,2.248940739450926,1.0,109.0,0.023964353726961752,0.00020885467529296875,76.45081758499146
+Mixtral-8x7B-Instruct-v0.1,japanese,markdown,9.979603266716003,431.71666666666664,5.983333333333333,1.0810483702683527,1.0,12.0,0.017558807731893653,0.0002148151397705078,2.994176149368286
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,call,,,,,,,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,json,1.9458302193674548,366.25,61.58965517241379,4.169774654522613,1.0,13.0,0.012296600931852907,0.00024318695068359375,0.17363858222961426
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,markdown,1.6830505162477494,282.55,47.1125,3.9572829131652663,1.0,14.0,0.012217252319600401,0.0003170967102050781,0.15227961540222168
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,call,,,,,,,,,
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,json,1.4167885013989039,72.81428571428572,8.8,2.0274463007159906,1.0,13.0,0.014790566363975749,0.00031280517578125,0.3310587406158447
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,markdown,1.336316986878713,89.78333333333333,12.716666666666667,2.432054176072235,1.0,13.0,0.013910464400900675,0.0005300045013427734,0.176621675491333
+chat-bison,english,call,,,,,,,,,
+chat-bison,english,json,3.446989095211029,550.05,86.0,89.4390243902439,1.0,138.0,0.4258628240445765,0.0006544589996337891,2.2015340328216553
+chat-bison,japanese,call,,,,,,,,,
+chat-bison,japanese,json,4.333498167991638,246.1,12.0,33.48299319727891,3.0,124.0,0.47770044917152044,0.0005426406860351562,3.5425846576690674
+chat-bison (PaLM 2),english,call,,,,,,,,,
+chat-bison (PaLM 2),english,json,2.5406186148524283,381.7267857142857,60.88125,78.31727422604872,1.0,142.0,0.3495202942369709,0.0003688335418701172,7.334931373596191
+chat-bison (PaLM 2),english,markdown,2.9635989278554917,490.4875,78.5375,84.20386266094421,1.0,138.0,0.3641034907025841,0.0004639625549316406,4.600815534591675
+chat-bison (PaLM 2),japanese,call,,,,,,,,,
+chat-bison (PaLM 2),japanese,json,3.0616454005241396,232.68,10.48,34.21764705882353,3.0,124.0,0.326771635167739,0.0005857944488525391,2.7724690437316895
+chat-bison (PaLM 2),japanese,markdown,4.669546687602997,431.725,13.85,31.74448529411765,2.0,124.0,0.27883222348549785,0.0005550384521484375,7.114030122756958
+chat-bison-32k,english,call,,,,,,,,,
+chat-bison-32k,english,json,3.74513863325119,538.4,84.25,88.26229508196721,1.0,134.0,0.47832588094179745,0.0006556510925292969,3.0114917755126953
+chat-bison-32k,japanese,call,,,,,,,,,
+chat-bison-32k,japanese,json,4.5112169027328495,241.45,11.8,35.77037037037037,6.0,124.0,0.5467217816246881,0.0005869865417480469,4.387756824493408
+chat-bison-32k (PaLM 2 32K),english,call,,,,,,,,,
+chat-bison-32k (PaLM 2 32K),english,json,7.142214598400252,335.47232142857143,53.799107142857146,77.18344289235826,1.0,133.0,1.4512595853652734,0.00041985511779785156,101.5493803024292
+chat-bison-32k (PaLM 2 32K),english,markdown,3.44630788564682,429.95,69.35,82.6826923076923,1.0,133.0,0.5048719226167753,0.0005412101745605469,18.035786390304565
+chat-bison-32k (PaLM 2 32K),japanese,call,,,,,,,,,
+chat-bison-32k (PaLM 2 32K),japanese,json,5.072022783756256,272.85,10.35,34.320754716981135,4.0,124.0,0.5336985453119818,0.0005052089691162109,52.47814655303955
+chat-bison-32k (PaLM 2 32K),japanese,markdown,6.1971998631954195,446.45,21.4,32.118705035971225,5.0,124.0,0.3856057263106751,0.0005400180816650391,7.165225505828857
+gemini-pro,english,call,,,,,,,,,
+gemini-pro,english,json,2.7415290858475547,381.48706896551727,60.894827586206894,204.58853444290338,1.0,988.0,1.4202116817663073,0.0004703998565673828,13.73231840133667
+gemini-pro,english,markdown,3.0089986324310303,467.66,77.45,183.39607843137256,12.0,472.0,1.1130499073103362,0.0006177425384521484,4.875006198883057
+gemini-pro,japanese,call,,,,,,,,,
+gemini-pro,japanese,json,4.5300196591070145,233.19285714285715,5.885714285714286,91.1927374301676,4.0,333.0,1.4752027082709627,0.0006062984466552734,8.291066646575928
+gemini-pro,japanese,markdown,6.166934008781727,366.3666666666667,7.883333333333334,90.83471074380165,9.0,266.0,1.3123543331445742,0.0006639957427978516,13.754565954208374
+gpt-3.5-turbo,english,call,2.908525016600803,456.85911016949154,72.26059322033899,,,,,,
+gpt-3.5-turbo,english,json,2.566863525494622,398.56960556844547,65.32366589327147,4.806746320443226,1.0,16.0,0.015585779219764062,0.0005395412445068359,1.4500916004180908
+gpt-3.5-turbo,english,markdown,2.3388922008184285,506.90384615384613,81.99038461538461,5.050584403142365,1.0,16.0,0.01715806382989308,0.0005617141723632812,1.127073049545288
+gpt-3.5-turbo,japanese,call,7.255881716940138,381.2555309734513,6.323008849557522,,,,,,
+gpt-3.5-turbo,japanese,json,5.226112197780009,232.71604938271605,3.493827160493827,1.1291481969569905,1.0,7.0,0.02129129201220198,0.0005452632904052734,13.75982117652893
+gpt-3.5-turbo,japanese,markdown,7.29901300883684,352.93548387096774,5.967741935483871,1.1894977168949772,1.0,15.0,0.021750167313542772,0.0005612373352050781,13.71114993095398
+gpt-4,english,call,9.15708539162022,373.33666666666664,59.528888888888886,,,,,,
+gpt-4,english,json,7.963626848197565,383.3707317073171,61.69512195121951,4.7834568389659005,1.0,16.0,0.08687007305138472,0.000537872314453125,3.531284809112549
+gpt-4,english,markdown,10.02310108244419,428.35,67.825,4.990243192078054,1.0,16.0,0.10377185442257747,0.0005447864532470703,2.087376117706299
+gpt-4,japanese,call,24.259297592319243,213.65813953488373,1.1627906976744187,,,,,,
+gpt-4,japanese,json,22.703774025460252,209.86666666666667,4.133333333333334,1.1685226429101707,1.0,7.0,0.11835955503744466,0.0005507469177246094,3.5608017444610596
+gpt-4,japanese,markdown,31.99519391655922,275.1,2.925,1.1327980234712787,1.0,9.0,0.12606545905407157,0.0005619525909423828,2.9794347286224365
+gpt-4-turbo,english,call,7.319288367695279,372.86,58.532222222222224,,,,,,
+gpt-4-turbo,english,json,6.457074524425879,407.0243902439024,64.18414634146342,4.735998183701559,1.0,17.0,0.06474667943090798,0.0006806850433349609,5.431884765625
+gpt-4-turbo,english,markdown,8.325546002388,462.1125,71.925,4.984360253471754,1.0,16.0,0.07950969425127868,0.0006999969482421875,2.0926766395568848
+gpt-4-turbo,japanese,call,32.67405146055443,372.7860465116279,24.886046511627907,,,,,,
+gpt-4-turbo,japanese,json,19.359180968999862,250.44166666666666,6.183333333333334,1.184261339007763,1.0,8.0,0.08582140138847597,0.0006830692291259766,13.252992153167725
+gpt-4-turbo,japanese,markdown,20.977174520492554,283.55,4.45,1.1460038395473375,1.0,10.0,0.07991695286011809,0.0007085800170898438,2.3845231533050537
+llama-2-70b-chat,english,call,,,,,,,,,
+llama-2-70b-chat,english,json,2.917410132025375,314.4871134020619,48.49828178694158,4.02267032967033,1.0,15.0,0.019103390617685005,0.0001990795135498047,3.8762006759643555
+llama-2-70b-chat,english,markdown,4.99397261045417,795.1176470588235,121.01960784313725,4.177931176591799,1.0,16.0,0.018161741699259625,0.0002124309539794922,0.7439091205596924
+llama-2-70b-chat,japanese,call,,,,,,,,,
+llama-2-70b-chat,japanese,json,5.265992468681889,358.9718309859155,41.36619718309859,2.3362207250561435,1.0,15.0,0.023359512199899433,0.00020837783813476562,0.9314718246459961
+llama-2-70b-chat,japanese,markdown,11.084085723449444,1290.6774193548388,147.82258064516128,2.2644745033674796,1.0,16.0,0.01566780037868879,0.0002117156982421875,76.44498753547668
+zephyr-7b-beta,english,call,,,,,,,,,
+zephyr-7b-beta,english,json,3.94231437737087,365.275,59.88125,4.262252042007002,1.0,14.0,0.03629181767914927,0.0003070831298828125,0.4896988868713379
+zephyr-7b-beta,english,markdown,4.959704548120499,358.9375,58.3625,4.390672782874618,1.0,15.0,0.05067269182351022,0.00040459632873535156,75.15552616119385
+zephyr-7b-beta,japanese,call,,,,,,,,,
+zephyr-7b-beta,japanese,json,2.339889786640803,41.583333333333336,3.6333333333333333,1.1566991191469633,1.0,7.0,0.042134473762636,0.018090486526489258,0.4599947929382324
+zephyr-7b-beta,japanese,markdown,2.3176693956057233,38.1,2.3833333333333333,1.0343891402714933,1.0,4.0,0.041092377011053165,0.02583909034729004,0.4606599807739258

data/2024-02-03 22:42:30.259126_general_plot.csv

@@ -0,0 +1,428 @@
+path,plot_object,description,comment,plot_json
+./html/plots/response_times.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.27102681]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.69058203]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.79273835]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison',
+              'offsetgroup': 'chat-bison',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.89024363]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.66830386]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k',
+              'offsetgroup': 'chat-bison-32k',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.12817777]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.73886213]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.03678889]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([4.29307391]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([14.32373439]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([15.5354884]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.6742315]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([3.44499114]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model endpoints response times'},
+               'xaxis': {'anchor': 'y',
+                         'categoryarray': [Mistral-7B-Instruct-v0.2,
+                                           Mixtral-8x7B-Instruct-v0.1,
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           chat-bison, chat-bison (PaLM 2), chat-
+                                           bison-32k, chat-bison-32k (PaLM 2 32K),
+                                           gemini-pro, gpt-3.5-turbo, gpt-4,
+                                           gpt-4-turbo, llama-2-70b-chat,
+                                           zephyr-7b-beta],
+                         'categoryorder': 'array',
+                         'domain': [0.0, 1.0],
+                         'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
+})",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[4.690582027777991],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.7927383464436197],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.668303858522159],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.738862129823485],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.036788886637026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.293073908173212],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[14.323734385105828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.535488395149827],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.6742314998615178],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.4449911407752936],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
+./html/plots/execution_costs.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-3.5-turbo',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-3.5-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.22948]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4-turbo',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'gpt-4-turbo',
+              'offsetgroup': 'gpt-4-turbo',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4-turbo'], dtype=object),
+              'xaxis': 'x',
+              'y': array([2.9468]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gpt-4',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'gpt-4',
+              'offsetgroup': 'gpt-4',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gpt-4'], dtype=object),
+              'xaxis': 'x',
+              'y': array([6.342]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['llama-2-70b-chat'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.02828]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.12708]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.04109804]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.10571385]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.00933233]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['gemini-pro'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.05789]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison (PaLM 2)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison (PaLM 2)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.04731]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'xaxis': 'x',
+              'y': array([0.04454]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Costs of execution of 800 test queries per model'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
+})",Costs of execution of 800 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.22948],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[2.9467999999999996],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[6.3420000000000005],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.02828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.12708],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.0410980390045378],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.1057138456636005],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.009332333167394003],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.05789],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.04731],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.044539999999999996],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 800 test queries per model""},""barmode"":""relative""}}"
+./html/plots/model_sizes.html,"Figure({
+    'data': [{'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'xaxis': 'x',
+              'y': array([56.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['zephyr-7b-beta'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'xaxis': 'x',
+              'y': array([7.]),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'v',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'xaxis': 'x',
+              'y': array([1.1]),
+              'yaxis': 'y'}],
+    'layout': {'barmode': 'relative',
+               'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+               'template': '...',
+               'title': {'text': 'Model sizes in billions of parameters'},
+               'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
+               'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
+})",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"

data/time_of_day_plot.json

@@ -1 +1 @@
-{"data":[{"alignmentgroup":"True","hovertemplate":"Model=Mistral-7B-Instruct-v0.2\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mistral-7B-Instruct-v0.2","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"Mistral-7B-Instruct-v0.2","offsetgroup":"Mistral-7B-Instruct-v0.2","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[null,null,3.174540030956268],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=Mixtral-8x7B-Instruct-v0.1\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mixtral-8x7B-Instruct-v0.1","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"Mixtral-8x7B-Instruct-v0.1","offsetgroup":"Mixtral-8x7B-Instruct-v0.1","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[7.142197625471814],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","marker":{"color":"#00cc96","pattern":{"shape":""}},"name":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","offsetgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[1.2902645373344421],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison","marker":{"color":"#ab63fa","pattern":{"shape":""}},"name":"chat-bison","offsetgroup":"chat-bison","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[3.890243631601334],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison-32k\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison-32k","marker":{"color":"#FFA15A","pattern":{"shape":""}},"name":"chat-bison-32k","offsetgroup":"chat-bison-32k","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[4.128177767992019],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gemini-pro\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gemini-pro","marker":{"color":"#19d3f3","pattern":{"shape":""}},"name":"gemini-pro","offsetgroup":"gemini-pro","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[4.72138064004937],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-3.5-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-3.5-turbo","marker":{"color":"#FF6692","pattern":{"shape":""}},"name":"gpt-3.5-turbo","offsetgroup":"gpt-3.5-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[4.666725277900696,4.351089119911194,5.336839800789242],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4","marker":{"color":"#B6E880","pattern":{"shape":""}},"name":"gpt-4","offsetgroup":"gpt-4","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[15.5218456586202],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4-turbo","marker":{"color":"#FF97FF","pattern":{"shape":""}},"name":"gpt-4-turbo","offsetgroup":"gpt-4-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[14.008529031276703],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=llama-2-70b-chat\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"llama-2-70b-chat","marker":{"color":"#FECB52","pattern":{"shape":""}},"name":"llama-2-70b-chat","offsetgroup":"llama-2-70b-chat","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[2.1692867279052734,5.552149415016174,6.175082007679371],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=zephyr-7b-beta\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"zephyr-7b-beta","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"zephyr-7b-beta","offsetgroup":"zephyr-7b-beta","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[null,null,3.2387993240356447],"yaxis":"y","type":"bar"}],"layout":{"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Time of day"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Execution Time (s)"}},"legend":{"title":{"text":"Model"},"tracegroupgap":0},"title":{"text":"Execution time for different times of the day"},"barmode":"group"}}
+{"data":[{"alignmentgroup":"True","hovertemplate":"Model=Mistral-7B-Instruct-v0.2\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mistral-7B-Instruct-v0.2","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"Mistral-7B-Instruct-v0.2","offsetgroup":"Mistral-7B-Instruct-v0.2","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[1.9149879813194275,3.228973722457886,3.2048643112182615,3.397640073299408,3.389284573495388,null,null,3.734026002883911],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=Mixtral-8x7B-Instruct-v0.1\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mixtral-8x7B-Instruct-v0.1","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"Mixtral-8x7B-Instruct-v0.1","offsetgroup":"Mixtral-8x7B-Instruct-v0.1","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[3.8265925436529495,5.362903979589355,5.80184749175942,3.6435119574237023,5.62397656769588,3.010503646099206,3.2748278152374994,3.9661054956285575],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","marker":{"color":"#00cc96","pattern":{"shape":""}},"name":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","offsetgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[1.6341248273849487,1.8381905496120452,1.7547113946505954,1.7984187936782836,1.5875422928068372,2.0128010153770446,2.0273348593711855,1.9041210174560548],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison","marker":{"color":"#ab63fa","pattern":{"shape":""}},"name":"chat-bison","offsetgroup":"chat-bison","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight"],"xaxis":"x","y":[3.890243631601334],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison (PaLM 2)\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison (PaLM 2)","marker":{"color":"#FFA15A","pattern":{"shape":""}},"name":"chat-bison (PaLM 2)","offsetgroup":"chat-bison (PaLM 2)","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[2.468383938074112,2.6460144804074215,3.0387172081253744,2.688272579908371,2.7250528037548065,2.460119960308075,2.4665334616388592,2.488477897644043],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison-32k\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison-32k","marker":{"color":"#19d3f3","pattern":{"shape":""}},"name":"chat-bison-32k","offsetgroup":"chat-bison-32k","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight"],"xaxis":"x","y":[4.128177767992019],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison-32k (PaLM 2 32K)\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison-32k (PaLM 2 32K)","marker":{"color":"#FF6692","pattern":{"shape":""}},"name":"chat-bison-32k (PaLM 2 32K)","offsetgroup":"chat-bison-32k (PaLM 2 32K)","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[7.7498266498247785,5.014458654477046,4.5047362284226855,7.7332194912433625,7.796841062307358,6.491292915344238,9.141417106560299,7.7035503840446475],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gemini-pro\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gemini-pro","marker":{"color":"#B6E880","pattern":{"shape":""}},"name":"gemini-pro","offsetgroup":"gemini-pro","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[3.338477972348531,2.816922114008949,2.9268629640903114,2.86901999375759,3.7189874940246117,2.8375814715210272,2.7453590130460435,2.497767536007628],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-3.5-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-3.5-turbo","marker":{"color":"#FF97FF","pattern":{"shape":""}},"name":"gpt-3.5-turbo","offsetgroup":"gpt-3.5-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[3.822115447632102,3.9458200880459375,4.384064777692159,3.5153889304115657,5.110168156187617,6.062970260473398,3.787998208734724,3.126271222697364],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4","marker":{"color":"#FECB52","pattern":{"shape":""}},"name":"gpt-4","offsetgroup":"gpt-4","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[12.3578163115329,16.002364798386893,16.80607506932254,13.841120740345547,14.298642643005493,12.813134506115546,14.348626694414351,12.759107512468733],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4-turbo","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"gpt-4-turbo","offsetgroup":"gpt-4-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[13.770663784850727,13.997754749229976,22.320911452883767,16.550320884159632,14.592236209392548,14.254795966698573,11.555620827939775,13.686854598257277],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=llama-2-70b-chat\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"llama-2-70b-chat","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"llama-2-70b-chat","offsetgroup":"llama-2-70b-chat","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[3.249819871626402,4.234376892130426,3.6833307104881365,2.9706250462084185,4.719581684340602,2.8264514451677147,2.8660141522424265,2.872361832027194],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=zephyr-7b-beta\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"zephyr-7b-beta","marker":{"color":"#00cc96","pattern":{"shape":""}},"name":"zephyr-7b-beta","offsetgroup":"zephyr-7b-beta","orientation":"v","showlegend":true,"textposition":"auto","x":["0-2 midnight","12-14 afternoon","15-17 late afternoon","18-20 evening","21-23 late evening","3-5 night","6-8 early morning","9-11 morning"],"xaxis":"x","y":[1.9264068841934203,3.5789777278900146,3.4592524923459447,3.5673056403795878,3.45343524068594,null,null,4.056525647640228],"yaxis":"y","type":"bar"}],"layout":{"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Time of day"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Execution Time (s)"}},"legend":{"title":{"text":"Model"},"tracegroupgap":0},"title":{"text":"Execution time for different times of the day"},"barmode":"group"}}

pipeline/config.py

@@ -35,11 +35,12 @@ class QueriesConfig(Config):
 
 class OpenAIConfig(Config):
     mock: bool = False
+    remove_old_measurements: bool = False
 
 
 class QueriesDatasetConfig(Config):
     dataset_name: str = "GEM/xlsum"
-    samples_per_measurement: int = 2
+    samples_per_measurement: int = 20
     languages: List[str] = ["english", "japanese"]
     query_config: QueriesConfig = QueriesConfig()
 

pipeline/models.py

@@ -1,10 +1,8 @@
 import os
 from dataclasses import dataclass
 
-import pandas as pd
 
-
-@dataclass
+@dataclass(unsafe_hash=True)
 class Model(object):
     model_display_name: str
     model_name: str
@@ -17,6 +15,7 @@ class Model(object):
     cost_per_million_tokens: int = None
     cost_per_million_input_tokens: int = None
     cost_per_million_output_tokens: int = None
+    input_size: int = None
 
     def __post_init__(self):
         self.cost_per_million_input_tokens = self.cost_per_million_input_tokens or self.cost_per_million_tokens
@@ -47,6 +46,8 @@ MODELS = [
         supports_functions=True,
         cost_per_million_input_tokens=1,
         cost_per_million_output_tokens=2,
+        # https://learn.microsoft.com/en-us/answers/questions/1356487/what-is-the-exact-maximum-input-tokens-of-azure-gp
+        input_size=4096
     ),
     Model(
         "gpt-4-turbo",
@@ -56,6 +57,8 @@ MODELS = [
         supports_functions=True,
         cost_per_million_input_tokens=10,
         cost_per_million_output_tokens=30,
+        # https://writesonic.com/blog/gpt-4-turbo-vs-gpt-4
+        input_size=128_000,
     ),
     Model(
         "gpt-4",
@@ -65,16 +68,7 @@ MODELS = [
         supports_functions=True,
         cost_per_million_input_tokens=30,
         cost_per_million_output_tokens=60,
-    ),
-    # we don't test gpt-4-32k because the tasks don't reach gpt-4 limitations
-    Model(
-        "gpt-3.5-turbo",
-        "gpt-3.5-turbo",
-        None,
-        "OpenAI",
-        supports_functions=True,
-        cost_per_million_input_tokens=1,
-        cost_per_million_output_tokens=2,
+        input_size=32_000,
     ),
     # source: https://www.together.ai/pricing
     Model(
@@ -83,6 +77,8 @@ MODELS = [
         None,
         "Together AI",
         cost_per_million_tokens=0.2,
+        # https://github.com/facebookresearch/llama/issues/148
+        input_size=2048,
     ),
     Model(
         "Mixtral-8x7B-Instruct-v0.1",
@@ -135,6 +131,8 @@ MODELS = [
         # https://cloud.google.com/vertex-ai/docs/generative-ai/pricing
         cost_per_million_input_tokens=0.25,
         cost_per_million_output_tokens=0.5,
+        # https://ai.google.dev/models/palm
+        input_size=8196,
     ),
     Model(
         "chat-bison-32k (PaLM 2 32K)",