2024-02-08 14:48:57 Publish script update

app.py

@@ -32,17 +32,27 @@ We used litellm python library for all of  the models which naturally adds some
 We also attempted to use HF Endpoints as much as possible due to their popularity and transparency of how the model is executed.
 """
 
-time_periods_explanation_df = pd.DataFrame({
-    'time_of_day': ["early morning", "morning", "afternoon", "late afternoon", "evening", "late evening", "midnight", "night"],
-    'hour_range': ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"]
-})
+time_periods_explanation_df = pd.DataFrame(
+    {
+        "time_of_day": [
+            "early morning",
+            "morning",
+            "afternoon",
+            "late afternoon",
+            "evening",
+            "late evening",
+            "midnight",
+            "night",
+        ],
+        "hour_range": ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"],
+    }
+)
 
 queries_config = QueriesConfig()
 
-output_types_df = pd.DataFrame({
-    "Output Type": queries_config.query_template.keys(),
-    "Added text": queries_config.query_template.values()
-})
+output_types_df = pd.DataFrame(
+    {"Output Type": queries_config.query_template.keys(), "Added text": queries_config.query_template.values()}
+)
 
 summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
 time_of_day_comparison_df = pd.read_csv("data/2024-02-06 09:49:19.637072_time_of_day_comparison.csv")
@@ -140,12 +150,13 @@ with gr.Blocks() as demo:
     with gr.Row():
         filter_textbox.render()
         filter_button.render()
-    
+
     with gr.Tab("About this project"):
-        gr.Markdown(README.format(
-            queries_config.base_query_template.replace("```", "'''"),
-            output_types_df.to_markdown(index=False)
-        ))
+        gr.Markdown(
+            README.format(
+                queries_config.base_query_template.replace("```", "'''"), output_types_df.to_markdown(index=False)
+            )
+        )
     with gr.Tab("General plots"):
         for index, row in general_plots.iterrows():
             plot = plotly.io.from_json(row["plot_json"])
@@ -158,14 +169,16 @@ with gr.Blocks() as demo:
             collapse_languages_button.render()
             collapse_output_method_button.render()
         summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Output characteristics")
-        gr.Markdown("""\
+        gr.Markdown(
+            """\
 This table compares output characteristics of different models which include execution time, output size and chunking of the output. Some providers and models don't support output chunking, in this case chunk related fields are left empty.
 
 Execution time refers to averaged time needed to execute one query.
 
 To count words we split the output string by whitespace `\w` regex character.
 
-Chunk sizes are measured in the characters count.""")
+Chunk sizes are measured in the characters count."""
+        )
         for index, row in output_plots.iterrows():
             plot = plotly.io.from_json(row["plot_json"])
             plot.update_layout(autosize=True)
@@ -177,9 +190,12 @@ Chunk sizes are measured in the characters count.""")
             plot = plotly.io.from_json(row["plot_json"])
             plot.update_layout(autosize=True)
             gr.Plot(plot, label=row["header"], scale=1)
-        time_periods_explanation_ui = gr.DataFrame(dataframe_style(time_periods_explanation_df), label="Times of day ranges")
+        time_periods_explanation_ui = gr.DataFrame(
+            dataframe_style(time_periods_explanation_df), label="Times of day ranges"
+        )
         time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day")
-        gr.Markdown("""\
+        gr.Markdown(
+            """\
 These measurements were made by testing the models using the same dataset as in the other comparisons every hour for 24 hours.
                     
 Execution time refers to averaged time needed to execute one query.
@@ -187,8 +203,9 @@ Execution time refers to averaged time needed to execute one query.
 Hours and times of day in the table and in the plot are based on Central European Time.
                     
 Measurements were made during a normal work week.
-""")
-        # display rest of the plots 
+"""
+        )
+        # display rest of the plots
         for index, row in time_of_day_plots[1:].iterrows():
             plot = plotly.io.from_json(row["plot_json"])
             plot.update_layout(autosize=True)
@@ -204,7 +221,8 @@ Hugging Face Inference Endpoints are charged by hour so to compare different pro
 for models hosted this way we calculated "Cost Per Token" column using data collected during the experiment.
 
 Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
-            """)
+            """
+        )
     filter_button.click(
         fn=filter_dataframes,
         inputs=filter_textbox,

data/2024-02-08 14:18:16.852773_time_of_day_comparison.csv

@@ -0,0 +1,87 @@
+model,time_of_day,execution_time,characters_count,words_count,provider
+Mistral-7B-Instruct-v0.2,morning,3.734026002883911,362.9,58.35,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,afternoon,3.228973722457886,222.0625,32.25,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late afternoon,3.2048643112182615,219.8625,31.8,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,evening,3.397640073299408,261.18333333333334,40.1,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late evening,3.389284573495388,175.79375,25.68125,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,midnight,1.9149879813194275,37.8,2.95,Hugging Face Inference Endpoint
+Mixtral-8x7B-Instruct-v0.1,early morning,4.526968242530536,285.045,41.86,Together AI
+Mixtral-8x7B-Instruct-v0.1,morning,3.9661054956285575,304.82,47.28,Together AI
+Mixtral-8x7B-Instruct-v0.1,afternoon,5.362903979589355,369.3192307692308,54.353846153846156,Together AI
+Mixtral-8x7B-Instruct-v0.1,late afternoon,5.80184749175942,347.9681818181818,47.27272727272727,Together AI
+Mixtral-8x7B-Instruct-v0.1,evening,3.6435119574237023,326.69,48.545,Together AI
+Mixtral-8x7B-Instruct-v0.1,late evening,5.62397656769588,395.15714285714284,49.02857142857143,Together AI
+Mixtral-8x7B-Instruct-v0.1,midnight,4.639010797279158,323.0394736842105,42.69210526315789,Together AI
+Mixtral-8x7B-Instruct-v0.1,night,4.009439338194697,301.24545454545455,42.21818181818182,Together AI
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning,2.0273348593711855,372.13,62.53,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.9041210174560548,372.05,62.6,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.8381905496120452,308.795,51.08,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon,1.7547113946505954,285.17857142857144,46.9,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.7984187936782836,313.99,51.96,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late evening,1.5875422928068372,192.78333333333333,31.261111111111113,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,1.6341248273849487,210.2,34.2,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night,2.0128010153770446,372.05,62.6,Hugging Face Inference Endpoint
+chat-bison (PaLM 2),early morning,2.4665334616388592,381.9214285714286,60.892857142857146,Google VertexAI
+chat-bison (PaLM 2),morning,2.488477897644043,381.73,60.88,Google VertexAI
+chat-bison (PaLM 2),afternoon,2.6460144804074215,376.28076923076924,55.965384615384615,Google VertexAI
+chat-bison (PaLM 2),late afternoon,3.0387172081253744,381.1363636363636,53.35454545454545,Google VertexAI
+chat-bison (PaLM 2),evening,2.688272579908371,367.07,55.89,Google VertexAI
+chat-bison (PaLM 2),late evening,2.7250528037548065,382.725,52.95,Google VertexAI
+chat-bison (PaLM 2),midnight,2.468383938074112,381.95,60.9,Google VertexAI
+chat-bison (PaLM 2),night,2.460119960308075,381.92,60.94,Google VertexAI
+chat-bison-32k (PaLM 2 32K),early morning,9.141417106560299,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),morning,7.7035503840446475,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),afternoon,5.014458654477046,339.3692307692308,49.323076923076925,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late afternoon,4.5047362284226855,348.65909090909093,48.35454545454545,Google VertexAI
+chat-bison-32k (PaLM 2 32K),evening,7.7332194912433625,329.46,49.5,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late evening,7.796841062307358,349.95,47.805,Google VertexAI
+chat-bison-32k (PaLM 2 32K),midnight,7.7498266498247785,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),night,6.491292915344238,335.75,53.85,Google VertexAI
+gemini-pro,early morning,2.7453590130460435,381.2214285714286,60.964285714285715,Google VertexAI
+gemini-pro,morning,2.497767536007628,371.93,59.39,Google VertexAI
+gemini-pro,afternoon,2.816922114008949,360.62307692307695,53.25769230769231,Google VertexAI
+gemini-pro,late afternoon,2.9268629640903114,364.57272727272726,50.1,Google VertexAI
+gemini-pro,evening,2.86901999375759,366.4,55.045,Google VertexAI
+gemini-pro,late evening,3.7189874940246117,390.9142857142857,51.35,Google VertexAI
+gemini-pro,midnight,3.338477972348531,369.825,55.9125,Google VertexAI
+gemini-pro,night,2.8375814715210272,374.8,60.04,Google VertexAI
+gpt-3.5-turbo,early morning,3.787998208734724,403.74444444444447,47.34444444444444,OpenAI
+gpt-3.5-turbo,morning,3.126271222697364,389.9888888888889,50.93888888888889,OpenAI
+gpt-3.5-turbo,afternoon,3.9458200880459375,381.4428571428571,43.76190476190476,OpenAI
+gpt-3.5-turbo,late afternoon,4.384064777692159,396.4357142857143,43.28333333333333,OpenAI
+gpt-3.5-turbo,evening,3.5153889304115657,385.18095238095236,46.01428571428571,OpenAI
+gpt-3.5-turbo,late evening,5.110168156187617,422.73510971786834,45.358934169278996,OpenAI
+gpt-3.5-turbo,midnight,3.822115447632102,426.71923076923076,49.05,OpenAI
+gpt-3.5-turbo,night,6.062970260473398,418.6192307692308,44.926923076923075,OpenAI
+gpt-4,early morning,14.348626694414351,323.5388888888889,40.544444444444444,OpenAI
+gpt-4,morning,12.759107512468733,338.18333333333334,46.85,OpenAI
+gpt-4,afternoon,16.002364798386893,318.3095238095238,38.77142857142857,OpenAI
+gpt-4,late afternoon,16.80607506932254,313.3595238095238,37.49285714285714,OpenAI
+gpt-4,evening,13.841120740345547,318.0809523809524,40.19285714285714,OpenAI
+gpt-4,late evening,14.298642643005493,314.336,37.012,OpenAI
+gpt-4,midnight,12.3578163115329,334.075,41.35,OpenAI
+gpt-4,night,12.813134506115546,316.93461538461537,37.93076923076923,OpenAI
+gpt-4-turbo,early morning,11.555620827939775,357.65555555555557,47.21666666666667,OpenAI
+gpt-4-turbo,morning,13.686854598257277,381.8888888888889,55.02777777777778,OpenAI
+gpt-4-turbo,afternoon,13.997754749229976,351.01190476190476,46.03333333333333,OpenAI
+gpt-4-turbo,late afternoon,22.320911452883767,381.65714285714284,47.35476190476191,OpenAI
+gpt-4-turbo,evening,16.550320884159632,382.31666666666666,48.45,OpenAI
+gpt-4-turbo,late evening,14.592236209392548,413.61,44.8,OpenAI
+gpt-4-turbo,midnight,13.770663784850727,382.7613636363636,47.61818181818182,OpenAI
+gpt-4-turbo,night,14.254795966698573,352.54615384615386,46.37692307692308,OpenAI
+llama-2-70b-chat,early morning,2.8660141522424265,289.6642857142857,44.614285714285714,Together AI
+llama-2-70b-chat,morning,2.872361832027194,283.35,43.45,Together AI
+llama-2-70b-chat,afternoon,4.234376892130426,375.44615384615383,55.238461538461536,Together AI
+llama-2-70b-chat,late afternoon,3.6833307104881365,433.6636363636364,59.445454545454545,Together AI
+llama-2-70b-chat,evening,2.9706250462084185,317.245,47.4,Together AI
+llama-2-70b-chat,late evening,4.719581684340602,572.1689655172414,79.83103448275862,Together AI
+llama-2-70b-chat,midnight,3.249819871626402,346.8875,52.06875,Together AI
+llama-2-70b-chat,night,2.8264514451677147,313.71,48.42,Together AI
+zephyr-7b-beta,early morning,3.937663261095683,273.03333333333336,43.45,Hugging Face Inference Endpoint
+zephyr-7b-beta,morning,4.056525647640228,386.3,63.35,Hugging Face Inference Endpoint
+zephyr-7b-beta,afternoon,3.5789777278900146,277.6,44.016666666666666,Hugging Face Inference Endpoint
+zephyr-7b-beta,late afternoon,3.4592524923459447,248.59,38.9,Hugging Face Inference Endpoint
+zephyr-7b-beta,evening,3.5673056403795878,273.3666666666667,43.5,Hugging Face Inference Endpoint
+zephyr-7b-beta,late evening,3.45343524068594,176.23125,27.21875,Hugging Face Inference Endpoint
+zephyr-7b-beta,midnight,3.7253047794103624,217.82083333333333,33.67916666666667,Hugging Face Inference Endpoint
+zephyr-7b-beta,night,3.6954557319482166,216.55833333333334,33.46666666666667,Hugging Face Inference Endpoint

data/2024-02-08 14:18:58.194844_time_of_day_comparison.csv

@@ -0,0 +1,87 @@
+model,time_of_day,execution_time,characters_count,words_count,provider
+Mistral-7B-Instruct-v0.2,morning,3.734026002883911,362.9,58.35,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,afternoon,3.228973722457886,222.0625,32.25,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late afternoon,3.2048643112182615,219.8625,31.8,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,evening,3.397640073299408,261.18333333333334,40.1,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late evening,3.389284573495388,175.79375,25.68125,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,midnight,1.9149879813194275,37.8,2.95,Hugging Face Inference Endpoint
+Mixtral-8x7B-Instruct-v0.1,early morning,4.526968242530536,285.045,41.86,Together AI
+Mixtral-8x7B-Instruct-v0.1,morning,3.9661054956285575,304.82,47.28,Together AI
+Mixtral-8x7B-Instruct-v0.1,afternoon,5.362903979589355,369.3192307692308,54.353846153846156,Together AI
+Mixtral-8x7B-Instruct-v0.1,late afternoon,5.80184749175942,347.9681818181818,47.27272727272727,Together AI
+Mixtral-8x7B-Instruct-v0.1,evening,3.6435119574237023,326.69,48.545,Together AI
+Mixtral-8x7B-Instruct-v0.1,late evening,5.62397656769588,395.15714285714284,49.02857142857143,Together AI
+Mixtral-8x7B-Instruct-v0.1,midnight,4.639010797279158,323.0394736842105,42.69210526315789,Together AI
+Mixtral-8x7B-Instruct-v0.1,night,4.009439338194697,301.24545454545455,42.21818181818182,Together AI
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning,2.0273348593711855,372.13,62.53,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.9041210174560548,372.05,62.6,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.8381905496120452,308.795,51.08,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon,1.7547113946505954,285.17857142857144,46.9,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.7984187936782836,313.99,51.96,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late evening,1.5875422928068372,192.78333333333333,31.261111111111113,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,1.6341248273849487,210.2,34.2,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night,2.0128010153770446,372.05,62.6,Hugging Face Inference Endpoint
+chat-bison (PaLM 2),early morning,2.4665334616388592,381.9214285714286,60.892857142857146,Google VertexAI
+chat-bison (PaLM 2),morning,2.488477897644043,381.73,60.88,Google VertexAI
+chat-bison (PaLM 2),afternoon,2.6460144804074215,376.28076923076924,55.965384615384615,Google VertexAI
+chat-bison (PaLM 2),late afternoon,3.0387172081253744,381.1363636363636,53.35454545454545,Google VertexAI
+chat-bison (PaLM 2),evening,2.688272579908371,367.07,55.89,Google VertexAI
+chat-bison (PaLM 2),late evening,2.7250528037548065,382.725,52.95,Google VertexAI
+chat-bison (PaLM 2),midnight,2.468383938074112,381.95,60.9,Google VertexAI
+chat-bison (PaLM 2),night,2.460119960308075,381.92,60.94,Google VertexAI
+chat-bison-32k (PaLM 2 32K),early morning,9.141417106560299,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),morning,7.7035503840446475,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),afternoon,5.014458654477046,339.3692307692308,49.323076923076925,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late afternoon,4.5047362284226855,348.65909090909093,48.35454545454545,Google VertexAI
+chat-bison-32k (PaLM 2 32K),evening,7.7332194912433625,329.46,49.5,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late evening,7.796841062307358,349.95,47.805,Google VertexAI
+chat-bison-32k (PaLM 2 32K),midnight,7.7498266498247785,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),night,6.491292915344238,335.75,53.85,Google VertexAI
+gemini-pro,early morning,2.7453590130460435,381.2214285714286,60.964285714285715,Google VertexAI
+gemini-pro,morning,2.497767536007628,371.93,59.39,Google VertexAI
+gemini-pro,afternoon,2.816922114008949,360.62307692307695,53.25769230769231,Google VertexAI
+gemini-pro,late afternoon,2.9268629640903114,364.57272727272726,50.1,Google VertexAI
+gemini-pro,evening,2.86901999375759,366.4,55.045,Google VertexAI
+gemini-pro,late evening,3.7189874940246117,390.9142857142857,51.35,Google VertexAI
+gemini-pro,midnight,3.338477972348531,369.825,55.9125,Google VertexAI
+gemini-pro,night,2.8375814715210272,374.8,60.04,Google VertexAI
+gpt-3.5-turbo,early morning,3.787998208734724,403.74444444444447,47.34444444444444,OpenAI
+gpt-3.5-turbo,morning,3.126271222697364,389.9888888888889,50.93888888888889,OpenAI
+gpt-3.5-turbo,afternoon,3.9458200880459375,381.4428571428571,43.76190476190476,OpenAI
+gpt-3.5-turbo,late afternoon,4.384064777692159,396.4357142857143,43.28333333333333,OpenAI
+gpt-3.5-turbo,evening,3.5153889304115657,385.18095238095236,46.01428571428571,OpenAI
+gpt-3.5-turbo,late evening,5.110168156187617,422.73510971786834,45.358934169278996,OpenAI
+gpt-3.5-turbo,midnight,3.822115447632102,426.71923076923076,49.05,OpenAI
+gpt-3.5-turbo,night,6.062970260473398,418.6192307692308,44.926923076923075,OpenAI
+gpt-4,early morning,14.348626694414351,323.5388888888889,40.544444444444444,OpenAI
+gpt-4,morning,12.759107512468733,338.18333333333334,46.85,OpenAI
+gpt-4,afternoon,16.002364798386893,318.3095238095238,38.77142857142857,OpenAI
+gpt-4,late afternoon,16.80607506932254,313.3595238095238,37.49285714285714,OpenAI
+gpt-4,evening,13.841120740345547,318.0809523809524,40.19285714285714,OpenAI
+gpt-4,late evening,14.298642643005493,314.336,37.012,OpenAI
+gpt-4,midnight,12.3578163115329,334.075,41.35,OpenAI
+gpt-4,night,12.813134506115546,316.93461538461537,37.93076923076923,OpenAI
+gpt-4-turbo,early morning,11.555620827939775,357.65555555555557,47.21666666666667,OpenAI
+gpt-4-turbo,morning,13.686854598257277,381.8888888888889,55.02777777777778,OpenAI
+gpt-4-turbo,afternoon,13.997754749229976,351.01190476190476,46.03333333333333,OpenAI
+gpt-4-turbo,late afternoon,22.320911452883767,381.65714285714284,47.35476190476191,OpenAI
+gpt-4-turbo,evening,16.550320884159632,382.31666666666666,48.45,OpenAI
+gpt-4-turbo,late evening,14.592236209392548,413.61,44.8,OpenAI
+gpt-4-turbo,midnight,13.770663784850727,382.7613636363636,47.61818181818182,OpenAI
+gpt-4-turbo,night,14.254795966698573,352.54615384615386,46.37692307692308,OpenAI
+llama-2-70b-chat,early morning,2.8660141522424265,289.6642857142857,44.614285714285714,Together AI
+llama-2-70b-chat,morning,2.872361832027194,283.35,43.45,Together AI
+llama-2-70b-chat,afternoon,4.234376892130426,375.44615384615383,55.238461538461536,Together AI
+llama-2-70b-chat,late afternoon,3.6833307104881365,433.6636363636364,59.445454545454545,Together AI
+llama-2-70b-chat,evening,2.9706250462084185,317.245,47.4,Together AI
+llama-2-70b-chat,late evening,4.719581684340602,572.1689655172414,79.83103448275862,Together AI
+llama-2-70b-chat,midnight,3.249819871626402,346.8875,52.06875,Together AI
+llama-2-70b-chat,night,2.8264514451677147,313.71,48.42,Together AI
+zephyr-7b-beta,early morning,3.937663261095683,273.03333333333336,43.45,Hugging Face Inference Endpoint
+zephyr-7b-beta,morning,4.056525647640228,386.3,63.35,Hugging Face Inference Endpoint
+zephyr-7b-beta,afternoon,3.5789777278900146,277.6,44.016666666666666,Hugging Face Inference Endpoint
+zephyr-7b-beta,late afternoon,3.4592524923459447,248.59,38.9,Hugging Face Inference Endpoint
+zephyr-7b-beta,evening,3.5673056403795878,273.3666666666667,43.5,Hugging Face Inference Endpoint
+zephyr-7b-beta,late evening,3.45343524068594,176.23125,27.21875,Hugging Face Inference Endpoint
+zephyr-7b-beta,midnight,3.7253047794103624,217.82083333333333,33.67916666666667,Hugging Face Inference Endpoint
+zephyr-7b-beta,night,3.6954557319482166,216.55833333333334,33.46666666666667,Hugging Face Inference Endpoint

data/2024-02-08 14:20:56.432044_time_of_day_comparison.csv

@@ -0,0 +1,87 @@
+provider,model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,morning,3.734026002883911,362.9,58.35,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,afternoon,3.228973722457886,222.0625,32.25,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late afternoon,3.2048643112182615,219.8625,31.8,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,evening,3.397640073299408,261.18333333333334,40.1,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,late evening,3.389284573495388,175.79375,25.68125,Hugging Face Inference Endpoint
+Mistral-7B-Instruct-v0.2,midnight,1.9149879813194275,37.8,2.95,Hugging Face Inference Endpoint
+Mixtral-8x7B-Instruct-v0.1,early morning,4.526968242530536,285.045,41.86,Together AI
+Mixtral-8x7B-Instruct-v0.1,morning,3.9661054956285575,304.82,47.28,Together AI
+Mixtral-8x7B-Instruct-v0.1,afternoon,5.362903979589355,369.3192307692308,54.353846153846156,Together AI
+Mixtral-8x7B-Instruct-v0.1,late afternoon,5.80184749175942,347.9681818181818,47.27272727272727,Together AI
+Mixtral-8x7B-Instruct-v0.1,evening,3.6435119574237023,326.69,48.545,Together AI
+Mixtral-8x7B-Instruct-v0.1,late evening,5.62397656769588,395.15714285714284,49.02857142857143,Together AI
+Mixtral-8x7B-Instruct-v0.1,midnight,4.639010797279158,323.0394736842105,42.69210526315789,Together AI
+Mixtral-8x7B-Instruct-v0.1,night,4.009439338194697,301.24545454545455,42.21818181818182,Together AI
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning,2.0273348593711855,372.13,62.53,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.9041210174560548,372.05,62.6,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.8381905496120452,308.795,51.08,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon,1.7547113946505954,285.17857142857144,46.9,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.7984187936782836,313.99,51.96,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late evening,1.5875422928068372,192.78333333333333,31.261111111111113,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,1.6341248273849487,210.2,34.2,Hugging Face Inference Endpoint
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night,2.0128010153770446,372.05,62.6,Hugging Face Inference Endpoint
+chat-bison (PaLM 2),early morning,2.4665334616388592,381.9214285714286,60.892857142857146,Google VertexAI
+chat-bison (PaLM 2),morning,2.488477897644043,381.73,60.88,Google VertexAI
+chat-bison (PaLM 2),afternoon,2.6460144804074215,376.28076923076924,55.965384615384615,Google VertexAI
+chat-bison (PaLM 2),late afternoon,3.0387172081253744,381.1363636363636,53.35454545454545,Google VertexAI
+chat-bison (PaLM 2),evening,2.688272579908371,367.07,55.89,Google VertexAI
+chat-bison (PaLM 2),late evening,2.7250528037548065,382.725,52.95,Google VertexAI
+chat-bison (PaLM 2),midnight,2.468383938074112,381.95,60.9,Google VertexAI
+chat-bison (PaLM 2),night,2.460119960308075,381.92,60.94,Google VertexAI
+chat-bison-32k (PaLM 2 32K),early morning,9.141417106560299,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),morning,7.7035503840446475,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),afternoon,5.014458654477046,339.3692307692308,49.323076923076925,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late afternoon,4.5047362284226855,348.65909090909093,48.35454545454545,Google VertexAI
+chat-bison-32k (PaLM 2 32K),evening,7.7332194912433625,329.46,49.5,Google VertexAI
+chat-bison-32k (PaLM 2 32K),late evening,7.796841062307358,349.95,47.805,Google VertexAI
+chat-bison-32k (PaLM 2 32K),midnight,7.7498266498247785,335.75,53.85,Google VertexAI
+chat-bison-32k (PaLM 2 32K),night,6.491292915344238,335.75,53.85,Google VertexAI
+gemini-pro,early morning,2.7453590130460435,381.2214285714286,60.964285714285715,Google VertexAI
+gemini-pro,morning,2.497767536007628,371.93,59.39,Google VertexAI
+gemini-pro,afternoon,2.816922114008949,360.62307692307695,53.25769230769231,Google VertexAI
+gemini-pro,late afternoon,2.9268629640903114,364.57272727272726,50.1,Google VertexAI
+gemini-pro,evening,2.86901999375759,366.4,55.045,Google VertexAI
+gemini-pro,late evening,3.7189874940246117,390.9142857142857,51.35,Google VertexAI
+gemini-pro,midnight,3.338477972348531,369.825,55.9125,Google VertexAI
+gemini-pro,night,2.8375814715210272,374.8,60.04,Google VertexAI
+gpt-3.5-turbo,early morning,3.787998208734724,403.74444444444447,47.34444444444444,OpenAI
+gpt-3.5-turbo,morning,3.126271222697364,389.9888888888889,50.93888888888889,OpenAI
+gpt-3.5-turbo,afternoon,3.9458200880459375,381.4428571428571,43.76190476190476,OpenAI
+gpt-3.5-turbo,late afternoon,4.384064777692159,396.4357142857143,43.28333333333333,OpenAI
+gpt-3.5-turbo,evening,3.5153889304115657,385.18095238095236,46.01428571428571,OpenAI
+gpt-3.5-turbo,late evening,5.110168156187617,422.73510971786834,45.358934169278996,OpenAI
+gpt-3.5-turbo,midnight,3.822115447632102,426.71923076923076,49.05,OpenAI
+gpt-3.5-turbo,night,6.062970260473398,418.6192307692308,44.926923076923075,OpenAI
+gpt-4,early morning,14.348626694414351,323.5388888888889,40.544444444444444,OpenAI
+gpt-4,morning,12.759107512468733,338.18333333333334,46.85,OpenAI
+gpt-4,afternoon,16.002364798386893,318.3095238095238,38.77142857142857,OpenAI
+gpt-4,late afternoon,16.80607506932254,313.3595238095238,37.49285714285714,OpenAI
+gpt-4,evening,13.841120740345547,318.0809523809524,40.19285714285714,OpenAI
+gpt-4,late evening,14.298642643005493,314.336,37.012,OpenAI
+gpt-4,midnight,12.3578163115329,334.075,41.35,OpenAI
+gpt-4,night,12.813134506115546,316.93461538461537,37.93076923076923,OpenAI
+gpt-4-turbo,early morning,11.555620827939775,357.65555555555557,47.21666666666667,OpenAI
+gpt-4-turbo,morning,13.686854598257277,381.8888888888889,55.02777777777778,OpenAI
+gpt-4-turbo,afternoon,13.997754749229976,351.01190476190476,46.03333333333333,OpenAI
+gpt-4-turbo,late afternoon,22.320911452883767,381.65714285714284,47.35476190476191,OpenAI
+gpt-4-turbo,evening,16.550320884159632,382.31666666666666,48.45,OpenAI
+gpt-4-turbo,late evening,14.592236209392548,413.61,44.8,OpenAI
+gpt-4-turbo,midnight,13.770663784850727,382.7613636363636,47.61818181818182,OpenAI
+gpt-4-turbo,night,14.254795966698573,352.54615384615386,46.37692307692308,OpenAI
+llama-2-70b-chat,early morning,2.8660141522424265,289.6642857142857,44.614285714285714,Together AI
+llama-2-70b-chat,morning,2.872361832027194,283.35,43.45,Together AI
+llama-2-70b-chat,afternoon,4.234376892130426,375.44615384615383,55.238461538461536,Together AI
+llama-2-70b-chat,late afternoon,3.6833307104881365,433.6636363636364,59.445454545454545,Together AI
+llama-2-70b-chat,evening,2.9706250462084185,317.245,47.4,Together AI
+llama-2-70b-chat,late evening,4.719581684340602,572.1689655172414,79.83103448275862,Together AI
+llama-2-70b-chat,midnight,3.249819871626402,346.8875,52.06875,Together AI
+llama-2-70b-chat,night,2.8264514451677147,313.71,48.42,Together AI
+zephyr-7b-beta,early morning,3.937663261095683,273.03333333333336,43.45,Hugging Face Inference Endpoint
+zephyr-7b-beta,morning,4.056525647640228,386.3,63.35,Hugging Face Inference Endpoint
+zephyr-7b-beta,afternoon,3.5789777278900146,277.6,44.016666666666666,Hugging Face Inference Endpoint
+zephyr-7b-beta,late afternoon,3.4592524923459447,248.59,38.9,Hugging Face Inference Endpoint
+zephyr-7b-beta,evening,3.5673056403795878,273.3666666666667,43.5,Hugging Face Inference Endpoint
+zephyr-7b-beta,late evening,3.45343524068594,176.23125,27.21875,Hugging Face Inference Endpoint
+zephyr-7b-beta,midnight,3.7253047794103624,217.82083333333333,33.67916666666667,Hugging Face Inference Endpoint
+zephyr-7b-beta,night,3.6954557319482166,216.55833333333334,33.46666666666667,Hugging Face Inference Endpoint

data/2024-02-08 14:22:42.700060_time_of_day_comparison.csv

@@ -0,0 +1,89 @@
+model,time_of_day,execution_time,characters_count,words_count
+Mistral-7B-Instruct-v0.2,morning,3.734026002883911,362.9,58.35
+Mistral-7B-Instruct-v0.2,afternoon,3.228973722457886,222.0625,32.25
+Mistral-7B-Instruct-v0.2,late afternoon,3.2048643112182615,219.8625,31.8
+Mistral-7B-Instruct-v0.2,evening,3.397640073299408,261.18333333333334,40.1
+Mistral-7B-Instruct-v0.2,late evening,3.389284573495388,175.79375,25.68125
+Mistral-7B-Instruct-v0.2,midnight,1.9149879813194275,37.8,2.95
+Mixtral-8x7B-Instruct-v0.1,early morning,4.526968242530536,285.045,41.86
+Mixtral-8x7B-Instruct-v0.1,morning,3.9661054956285575,304.82,47.28
+Mixtral-8x7B-Instruct-v0.1,afternoon,5.362903979589355,369.3192307692308,54.353846153846156
+Mixtral-8x7B-Instruct-v0.1,late afternoon,5.80184749175942,347.9681818181818,47.27272727272727
+Mixtral-8x7B-Instruct-v0.1,evening,3.6435119574237023,326.69,48.545
+Mixtral-8x7B-Instruct-v0.1,late evening,5.62397656769588,395.15714285714284,49.02857142857143
+Mixtral-8x7B-Instruct-v0.1,midnight,4.639010797279158,323.0394736842105,42.69210526315789
+Mixtral-8x7B-Instruct-v0.1,night,4.009439338194697,301.24545454545455,42.21818181818182
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,early morning,2.0273348593711855,372.13,62.53
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,morning,1.9041210174560548,372.05,62.6
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,afternoon,1.8381905496120452,308.795,51.08
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late afternoon,1.7547113946505954,285.17857142857144,46.9
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,evening,1.7984187936782836,313.99,51.96
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,late evening,1.5875422928068372,192.78333333333333,31.261111111111113
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,midnight,1.6341248273849487,210.2,34.2
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,night,2.0128010153770446,372.05,62.6
+chat-bison,midnight,3.890243631601334,398.075,49.0
+chat-bison (PaLM 2),early morning,2.4665334616388592,381.9214285714286,60.892857142857146
+chat-bison (PaLM 2),morning,2.488477897644043,381.73,60.88
+chat-bison (PaLM 2),afternoon,2.6460144804074215,376.28076923076924,55.965384615384615
+chat-bison (PaLM 2),late afternoon,3.0387172081253744,381.1363636363636,53.35454545454545
+chat-bison (PaLM 2),evening,2.688272579908371,367.07,55.89
+chat-bison (PaLM 2),late evening,2.7250528037548065,382.725,52.95
+chat-bison (PaLM 2),midnight,2.468383938074112,381.95,60.9
+chat-bison (PaLM 2),night,2.460119960308075,381.92,60.94
+chat-bison-32k,midnight,4.128177767992019,389.925,48.025
+chat-bison-32k (PaLM 2 32K),early morning,9.141417106560299,335.75,53.85
+chat-bison-32k (PaLM 2 32K),morning,7.7035503840446475,335.75,53.85
+chat-bison-32k (PaLM 2 32K),afternoon,5.014458654477046,339.3692307692308,49.323076923076925
+chat-bison-32k (PaLM 2 32K),late afternoon,4.5047362284226855,348.65909090909093,48.35454545454545
+chat-bison-32k (PaLM 2 32K),evening,7.7332194912433625,329.46,49.5
+chat-bison-32k (PaLM 2 32K),late evening,7.796841062307358,349.95,47.805
+chat-bison-32k (PaLM 2 32K),midnight,7.7498266498247785,335.75,53.85
+chat-bison-32k (PaLM 2 32K),night,6.491292915344238,335.75,53.85
+gemini-pro,early morning,2.7453590130460435,381.2214285714286,60.964285714285715
+gemini-pro,morning,2.497767536007628,371.93,59.39
+gemini-pro,afternoon,2.816922114008949,360.62307692307695,53.25769230769231
+gemini-pro,late afternoon,2.9268629640903114,364.57272727272726,50.1
+gemini-pro,evening,2.86901999375759,366.4,55.045
+gemini-pro,late evening,3.7189874940246117,390.9142857142857,51.35
+gemini-pro,midnight,3.338477972348531,369.825,55.9125
+gemini-pro,night,2.8375814715210272,374.8,60.04
+gpt-3.5-turbo,early morning,3.787998208734724,403.74444444444447,47.34444444444444
+gpt-3.5-turbo,morning,3.126271222697364,389.9888888888889,50.93888888888889
+gpt-3.5-turbo,afternoon,3.9458200880459375,381.4428571428571,43.76190476190476
+gpt-3.5-turbo,late afternoon,4.384064777692159,396.4357142857143,43.28333333333333
+gpt-3.5-turbo,evening,3.5153889304115657,385.18095238095236,46.01428571428571
+gpt-3.5-turbo,late evening,5.110168156187617,422.73510971786834,45.358934169278996
+gpt-3.5-turbo,midnight,3.822115447632102,426.71923076923076,49.05
+gpt-3.5-turbo,night,6.062970260473398,418.6192307692308,44.926923076923075
+gpt-4,early morning,14.348626694414351,323.5388888888889,40.544444444444444
+gpt-4,morning,12.759107512468733,338.18333333333334,46.85
+gpt-4,afternoon,16.002364798386893,318.3095238095238,38.77142857142857
+gpt-4,late afternoon,16.80607506932254,313.3595238095238,37.49285714285714
+gpt-4,evening,13.841120740345547,318.0809523809524,40.19285714285714
+gpt-4,late evening,14.298642643005493,314.336,37.012
+gpt-4,midnight,12.3578163115329,334.075,41.35
+gpt-4,night,12.813134506115546,316.93461538461537,37.93076923076923
+gpt-4-turbo,early morning,11.555620827939775,357.65555555555557,47.21666666666667
+gpt-4-turbo,morning,13.686854598257277,381.8888888888889,55.02777777777778
+gpt-4-turbo,afternoon,13.997754749229976,351.01190476190476,46.03333333333333
+gpt-4-turbo,late afternoon,22.320911452883767,381.65714285714284,47.35476190476191
+gpt-4-turbo,evening,16.550320884159632,382.31666666666666,48.45
+gpt-4-turbo,late evening,14.592236209392548,413.61,44.8
+gpt-4-turbo,midnight,13.770663784850727,382.7613636363636,47.61818181818182
+gpt-4-turbo,night,14.254795966698573,352.54615384615386,46.37692307692308
+llama-2-70b-chat,early morning,2.8660141522424265,289.6642857142857,44.614285714285714
+llama-2-70b-chat,morning,2.872361832027194,283.35,43.45
+llama-2-70b-chat,afternoon,4.234376892130426,375.44615384615383,55.238461538461536
+llama-2-70b-chat,late afternoon,3.6833307104881365,433.6636363636364,59.445454545454545
+llama-2-70b-chat,evening,2.9706250462084185,317.245,47.4
+llama-2-70b-chat,late evening,4.719581684340602,572.1689655172414,79.83103448275862
+llama-2-70b-chat,midnight,3.249819871626402,346.8875,52.06875
+llama-2-70b-chat,night,2.8264514451677147,313.71,48.42
+zephyr-7b-beta,early morning,3.937663261095683,273.03333333333336,43.45
+zephyr-7b-beta,morning,4.056525647640228,386.3,63.35
+zephyr-7b-beta,afternoon,3.5789777278900146,277.6,44.016666666666666
+zephyr-7b-beta,late afternoon,3.4592524923459447,248.59,38.9
+zephyr-7b-beta,evening,3.5673056403795878,273.3666666666667,43.5
+zephyr-7b-beta,late evening,3.45343524068594,176.23125,27.21875
+zephyr-7b-beta,midnight,3.7253047794103624,217.82083333333333,33.67916666666667
+zephyr-7b-beta,night,3.6954557319482166,216.55833333333334,33.46666666666667

pipeline/models.py

@@ -47,7 +47,7 @@ MODELS = [
         cost_per_million_input_tokens=1,
         cost_per_million_output_tokens=2,
         # https://learn.microsoft.com/en-us/answers/questions/1356487/what-is-the-exact-maximum-input-tokens-of-azure-gp
-        input_size=4096
+        input_size=4096,
     ),
     Model(
         "gpt-4-turbo",
@@ -150,4 +150,4 @@ MODELS = [
 MODELS = [model for model in MODELS 
           if model.model_name=="together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1"
           or model.model_name=="huggingface/HuggingFaceH4/zephyr-7b-beta"]
-"""
+"""