Update

app.py

@@ -0,0 +1,162 @@
+import re
+
+import gradio as gr
+import pandas as pd
+import plotly
+from pandas.api.types import is_numeric_dtype
+
+from pipeline.config import QueriesConfig, LLMBoardConfig
+from pipeline.models import models_costs
+
+README = """
+<br/><h2>About this project</h2>
+<p>
+This project analyses different models and providers from the perspective of an application developer.
+<br/>
+Models are asked to summarize a text in different languages and using different output formats with following prompt:
+</p>
+<pre>
+<code>{}</code>
+</pre>
+"""
+
+summary_df: pd.DataFrame = pd.read_csv("data/2024-01-25 13:30:17.207984_summary.csv")
+time_of_day_comparison_df = pd.read_csv("data/2024-01-25 13:30:20.959750_time_of_day_comparison.csv")
+general_plots = pd.read_csv("data/2024-01-25 12:22:00.759762_general_plot.csv")
+
+
+with open("data/time_of_day_plot.json", "r") as f:
+    time_of_day_plot = plotly.io.from_json(f.read())
+
+model_costs_df = models_costs()
+
+
+searched_model_name = ""
+collapse_languages = False
+collapse_output_method = False
+
+
+def filter_dataframes(input: str):
+    global searched_model_name
+    input = input.lower()
+    searched_model_name = input
+    return dataframes()
+
+
+def collapse_languages_toggle():
+    global collapse_languages
+    if collapse_languages:
+        collapse_languages = False
+        button_text = "Collapse languages"
+    else:
+        collapse_languages = True
+        button_text = "Un-collapse languages"
+    return dataframes()[0], button_text
+
+
+def collapse_output_method_toggle():
+    global collapse_output_method
+    if collapse_output_method:
+        collapse_output_method = False
+        button_text = "Collapse output method"
+    else:
+        collapse_output_method = True
+        button_text = "Un-collapse output method"
+    return dataframes()[0], button_text
+
+
+def dataframes():
+    global collapse_languages, collapse_output_method, searched_model_name, summary_df, time_of_day_comparison_df, model_costs_df
+
+    summary_df_columns = summary_df.columns.to_list()
+    group_columns = LLMBoardConfig().group_columns.copy()
+    if collapse_languages:
+        summary_df_columns.remove("language")
+        group_columns.remove("language")
+    if collapse_output_method:
+        summary_df_columns.remove("template_name")
+        group_columns.remove("template_name")
+
+    summary_df_processed = summary_df[summary_df_columns].groupby(by=group_columns).mean().reset_index()
+    return (
+        dataframe_style(summary_df_processed[summary_df_processed.model.str.lower().str.contains(searched_model_name)]),
+        dataframe_style(
+            time_of_day_comparison_df[time_of_day_comparison_df.model.str.lower().str.contains(searched_model_name)]
+        ),
+        dataframe_style(model_costs_df[model_costs_df.Model.str.lower().str.contains(searched_model_name)]),
+    )
+
+
+def dataframe_style(df: pd.DataFrame):
+    df = df.copy()
+    df.columns = [snake_case_to_title(column) for column in df.columns]
+    column_formats = {}
+    for column in df.columns:
+        if is_numeric_dtype(df[column]):
+            if column == "execution_time":
+                column_formats[column] = "{:.4f}"
+            else:
+                column_formats[column] = "{:.2f}"
+    df = df.style.format(column_formats, na_rep="")
+    return df
+
+
+def snake_case_to_title(text):
+    # Convert snake_case to title-case
+    words = re.split(r"_", text)
+    title_words = [word.capitalize() for word in words]
+    return " ".join(title_words)
+
+
+filter_textbox = gr.Textbox(label="Model name part")
+filter_button = gr.Button("Filter dataframes by model name")
+collapse_languages_button = gr.Button("Collapse languages")
+collapse_output_method_button = gr.Button("Collapse output method")
+last_textbox = 0
+
+with gr.Blocks() as demo:
+    gr.HTML("<h1>LLM Board</h1>" + README.format(QueriesConfig().base_query_template))
+
+    with gr.Row():
+        filter_textbox.render()
+        filter_button.render()
+    with gr.Tab("Basic information"):
+        for index, row in general_plots.iterrows():
+            gr.Plot(plotly.io.from_json(row["plot_json"]), label=row["description"])
+            gr.Markdown(str(row["comment"]))
+    with gr.Tab("Output characteristics"):
+        with gr.Row():
+            collapse_languages_button.render()
+            collapse_output_method_button.render()
+        summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Statistics")
+
+    with gr.Tab("Preformance by time of the day"):
+        time_of_day_comparison_ui = gr.DataFrame(time_of_day_comparison_df, label="Time of day")
+        time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot")
+
+    with gr.Tab("Costs comparison"):
+        models_costs_ui = gr.DataFrame(dataframe_style(models_costs()), label="Costs comparison")
+    filter_button.click(
+        fn=filter_dataframes,
+        inputs=filter_textbox,
+        outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui],
+        api_name="filter_dataframes",
+    )
+    filter_textbox.submit(
+        fn=filter_dataframes,
+        inputs=filter_textbox,
+        outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui],
+        api_name="filter_dataframes",
+    )
+    collapse_languages_button.click(
+        fn=collapse_languages_toggle,
+        outputs=[summary_ui, collapse_languages_button],
+        api_name="collapse_languages_toggle",
+    )
+    collapse_output_method_button.click(
+        fn=collapse_output_method_toggle,
+        outputs=[summary_ui, collapse_output_method_button],
+        api_name="collapse_output_method_toggle",
+    )
+
+demo.launch()

data

@@ -0,0 +1 @@
+{"data":[{"alignmentgroup":"True","hovertemplate":"Model=Mistral-7B-Instruct-v0.2\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mistral-7B-Instruct-v0.2","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"Mistral-7B-Instruct-v0.2","offsetgroup":"Mistral-7B-Instruct-v0.2","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[null,null,3.174540030956268],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=Mixtral-8x7B-Instruct-v0.1\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"Mixtral-8x7B-Instruct-v0.1","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"Mixtral-8x7B-Instruct-v0.1","offsetgroup":"Mixtral-8x7B-Instruct-v0.1","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[7.142197625471814],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","marker":{"color":"#00cc96","pattern":{"shape":""}},"name":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","offsetgroup":"TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[1.2902645373344421],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison","marker":{"color":"#ab63fa","pattern":{"shape":""}},"name":"chat-bison","offsetgroup":"chat-bison","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[3.890243631601334],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=chat-bison-32k\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"chat-bison-32k","marker":{"color":"#FFA15A","pattern":{"shape":""}},"name":"chat-bison-32k","offsetgroup":"chat-bison-32k","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[4.128177767992019],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gemini-pro\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gemini-pro","marker":{"color":"#19d3f3","pattern":{"shape":""}},"name":"gemini-pro","offsetgroup":"gemini-pro","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[4.72138064004937],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-3.5-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-3.5-turbo","marker":{"color":"#FF6692","pattern":{"shape":""}},"name":"gpt-3.5-turbo","offsetgroup":"gpt-3.5-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[4.666725277900696,4.351089119911194,5.336839800789242],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4","marker":{"color":"#B6E880","pattern":{"shape":""}},"name":"gpt-4","offsetgroup":"gpt-4","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[15.5218456586202],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=gpt-4-turbo\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"gpt-4-turbo","marker":{"color":"#FF97FF","pattern":{"shape":""}},"name":"gpt-4-turbo","offsetgroup":"gpt-4-turbo","orientation":"v","showlegend":true,"textposition":"auto","x":["morning"],"xaxis":"x","y":[14.008529031276703],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=llama-2-70b-chat\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"llama-2-70b-chat","marker":{"color":"#FECB52","pattern":{"shape":""}},"name":"llama-2-70b-chat","offsetgroup":"llama-2-70b-chat","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[2.1692867279052734,5.552149415016174,6.175082007679371],"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"Model=zephyr-7b-beta\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"zephyr-7b-beta","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"zephyr-7b-beta","offsetgroup":"zephyr-7b-beta","orientation":"v","showlegend":true,"textposition":"auto","x":["afternoon","evening","morning"],"xaxis":"x","y":[null,null,3.2387993240356447],"yaxis":"y","type":"bar"}],"layout":{"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Time of day"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Execution Time (s)"}},"legend":{"title":{"text":"Model"},"tracegroupgap":0},"title":{"text":"Execution time for different times of the day"},"barmode":"group"}}

pipeline/config.py

@@ -0,0 +1,62 @@
+from typing import List
+
+# this try/except is important for publishing to Hugging Face
+try:
+    from dagster import Config
+except ImportError:
+    Config = object
+
+class LLMBoardConfig(Config):
+    group_columns: List[str] = ["model", "language", "template_name"]
+    single_values_columns: List[str] = ["execution_time", "characters_count", "words_count"]
+    list_columns: List[str] = ["chunk_sizes", "chunk_generation_times", "chunk_generation_times_by_chunk_sizes"]
+    plot_dir: str = "./html/plots/"
+    plot_json_dir: str = "./data/"
+    saving_path: str = "data/"
+
+
+class QueriesConfig(Config):
+    base_query_template: str = """Summarize me this text, the summary should be in {language}
+```
+{text}
+```
+"""
+
+    query_template: dict = {
+        "markdown": """Return output as markdown""",
+        "json": """Return output as json in format:
+{
+"summary": "<summary">
+}""",
+        "call": """Return output by calling summary_result()""",
+    }
+
+
+class OpenAIConfig(Config):
+    mock: bool = False
+
+
+class QueriesDatasetConfig(Config):
+    dataset_name: str = "GEM/xlsum"
+    samples_per_measurement: int = 2
+    languages: List[str] = ["english", "japanese"]
+    query_config: QueriesConfig = QueriesConfig()
+
+
+class SummaryConfig(Config):
+    saving_path: str = "data/"
+
+
+class TimeOfDayComparisonConfig(Config):
+    saving_path: str = "data/"
+
+
+class GeneralPlotConfig(Config):
+    plots_dir: str = "./html/plots/"
+    saving_path: str = "data/"
+    endpoint_startup_time_minutes: int = 2
+    endpoint_cleanup_time_minutes: int = 2
+    seconds_per_token: float = 184 / 6
+    input_size: int = 100
+    expected_output_size: int = 50
+    queries: int = 1000

pipeline/models.py

@@ -0,0 +1,150 @@
+import os
+from dataclasses import dataclass
+
+import pandas as pd
+
+
+@dataclass
+class Model(object):
+    model_display_name: str
+    model_name: str
+    api_url: str
+    provider: str
+    hourly_cost: int = None
+    cost: str = None
+    supports_functions: str = False
+    size_billion_parameters: int = None  # in billion paramters
+    cost_per_million_tokens: int = None
+    cost_per_million_input_tokens: int = None
+    cost_per_million_output_tokens: int = None
+
+    def __post_init__(self):
+        self.cost_per_million_input_tokens = self.cost_per_million_input_tokens or self.cost_per_million_tokens
+        self.cost_per_million_output_tokens = self.cost_per_million_output_tokens or self.cost_per_million_tokens
+        if not self.cost and self.hourly_cost:
+            self.cost = f"${self.hourly_cost} / hour"
+        if not self.cost and self.cost_per_million_tokens:
+            self.cost = f"${self.cost_per_million_tokens} / 1M tokens"
+        elif not self.cost and self.cost_per_million_input_tokens and self.cost_per_million_output_tokens:
+            self.cost = f"${self.cost_per_million_input_tokens} / 1M input tokens, ${self.cost_per_million_output_tokens} / 1M output tokens"
+
+
+env = os.environ
+
+MODELS = [
+    # source: https://openai.com/pricing
+    # converted costs from dollar/1K tokens to dollar/1M for readability and together_ai comparability
+    Model(
+        "gpt-3.5-turbo",
+        "gpt-3.5-turbo",
+        None,
+        "OpenAI",
+        supports_functions=True,
+        cost_per_million_input_tokens=1,
+        cost_per_million_output_tokens=2,
+    ),
+    Model(
+        "gpt-4-turbo",
+        "gpt-4-1106-preview",
+        None,
+        "OpenAI",
+        supports_functions=True,
+        cost_per_million_input_tokens=10,
+        cost_per_million_output_tokens=30,
+    ),
+    Model(
+        "gpt-4",
+        "gpt-4",
+        None,
+        "OpenAI",
+        supports_functions=True,
+        cost_per_million_input_tokens=30,
+        cost_per_million_output_tokens=60,
+    ),
+    # we don't test gpt-4-32k because the tasks don't reach gpt-4 limitations
+    Model(
+        "gpt-3.5-turbo",
+        "gpt-3.5-turbo",
+        None,
+        "OpenAI",
+        supports_functions=True,
+        cost_per_million_input_tokens=1,
+        cost_per_million_output_tokens=2,
+    ),
+    # source: https://www.together.ai/pricing
+    Model(
+        "llama-2-70b-chat",
+        "together_ai/togethercomputer/llama-2-70b-chat",
+        None,
+        "Together AI",
+        cost_per_million_tokens=0.2,
+    ),
+    Model(
+        "Mixtral-8x7B-Instruct-v0.1",
+        "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1",
+        None,
+        "Together AI",
+        size_billion_parameters=8 * 7,
+        cost_per_million_tokens=0.9,
+    ),
+    # taken from endpoint pages
+    Model(
+        "zephyr-7b-beta",
+        "huggingface/HuggingFaceH4/zephyr-7b-beta",
+        env["ZEPHYR_7B_BETA_URL"],
+        "Hugging Face Inference Endpoint",
+        hourly_cost=1.30,
+        size_billion_parameters=7,
+    ),
+    Model(
+        "Mistral-7B-Instruct-v0.2",
+        "huggingface/mistralai/Mistral-7B-Instruct-v0.2",
+        env["MISTRAL_7B_BETA_URL"],
+        "Hugging Face Inference Endpoint",
+        hourly_cost=1.30,
+        size_billion_parameters=7,
+    ),
+    Model(
+        "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+        "huggingface/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+        env["TINY_LLAMA_URL"],
+        "Hugging Face Inference Endpoint",
+        hourly_cost=0.60,
+        size_billion_parameters=1.1,
+    ),
+    Model(
+        "gemini-pro",
+        "gemini-pro",
+        None,
+        "Google VertexAI",
+        # https://ai.google.dev/pricing
+        cost="$0.25 / 1M input characters, $0.5 / 1K output characters (60 queries per minute are free)",
+        cost_per_million_input_tokens=0.25,
+        cost_per_million_output_tokens=0.5,
+    ),
+    Model(
+        "chat-bison",
+        "chat-bison",
+        None,
+        "Google VertexAI",
+        # https://cloud.google.com/vertex-ai/docs/generative-ai/pricing
+        cost_per_million_input_tokens=0.25,
+        cost_per_million_output_tokens=0.5,
+    ),
+    Model(
+        "chat-bison-32k",
+        "chat-bison-32k",
+        None,
+        "Google VertexAI",
+        # https://cloud.google.com/vertex-ai/docs/generative-ai/pricing
+        cost_per_million_input_tokens=0.25,
+        cost_per_million_output_tokens=0.5,
+    ),
+]
+
+
+def models_costs():
+    return pd.DataFrame(
+        [(model.model_display_name, model.provider, model.cost) for model in MODELS],
+        columns=["Model", "Provider", "Cost"],
+    )

requirements.txt

@@ -0,0 +1,2 @@
+pandas
+plotly