app.py

import re

import gradio as gr
import pandas as pd
import plotly
from pandas.api.types import is_numeric_dtype

from pipeline.config import LLMBoardConfig, QueriesConfig

README = """
This projects compares different large language models and their providers for real time applications and mass data processing. 
While other boards compare LLMs on different human intelligence tasks we focus on features related to business and engineering aspects such as response times, pricing and data streaming capabilities. 

We chose a task of newspaper articles summarization as it represents a very standard type of task where model has to understand unstructured natural language text, process it and output text in a specified format.
For this version we chose English, Polish and Japanese languages, with Japanese representing languages using logographic alphabets. This will verify the effectiveness of the LLM for different language groups.

We used the following prompt:

```
{}
```

Where language variable is original language of the text as we wanted to avoid the model translating the text to English during summarization.

The model was asked to return the output in three formats: markdown, json and function call. Note that currently function calls are only supported by Open AI API.
To do that we added following text to the query:

{}

When  measuring execution time we used `time.time()` result saved to variable  before making the call to API and compared it to `time.time()` result  after receiving the results. We used litellm python library for all of  the models which naturally adds some overhead compared to pure curl  calls.

In order to count tokens we split the output string by  whitespace `\w` regex character. For data which was impossible to obtain through the API, such as model sizes we only used official sources such  as developers' release blogs and their documentation.

When it comes to pricing most providers charge per token count, while HuggingFace Endpoints allow the user to choose machine type and host the model repository on it. The user is then charged by the running time of the machine. In this project we attempted to use HF Endpoints as much as possible due to their popularity and transparency of how the model is executed.
"""

time_periods_explanation_df = pd.DataFrame({
    'time_of_day': ["early morning", "morning", "afternoon", "late afternoon", "evening", "late evening", "midnight", "night"],
    'hour_range': ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"]
})

queries_config = QueriesConfig()

output_types_df = pd.DataFrame({
    "Output Type": queries_config.query_template.keys(),
    "Added text": queries_config.query_template.values()
})

summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
time_of_day_comparison_df = pd.read_csv("data/2024-02-06 09:49:19.637072_time_of_day_comparison.csv")
general_plots = pd.read_csv("data/general_plots.csv")
model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv")
time_of_day_plots = pd.read_csv("data/time_of_day_plots.csv")
output_plots = pd.read_csv("data/output_plots.csv")

searched_model_name = ""
collapse_languages = False
collapse_output_method = False


def filter_dataframes(input: str):
    global searched_model_name
    input = input.lower()
    searched_model_name = input
    return dataframes()


def collapse_languages_toggle():
    global collapse_languages
    if collapse_languages:
        collapse_languages = False
        button_text = "Collapse languages"
    else:
        collapse_languages = True
        button_text = "Un-collapse languages"
    return dataframes()[0], button_text


def collapse_output_method_toggle():
    global collapse_output_method
    if collapse_output_method:
        collapse_output_method = False
        button_text = "Collapse output method"
    else:
        collapse_output_method = True
        button_text = "Un-collapse output method"
    return dataframes()[0], button_text


def dataframes():
    global collapse_languages, collapse_output_method, searched_model_name, summary_df, time_of_day_comparison_df, model_costs_df

    summary_df_columns = summary_df.columns.to_list()
    group_columns = LLMBoardConfig().group_columns.copy()
    if collapse_languages:
        summary_df_columns.remove("language")
        group_columns.remove("language")
    if collapse_output_method:
        summary_df_columns.remove("template_name")
        group_columns.remove("template_name")

    summary_df_processed = summary_df[summary_df_columns].groupby(by=group_columns).mean().reset_index()
    return (
        dataframe_style(summary_df_processed[summary_df_processed.model.str.lower().str.contains(searched_model_name)]),
        dataframe_style(
            time_of_day_comparison_df[time_of_day_comparison_df.model.str.lower().str.contains(searched_model_name)]
        ),
        dataframe_style(model_costs_df[model_costs_df.model.str.lower().str.contains(searched_model_name)]),
    )


def dataframe_style(df: pd.DataFrame):
    df = df.copy()
    df.columns = [snake_case_to_title(column) for column in df.columns]
    column_formats = {}
    for column in df.columns:
        if is_numeric_dtype(df[column]):
            if column == "execution_time":
                column_formats[column] = "{:.4f}"
            else:
                column_formats[column] = "{:.2f}"
    df = df.style.format(column_formats, na_rep="")
    return df


def snake_case_to_title(text):
    # Convert snake_case to title-case
    words = re.split(r"_", text)
    title_words = [word.capitalize() for word in words]
    return " ".join(title_words)


filter_textbox = gr.Textbox(label="Model name part", scale=2)
filter_button = gr.Button("Filter dataframes by model name", scale=1)
collapse_languages_button = gr.Button("Collapse languages")
collapse_output_method_button = gr.Button("Collapse output method")
last_textbox = 0

with gr.Blocks() as demo:
    gr.HTML("<h1>LLM Board</h1>")

    with gr.Row():
        filter_textbox.render()
        filter_button.render()
    
    with gr.Tab("About this project"):
        gr.Markdown(README.format(
            queries_config.base_query_template.replace("```", "'''"),
            output_types_df.to_markdown(index=False)
        ))
    with gr.Tab("General plots"):
        for index, row in general_plots.iterrows():
            plot = plotly.io.from_json(row["plot_json"])
            plot.update_layout(autosize=True)
            gr.Plot(plot, label=row["header"], scale=1)
            if pd.notna(row["description"]):
                gr.Markdown(str(row["description"]))
    with gr.Tab("Output characteristics"):
        with gr.Row():
            collapse_languages_button.render()
            collapse_output_method_button.render()
        summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Output characteristics")
        gr.Markdown("""\
This table compares output characteristics of different models which include execution time, output size and chunking of the output. Some providers and models don't support output chunking, in this case chunk related fields are left empty.

Execution time refers to averaged time needed to execute one query.

To count words we split the output string by whitespace `\w` regex character.

Chunk sizes are measured in the characters count.""")
        for index, row in output_plots.iterrows():
            plot = plotly.io.from_json(row["plot_json"])
            plot.update_layout(autosize=True)
            gr.Plot(plot, label=row["header"], scale=1)

    with gr.Tab("Preformance by time of the day"):
        # display only first plot for all models
        for index, row in time_of_day_plots[0:1].iterrows():
            plot = plotly.io.from_json(row["plot_json"])
            plot.update_layout(autosize=True)
            gr.Plot(plot, label=row["header"], scale=1)
        time_periods_explanation_ui = gr.DataFrame(dataframe_style(time_periods_explanation_df), label="Times of day ranges")
        time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day")
        gr.Markdown("""\
These measurements were made by testing the models using the same dataset as in the other comparisons every hour for 24 hours.
                    
Execution time refers to averaged time needed to execute one query.

Hours and times of day in the table and in the plot are based on Central European Time.
                    
Measurements were made during a normal work week.
""")
        # display rest of the plots 
        for index, row in time_of_day_plots[1:].iterrows():
            plot = plotly.io.from_json(row["plot_json"])
            plot.update_layout(autosize=True)
            gr.Plot(plot, label=row["header"], scale=1)

    with gr.Tab("Costs comparison"):
        models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")
        gr.Markdown(
            """\
Provider pricing column contains pricing from the website of the provider.

Hugging Face Inference Endpoints are charged by hour so to compare different providers together, 
for models hosted this way we calculated "Cost Per Token" column using data collected during the experiment.

Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
            """)
    filter_button.click(
        fn=filter_dataframes,
        inputs=filter_textbox,
        outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui],
        api_name="filter_dataframes",
    )
    filter_textbox.submit(
        fn=filter_dataframes,
        inputs=filter_textbox,
        outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui],
        api_name="filter_dataframes",
    )
    collapse_languages_button.click(
        fn=collapse_languages_toggle,
        outputs=[summary_ui, collapse_languages_button],
        api_name="collapse_languages_toggle",
    )
    collapse_output_method_button.click(
        fn=collapse_output_method_toggle,
        outputs=[summary_ui, collapse_output_method_button],
        api_name="collapse_output_method_toggle",
    )

demo.launch()