import re import gradio as gr import pandas as pd import plotly from pandas.api.types import is_numeric_dtype from pipeline.config import LLMBoardConfig README = """ This projects compares different large language models and their providers for real time applications and mass data processing. While other boards compare LLMs on different human intelligence tasks we focus on features related to business and engineering aspects such as response times, pricing and data streaming capabilities. We chose a task of newspaper articles summarization as it represents a very standard type of task where model has to understand unstructured natural language text, process it and output text in a specified format. For this version we chose English, Polish and Japanese languages, with Japanese representing languages using logographic alphabets. This will verify the effectiveness of the LLM for different language groups. We used the following prompt: ``` Summarize me this text, the summary should be in {language} ``` Where language variable is original language of the text as we wanted to avoid the model translating the text to English during summarization. The model was asked to return the output in three formats: markdown, json and function call. Note that currently function calls are only supported by Open AI API. To do that we added following text to the query: ``` ... ``` When measuring execution time we used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results. We used litellm python library for all of the models which naturally adds some overhead compared to pure curl calls. In order to count tokens we split the output string by whitespace \w regex character. For data which was impossible to obtain through the API, such as model sizes we only used official sources such as developers' release blogs and their documentation. When it comes to pricing most providers charge per token count, while HuggingFace Endpoints allow the user to choose machine type and host the model repository on it. The user is then charged by the running time of the machine. In this project we attempted to use HF Endpoints as much as possible due to their popularity and transparency of how the model is executed. """ time_periods_explanation_df = pd.DataFrame({ 'name': ["early morning", "morning", "afternoon", "late afternoon", "evening", "late evening", "midnight", "night"], 'hour_range': ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"] }) summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv") time_of_day_comparison_df = pd.read_csv("data/2024-02-06 09:49:19.637072_time_of_day_comparison.csv") general_plots = pd.read_csv("data/2024-02-05 12:03:42.452218_general_plot.csv") model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv") with open("data/time_of_day_plot.json", "r") as f: time_of_day_plot = plotly.io.from_json(f.read()) time_of_day_plot.update_layout(autosize=True) searched_model_name = "" collapse_languages = False collapse_output_method = False def filter_dataframes(input: str): global searched_model_name input = input.lower() searched_model_name = input return dataframes() def collapse_languages_toggle(): global collapse_languages if collapse_languages: collapse_languages = False button_text = "Collapse languages" else: collapse_languages = True button_text = "Un-collapse languages" return dataframes()[0], button_text def collapse_output_method_toggle(): global collapse_output_method if collapse_output_method: collapse_output_method = False button_text = "Collapse output method" else: collapse_output_method = True button_text = "Un-collapse output method" return dataframes()[0], button_text def dataframes(): global collapse_languages, collapse_output_method, searched_model_name, summary_df, time_of_day_comparison_df, model_costs_df summary_df_columns = summary_df.columns.to_list() group_columns = LLMBoardConfig().group_columns.copy() if collapse_languages: summary_df_columns.remove("language") group_columns.remove("language") if collapse_output_method: summary_df_columns.remove("template_name") group_columns.remove("template_name") summary_df_processed = summary_df[summary_df_columns].groupby(by=group_columns).mean().reset_index() return ( dataframe_style(summary_df_processed[summary_df_processed.model.str.lower().str.contains(searched_model_name)]), dataframe_style( time_of_day_comparison_df[time_of_day_comparison_df.model.str.lower().str.contains(searched_model_name)] ), dataframe_style(model_costs_df[model_costs_df.model.str.lower().str.contains(searched_model_name)]), ) def dataframe_style(df: pd.DataFrame): df = df.copy() df.columns = [snake_case_to_title(column) for column in df.columns] column_formats = {} for column in df.columns: if is_numeric_dtype(df[column]): if column == "execution_time": column_formats[column] = "{:.4f}" else: column_formats[column] = "{:.2f}" df = df.style.format(column_formats, na_rep="") return df def snake_case_to_title(text): # Convert snake_case to title-case words = re.split(r"_", text) title_words = [word.capitalize() for word in words] return " ".join(title_words) filter_textbox = gr.Textbox(label="Model name part") filter_button = gr.Button("Filter dataframes by model name") collapse_languages_button = gr.Button("Collapse languages") collapse_output_method_button = gr.Button("Collapse output method") last_textbox = 0 with gr.Blocks() as demo: gr.HTML("

LLM Board

") with gr.Row(): filter_textbox.render() filter_button.render() with gr.Tab("About this project"): gr.Markdown(README) with gr.Tab("General plots"): for index, row in general_plots.iterrows(): plot = plotly.io.from_json(row["plot_json"]) plot.update_layout(autosize=True) gr.Plot(plot, label=row["description"], scale=1) if pd.notna(row["comment"]): gr.Markdown(str(row["comment"])) with gr.Tab("Output characteristics"): with gr.Row(): collapse_languages_button.render() collapse_output_method_button.render() summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Statistics") with gr.Tab("Preformance by time of the day"): time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day") time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot", scale=1) time_periods_explanation_ui = gr.DataFrame(dataframe_style(time_periods_explanation_df), label="Times of day ranges") gr.Markdown(""" These measurements were made by testing the models using the same dataset as in the other comparisons every hour for 24 hours. Hours and times of day in the table and in the plot are based on Central European Time. Measurements were made during a normal work week. """) with gr.Tab("Costs comparison"): models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison") gr.Markdown( 'Note that pause and resume time cost was not included in the "Cost per token" column calculation for the models billed hourly.' ) filter_button.click( fn=filter_dataframes, inputs=filter_textbox, outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui], api_name="filter_dataframes", ) filter_textbox.submit( fn=filter_dataframes, inputs=filter_textbox, outputs=[summary_ui, time_of_day_comparison_ui, models_costs_ui], api_name="filter_dataframes", ) collapse_languages_button.click( fn=collapse_languages_toggle, outputs=[summary_ui, collapse_languages_button], api_name="collapse_languages_toggle", ) collapse_output_method_button.click( fn=collapse_output_method_toggle, outputs=[summary_ui, collapse_output_method_button], api_name="collapse_output_method_toggle", ) demo.launch()