Last commit not found
| # %% | |
| import json | |
| import pandas as pd | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| import plotly.io as pio | |
| from plotly.subplots import make_subplots | |
| # %% | |
| # Read the json file | |
| # This file processes the llm_gpu_benchmark.json file in the tmp/inputs folder | |
| # File is generated using the command | |
| # curl -sSL https://raw.githubusercontent.com/h2oai/h2ogpt/main/benchmarks/perf.json | jq -s '.' > llm_gpu_benchmarks.json | |
| with open('llm_gpu_benchmarks.json') as f: | |
| data = json.load(f) | |
| del f | |
| # %% | |
| # Read the json file into a dataframe | |
| df = pd.json_normalize(data) | |
| del data | |
| # %% | |
| # Process the dataframe | |
| # Drop columns that are not needed | |
| df.drop(columns=['task', 'ngpus', 'reps', 'date', 'git_sha', 'transformers', 'bitsandbytes', 'cuda', 'hostname', | |
| 'summarize_input_len_bytes'], inplace=True) | |
| # Rename columns | |
| df.rename(columns={'n_gpus': 'gpu_count'}, inplace=True) | |
| # Split the gpu column into gpu and gpu_memory | |
| df["gpu_name"] = df.gpus.str.extract(r'[1-9] x ([\w\- ]+) .+') | |
| df["gpu_memory_gb"] = round( | |
| pd.to_numeric(df.gpus.str.extract(r'[\w ]+ \(([\d]+) .+', expand=False), errors='coerce') / 1024) | |
| df["gpu_memory_gb"] = df["gpu_memory_gb"].astype('Int64') | |
| df.drop(columns=['gpus'], inplace=True) | |
| # Manage gpu_names | |
| df.gpu_name = df.gpu_name.str.replace('NVIDIA ', '') | |
| df.gpu_name = df.gpu_name.str.replace('GeForce ', '') | |
| df.gpu_name = df.gpu_name.str.replace('A100-SXM4-80GB', 'A100 SXM4') | |
| df.gpu_name = df.gpu_memory_gb.astype(str) + "-" + df.gpu_name | |
| # Remove CPUs | |
| df.drop(df[df.gpu_name.isnull()].index, inplace=True) | |
| # %% | |
| # Remove duplicate rows | |
| df.drop_duplicates(['backend', 'base_model', 'bits', 'gpu_count', 'gpu_name'], inplace=True) | |
| # %% Add baseline comparison columns | |
| # Looking at the CPU data for 4, 8, and 16 bit quantization values for the benchmark we are simplifying it to a single | |
| # value | |
| cpu_summary_out_throughput = 1353 / 1216 # bytes/second (calculated from summarize_output_len_bytes / summarize_time) | |
| cpu_generate_out_throughput = 849 / 180 # bytes/second (calculated from generate_output_len_bytes / generate_time) | |
| # add GPU throughput columns | |
| df["summary_out_throughput"] = df.summarize_output_len_bytes / df.summarize_time | |
| df["generate_out_throughput"] = df.generate_output_len_bytes / df.generate_time | |
| # add GPU throughput boost columns | |
| df["summary_out_throughput_normalize"] = df.summary_out_throughput / cpu_summary_out_throughput | |
| df["generate_out_throughput_normalize"] = df.generate_out_throughput / cpu_generate_out_throughput | |
| # %% | |
| # df.to_excel('tmp/scratchpad/output/llm_gpu_benchmarks.xlsx', index=False) | |
| # %% | |
| pio.renderers.default = "browser" | |
| # %% | |
| bits_bar_colors = {'4': px.colors.qualitative.D3[0], | |
| '8': px.colors.qualitative.D3[1], | |
| '16': px.colors.qualitative.D3[2]} | |
| backends = list(df.backend.unique()) | |
| base_models = list(df.base_model.unique()) | |
| n_gpus = list(df.gpu_count.unique()) | |
| # %% | |
| for backend in backends: | |
| # for backend in ['transformers']: | |
| fig_bar = make_subplots(rows=len(n_gpus), | |
| cols=len(base_models) * 2, | |
| shared_xaxes='all', | |
| shared_yaxes='columns', | |
| start_cell="top-left", | |
| vertical_spacing=0.1, | |
| print_grid=False, | |
| row_titles=[f'{gpu_count} GPUs' for gpu_count in n_gpus], | |
| column_titles=['llama2-7b-chat Summarization', 'llama2-7b-chat Generation', | |
| 'llama2-13b-chat Summarization', 'llama2-13b-chat Generation', | |
| 'llama2-70b-chat Summarization', 'llama2-70b-chat Generation'],) | |
| # for base_model in ['h2oai/h2ogpt-4096-llama2-7b-chat']: | |
| for base_model in base_models: | |
| for gpu_count in n_gpus: | |
| for bits in sorted(df.bits.unique()): | |
| sub_df = df[(df.backend == backend) & | |
| (df.base_model == base_model) & | |
| (df.gpu_count == gpu_count) & | |
| (df.bits == bits)].sort_values(by='gpu_name') | |
| fig_bar.add_trace(go.Bar(x=sub_df.summary_out_throughput_normalize, | |
| y=sub_df.gpu_name, | |
| name=f'sum-{bits} bits', | |
| legendgroup=f'sum-{bits} bits', | |
| marker=dict(color=bits_bar_colors[f'{bits}']), | |
| orientation='h'), | |
| row=n_gpus.index(gpu_count) + 1, | |
| col=base_models.index(base_model) * 2 + 1) | |
| fig_bar.add_trace(go.Bar(x=sub_df.generate_out_throughput_normalize, | |
| y=sub_df.gpu_name, | |
| name=f'gen-{bits} bits', | |
| legendgroup=f'gen-{bits} bits', | |
| marker=dict(color=bits_bar_colors[f'{bits}']), | |
| orientation='h'), | |
| row=list(n_gpus).index(gpu_count) + 1, | |
| col=list(base_models).index(base_model) * 2 + 2) | |
| fig_bar.update_layout(plot_bgcolor='rgb(250,250,250)', | |
| showlegend=True, | |
| barmode="group") | |
| # fig_bar.show() | |
| fig_bar.write_html(f'llm_gpu_benchmark_{backend}.html', include_plotlyjs='cdn') |