Spaces:
Sleeping
Sleeping
import pandas as pd | |
import numpy as np | |
data = { | |
"Models": [ | |
"CodeGen-16B-Multi", | |
"StarCoder-15B", | |
"StarCoderBase-15B", | |
"StarCoderBase-7B", | |
"StarCoderBase-3B", | |
"Replit-2.7B", | |
"SantaCoder-1.1B", | |
"StarCoderBase-1.1B", | |
"CodeGen25-7B-mono", | |
"CodeGen25-7B-multi", | |
], | |
"Size (B)": [16, 15, 15, 7, 3, 2.7, 1.1, 1.1, 7, 7], | |
"Throughput (tokens/s)": [17.20, 38.60, 44.20, 43.10, 50.00, 42.20, 50.80, 71.40, 34.10, 32.60], | |
"Throughput (tokens/s) bs=50": [0.00, 1490.00, 1460.00, 1700.00, 1770.00, 577.00, 2270.00, 2360.00, 687.00, 680.00], | |
"Seq_length": [2048, 8192, 8192, 8192, 8192, 2048, 2048, 8192, 2048, 2048], | |
"#Languages": [6, 86, 86, 86, 86, 20, 3, 86, 86, 86], | |
"humaneval-python": [19.26, 33.57, 30.35, 28.37, 21.50, 20.12, 18.12, 15.17, 33.08, 28.70], | |
"java": [22.20, 30.22, 28.53, 24.44, 19.25, 21.39, 15.00, 14.20, 19.75, 26.01], | |
"javascript": [19.15, 30.79, 31.70, 27.35, 21.32, 20.18, 15.47, 13.38, 23.22, 26.27], | |
"cpp": [21.00, 31.55, 30.56, 23.30, 19.43, 20.37, 6.20, 11.68, 18.62, 25.75], | |
"php": [8.37, 26.08, 26.75, 22.12, 18.55, 16.14, 1.50, 9.94, 16.75, 21.98], | |
"julia": [0.00, 23.02, 21.09, 21.77, 16.10, 1.24, 0.00, 11.31, 4.65, 19.11], | |
"d": [7.68, 13.57, 10.01, 8.10, 4.97, 6.41, 0.00, 4.65, 4.32, 8.84], | |
"lua": [8.50, 23.89, 26.61, 23.35, 18.04, 2.11, 0.10, 12.52, 6.75, 23.44], | |
"r": [6.45, 15.50, 10.18, 14.51, 10.10, 7.20, 0.00, 5.73, 4.41, 11.59], | |
"ruby": [0.00, 1.24, 17.25, 18.39, 3.93, 10.75, 0.00, 0.31, 0.00, 17.72], | |
"racket": [0.66, 0.07, 11.77, 11.08, 7.87, 3.22, 0.00, 5.03, 4.07, 10.37], | |
"rust": [4.21, 21.84, 24.46, 22.60, 16.32, 15.19, 2.00, 10.24, 7.83, 21.84], | |
"swift": [1.25, 22.74, 16.74, 15.10, 9.98, 5.88, 0.70, 3.92, 1.71, 16.62], | |
"Peak Memory (MB)": [32890, 33461, 32366, 16512, 8414, 7176, 4602, 4586, 15336, 15336], | |
} | |
df = pd.DataFrame(data).set_index("Models") | |
df = df.reset_index().rename(columns={"index": "Language"}) | |
temp_df = df.copy() | |
temp_df = temp_df.apply(pd.to_numeric, errors="coerce") | |
temp_df[temp_df <= 1] = np.nan | |
# get average over all columns from index 4 until -1 not included | |
temp_ = temp_df.iloc[:, 6:-1] | |
print(temp_) | |
df.insert(2, "Average score", temp_.mean(axis=1).round(2)) | |
#print average score | |
print(df["Average score"]) | |
# sort with regard to column average | |
df = df.sort_values(by=["Average score"], ascending=False) | |
df.to_csv("/fsx/loubna/code/code-leaderboard/starcoder-models-eval/code_eval_board.csv", index=False) | |
print(df) | |