from dataclasses import dataclass, make_dataclass from enum import Enum import pandas as pd def fields(raw_class): return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] # These classes are for user facing column names, # to avoid having to change them all around the code # when a modif is needed @dataclass class ColumnContent: name: str type: str displayed_by_default: bool hidden: bool = False never_hidden: bool = False ## Leaderboard columns auto_eval_column_dict = [] # Init auto_eval_column_dict.append( ["method_name", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)] ) auto_eval_column_dict.append(["model_name", ColumnContent, ColumnContent("Base Model", "markdown", True)]) # Scores auto_eval_column_dict.append(["score", ColumnContent, ColumnContent("Score", "number", True)]) auto_eval_column_dict.append(["full_pass_count", ColumnContent, ColumnContent("Repaired", "number", True)]) auto_eval_column_dict.append(["fast_pass_count", ColumnContent, ColumnContent("Repaired (Fast)", "number", False)]) auto_eval_column_dict.append(["with_hint", ColumnContent, ColumnContent("Hint", "str", True)]) auto_eval_column_dict.append(["attempts", ColumnContent, ColumnContent("Number of Attempts", "number", False)]) auto_eval_column_dict.append( ["full_pass_count_crash", ColumnContent, ColumnContent("Repaired (Crash)", "number", True)] ) auto_eval_column_dict.append( ["full_pass_count_miscompilation", ColumnContent, ColumnContent("Repaired (Miscompilation)", "number", True)] ) auto_eval_column_dict.append(["full_pass_count_hang", ColumnContent, ColumnContent("Repaired (Hang)", "number", True)]) auto_eval_column_dict.append( ["build_success_rate", ColumnContent, ColumnContent("Build Success Rate (%)", "number", False)] ) auto_eval_column_dict.append(["mttr", ColumnContent, ColumnContent("MTTR (min)", "number", True)]) auto_eval_column_dict.append(["sample_count", ColumnContent, ColumnContent("Average Sample Count", "number", False)]) # We use make dataclass to dynamically fill the scores from Tasks AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True) # Column selection COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]