update

app.py

@@ -11,17 +11,23 @@ from src.about import (
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
+    nc_tasks
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
+    COLS_NC,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
     TYPES,
-    AutoEvalColumn,
+    AutoEvalColumn_NodeClassification,
+    #AutoEvalColumn,
     ModelType,
+    TASK_LIST,
+    OFFICIAL,
+    HONOR,
     fields,
     WeightType,
     Precision
@@ -50,44 +56,32 @@ except Exception:
     restart_space()
 
 
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+original_df = get_leaderboard_df(EVAL_REQUESTS_PATH, nc_tasks)
 leaderboard_df = original_df.copy()
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
+    #filtered_df = filter_models(hidden_df, size_query, show_deleted)
+    filtered_df = filter_queries(query, hidden_df)
     df = select_columns(filtered_df, columns)
     return df
 
 
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
+    return df[(df[AutoEvalColumn_NodeClassification.model.name].str.contains(query, case=False))]
 
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
+        "Model"
     ]
     # We use COLS to maintain sorting
     filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
+        always_here_cols + [c for c in COLS_NC if c in df.columns and c in columns] 
     ]
     return filtered_df
 
@@ -105,40 +99,39 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
             filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+                subset=[AutoEvalColumn_NodeClassification.model.name]
             )
 
     return filtered_df
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, size_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
         filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+        filtered_df = df[df[AutoEvalColumn_NodeClassification.still_on_hub.name] == True]
 
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+    #type_emoji = [t[0] for t in type_query]
+    #filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    #filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    params_column = pd.to_numeric(df[AutoEvalColumn_NodeClassification.params.name], errors="coerce")
     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
     filtered_df = filtered_df.loc[mask]
 
     return filtered_df
 
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Node Classification Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -151,52 +144,27 @@ with demo:
                         shown_columns = gr.CheckboxGroup(
                             choices=[
                                 c.name
-                                for c in fields(AutoEvalColumn)
+                                for c in fields(AutoEvalColumn_NodeClassification)
                                 if not c.hidden and not c.never_hidden
                             ],
                             value=[
                                 c.name
-                                for c in fields(AutoEvalColumn)
+                                for c in fields(AutoEvalColumn_NodeClassification)
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
                             label="Select columns to show",
                             elem_id="column-select",
                             interactive=True,
                         )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
+                    
+            print(leaderboard_df)
+            print(fields(AutoEvalColumn_NodeClassification))
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                    [c.name for c in fields(AutoEvalColumn_NodeClassification) if c.never_hidden]
                     + shown_columns.value
                 ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                headers=[c.name for c in fields(AutoEvalColumn_NodeClassification) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -205,8 +173,8 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
-                headers=COLS,
+                value=original_df[COLS_NC],
+                headers=COLS_NC,
                 datatype=TYPES,
                 visible=False,
             )
@@ -215,116 +183,86 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_columns]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
                         search_bar,
                     ],
                     leaderboard_table,
                     queue=True,
                 )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
+                    author_name_textbox = gr.Textbox(label="Your name")
+                    email_textbox = gr.Textbox(label="Your email")
+                    relbench_version_textbox = gr.Textbox(label="RelBench version")
+
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
+                    
+                    '''
+                    dataset_name_textbox = gr.Dropdown(
+                        choices=[t.value.name for t in TASK_LIST],
+                        label="Task name (e.g. rel-amazon-user-churn)",
                         multiselect=False,
                         value=None,
                         interactive=True,
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
+                    '''
+                    
+                    official_or_not = gr.Dropdown(
+                        choices=[i.value.name for i in OFFICIAL],
+                        label="Is it an official submission?",
                         multiselect=False,
-                        value="float16",
+                        value=None,
                         interactive=True,
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
+                    paper_url_textbox = gr.Textbox(label="Paper URL Link")
+                    github_url_textbox = gr.Textbox(label="GitHub URL Link")
+
+                with gr.Column():
+                    test_performance = gr.TextArea(label="Test set performance, use {task: [mean,std]} format e.g. {'rel-amazon/user-churn': [0.352,0.023], 'rel-amazon/user-ltv': [0.304,0.022], ...}")
+                    valid_performance = gr.TextArea(label="Validation set performance, use {task: [mean,std]} format e.g. {'rel-amazon/user-churn': [0.352,0.023], 'rel-amazon/user-ltv': [0.304,0.022], ...}")
+                    parameters_textbox = gr.Textbox(label="Number of parameters")
+                    honor_code = gr.Dropdown(
+                        choices=[i.value.name for i in HONOR],
+                        label="Click here to agree to the honor code",
                         multiselect=False,
-                        value="Original",
+                        value=None,
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
+                    author_name_textbox,
+                    email_textbox,
+                    relbench_version_textbox,
                     model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    official_or_not,
+                    test_performance,
+                    valid_performance,
+                    paper_url_textbox,
+                    github_url_textbox,
+                    parameters_textbox,
+                    honor_code,
                 ],
                 submission_result,
             )

src/about.py

@@ -18,55 +18,57 @@ class Tasks(Enum):
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 
-
+class nc_tasks(Enum):
+    task0 = Task("rel-amazon/user-churn", "auroc", "user-churn")
+    task1 = Task("rel-amazon/item-churn", "auroc", "item-churn")
+    task2 = Task("rel-avito/user-clicks", "auroc", "user-clicks")
+    task3 = Task("rel-avito/user-visits", "auroc", "user-visits")
+    task4 = Task("rel-hm/user-churn", "auroc", "hm-user-churn")
+    task5 = Task("rel-stack/user-badge", "auroc", "user-badge")
+    task6 = Task("rel-stack/user-engagement", "auroc", "user-engagement")
+    task7 = Task("rel-f1/driver-dnf", "auroc", "driver-dnf")
+    task8 = Task("rel-f1/driver-top3", "auroc", "driver-top3")
+    task9 = Task("rel-trial/study-outcome", "auroc", "study-outcome")
+    task10 = Task("rel-event/user-repeat", "auroc", "user-repeat")
+    task11 = Task("rel-event/user-ignore", "auroc", "user-ignore")
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<p align="center"><img src="https://relbench.stanford.edu/img/logo.png" alt="logo" width="400px" /></p>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+Relational Deep Learning is a new approach for end-to-end representation learning on data spread across multiple tables, such as in a relational database (see our vision paper). RelBench is the accompanying benchmark which seeks to facilitate efficient, robust and reproducible research in this direction. It comprises of a collection of realistic, large-scale, and diverse datasets structured as relational tables, along with machine learning tasks defined on them. It provides full support for data downloading, task specification and standardized evaluation in an ML-framework-agnostic manner. Additionally, there is seamless integration with PyTorch Geometric to load the data as a graph and train GNN models, and with PyTorch Frame to encode the various types of table columns. Finally, there is a leaderboard for tracking progress.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
+## Overview of RelBench
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+## Instruction to submit your model
+
+Once you have developed your model and got results, you can submit your test results to our leaderboards. For each dataset, we require you to submit the following information.
+
+- **Your name**: Primary contact's name
+- **Your email**: Primary contact's email 
+- **RelBench version**: The RelBench version used to conduct the experiments. 
+- **Model name**: The name of the method. This is an unique identifier of the model. Please be distinct with any existing model names. It will be overriden if the same model name is submitted.
+- **Task name**: The name of an Relbench dataset that you use to evaluate the method. Choose from the dropdown menus.
+- **Is it an official submission**: Whether the implementation is official (implementation by authors who proposed the method) or unofficial (re-implementation of the method by non-authors).
+- **Test performance**: Raw test performance output by RelBench model evaluators, where average and unbiased standard deviation must be taken over 5 different random seeds. You can either not fix random seeds at all, or use the random seeds from 0 to 4. We highly discourage you to tune the random seeds.
+- **Validation performance**: Validation performance of the model that is used to report the test performance above.
+- **Paper URL Link**: The original paper describing the method (arXiv link is recommended. paper needs not be peer-reviewed). If your method has any original component (e.g., even just combining existing methods XXX and YYY), you have to write a technical report describing it (e.g., how you exactly combined XXX and YYY).
+- **GitHub URL Link**: The Github repository or directory containining all code to reproduce the result. A placeholder repository is not allowed.
+- **Number of Parameters**: The number of parameters of your model, which can be calculated by sum(p.numel() for p in model.parameters()). If you use multi-stage training (e.g., apply node2vec and then MLP), please sum up all the parameters (both node2vec and MLP parameters).
+- **Honor code**: Please acknowledge that your submission adheres to all the ethical policies and your result is reproducible.
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@article{relbench,
+  title={Relational Deep Learning: Graph Representation Learning on Relational Tables},
+  author={Matthias Fey, Weihua Hu, Kexin Huang, Jan Eric Lenssen, Rishabh Ranjan, Joshua Robinson, Rex Ying, Jiaxuan You, Jure Leskovec},
+  year={2023}
+}
 """

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, nc_tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -43,6 +43,21 @@ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sh
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
+auto_eval_column_dict_nc = []
+auto_eval_column_dict_nc.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict_nc.append(["average_rank", ColumnContent, ColumnContent("Average Rank⬆️", "number", True)])
+for task in nc_tasks:
+    auto_eval_column_dict_nc.append(['_'.join(task.value.col_name.split('-')), ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+auto_eval_column_dict_nc.append(["author", ColumnContent, ColumnContent("Author", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nc.append(["email", ColumnContent, ColumnContent("Email", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nc.append(["Paper_URL", ColumnContent, ColumnContent("Paper URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nc.append(["Github_URL", ColumnContent, ColumnContent("Github URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nc.append(["Time", ColumnContent, ColumnContent("Time", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nc.append(["num_of_Params", ColumnContent, ColumnContent("# of Params", "markdown", True, never_hidden=False)])
+
+AutoEvalColumn_NodeClassification = make_dataclass("AutoEvalColumn_NodeClassification", auto_eval_column_dict_nc, frozen=True)
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -83,6 +98,58 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+class OFFICIAL(Enum):
+    official = ModelDetails("Official")
+    unofficial = ModelDetails("Unofficial")
+
+class HONOR(Enum):
+    yes = ModelDetails("Yes")
+    no = ModelDetails("No")
+
+class TASK_LIST(Enum):
+    amazon_user_churn = ModelDetails("rel-amazon-user-churn")
+    amazon_item_churn = ModelDetails("rel-amazon-item-churn")
+    amazon_user_ltv = ModelDetails("rel-amazon-user-ltv")
+    amazon_item_ltv = ModelDetails("rel-amazon-item-ltv")
+    amazon_user_item_purchase = ModelDetails("rel-amazon-user-item-purchase")
+    amazon_user_item_rate = ModelDetails("rel-amazon-user-item-rate")
+    amazon_user_item_review = ModelDetails("rel-amazon-user-item-review")
+
+    # rel-stack
+    stack_user_engagement = ModelDetails("rel-stack-user-engagement")
+    stack_user_badge = ModelDetails("rel-stack-user-badge")
+    stack_post_votes = ModelDetails("rel-stack-post-votes")
+    stack_user_post_comment = ModelDetails("rel-stack-user-post-comment")
+    stack_user_post_related = ModelDetails("rel-stack-user-post-related")
+
+    # rel-trial
+    trial_study_outcome = ModelDetails("rel-trial-study-outcome")
+    trial_study_adverse = ModelDetails("rel-trial-study-adverse")
+    trial_site_success = ModelDetails("rel-trial-site-success")
+    trial_condition_sponsor_run = ModelDetails("rel-trial-condition-sponsor-run")
+    trial_site_sponsor_run = ModelDetails("rel-trial-site-sponsor-run")
+
+    # rel-f1
+    f1_driver_position = ModelDetails("rel-f1-driver-position")
+    f1_driver_dnf = ModelDetails("rel-f1-driver-dnf")
+    f1_driver_top3 = ModelDetails("rel-f1-driver-top3")
+
+    # rel-hm
+    hm_user_churn = ModelDetails("rel-hm-user-churn")
+    hm_item_sales = ModelDetails("rel-hm-item-sales")
+    hm_user_item_purchase = ModelDetails("rel-hm-user-item-purchase")
+
+    # rel-event
+    event_user_repeat = ModelDetails("rel-event-user-repeat")
+    event_user_ignore = ModelDetails("rel-event-user-ignore")
+    event_user_attendance = ModelDetails("rel-event-user-attendance")
+
+    # rel-avito
+    avito_user_visits = ModelDetails("rel-avito-user-visits")
+    avito_user_clicks = ModelDetails("rel-avito-user-clicks")
+    avito_ads_clicks = ModelDetails("rel-avito-ads-clicks")
+    avito_user_ad_visit = ModelDetails("rel-avito-user-ad-visit")
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
@@ -114,6 +181,7 @@ class Precision(Enum):
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_NC = [c.name for c in fields(AutoEvalColumn_NodeClassification) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]

src/envs.py

@@ -6,7 +6,7 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "relbench" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"

src/populate.py

@@ -1,6 +1,6 @@
 import json
 import os
-
+from ast import literal_eval
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
@@ -8,19 +8,55 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
+'''
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    #df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    #df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    #df = df[has_no_nan_values(df, benchmark_cols)]
     return raw_data, df
+'''
+
+def get_leaderboard_df(EVAL_REQUESTS_PATH, tasks) -> pd.DataFrame:
+
+    model_result_filepaths = []
+    for root,_, files in os.walk(EVAL_REQUESTS_PATH):
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+    
+    model_res = []
+    for model in model_result_filepaths:
+        import json
+        with open(model) as f:
+            model_res.append(json.load(f))
+
+    for model in model_res:
+        model["test"] = literal_eval(model["test"])
+        model["valid"] = literal_eval(model["valid"])
+        model["params"] = int(model["params"])
+        model['submitted_time'] = model['submitted_time'].split('T')[0]
+        #model['paper_url'] = '[Link](' + model['paper_url'] + ')'
+        #model['github_url'] = '[Link](' + model['github_url'] + ')'
+
+    name2short_name = {task.value.benchmark: task.value.col_name for task in tasks}
+    for model in model_res:
+        model.update({name2short_name[i]: str(model['test'][i][0])[:4] + '±' + str(model['test'][i][1])[:4] if i in model['test'] else '-' for i in name2short_name})
 
+    columns_to_show = ['model', 'author', 'email', 'paper_url', 'github_url', 'submitted_time', 'params'] + list(name2short_name.values())
+    df_res = pd.DataFrame([{col: model[col] for col in columns_to_show} for model in model_res])
+    ranks = df_res[list(name2short_name.values())].rank()
+    df_res.rename(columns={'model': 'Model', 'author': 'Author', 'email': 'Email', 'paper_url': 'Paper URL', 'github_url': 'Github URL', 'submitted_time': 'Time', 'params': '# of Params'}, inplace=True)
+    df_res['Average Rank⬆️'] = ranks.mean(axis=1)
+    df_res.sort_values(by='Average Rank⬆️', ascending=True, inplace=True)
+    return df_res
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""

src/submission/check_validity.py

@@ -88,7 +88,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(f"{info['model']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -14,93 +14,77 @@ from src.submission.check_validity import (
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        try:
+            return super().default(obj)
+        except TypeError:
+            return str(obj)  # Convert non-serializable object to string
+
+def add_new_eval_json(eval_entry, out_path):
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry, cls=CustomJSONEncoder))
+
 def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    author,
+    email,
+    relbench_version,
+    model,
+    official_or_not,
+    test_performance,
+    valid_performance,
+    paper_url,
+    github_url,
+    parameters,
+    honor_code,
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
     model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
 
-    precision = precision.split(" ")[0]
+    #precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
+    model_size = parameters
 
     # Seems good, creating the eval
     print("Adding new eval")
 
     eval_entry = {
         "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+        "author": author,
+        "email": email,
+        "relbench_version": relbench_version,
+        "official_or_not": official_or_not,
+        "test": test_performance,
+        "valid": valid_performance,
+        "paper_url": paper_url,
+        "github_url": github_url,
+        "honor_code": honor_code,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
         "params": model_size,
-        "license": license,
         "private": False,
     }
 
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
+    # TODO: Check for duplicate submission
+    #if f"{model}_{author}_{precision}" in REQUESTED_MODELS:
+    #    return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False.json"
 
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
+    print(eval_entry)
+
+    #with open(out_path, "w") as f:
+    #    f.write(json.dumps(eval_entry))
+
+    add_new_eval_json(eval_entry, out_path)
 
     print("Uploading eval file")
     API.upload_file(