update full instructions V1

app.py

@@ -287,7 +287,7 @@ with demo:
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="QA Model name")
+                    model_name_textbox = gr.Textbox(label="QA model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
@@ -330,15 +330,15 @@ with demo:
                 submission_result,
             )
 
-    with gr.Row():
-        with gr.Accordion("📙 More about the task", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    # with gr.Row():
+    #     with gr.Accordion("📙 More about the task", open=False):
+    #         citation_button = gr.Textbox(
+    #             value=CITATION_BUTTON_TEXT,
+    #             label=CITATION_BUTTON_LABEL,
+    #             lines=20,
+    #             elem_id="citation-button",
+    #             show_copy_button=True,
+    #         )
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)

src/about.py

@@ -25,20 +25,88 @@ TITLE = """<h1 align="center" id="space-title">Grounded QA leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Build an open-domain QA system that can answer any question posed by humans!
+Build an open-domain QA system that can answer any question posed by humans! For more: https://sites.google.com/view/qanta/home
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## Generative QA
-If you are working on a generative QA model, you are expected to submit your system by filling the `Model name`.
+## QA variants
 
-## Extractive QA
-If you are working on a generative QA model, you are expected to submit your QA system by filling the `Model name` and your retriever in `Base model`.
+### Generative QA
+This type of QA system aims to generate an answer to a given question directly.
+
+#### Input
+(1) `question` string
+
+```
+E.g. qa_pipe(question)
+```
+
+#### Output
+Return in a JSON format: (1) `guess` string, (2) `confidence` score which should be a float number representing the probability (0-1) of your guess.
+
+```
+E.g. {'guess': 'Apple', 'confidence': 0.02}
+```
+
+Reminder: Feel free to check the tutorial provided to see how you could calculate the probability of the generated tokens!
+
+### Extractive QA
+This type of QA system aims to extract an answer span from a context passage for a given question.
+
+#### Input
+(1) `question` string, and (2) `context` string
+
+```
+E.g. qa_pipe(question=question, context=context)
+```
+
+#### Output
+Return in a JSON format: (1) `guess` string, (2) `confidence` score which should be a float number representing the probability (0-1) of your guess.
+
+```
+E.g. {'guess': 'Apple', 'confidence': 0.02}
+```
+
+Reminder: If you are playing around with an extractive QA model already, HF QA models output the `score` already, so you only need to wrap the `score` to `confidence`.
+
+#### Customized retriever
+If you didn’t submit anything for retriever, we will feed the `context` string with our pre-loaded context. However, we do provide the option for you to customize your retriever model with the dataset you wish to do retrieval. Please check the tutorial example for more details.
+
+## Evaluation Metric
+For each question in the test set, we parsed it into multiple runs and fed each run as the question to your pipeline. Then we use the confidence scores calculated for all runs to get the Buzz Confidence.
+
+## FAQ
+What if my system type is not specified here or not supported yet?
+- Please have a private post to instructors so we could check how we could adapt the leaderboard for your purpose. Thanks!
+
+I don’t understand where I could start to build a QA system for submission.
+- Please check our submission tutorials. From there, you could fine-tune or do anything above the base models.
 """
 
 EVALUATION_QUEUE_TEXT = """
-We have the generative QA tutorial [here](https://docs.google.com/document/d/1ZwyEnrF7OjkDdNpEsiSU1RPEilepAmmFJZgVypzm9Es)!
+**Step 1: Make sure it could work locally**
+After you have a QA system uploaded to HuggingFace (with license specified), please check with the following example code to see if your pipe could return the guess and confidence score in a **JSON** format.
+
+```
+from transformers import pipeline
+qa_pipe = pipeline(model="...", trust_remote_code=True)
+
+# If it is a Generative QA pipeline
+qa_pipe(“Where is UMD?”)
+
+# If it is a Extractive QA pipeline
+qa_pipe(question=“Where is UMD?”, context=”UMD is in Maryland.”)
+```
+
+**Step 2: Fill in the submission form**
+(1) Fill in the `QA model name`
+(2) Fill in the `Revision commit`: if you leave it empty, by default it will be `main`.
+(3) Fill in the `Model type`
+(4) `Precision` by default is `float16 `. You could update it as needed.
+(5) If you have a trained retriever and want to submit an Extractive QA system, please also fill in the `Retrieved dataset name` and `Retriever model`.
+
+Here is a tutorial on how you could make pipe wrappers for submissions: [Colab](https://colab.research.google.com/drive/1bCt2870SdY6tI4uE3JPG8_3nLmNJXX6_?usp=sharing)
 """
 
 CITATION_BUTTON_LABEL = "Copy the following link to check more details"