Add previous results

datacards/considerations.py

@@ -1,13 +1,13 @@
 import streamlit as st
 
-from .streamlit_utils import (
-    make_text_input
-)
+from .streamlit_utils import make_text_input
 
 N_FIELDS = 1
 
+
 def considerations_page():
     return None
 
+
 def considerations_summary():
     return None

datacards/context.py

@@ -1,13 +1,13 @@
 import streamlit as st
 
-from .streamlit_utils import (
-    make_text_input
-)
+from .streamlit_utils import make_text_input
 
 N_FIELDS = 1
 
+
 def context_page():
     return None
 
+
 def context_summary():
     return None

datacards/curation.py

@@ -1,8 +1,6 @@
 import streamlit as st
 
-from .streamlit_utils import (
-    make_text_input
-)
+from .streamlit_utils import make_text_input
 
 from .streamlit_utils import (
     make_multiselect,
@@ -20,33 +18,40 @@ N_FIELDS_PII = 0
 N_FIELDS_MAINTENANCE = 0
 N_FIELDS_GEM = 0
 
-N_FIELDS = N_FIELDS_ORIGINAL + \
-    N_FIELDS_LANGUAGE + \
-    N_FIELDS_ANNOTATIONS + \
-    N_FIELDS_CONSENT + \
-    N_FIELDS_PII + \
-    N_FIELDS_MAINTENANCE + \
-    N_FIELDS_GEM
+N_FIELDS = (
+    N_FIELDS_ORIGINAL
+    + N_FIELDS_LANGUAGE
+    + N_FIELDS_ANNOTATIONS
+    + N_FIELDS_CONSENT
+    + N_FIELDS_PII
+    + N_FIELDS_MAINTENANCE
+    + N_FIELDS_GEM
+)
 
 
 """
 What was the selection criteria? [Describe the process for selecting instances to include in the dataset, including any tools used.]
 """
 
+
 def curation_page():
-    st.session_state.card_dict["curation"] = st.session_state.card_dict.get("curation", {})
+    st.session_state.card_dict["curation"] = st.session_state.card_dict.get(
+        "curation", {}
+    )
     with st.expander("Original Curation", expanded=False):
         key_pref = ["curation", "original"]
-        st.session_state.card_dict["curation"]["original"] = st.session_state.card_dict["curation"].get("original", {})
+        st.session_state.card_dict["curation"]["original"] = st.session_state.card_dict[
+            "curation"
+        ].get("original", {})
         make_text_area(
             label="Original curation rationale",
             key_list=key_pref + ["rationale"],
-            help="Describe the curation rationale behind the original dataset(s)."
+            help="Describe the curation rationale behind the original dataset(s).",
         )
         make_text_area(
             label="What was the communicative goal?",
             key_list=key_pref + ["communicative"],
-            help="Describe the communicative goal that the original dataset(s) was trying to represent."
+            help="Describe the communicative goal that the original dataset(s) was trying to represent.",
         )
         make_radio(
             label="Is the dataset aggregated from different data sources?",
@@ -57,95 +62,130 @@ def curation_page():
         make_text_area(
             label="If yes, list the sources",
             key_list=key_pref + ["aggregated-sources"],
-            help="Otherwise, type N/A"
+            help="Otherwise, type N/A",
         )
     with st.expander("Language Data", expanded=False):
         key_pref = ["curation", "language"]
-        st.session_state.card_dict["curation"]["language"] = st.session_state.card_dict["curation"].get("language", {})
+        st.session_state.card_dict["curation"]["language"] = st.session_state.card_dict[
+            "curation"
+        ].get("language", {})
         make_multiselect(
             label="How was the language data obtained?",
-            options=["found", "created for the dataset", "crowdsourced", "machine-generated", "other"],
-            key_list=key_pref+["obtained"],
+            options=[
+                "found",
+                "created for the dataset",
+                "crowdsourced",
+                "machine-generated",
+                "other",
+            ],
+            key_list=key_pref + ["obtained"],
         )
         make_multiselect(
             label="If found, where from?",
             options=["website", "offline media collection", "other", "N/A"],
-            key_list=key_pref+["found"],
-            help="select N/A if none of the language data was found"
+            key_list=key_pref + ["found"],
+            help="select N/A if none of the language data was found",
         )
         make_multiselect(
             label="If crowdsourced, where from?",
-            options=["Amazon Mechanical Turk", "other crowdworker platform", "participatory experiment", "other", "N/A"],
-            key_list=key_pref+["crowdsourced"],
-            help="select N/A if none of the language data was crowdsourced"
+            options=[
+                "Amazon Mechanical Turk",
+                "other crowdworker platform",
+                "participatory experiment",
+                "other",
+                "N/A",
+            ],
+            key_list=key_pref + ["crowdsourced"],
+            help="select N/A if none of the language data was crowdsourced",
         )
         make_text_area(
             label="If created for the dataset, describe the creation process.",
-            key_list=key_pref+["created"],
+            key_list=key_pref + ["created"],
         )
         make_text_area(
             label="What further information do we have on the language producers?",
-            key_list=key_pref+["producers-description"],
+            key_list=key_pref + ["producers-description"],
             help="Provide a description of the context in which the language was produced and who produced it.",
         )
         make_text_input(
             label="If text was machine-generated for the dataset, provide a link to the generation method if available (N/A otherwise).",
-            key_list=key_pref+["machine-generated"],
+            key_list=key_pref + ["machine-generated"],
             help="if the generation code is unavailable, enter N/A",
         )
         make_selectbox(
             label="Was the text validated by a different worker or a data curator?",
-            options=["not validated", "validated by crowdworker", "validated by data curator", "other"],
-            key_list=key_pref+["validated"],
-            help="this question is about human or human-in-the-loop validation only"
+            options=[
+                "not validated",
+                "validated by crowdworker",
+                "validated by data curator",
+                "other",
+            ],
+            key_list=key_pref + ["validated"],
+            help="this question is about human or human-in-the-loop validation only",
         )
         make_multiselect(
             label="In what kind of organization did the curation happen?",
-            options= ["industry",  "academic", "independent", "other"],
-            key_list=key_pref+["organization-type"],
+            options=["industry", "academic", "independent", "other"],
+            key_list=key_pref + ["organization-type"],
         )
         make_text_input(
             label="Name the organization(s).",
-            key_list=key_pref+["organization-names"],
+            key_list=key_pref + ["organization-names"],
             help="comma-separated",
         )
         make_text_area(
             label="How was the text data pre-processed? (Enter N/A if the text was not pre-processed)",
-            key_list=key_pref+["pre-processed"],
-            help="List the steps in preprocessing the data for the dataset. Enter N/A if no steps were taken."
+            key_list=key_pref + ["pre-processed"],
+            help="List the steps in preprocessing the data for the dataset. Enter N/A if no steps were taken.",
         )
         make_selectbox(
             label="Were text instances selected or filtered?",
             options=["not filtered", "manually", "algorithmically", "hybrid"],
-            key_list=key_pref+["is-filtered"],
+            key_list=key_pref + ["is-filtered"],
         )
         make_text_area(
             label="What were the selection criteria?",
-            key_list=key_pref+["filtered-criteria"],
-            help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A."
+            key_list=key_pref + ["filtered-criteria"],
+            help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
         )
     with st.expander("Structured Annotations", expanded=False):
         key_pref = ["curation", "annotations"]
-        st.session_state.card_dict["curation"]["annotations"] = st.session_state.card_dict["curation"].get("annotations", {})
+        st.session_state.card_dict["curation"][
+            "annotations"
+        ] = st.session_state.card_dict["curation"].get("annotations", {})
     with st.expander("Consent", expanded=False):
         key_pref = ["curation", "consent"]
-        st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict["curation"].get("consent", {})
+        st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
+            "curation"
+        ].get("consent", {})
     with st.expander("Private Identifying Information (PII)", expanded=False):
         key_pref = ["curation", "pii"]
-        st.session_state.card_dict["curation"]["pii"] = st.session_state.card_dict["curation"].get("pii", {})
+        st.session_state.card_dict["curation"]["pii"] = st.session_state.card_dict[
+            "curation"
+        ].get("pii", {})
     with st.expander("Maintenance", expanded=False):
         key_pref = ["curation", "maintenance"]
-        st.session_state.card_dict["curation"]["maintenance"] = st.session_state.card_dict["curation"].get("maintenance", {})
+        st.session_state.card_dict["curation"][
+            "maintenance"
+        ] = st.session_state.card_dict["curation"].get("maintenance", {})
     with st.expander("GEM Additional Curation", expanded=False):
         key_pref = ["curation", "gem"]
-        st.session_state.card_dict["curation"]["gem"] = st.session_state.card_dict["curation"].get("gem", {})
+        st.session_state.card_dict["curation"]["gem"] = st.session_state.card_dict[
+            "curation"
+        ].get("gem", {})
 
 
 def curation_summary():
-    total_filled = sum([len(dct) for dct in st.session_state.card_dict.get('curation', {}).values()])
-    with st.expander(f"Dataset Curation Completion - {total_filled} of {N_FIELDS}", expanded=False):
+    total_filled = sum(
+        [len(dct) for dct in st.session_state.card_dict.get("curation", {}).values()]
+    )
+    with st.expander(
+        f"Dataset Curation Completion - {total_filled} of {N_FIELDS}", expanded=False
+    ):
         completion_markdown = ""
-        completion_markdown += f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        completion_markdown += (
+            f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        )
         completion_markdown += f"- **Sub-section - Original Curation:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('original', {}))} of {N_FIELDS_ORIGINAL} fields\n"
         completion_markdown += f"- **Sub-section - Language Data:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('language', {}))} of {N_FIELDS_LANGUAGE} fields\n"
         completion_markdown += f"- **Sub-section - Structured Annotations:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('annotations', {}))} of {N_FIELDS_ANNOTATIONS} fields\n"

datacards/gem.py

@@ -1,8 +1,6 @@
 import streamlit as st
 
-from .streamlit_utils import (
-    make_text_input
-)
+from .streamlit_utils import make_text_input
 
 from .streamlit_utils import (
     make_text_area,
@@ -12,15 +10,16 @@ from .streamlit_utils import (
 N_FIELDS_RATIONALE = 5
 N_FIELDS_STARTING = 2
 
-N_FIELDS = N_FIELDS_RATIONALE + \
-    N_FIELDS_STARTING
+N_FIELDS = N_FIELDS_RATIONALE + N_FIELDS_STARTING
 
 
 def gem_page():
     st.session_state.card_dict["gem"] = st.session_state.card_dict.get("gem", {})
     with st.expander("Rationale", expanded=False):
         key_pref = ["gem", "rationale"]
-        st.session_state.card_dict["gem"]["rationale"] = st.session_state.card_dict["gem"].get("rationale", {})
+        st.session_state.card_dict["gem"]["rationale"] = st.session_state.card_dict[
+            "gem"
+        ].get("rationale", {})
         make_text_area(
             label="What does this dataset contribute toward better generation evaluation and why is it part of GEM?",
             key_list=key_pref + ["contribution"],
@@ -50,7 +49,9 @@ def gem_page():
         )
     with st.expander("Getting Started", expanded=False):
         key_pref = ["gem", "starting"]
-        st.session_state.card_dict["gem"]["starting"] = st.session_state.card_dict["gem"].get("starting", {})
+        st.session_state.card_dict["gem"]["starting"] = st.session_state.card_dict[
+            "gem"
+        ].get("starting", {})
         make_text_area(
             label="Getting started with in-depth research on the task. Add relevant pointers to resources that researchers can consult when they want to get started digging deeper into the task.",
             key_list=key_pref + ["research-pointers"],
@@ -64,10 +65,16 @@ def gem_page():
 
 
 def gem_summary():
-    total_filled = sum([len(dct) for dct in st.session_state.card_dict.get('gem', {}).values()])
-    with st.expander(f"Dataset in GEM Completion - {total_filled} of {N_FIELDS}", expanded=False):
+    total_filled = sum(
+        [len(dct) for dct in st.session_state.card_dict.get("gem", {}).values()]
+    )
+    with st.expander(
+        f"Dataset in GEM Completion - {total_filled} of {N_FIELDS}", expanded=False
+    ):
         completion_markdown = ""
-        completion_markdown += f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        completion_markdown += (
+            f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        )
         completion_markdown += f"- **Sub-section - Rationale:**\n  - {len(st.session_state.card_dict.get('gem', {}).get('rationale', {}))} of {N_FIELDS_RATIONALE} fields\n"
         completion_markdown += f"- **Sub-section - Getting Started:**\n  - {len(st.session_state.card_dict.get('gem', {}).get('starting', {}))} of {N_FIELDS_STARTING} fields\n"
         st.markdown(completion_markdown)

datacards/overview.py

@@ -16,25 +16,29 @@ N_FIELDS_LANGUAGES = 8
 N_FIELDS_CREDIT = 3
 N_FIELDS_STRUCTURE = 7
 
-N_FIELDS = N_FIELDS_WHERE + \
-    N_FIELDS_LANGUAGES + \
-    N_FIELDS_CREDIT + \
-    N_FIELDS_STRUCTURE
+N_FIELDS = N_FIELDS_WHERE + N_FIELDS_LANGUAGES + N_FIELDS_CREDIT + N_FIELDS_STRUCTURE
 
 
 languages_bcp47 = [
     x
-    for x in json.load(open(pjoin("resources", "bcp47.json"), encoding="utf-8"))["subtags"]
+    for x in json.load(open(pjoin("resources", "bcp47.json"), encoding="utf-8"))[
+        "subtags"
+    ]
     if x["type"] == "language"
 ]
 
 license_list = json.load(open(pjoin("resources", "licenses.json"), encoding="utf-8"))
 
+
 def overview_page():
-    st.session_state.card_dict["overview"] = st.session_state.card_dict.get("overview", {})
+    st.session_state.card_dict["overview"] = st.session_state.card_dict.get(
+        "overview", {}
+    )
     with st.expander("Where to find the data and its documentation", expanded=False):
         key_pref = ["overview", "where"]
-        st.session_state.card_dict["overview"]["where"] = st.session_state.card_dict["overview"].get("where", {})
+        st.session_state.card_dict["overview"]["where"] = st.session_state.card_dict[
+            "overview"
+        ].get("where", {})
         make_text_input(
             label="What is the webpage for the dataset (if it exists)?",
             key_list=key_pref + ["website"],
@@ -83,7 +87,9 @@ def overview_page():
         )
     with st.expander("Languages and Intended Use", expanded=False):
         key_pref = ["overview", "languages"]
-        st.session_state.card_dict["overview"]["languages"] = st.session_state.card_dict["overview"].get("languages", {})
+        st.session_state.card_dict["overview"][
+            "languages"
+        ] = st.session_state.card_dict["overview"].get("languages", {})
         make_radio(
             label="Is the dataset multilingual?",
             options=["no", "yes"],
@@ -93,9 +99,7 @@ def overview_page():
         make_multiselect(
             label="What languages/dialects are covered in the dataset?",
             key_list=key_pref + ["language-names"],
-            options=[
-                ", ".join(x["description"]) for x in languages_bcp47
-            ],
+            options=[", ".join(x["description"]) for x in languages_bcp47],
             help="This is a comprehensive list of languages obtained from the BCP-47 standard list.",
         )
         make_text_area(
@@ -117,20 +121,23 @@ def overview_page():
             label="What is the license of the dataset?",
             key_list=key_pref + ["license"],
             options=license_list,
-            help="select `other` if missing from list, `unkown` if not provided."
+            help="select `other` if missing from list, `unkown` if not provided.",
         )
         make_selectbox(
             label="What primary task does the dataset support?",
             key_list=key_pref + ["task"],
-            options=["Content Transfer", "Data-to-Text",
-                     "Dialog Response Generation",
-                     "Paraphrasing", "Question Generation",
-                     "Reasoning",
-                     "Simplification",
-                     "Style Transfer",
-                     "Summarization",
-                     "Text-to-Slide",
-                     ],
+            options=[
+                "Content Transfer",
+                "Data-to-Text",
+                "Dialog Response Generation",
+                "Paraphrasing",
+                "Question Generation",
+                "Reasoning",
+                "Simplification",
+                "Style Transfer",
+                "Summarization",
+                "Text-to-Slide",
+            ],
             help="Select `other` if the task is not included in the list.",
         )
         make_text_area(
@@ -140,7 +147,9 @@ def overview_page():
         )
     with st.expander("Credit", expanded=False):
         key_pref = ["overview", "credit"]
-        st.session_state.card_dict["overview"]["credit"] = st.session_state.card_dict.get("credit", {})
+        st.session_state.card_dict["overview"][
+            "credit"
+        ] = st.session_state.card_dict.get("credit", {})
         make_text_input(
             label="Who created the original dataset? List the people involved in collecting the dataset and their affiliation(s).",
             key_list=key_pref + ["creators"],
@@ -158,7 +167,9 @@ def overview_page():
         )
     with st.expander("Structure", expanded=False):
         key_pref = ["overview", "structure"]
-        st.session_state.card_dict["overview"]["structure"] = st.session_state.card_dict.get("structure", {})
+        st.session_state.card_dict["overview"][
+            "structure"
+        ] = st.session_state.card_dict.get("structure", {})
         data_fields_help = """
         [free text; paragraphs]
         - Mention their data type, and whether and how they are used as part of the generation pipeline.
@@ -203,10 +214,16 @@ def overview_page():
 
 
 def overview_summary():
-    total_filled = sum([len(dct) for dct in st.session_state.card_dict.get('overview', {}).values()])
-    with st.expander(f"Dataset Overview Completion - {total_filled} of {N_FIELDS}", expanded=False):
+    total_filled = sum(
+        [len(dct) for dct in st.session_state.card_dict.get("overview", {}).values()]
+    )
+    with st.expander(
+        f"Dataset Overview Completion - {total_filled} of {N_FIELDS}", expanded=False
+    ):
         completion_markdown = ""
-        completion_markdown += f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        completion_markdown += (
+            f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        )
         completion_markdown += f"- **Sub-section - Where to find:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('where', {}))} of {N_FIELDS_WHERE} fields\n"
         completion_markdown += f"- **Sub-section - Languages and Intended Use:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('languages', {}))} of {N_FIELDS_LANGUAGES} fields\n"
         completion_markdown += f"- **Sub-section - Credit:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('credit', {}))} of {N_FIELDS_CREDIT} fields\n"

datacards/results.py

@@ -1,13 +1,72 @@
 import streamlit as st
 
 from .streamlit_utils import (
-    make_text_input
+    make_multiselect,
+    make_selectbox,
+    make_text_area,
+    make_text_input,
+    make_radio,
 )
 
 N_FIELDS = 1
 
+
 def results_page():
-    return None
+    with st.expander("Previous Results", expanded=False):
+        key_pref = ["results", "results"]
+        st.session_state.card_dict["results"]["results"] = st.session_state.card_dict[
+            "results"
+        ].get("results", {})
+        make_multiselect(
+            label="What metrics are typically used for this task?",
+            key_list=key_pref + ["metrics"],
+            options=[
+                "BERT-Score",
+                "BLEU",
+                "BLEURT",
+                "ChrF",
+                "Entailment",
+                "FeQA",
+                "METEOR" "MoverScore",
+                "QAGS",
+                "ROUGE",
+                "WER",
+            ],
+            help="Select all metrics that are typically used when evaluating models for this task.",
+        )
+        make_text_area(
+            label="Describe the metrics and evaluation methodology that the dataset creators used when introducing this task.",
+            key_list=key_pref + ["original-evaluation"],
+            help="When the generation task was not evaluated when this dataset was introduced, write N/A.",
+        )
+        make_radio(
+            label="Are previous results available?",
+            options=["no", "yes"],
+            key_list=key_pref + ["has-previous-results"],
+            help="Have papers evaluated models on this task? If no, write N/A for the following three questions.",
+        )
+        make_text_area(
+            label="What evaluation approaches have others used?",
+            key_list=key_pref + ["modern-evaluation"],
+            help="If the modern evaluation strategy diverts from the original, describe how models are being evaluated.",
+        )
+        make_text_area(
+            label="What are previous results",
+            key_list=key_pref + ["previous-results"],
+            help="List the source and performance metrics for models on this dataset.",
+        )
+        make_text_area(
+            label="Definitions",
+            key_list=key_pref + ["definitions"],
+            help="If the evaluation strategies in the previous questions go beyond the list of metrics above, add descriptions and/or definitions for each metric.",
+        )
+        make_text_area(
+            label="What aspect of model ability can be measured with this dataset?",
+            key_list=key_pref + ["model-abilities"],
+            help="What kind of abilities should a model exhibit that performs well on the task of this dataset (e.g., reasoning capability, morphological inflection)?.",
+        )
+
+
 
 def results_summary():
     return None

datacards/streamlit_utils.py

@@ -4,10 +4,13 @@ import streamlit as st
 # Streamlit widgets with persistence
 def is_filled(key_list):
     state_filled_key = "_".join(key_list) + "_filled"
+
     def on_change_action():
         st.session_state.save_state[state_filled_key] = True
+
     return on_change_action
 
+
 def update_card_dict(key_list, use_default=None):
     state_key = "_".join(key_list)
     if st.session_state.save_state.get(state_key + "_filled", False) or use_default:
@@ -39,7 +42,9 @@ def make_multiselect(
     return res
 
 
-def make_selectbox(key_list, label, options, format_func=lambda x: x, help="", index=None):
+def make_selectbox(
+    key_list, label, options, format_func=lambda x: x, help="", index=None
+):
     key = "_".join(key_list)
     if key in st.session_state:
         st.session_state.save_state[key] = st.session_state[key]
@@ -56,7 +61,9 @@ def make_selectbox(key_list, label, options, format_func=lambda x: x, help="", i
         on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list, use_default=st.session_state.save_state.get(key, options[0]))  # use the default value even without interactions
+    update_card_dict(
+        key_list, use_default=st.session_state.save_state.get(key, options[0])
+    )  # use the default value even without interactions
     return res
 
 
@@ -75,7 +82,9 @@ def make_radio(key_list, label, options, format_func=lambda x: x, help="", index
         on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list, use_default=st.session_state.save_state.get(key, options[0]))  # use the default value even without interactions
+    update_card_dict(
+        key_list, use_default=st.session_state.save_state.get(key, options[0])
+    )  # use the default value even without interactions
     return res
 
 
@@ -102,7 +111,7 @@ def make_text_area(key_list, label, help="", value=None):
         st.session_state.save_state[key] = st.session_state[key]
     elif value is not None:
         st.session_state.save_state[key] = value
-    res =  st.text_area(
+    res = st.text_area(
         label=label,
         key=key,
         value=st.session_state.save_state.get(key, ""),