data upload script, support both splits

app.py

@@ -158,10 +158,10 @@ with demo:
                     org_textbox = gr.Textbox(label=AutoEvalColumn.organization.name)
                     # revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     sys_type_dropdown = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType],
+                        choices=[t.to_str(" ") for t in ModelType],
                         label=AutoEvalColumn.system_type.name,
                         multiselect=False,
-                        value=ModelType.LLM.to_str(" : "),
+                        value=ModelType.LLM.to_str(" "),
                         interactive=True,
                     )
 

scripts/upload_f1_dataset.py

@@ -0,0 +1,46 @@
+import argparse
+import fnmatch
+import json
+import os
+
+from datasets import Dataset
+
+from src.envs import CODE_PROBLEMS_REPO
+from src.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("--input_dir", type=str, help="Dir with .json files", required=True)
+    parser.add_argument("--dataset_name", type=str, default=f"{CODE_PROBLEMS_REPO}")
+    parser.add_argument("--split", type=str, choices=["hard", "warmup"], default="hard")
+    return parser.parse_args()
+
+
+def main(args: argparse.Namespace) -> None:
+    logger.info("Reading problem files from %s", args.input_dir)
+    input_files = fnmatch.filter(os.listdir(args.input_dir), "*.json")
+    if len(input_files) == 0:
+        raise ValueError(f"No .json files in input dir {args.input_dir}")
+    logger.info("Found %d code problems in %s", len(input_files), args.input_dir)
+
+    def ds_generator():
+        for fname in sorted(input_files):
+            formula_name = os.path.splitext(fname)[0]
+            cp_path = os.path.join(args.input_dir, fname)
+            with open(cp_path, "r", encoding="utf-8") as f:
+                code_problem = json.load(f)
+            logger.info("Read code problem for formula %s from %s", formula_name, cp_path)
+            yield dict(id=code_problem["id"], code_problem=code_problem)
+
+    ds = Dataset.from_generator(ds_generator)
+    logger.info("Created dataset")
+
+    ds.push_to_hub(args.dataset_name, split=args.split, private=True)
+    logger.info("Saved dataset to repo %s", args.dataset_name)
+
+
+if __name__ == "__main__":
+    main(get_args())

src/datamodel/data.py

@@ -3,26 +3,40 @@ import time
 
 from datasets import load_dataset
 
-from src.envs import TOKEN
+from src.envs import TOKEN, CODE_PROBLEMS_REPO, RESULTS_REPO, SUBMISSIONS_REPO
 from src.logger import get_logger
 
 logger = get_logger(__name__)
 
+
 class F1Data:
-    def __init__(self, cp_ds_name: str, sub_ds_name: str, res_ds_name: str):
+    def __init__(self, cp_ds_name: str, sub_ds_name: str, res_ds_name: str, split: str = "hard"):
         self.cp_dataset_name = cp_ds_name
         self.submissions_dataset_name = sub_ds_name
         self.results_dataset_name = res_ds_name
-        self.initialize()
-    
-    @functools.cached_property
-    def code_problem_formulas(self) -> set[str]:
-        return set(self.code_problems.keys())
+        self.split = split
+        self.code_problems = None
+        self._initialize()
 
-    def initialize(self):
+    def _initialize(self):
         logger.info("Initialize F1Data TOMEN='%s'", TOKEN)
         start_time = time.monotonic()
-        cp_ds = load_dataset(self.cp_dataset_name, split="hard", token=TOKEN)
-        logger.info("Loaded code-problems dataset from %s in %f sec", self.cp_dataset_name, time.monotonic() - start_time)
-        self.code_problems: dict[str, str] = {r["formula_name"]: r["code_problem"]["problem_description"] for r in cp_ds}
+        cp_ds = load_dataset(self.cp_dataset_name, split=self.split, token=TOKEN)
+        logger.info(
+            "Loaded code-problems dataset from %s in %f sec",
+            self.cp_dataset_name,
+            time.monotonic() - start_time,
+        )
+        self.code_problems: dict[str, str] = {r["id"]: r["code_problem"]["problem_description"] for r in cp_ds}
         logger.info("Code problems info: %s", self.code_problems)
+
+    @functools.cached_property
+    def code_problem_formulas(self) -> set[str]:
+        return set(self.code_problems.keys())
+
+
+if __name__ == "__main__":
+    split = "hard"
+    f1_data = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=split)
+
+    print(f"Found {len(f1_data.code_problem_formulas)} code problems in {split} split of {f1_data.cp_dataset_name}")

src/submission/submit.py

@@ -33,7 +33,7 @@ def validate_submission(lbdb: F1Data, pd_ds: pd.DataFrame) -> str | None:
     if submitted_formulas != lbdb.code_problem_formulas:
         missing = lbdb.code_problem_formulas - submitted_formulas
         unknown = submitted_formulas - lbdb.code_problem_formulas
-        return f"Mismatched formula names: missing {len(missing)} unknown {len(unknown)}"
+        return f"Mismatched formula names: {len(missing)} missing, {len(unknown)} unknown"
     if len(pd_ds) > len(lbdb.code_problem_formulas):
         return "Duplicate formula solutions exist in uploaded file"
     return None