Update scores

src/about.py

@@ -17,14 +17,15 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key, metric_key, title
-    task00 = Task("naive_judge", "score", "NaïveJudge (11)")
-    task01 = Task("human_eval_solidity", "score", "HumanEval for Solidity (9)")
-    task02 = Task("rouge1", "score", "ROUGE-unigrams")
-    task03 = Task("rouge2", "score", "ROUGE-bigrams")
-    task04 = Task("rougeL", "score", "ROUGE-Longest Common Subsequence")
-    task05 = Task("rougeLsum", "score", "ROUGE-Lsum")
-    task06 = Task("bleu", "score", "Bleu")
-    task07 = Task("brevity_penalty", "score", "Brevity Penalty")
+    task00 = Task("naive_judge", "score", "NaïveJudge")
+    task01 = Task("human_eval_solidity_pass@1", "score", "HumanEval for Solidity (pass@1)")
+    task02 = Task("human_eval_solidity_pass@3", "score", "HumanEval for Solidity (pass@3)")
+    task03 = Task("rouge1", "score", "ROUGE-unigrams")
+    task04 = Task("rouge2", "score", "ROUGE-bigrams")
+    task05 = Task("rougeL", "score", "ROUGE-Longest Common Subsequence")
+    task06 = Task("rougeLsum", "score", "ROUGE-Lsum")
+    task07 = Task("bleu", "score", "Bleu")
+    task08 = Task("brevity_penalty", "score", "Brevity Penalty")
 # ---------------------------------------------------
 
 # Your leaderboard name

src/leaderboard/read_evals.py

@@ -117,7 +117,8 @@ class EvalResult:
         # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         solbench = (
             self.results.get('naive_judge', 0) * 0.3 +
-            self.results.get('human_eval_solidity', 0) * 0.7
+            self.results.get('human_eval_solidity_pass@1', 0) * 0.5 +
+            self.results.get('human_eval_solidity_pass@3', 0) * 0.2
         )
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,