Debug

src/backend/run_eval_suite_harness.py

@@ -61,6 +61,8 @@ def run_evaluation(eval_request: EvalRequest, task_names: list, num_fewshot: int
     results["config"]["model_name"] = eval_request.model
     results["config"]["model_sha"] = eval_request.revision
 
+    print("Results is")
+    print(results)
     dumped = json.dumps(results, indent=2)
     logger.info(dumped)
 

src/envs.py

@@ -10,7 +10,7 @@ OWNER = "meg" # Change to your org - don't forget to create a results and reques
 
 # For harness evaluations
 DEVICE = "cuda:0" #if you add compute, for harness evaluations
-LIMIT = 20 # !!!! For testing, should be None for actual evaluations!!!
+LIMIT = 1 # !!!! For testing, should be None for actual evaluations!!!
 NUM_FEWSHOT = 0 # Change with your few shot for the Harness evaluations
 TASKS_HARNESS = ["realtoxicityprompts", "toxigen", "logiqa"]