limited max new tokens to 2

src/deepeval/base_task.py

@@ -77,7 +77,7 @@ class BaseTask(ABC):
 
         return answer
     
-    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=5, choices: list = []):
+    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=2, choices: list = []):
         """
         Handles multiple-choice questions where answers might have multiple tokens.
         """

src/deepeval/commonsense_reasoning_task.py

@@ -57,7 +57,7 @@ class CommonsenseReasoningTask(BaseTask):
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
 

src/deepeval/deepeval_task_manager.py

@@ -133,6 +133,6 @@ class DeepEvalTaskManager:
         return res
 
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["SUMMARIZATION"])
+    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["TOXICITY", "BIAS"])
     res = des.run_tasks()
     print(res)

src/deepeval/nli.py

@@ -48,7 +48,7 @@ class NLITask(BaseTask):
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()