update commonsense reasoning

src/deepeval/commonsense_reasoning_task.py

@@ -1,27 +1,47 @@
 from src.deepeval.base_task import BaseTask
+from collections import defaultdict
 from src.deepeval.utils import accuracy, accuracy_standard_error
 from typing import Any
 
+
 class CommonsenseReasoningTask(BaseTask):
     def __init__(self, model_name):
         super().__init__("metunlp/commonsense", model_name=model_name)
 
     def load_dataset_from_hf(self):
-        print("Loading the dataset")
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(1, len(dataset))))
+        return dataset.select(range(min(2, len(dataset))))
 
 
     def evaluate(self) -> dict[str, Any]:
         responses = []
-        total_count = len(self.dataset)
-        n_correct = 0
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
         for row in self.dataset:
-            print(row)
+            total_count += 1
+
+            # Get values from row
             label = row["label"]
             choices=[row["choice1"], row["choice2"]]
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            category = row["difficulty"]
+            answer = row["answer"]
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+            print("Type of answer:", type(answer))
 
+            # Get answer index (starting from 0)
+            if type(answer) == int:
+                answer_index = answer - 1 # 1 or 2
+            else:
+                answer_index = int(answer) - 1
+            correct_answer_letter = chr(65 + answer_index)
+
+            # Get question based on label
             if label == "effect":
                 question = "Seçeneklerden hangisi verilen önermenin bir sonucu veya etkisi olabilir?"
             elif label == "cause":
@@ -29,21 +49,36 @@ class CommonsenseReasoningTask(BaseTask):
             else:
                 question = "Seçeneklerden hangisi uygun?"  # Alternatif
 
-            prompt = f"Bağlam:\n{row["text"]}\nÖnerme:\n{row["context"]}\nSoru:{question}\nSeçenekler:\n{formatted_choices}"
-
-            messages = prompt
+            # Construct the prompt/message
+            instruction = ""
+            prompt = f"Bağlam:\n{row["text"]}\nÖnerme:\n{row["context"]}\nSoru:{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
 
-            model_answer = self.generate_response_mcqa_multi_token(messages, choices=choices)
-
-            correct_answer_letter = "A" if row["answer"] == 1 else "B" if row["answer"] == 2 else None
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
-            if correct_answer_letter == model_answer_cleaned:
-                n_correct += 1
+
+            # Print answers
             print(f"Correct Answer: {correct_answer_letter}")
             print(f"Model Answer: {model_answer}")
             print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
-        acc = accuracy(n_correct, total_count)
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
         acc_stderr = accuracy_standard_error(acc, total_count)
         return {"acc": acc, "acc_stderr": acc_stderr}