Add changes to files

src/deepeval/commonsense_reasoning_task.py

@@ -10,7 +10,7 @@ class CommonsenseReasoningTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(2, len(dataset))))
+        return dataset.select(range(min(10, len(dataset))))
 
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/complex_reasoning.py

@@ -0,0 +1,61 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class ComplexReasoningTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/complex-ales", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        correct_answers = []
+
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            choices = ast.literal_eval(row["choices"]) # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            correct_answer_letter = row["answer_choice"]
+            correct_answers.append(correct_answer_letter)
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+
+
+            # Construct the prompt/message
+            instruction = ""
+            prompt = f"Soru:\n{row["narrative"]}\n{row["question"]}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+
+        print("Answers:", correct_answers)
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/deepeval_task_manager.py

@@ -10,6 +10,9 @@ from src.deepeval.toxicity_task import ToxicityTask
 from src.deepeval.bias_task import BiasTask
 from src.deepeval.instruction_following_task import InstructionFollowingTask
 from src.deepeval.reading_comprehension_task import ReadingComprehensionTask
+from src.deepeval.reading_comp_mc import ReadingComprehensionMCTask
+from src.deepeval.complex_reasoning import ComplexReasoningTask
+from src.deepeval.nli import NLITask
 from typing import List
 load_dotenv()
 
@@ -31,6 +34,9 @@ class Task(Enum):
     INSTRUCTION_FOLLOWING = "instruction_following_tr"
     READING_COMPREHENSION = "reading_comprehension_tr"
     COMMONSENSE_REASONING = "commonsense_reasoning"
+    READING_COMPREHENSION_MC = "reading_comprehension_mc"
+    COMPLEX_REASONING = "complex_reasoning"
+    NLI = "nli"
 
 
 class DeepEvalTaskManager:
@@ -71,11 +77,6 @@ class DeepEvalTaskManager:
         res = turkish_general_knowledge_task.evaluate()
         return res
 
-    def commonsense_reasoning(self):
-        commonsense_reasoning_task = CommonsenseReasoningTask(self.model_name)
-        res = commonsense_reasoning_task.evaluate()
-        return res
-
     def summarization_tr(self):
         task = SummarizationTask(self.model_name)
         return task.evaluate()
@@ -100,7 +101,27 @@ class DeepEvalTaskManager:
         task = ReadingComprehensionTask(self.model_name)
         return task.evaluate()
 
+    def commonsense_reasoning(self):
+        commonsense_reasoning_task = CommonsenseReasoningTask(self.model_name)
+        res = commonsense_reasoning_task.evaluate()
+        return res
+
+    def reading_comprehension_mc(self):
+        reading_comprehension_mc_task = ReadingComprehensionMCTask(self.model_name)
+        res = reading_comprehension_mc_task.evaluate()
+        return res
+
+    def complex_reasoning(self):
+        complex_reasoning_task = ComplexReasoningTask(self.model_name)
+        res = complex_reasoning_task.evaluate()
+        return res
+
+    def nli(self):
+        nli_task = NLITask(self.model_name)
+        res = nli_task.evaluate()
+        return res
+
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("meta-llama/Llama-3.2-1B-Instruct", ["SENTIMENT_ANALYSIS", "SUMMARIZATION", "FAITHFULNESS", "TOXICITY", "BIAS", "INSTRUCTION_FOLLOWING","READING_COMPREHENSION"])
+    des = DeepEvalTaskManager("meta-llama/Llama-3.2-1B-Instruct", ["COMPLEX_REASONING","NLI"])
     res = des.run_tasks()
     print(res)

src/deepeval/nli.py

@@ -0,0 +1,74 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+
+
+class NLITask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/nli_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            label = row["label"].lower().replace(' ','')
+            choices=["entailment","contradiction","neutral"]
+            formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            category = row["difficulty"]
+            correct_answer_letter = "A" if label == "entailment" else \
+                                    "B" if label == "contradiction" else \
+                                    "C" if label == "neutral" else None
+
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+            print("Label:", label)
+
+            # Construct the prompt/message
+            instruction = ""
+            question = "Yukarıdaki cümleler arasındaki ilişki “entailment” (bir cümle diğerini ima eder), “neutral (cümleler birbirini ima etmez ve çelişmez) veya “contradiction (cümleler birbirleriyle çelişir) olarak karakterize edilebilir. Bu ilişkilerden hangisi olduğunu söyleyin."
+            context = f"Bağlam:\n{row["text"]}\n" # can add to prompt if needed
+            prompt = f"Cümle1:\n{row["premise"]}\nCümle2:{row["hypothesis"]}\nSoru:\n{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/reading_comp_mc.py

@@ -0,0 +1,77 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class ReadingComprehensionMCTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/reading_comp_mc", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            choices = ast.literal_eval(row["choices"]) # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            category = row["difficulty"].lower().replace(' ','')
+            answer = row["answer"]
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+            print("Type of answer:", type(answer))
+
+            # Get answer index (starting from 0)
+            if type(answer) == int:
+                answer_index = answer
+            else:
+                answer_index = int(answer)
+            correct_answer_letter = chr(65 + answer_index)
+
+
+            # Construct the prompt/message
+            instruction = ""
+            prompt = f"Paragraf:\n{row["text"]}\nSoru:{row["question_about_the_text"]}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+