Add remaining datasets except pos and ner

src/deepeval/base_task.py

@@ -77,52 +77,6 @@ class BaseTask(ABC):
 
         return answer
 
-    def generate_response_oeqa_multi_token(self, msg, max_new_tokens=-1, choices: list = []):
-        """
-        Handles multiple-choice questions where answers might have multiple tokens.
-        """
-        # Ensure tokenizer has proper special tokens set
-        if self.tokenizer.pad_token is None:
-            self.tokenizer.pad_token = self.tokenizer.eos_token
-
-        if self.model.config.pad_token_id is None:
-            self.model.config.pad_token_id = self.tokenizer.pad_token_id
-
-        chat = [
-                {"role": "user", "content": "You are a question-answering chatbot."},
-                {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
-                {"role": "user", "content": f"{msg}"},
-            ]
-        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-        print(formatted_chat)
-        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
-        input_ids = inputs.input_ids.to(self.model.device)
-        attention_mask = inputs.attention_mask.to(self.model.device)
-
-        # Custom LogitsProcessor to restrict generation
-        class RestrictToABCDLogitsProcessor(LogitsProcessor):
-            def __call__(self, input_ids, scores):
-                mask = torch.full_like(scores, float("-inf"))  # Block all tokens
-                return mask
-        logits_processor = LogitsProcessorList([RestrictToABCDLogitsProcessor()])
-
-        # Generate response
-        output = self.model.generate(
-            input_ids,
-            do_sample=True,
-            attention_mask=attention_mask,
-            #max_new_tokens=max_new_tokens,
-            eos_token_id=self.tokenizer.eos_token_id,
-            pad_token_id=self.tokenizer.pad_token_id,
-            temperature=0.4,
-            #logits_processor=logits_processor,
-        )
-        generated_ids = output[0]  # The generated sequence including the prompt
-        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
-        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
-        return generated_text
-
-
     def generate_response_mcqa_multi_token(self, msg, max_new_tokens=5, choices: list = []):
         """
         Handles multiple-choice questions where answers might have multiple tokens.

src/deepeval/bias.py

@@ -0,0 +1,98 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class BiasTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/sosyoloji_bias", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 2
+
+            # Get values from row
+
+            ## common
+            choices = ast.literal_eval(row["choices"])  # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+            ## ambiguous context
+            ambiguous_context = row["ambiguous_context"]
+            ambiguous_question = row["question_ambiguous"]
+            ambiguous_answer = row["answer_ambiguous"]
+            ambiguous_correct_answer_letter = chr(65 + ambiguous_answer)
+            ambiguous_prompt = f"Bağlam: {ambiguous_context}\nSoru: {ambiguous_question}\nSeçenekler:{formatted_choices}"
+
+            ## disambiguated context
+            disambiguated_context = row["disambiguated_context"]
+            disambiguated_question = row["question_disambiguated"]
+            disambiguated_answer = row["answer_disambiguated"]
+            disambiguated_correct_answer_letter = chr(65 + disambiguated_answer)
+            disambiguated_prompt = f"Bağlam: {disambiguated_context}\nSoru: {disambiguated_question}\nSeçenekler:{formatted_choices}"
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+
+            # STAGE 1
+            instruction = ""
+            prompt = f"Stage1:\n{ambiguous_prompt}\n{instruction}\n"
+            message = prompt
+            ambiguous_model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            ambiguous_model_answer_cleaned = ambiguous_model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+            ## Print answers
+            print(f"Correct Answer: {ambiguous_correct_answer_letter}")
+            print(f"Model Answer: {ambiguous_model_answer}")
+            print(f"Model Answer Cleaned: {ambiguous_model_answer_cleaned}")
+            print(f"Result: {ambiguous_correct_answer_letter == ambiguous_model_answer_cleaned}")
+            ## Check if correct based on metric
+            if ambiguous_correct_answer_letter == ambiguous_model_answer_cleaned:
+                true += 1
+                difficulty_results['correct'] += 1
+
+            difficulty_results['total'] += 1
+
+            # STAGE 2
+            instruction = ""
+            prompt = f"Stage2:\n{disambiguated_prompt}\n{instruction}\n"
+            message = prompt
+            disambiguated_model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            disambiguated_model_answer_cleaned = disambiguated_model_answer.strip().replace('\n', '').replace(' ','').upper().replace(':', '')
+            ## Print answers
+            print(f"Correct Answer: {disambiguated_correct_answer_letter}")
+            print(f"Model Answer: {disambiguated_model_answer}")
+            print(f"Model Answer Cleaned: {disambiguated_model_answer_cleaned}")
+            print(f"Result: {disambiguated_correct_answer_letter == disambiguated_model_answer_cleaned}")
+            responses.append((ambiguous_model_answer_cleaned,disambiguated_model_answer_cleaned))
+
+            ## Check if correct based on metric
+            if disambiguated_correct_answer_letter == disambiguated_model_answer_cleaned:
+                true += 1
+                difficulty_results['correct'] += 1
+
+            difficulty_results['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/deepeval_task_manager.py

@@ -15,6 +15,12 @@ from src.deepeval.complex_reasoning import ComplexReasoningTask
 from src.deepeval.truthfulness_task import TruthfulnessTask
 from src.deepeval.nli import NLITask
 from src.deepeval.math import MathTask
+from src.deepeval.turkish_vocabulary import TurkishVocabularyTask
+from src.deepeval.metaphors_and_idioms import MetaphorsAndIdiomsTask
+from src.deepeval.topic_detection import TopicDetectionTask
+from src.deepeval.sts import STSTask
+from src.deepeval.mmlu import MMLUTask
+from src.deepeval.bias import BiasTask
 from typing import List
 load_dotenv()
 HF_TOKEN=os.getenv("HF_TOKEN")
@@ -35,6 +41,12 @@ class Task(Enum):
     TRUTHFULNESS = "sosyoloji_truthfulness"
     NLI = "nli"
     MATH = "math"
+    TURKISH_VOCABULARY = "turkish_vocabulary"
+    METAPHORS_AND_IDIOMS = "metaphors_and_idioms"
+    TOPIC_DETECTION = "topic_detection"
+    STS = "sts"
+    MMLU = "mmlu"
+    BIAS_MC = "bias"
 
 
 class DeepEvalTaskManager:
@@ -139,7 +151,37 @@ class DeepEvalTaskManager:
         res = math_task.evaluate()
         return res
 
+    def turkish_vocabulary(self):
+        turkish_vocabulary_task = TurkishVocabularyTask(self.model_name)
+        res = turkish_vocabulary_task.evaluate()
+        return res
+
+    def metaphors_and_idioms(self):
+        metaphors_and_idioms_task = MetaphorsAndIdiomsTask(self.model_name)
+        res = metaphors_and_idioms_task.evaluate()
+        return res
+
+    def topic_detection(self):
+        topic_detection_task = TopicDetectionTask(self.model_name)
+        res = topic_detection_task.evaluate()
+        return res
+
+    def sts(self):
+        sts_task = STSTask(self.model_name)
+        res = sts_task.evaluate()
+        return res
+
+    def mmlu(self):
+        mmlu_task = MMLUTask(self.model_name)
+        res = mmlu_task.evaluate()
+        return res
+
+    def bias(self):
+        bias_task = BiasTask(self.model_name)
+        res = bias_task.evaluate()
+        return res
+
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["SUMMARIZATION"])
+    des = DeepEvalTaskManager("google/gemma-2b-it", ["MMLU"])
     res = des.run_tasks()
     print(res)

src/deepeval/math.py

@@ -10,7 +10,47 @@ class MathTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_oeqa_multi_token(self, msg,max_new_tokens: int = 128):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user", "content": "You are a question-answering chatbot."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+        # Generate response with proper token limits
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            max_new_tokens=max_new_tokens,
+        )
+
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+
+        return generated_text
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -31,23 +71,29 @@ class MathTask(BaseTask):
             print("Type of answer:", type(answer))
 
             # Construct the prompt/message
-            instruction = f"""Aşağıdaki matematik problemini verilen nihai cevap formatına uygun olacak şekilde çözün. Tüm adımları gösterdikten sonra, nihai cevabınızı sadece bir kez ve aşağıdaki kurallara uygun şekilde kutu içinde verin.
+            instruction = f"""Aşağıdaki matematik problemini verilen nihai cevap formatına uygun olacak şekilde çözün. Tüm adımları gösterdikten sonra, nihai cevabınızı sadece bir kez ve aşağıdaki kurallara uygun şekilde kutu (\\boxed{{}}) içinde verin.
 
 Nihai Cevap için Uyulması Gereken Format Kuralları:
 
-1. Nihai cevap, tek seferde \\boxed{{...}} içinde verilmeli. Örnek: Cevap 1 ise, "\\boxed{{1}}".
-2. Kesirleri her zaman en sade halde verilmeli.
+1. Kesirler her zaman en sade hallerinde verilmeli.
     - Matris içi kesirler: x/y biçiminde.
     - Diğer tüm kesirler: \\frac{{x}}{{y}} biçiminde.
-3. Çarpma işareti (*) kullanmayın. Örnek: 2x yazın, 2**x* değil.
-4. Birden çok değişken varsa alfabetik sıraya uyulmalı ve (x, y, z...), polinomları azalan derece sırasına göre yazılmalı.
-5. Her zaman aynı gösterim biçimi kullanılmalı. Ondalık yerine kesir kullanılmalı (ör. 0.5 yerine \\frac{{1}}{{2}} ).
-6. Faktörize polinomlar daima aynı faktör sırası ile verilsin; her sorguda aynı cevabı verecek şekilde tutarlılığı koruyun.
-7. Nihai cevabı kutu dışında tekrar etmeyin, biçimi değiştirmeyin. Aynı soru tekrarlandığında aynı formatı ve cevabı verin.
+2. Çarpma işareti (*) kullanılmamalı. Örnek: 2x yazın, 2**x* değil.
+3. Birden çok değişken varsa alfabetik sıraya uyulmalı ve (x, y, z...), polinomları azalan derece sırasına göre yazılmalı.
+4. Her zaman aynı gösterim biçimi kullanılmalı. Ondalık yerine kesir kullanılmalı (ör. 0.5 yerine \\frac{{1}}{{2}} ).
+5. Faktörize polinomlar daima aynı faktör sırası ile verilsin; her sorguda aynı cevabı verecek şekilde tutarlılığı koruyun.
+6. Nihai cevabı kutu dışında tekrar etmeyin, biçimi değiştirmeyin. Aynı soru tekrarlandığında aynı formatı ve cevabı verin.
+7. Nihai cevap, tek seferde \\boxed{{...}} içinde verilmeli. Örnek: Cevap x ise, "\\boxed{{x}}".
+
 
 Görev: Problemi çözün, son adımda yukarıdaki kurallara tam uyan tek bir kutu içinde nihai cevabı verin.
 
-            """
+
+Çözüm:
+
+
+Nihai cevap:
+                        """
             prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
             message = prompt
 

src/deepeval/metaphors_and_idioms.py

@@ -0,0 +1,87 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_split_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class MetaphorsAndIdiomsTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/metaphors_and_idioms", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset # dataset.select(range(min(10, len(dataset))))
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results =  defaultdict(lambda:  defaultdict(lambda: {'correct': 0, 'total': 0}))
+
+        total_count = 0
+        true = 0
+
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            category = "hard" if row["level"]== 1 else "easy" if row["level"] == 0 else None
+            answer_index = row["answer"]
+            correct_answer_letter = chr(65 + answer_index)
+            context = row["context"]
+            choices = ast.literal_eval(row["choices"])  # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+            subset = row["idiom_type"]
+
+            if subset == "atasözü":
+                question = "Aşağıda verilen durum hangi atasözü ile en iyi ifade edilebilir?"
+            elif subset == "deyim":
+                question = """Verilen bağlamda "[MASKED]" ile boş bırakılan yere hangi deyim getirilirse cümlenin akışı anlamlı olur?"""
+            else:
+                question = "Aşağıda verilen durum hangi atasözü ile en iyi ifade edilebilir?"
+
+            # Prints for debugging
+            print(f"Difficulty: {category}")
+            print("Type of difficulty:", type(category))
+            print(f"Answer: {correct_answer_letter}")
+            print("Type of answer:", type(answer_index))
+
+            # Construct the prompt/message
+            instruction = ""
+            prompt = f"Soru: {question}\nBağlam: {context}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[subset][category]['correct'] += 1
+
+            difficulty_results[subset][category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for subset in difficulty_results.keys():
+            subset_results = difficulty_results[subset]
+            for category, stats in subset_results.items():
+                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/mmlu.py

@@ -0,0 +1,87 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_config_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class MMLUTask(BaseTask):
+    def __init__(self, model_name):
+        self.subsets = get_dataset_config_names("metunlp/mmlu_tr")
+        print(self.subsets)
+        super().__init__("metunlp/mmlu_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        evaluate_count = 1
+        print("Loading dataset from Hugging Face.")
+        dataset_dict = {}
+        for subset in self.subsets:
+            subset_data = load_dataset(self.dataset_repo, subset, token=HF_TOKEN, split="train")
+            dataset_dict[subset] = subset_data.select(range(min(evaluate_count, len(subset_data))))
+        print("Dataset loaded.")
+        return dataset_dict
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+
+        total_count = 0
+        true = 0
+
+        for subset in self.subsets:
+            curr_dataset = self.dataset[subset]
+            print(curr_dataset[0])
+
+            for row in curr_dataset:
+                total_count += 1
+
+                # Get values from row
+                question = row["question"]
+                answer_index = row["answer"]
+                correct_answer_letter = chr(65 + answer_index)
+                choices = ast.literal_eval(row["choices"])  # Convert string to list
+                formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+
+                # Prints for debugging
+                print(f"Answer: {correct_answer_letter}")
+                print("Type of answer:", type(answer_index))
+
+                # Construct the prompt/message
+                instruction = f"Aşağıda {row["subject"]} konusunda çoktan seçmeli bir soru verilmiştir."
+                prompt = f"{instruction}\n\nSoru: {question}\nSeçenekler:\n{formatted_choices}\n\n"
+                message = prompt
+
+                # Get/format answer of the model
+                model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+                responses.append(model_answer)
+                model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+                # Print answers
+                print(f"Correct Answer: {correct_answer_letter}")
+                print(f"Model Answer: {model_answer}")
+                print(f"Model Answer Cleaned: {model_answer_cleaned}")
+                print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+                # Check if correct based on metric
+                if correct_answer_letter == model_answer_cleaned:
+                    true += 1
+                    difficulty_results[subset]['correct'] += 1
+
+                difficulty_results[subset]['total'] += 1
+
+        # Print results categorized by subset
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{subset.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/sts.py

@@ -0,0 +1,131 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import re
+from datasets import load_dataset
+import os
+from dotenv import load_dotenv
+import openai
+from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, LogitsProcessor
+import torch
+from typing import List
+
+class STSTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/sts_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_sts_multi_token(self, msg, max_new_tokens=5, choices: list = []):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user",
+             "content": "You are a sentence similarity scoring chatbot. Only respond with one of the given scores: 0, 1, 2, 3, 4, or 5."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+        # Generate the sequence of letters starting from 'A'
+        letters = ["0","1","2","3","4","5"]
+        encoded_choices = [self.tokenizer.encode(letter, add_special_tokens=False) for letter in letters]
+        flattened_encoded_choices = [item for sublist in encoded_choices for item in sublist]  # Flatten the list
+        print(flattened_encoded_choices)
+
+        allowed_tokens = flattened_encoded_choices
+        allowed_tokens += self.get_chat_template_tokens()  # Get the special chat tokens
+        allowed_token_ids = set(allowed_tokens)  # Ensure uniqueness
+
+        # Custom LogitsProcessor to restrict generation
+        class RestrictToABCDLogitsProcessor(LogitsProcessor):
+            def __call__(self, input_ids, scores):
+                mask = torch.full_like(scores, float("-inf"))  # Block all tokens
+                mask[:, list(allowed_token_ids)] = scores[:, list(allowed_token_ids)]  # Allow only A, B, C, D tokens
+                return mask
+
+        logits_processor = LogitsProcessorList([RestrictToABCDLogitsProcessor()])
+
+        # Generate response
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            max_new_tokens=max_new_tokens,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            logits_processor=logits_processor,
+        )
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        return generated_text
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = {'correct': 0, 'total': 0}
+
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            answer = row["score"]
+            choices = ["0","1","2","3","4","5"]
+
+            # Prints for debugging
+            print(f"Answer: {answer}")
+            print("Type of answer:", type(answer))
+
+            # Construct the prompt/message
+            instruction = f"Aşağıda verilen iki cümlenin birbirlerine olan anlamsal benzerliğini 0'dan 5'e kadar olan bir tam sayıyla söyleyin."
+            prompt = f"""{instruction}\nCümle 1: {row["sentence_1"]}\nCümle 2: {row["sentence_2"]}\nSadece tek bir tam sayı söyleyin, ek bir kelime ya da sembol kullanmayın."""
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_sts_multi_token(message, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {answer}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {answer == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if answer == model_answer_cleaned:
+                true += 1
+                difficulty_results['correct'] += 1
+
+            difficulty_results['total'] += 1
+
+        # Print results
+            stats = difficulty_results
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/topic_detection.py

@@ -0,0 +1,79 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class TopicDetectionTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/topic_detection_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            choices = ast.literal_eval(row["choices"]) # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            category = row["level"].lower().replace(' ','')
+            answer = row["answer"]
+            text = row["text"]
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+            print("Type of answer:", type(answer))
+
+            # Get answer index (starting from 0)
+            if type(answer) == int:
+                answer_index = answer
+            else:
+                answer_index = int(answer)
+            correct_answer_letter = chr(65 + answer_index)
+
+
+            # Construct the prompt/message
+            instruction = "Aşağıdaki metni analiz et ve seçeneklerden bu metnin en olası kategorisini belirle. Temaya ve detaylara dikkat ederek metnin ana fikrini göz önünde bulundurarak soruyu cevapla."
+            prompt = f"{instruction}\n\nMetin:\n{text}\nSeçenekler:\n{formatted_choices}\n\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/turkish_vocabulary.py

@@ -0,0 +1,100 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_split_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class TurkishVocabularyTask(BaseTask):
+    def __init__(self, model_name):
+        self.subsets = ["rare", "loan"]
+        super().__init__("metunlp/turkish_vocabulary", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        evaluate_count = 1
+        print("Loading dataset from Hugging Face.")
+        dataset_dict = {}
+        for subset in self.subsets:
+            subset_data = load_dataset(self.dataset_repo, subset, token=HF_TOKEN, split="train")
+            dataset_dict[subset] = subset_data.select(range(min(evaluate_count, len(subset_data))))
+        print("Dataset loaded.")
+        return dataset_dict
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: defaultdict(lambda: {'correct': 0, 'total': 0}))
+
+        total_count = 0
+        true = 0
+
+        for subset in self.subsets:
+            curr_dataset = self.dataset[subset]
+            print(curr_dataset[0])
+
+            # Determine the question based on the subset
+            if subset == "rare":
+                question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?"
+            elif subset == "loan":
+                question = "Verilen kelimenin Türkçe kökenli eş anlamlısı aşağıdakilerden hangisidir?"
+            else:
+                question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?"
+
+            for row in curr_dataset:
+                total_count += 1
+
+                # Get values from row
+                category = "hard" if row["level"]== 1 else "easy" if row["level"] == 0 else None
+                answer_index = row["answer"]
+                correct_answer_letter = chr(65 + answer_index)
+                word = row["word"]
+                choices = ast.literal_eval(row["choices"])  # Convert string to list
+                formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+
+
+                # Prints for debugging
+                print(f"Difficulty: {category}")
+                print("Type of difficulty:", type(category))
+                print(f"Answer: {correct_answer_letter}")
+                print("Type of answer:", type(answer_index))
+
+                # Construct the prompt/message
+                instruction = ""
+                prompt = f"Soru: {question}\nKelime: {word}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+                message = prompt
+
+                # Get/format answer of the model
+                model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+                responses.append(model_answer)
+                model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+                # Print answers
+                print(f"Correct Answer: {correct_answer_letter}")
+                print(f"Model Answer: {model_answer}")
+                print(f"Model Answer Cleaned: {model_answer_cleaned}")
+                print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+                # Check if correct based on metric
+                if correct_answer_letter == model_answer_cleaned:
+                    true += 1
+                    difficulty_results[subset][category]['correct'] += 1
+
+                difficulty_results[subset][category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for subset in self.subsets:
+            subset_results = difficulty_results[subset]
+            for category, stats in subset_results.items():
+                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+