add-aysu-tasks

Dockerfile

@@ -13,4 +13,4 @@ COPY --chown=user ./requirements.txt requirements.txt
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 COPY --chown=user . /app
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--loop", "asyncio"]

app.py

@@ -3,6 +3,16 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from svc.router import router
 
+import asyncio
+import sys
+
+# Disable uvloop by setting default asyncio policy
+if sys.platform == "win32":
+    # If running on Windows, you can skip applying the loop policy
+    pass
+else:
+    asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
+
 app = FastAPI(
     title="Resume Generator API",
     description="API for converting audio/text to structured resume with PDF generation",
@@ -27,4 +37,4 @@ async def health_check():
 
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8080)
+    uvicorn.run(app, host="0.0.0.0", port=8080, loop="asyncio")

auth/authentication.py

@@ -0,0 +1,33 @@
+from fastapi.security import OAuth2PasswordBearer
+from fastapi import HTTPException, Depends
+from jose import JWTError, jwt
+from datetime import datetime, timedelta
+
+
+SECRET_KEY = "llmbenchmark_tr" # your secret key
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 30
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="api/token")
+
+def create_access_token(data: dict):
+    to_encode = data.copy()
+    expire = datetime.now() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
+    to_encode.update({"exp": expire})
+    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+def get_current_user(token: str = Depends(oauth2_scheme)):
+    credentials_exception = HTTPException(
+        status_code=401,
+        detail="Could not validate credentials",
+        headers={"WWW-Authenticate": "Bearer"},
+    )
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        username: str = payload.get("sub")
+        if username is None:
+            raise credentials_exception
+        return username
+    except JWTError:
+        raise credentials_exception

requirements.txt

@@ -7,4 +7,6 @@ python-jose
 python-multipart
 deepeval
 --extra-index-url https://download.pytorch.org/whl/cu113
-torch
+huggingface-hub>=0.29.1
+torch
+sentencepiece

src/deepeval/base_task.py

@@ -2,11 +2,14 @@ from abc import ABC, abstractmethod
 from datasets import load_dataset
 import os
 from dotenv import load_dotenv
+import openai
 from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, LogitsProcessor
 import torch
 from typing import List
+from datetime import datetime
 load_dotenv()
 HF_TOKEN=os.getenv("HF_TOKEN")
+OPENAI_KEY = os.getenv("OPENAI_API_KEY")
 
 class BaseTask(ABC):
     _model_cache = {}  # Class-level cache for models and tokenizers
@@ -14,8 +17,9 @@ class BaseTask(ABC):
     def __init__(self, dataset_repo, model_name):
         self.dataset_repo = dataset_repo
         self.dataset = self.load_dataset_from_hf()
-        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        self.device = "auto" if torch.cuda.is_available() else "cpu"
         self.model, self.tokenizer = self.get_cached_model(model_name, self.device)
+        openai.api_key = OPENAI_KEY
 
 
     @classmethod
@@ -28,12 +32,17 @@ class BaseTask(ABC):
     @staticmethod
     def load_model(model_name: str, device):
         """Loads model and tokenizer once and caches it."""
+        print(f"Loading model: {model_name}")
+        start_time = datetime.now()
         model = AutoModelForCausalLM.from_pretrained(
             model_name,
             torch_dtype=torch.float16,
             device_map=device,
             token=HF_TOKEN,  # Replace with actual token
         )
+        end_time = datetime.now()
+        print(f"Model loaded in {(end_time - start_time).seconds} seconds.")
+        print("Model loaded.")
         tokenizer = AutoTokenizer.from_pretrained(model_name)
         return model, tokenizer
 
@@ -44,8 +53,8 @@ class BaseTask(ABC):
             self.tokenizer.pad_token = self.tokenizer.eos_token  # Use EOS token as PAD token
 
         inputs = self.tokenizer(msg, return_tensors="pt", padding=True, truncation=True)
-        input_ids = inputs.input_ids.to(self.model.device)
-        attention_mask = inputs.attention_mask.to(self.model.device)
+        input_ids = inputs.input_ids
+        attention_mask = inputs.attention_mask
 
         if self.model.config.pad_token_id is None:
             self.model.config.pad_token_id = self.tokenizer.eos_token_id
@@ -72,7 +81,7 @@ class BaseTask(ABC):
 
         return answer
     
-    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=5, choices: list = []):
+    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=2, choices: list = []):
         """
         Handles multiple-choice questions where answers might have multiple tokens.
         """
@@ -89,16 +98,16 @@ class BaseTask(ABC):
                 {"role": "user", "content": f"{msg}"},
             ]
         formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-        print(formatted_chat)
+        #print(formatted_chat)
         inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
-        input_ids = inputs.input_ids.to(self.model.device)
-        attention_mask = inputs.attention_mask.to(self.model.device)
+        input_ids = inputs.input_ids
+        attention_mask = inputs.attention_mask
 
         # Generate the sequence of letters starting from 'A'
         letters = [chr(ord('A') + i) for i in range(len(choices))]  # Create option letters A, B, C, D, E, ...
         encoded_choices = [self.tokenizer.encode(letter, add_special_tokens=False) for letter in letters]
         flattened_encoded_choices = [item for sublist in encoded_choices for item in sublist]  # Flatten the list
-        print(flattened_encoded_choices)
+        #print(flattened_encoded_choices)
 
         allowed_tokens = flattened_encoded_choices
         allowed_tokens += self.get_chat_template_tokens() # Get the special chat tokens
@@ -133,13 +142,25 @@ class BaseTask(ABC):
         if self.tokenizer.pad_token is None:
             self.tokenizer.pad_token = self.tokenizer.eos_token
 
-        inputs = self.tokenizer(prompt, return_tensors="pt", padding=True, truncation=True)
-        input_ids = inputs.input_ids.to(self.model.device)
-        attention_mask = inputs.attention_mask.to(self.model.device)
-
         if self.model.config.pad_token_id is None:
             self.model.config.pad_token_id = self.tokenizer.eos_token_id
 
+        chat = [
+            {"role": "user", "content": "You are a helpful AI assistant."},
+            {"role": "assistant", "content": "I am here to help you with any questions you may have."},
+            {"role": "user", "content": prompt},
+        ]
+        
+        formatted_chat = self.tokenizer.apply_chat_template(
+            chat,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids
+        attention_mask = inputs.attention_mask
+
         output = self.model.generate(
             input_ids,
             attention_mask=attention_mask,
@@ -147,7 +168,11 @@ class BaseTask(ABC):
             do_sample=True,
             temperature=0.7,
         )
-        result = self.tokenizer.decode(output[0], skip_special_tokens=True)
+
+        generated_ids = output[0]
+        prompt_len = input_ids.shape[1]
+        generated_tokens = generated_ids[prompt_len:]
+        result = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
         return result
 
     def get_chat_template_tokens(self):
@@ -164,7 +189,17 @@ class BaseTask(ABC):
         Define your own loading method if needed.
         :return: Dataset
         """
-        return load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
+        print("Loading dataset from Hugging Face.")
+        start_time = datetime.now()
+        dataset= load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
+        print("Dataset loaded.")
+        
+        # Load 50 from each dataset
+        if len(dataset) > 50:
+            dataset = dataset.shuffle(seed=42).select(range(50))
+        end_time = datetime.now()
+        print(f"Dataset loaded in {(end_time - start_time).seconds} seconds.")
+        return dataset
 
     @abstractmethod
     def evaluate(self):

src/deepeval/bias.py

@@ -0,0 +1,98 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class BiasTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/sosyoloji_bias", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 2
+
+            # Get values from row
+
+            ## common
+            choices = ast.literal_eval(row["choices"])  # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+            ## ambiguous context
+            ambiguous_context = row["ambiguous_context"]
+            ambiguous_question = row["question_ambiguous"]
+            ambiguous_answer = row["answer_ambiguous"]
+            ambiguous_correct_answer_letter = chr(64 + ambiguous_answer)  # 65 - 1 since we need the indexing to start from 0
+            ambiguous_prompt = f"Bağlam: {ambiguous_context}\nSoru: {ambiguous_question}\nSeçenekler:{formatted_choices}"
+
+            ## disambiguated context
+            disambiguated_context = row["disambiguated_context"]
+            disambiguated_question = row["question_disambiguated"]
+            disambiguated_answer = row["answer_disambiguated"]
+            disambiguated_correct_answer_letter = chr(64 + disambiguated_answer)  # 65 - 1 since we need the indexing to start from 0
+            disambiguated_prompt = f"Bağlam: {disambiguated_context}\nSoru: {disambiguated_question}\nSeçenekler:{formatted_choices}"
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+
+            # STAGE 1
+            instruction = ""
+            prompt = f"Stage1:\n{ambiguous_prompt}\n{instruction}\n"
+            message = prompt
+            ambiguous_model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            ambiguous_model_answer_cleaned = ambiguous_model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+            ## Print answers
+            print(f"Correct Answer: {ambiguous_correct_answer_letter}")
+            print(f"Model Answer: {ambiguous_model_answer}")
+            print(f"Model Answer Cleaned: {ambiguous_model_answer_cleaned}")
+            print(f"Result: {ambiguous_correct_answer_letter == ambiguous_model_answer_cleaned}")
+            ## Check if correct based on metric
+            if ambiguous_correct_answer_letter == ambiguous_model_answer_cleaned:
+                true += 1
+                difficulty_results["ambiguous"]['correct'] += 1
+
+            difficulty_results["ambiguous"]['total'] += 1
+
+            # STAGE 2
+            instruction = ""
+            prompt = f"Stage2:\n{disambiguated_prompt}\n{instruction}\n"
+            message = prompt
+            disambiguated_model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            disambiguated_model_answer_cleaned = disambiguated_model_answer.strip().replace('\n', '').replace(' ','').upper().replace(':', '')
+            ## Print answers
+            print(f"Correct Answer: {disambiguated_correct_answer_letter}")
+            print(f"Model Answer: {disambiguated_model_answer}")
+            print(f"Model Answer Cleaned: {disambiguated_model_answer_cleaned}")
+            print(f"Result: {disambiguated_correct_answer_letter == disambiguated_model_answer_cleaned}")
+            responses.append((ambiguous_model_answer_cleaned,disambiguated_model_answer_cleaned))
+
+            ## Check if correct based on metric
+            if disambiguated_correct_answer_letter == disambiguated_model_answer_cleaned:
+                true += 1
+                difficulty_results["disambiguated"]['correct'] += 1
+
+            difficulty_results["disambiguated"]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/bias_task.py

@@ -1,7 +1,6 @@
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import BiasMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
 
 class BiasTask(BaseTask):
@@ -10,7 +9,8 @@ class BiasTask(BaseTask):
         super().__init__("metunlp/sosyoloji_bias", model_name=model_name)
 
     def load_dataset_from_hf(self):
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
 
@@ -22,7 +22,6 @@ class BiasTask(BaseTask):
             disambiguated_context = row.get("disambiguated_context", "")
             nonnegative_question = row.get("question_disambiguated", "")
 
-
             prompt = (
                 f"Stage1:\nBağlam: {ambiguous_context}\n"
                 f"Soru: {negative_question}\nCevap:\n\n"
@@ -36,7 +35,7 @@ class BiasTask(BaseTask):
                 input=prompt,
                 actual_output=answer
             )
-            metric = BiasMetric(threshold=0.5)
+            metric = BiasMetric(threshold=0.0,model="gpt-4o-mini")
             metric.measure(test_case)
 
             results.append({
@@ -47,16 +46,6 @@ class BiasTask(BaseTask):
                 "prompt": prompt,
                 "answer": answer
             })
-
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Prompt ---")
-        #    print(res['prompt'])
-        #    print("--- Answer ---")
-        #    print(res['answer'])
-        #    print("\n---------------------------\n")
-
-        return {"results": results}
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        return {"results": overallScore}

src/deepeval/commonsense_reasoning_task.py

@@ -10,7 +10,7 @@ class CommonsenseReasoningTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -28,11 +28,13 @@ class CommonsenseReasoningTask(BaseTask):
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
             category = row["difficulty"]
             answer = row["answer"]
+            text = row["text"]
+            context = row["context"]
 
             # Prints for debugging
-            print(f"Choices: {choices}")
-            print("Type of choices:", type(choices))
-            print("Type of answer:", type(answer))
+            # print(f"Choices: {choices}")
+            # print("Type of choices:", type(choices))
+            # print("Type of answer:", type(answer))
 
             # Get answer index (starting from 0)
             if type(answer) == int:
@@ -51,18 +53,18 @@ class CommonsenseReasoningTask(BaseTask):
 
             # Construct the prompt/message
             instruction = ""
-            prompt = f"Bağlam:\n{row["text"]}\nÖnerme:\n{row["context"]}\nSoru:{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            prompt = f"Bağlam:\n{text}\nÖnerme:\n{context}\nSoru:{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
 
             # Print answers
-            print(f"Correct Answer: {correct_answer_letter}")
-            print(f"Model Answer: {model_answer}")
-            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            # print(f"Correct Answer: {correct_answer_letter}")
+            # print(f"Model Answer: {model_answer}")
+            # print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
             # Check if correct based on metric
             if correct_answer_letter == model_answer_cleaned:

src/deepeval/complex_reasoning.py

@@ -11,7 +11,7 @@ class ComplexReasoningTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -26,18 +26,20 @@ class ComplexReasoningTask(BaseTask):
 
             # Get values from row
             choices = ast.literal_eval(row["choices"]) # Convert string to list
+            narrative = row["narrative"]
+            question = row["question"]
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
             correct_answer_letter = row["answer_choice"]
             correct_answers.append(correct_answer_letter)
 
             # Prints for debugging
-            print(f"Choices: {choices}")
-            print("Type of choices:", type(choices))
+            # print(f"Choices: {choices}")
+            # print("Type of choices:", type(choices))
 
 
             # Construct the prompt/message
             instruction = ""
-            prompt = f"Soru:\n{row["narrative"]}\n{row["question"]}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            prompt = f"Soru:\n{narrative}\n{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
             message = prompt
 
             # Get/format answer of the model
@@ -48,9 +50,9 @@ class ComplexReasoningTask(BaseTask):
             if correct_answer_letter == model_answer_cleaned:
                 true += 1
             # Print answers
-            print(f"Correct Answer: {correct_answer_letter}")
-            print(f"Model Answer: {model_answer}")
-            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            # print(f"Correct Answer: {correct_answer_letter}")
+            # print(f"Model Answer: {model_answer}")
+            # print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
         print("Answers:", correct_answers)
         print("Results:", responses)

src/deepeval/deepeval_task_manager.py

@@ -12,15 +12,18 @@ from src.deepeval.instruction_following_task import InstructionFollowingTask
 from src.deepeval.reading_comprehension_task import ReadingComprehensionTask
 from src.deepeval.reading_comp_mc import ReadingComprehensionMCTask
 from src.deepeval.complex_reasoning import ComplexReasoningTask
+from src.deepeval.truthfulness_task import TruthfulnessTask
 from src.deepeval.nli import NLITask
+from src.deepeval.math import MathTask
+from src.deepeval.turkish_vocabulary import TurkishVocabularyTask
+from src.deepeval.metaphors_and_idioms import MetaphorsAndIdiomsTask
+from src.deepeval.topic_detection import TopicDetectionTask
+from src.deepeval.sts import STSTask
+from src.deepeval.mmlu import MMLUTask
+from src.deepeval.bias import BiasTask
 from typing import List
+from datetime import datetime
 load_dotenv()
-
-openai_configs = {
-    'OPENAI_API_KEY': 'OPENAI_KEY'
-}
-os.environ['OPENAI_API_KEY'] = openai_configs['OPENAI_API_KEY']
-
 HF_TOKEN=os.getenv("HF_TOKEN")
 
 class Task(Enum):
@@ -28,15 +31,23 @@ class Task(Enum):
     SENTIMENT_ANALYSIS = "sentiment_analysis_tr"
     TURKISH_GENERAL_KNOWLEDGE = "turkish_general_knowledge"
     SUMMARIZATION = "summarization_tr"
-    FAITHFULNESS = "faithfulness_tr"
-    TOXICITY = "toxicity_tr"
-    BIAS = "bias_tr"
+    FAITHFULNESS = "sosyoloji_faithfulness"
+    TOXICITY = "sosyoloji_toxicity"
+    BIAS = "sosyoloji_bias"
     INSTRUCTION_FOLLOWING = "instruction_following_tr"
-    READING_COMPREHENSION = "reading_comprehension_tr"
+    READING_COMPREHENSION = "reading_comprehension_mc"
+    READING_COMPREHENSION_OE = "reading_comp_oe"
     COMMONSENSE_REASONING = "commonsense_reasoning"
-    READING_COMPREHENSION_MC = "reading_comprehension_mc"
     COMPLEX_REASONING = "complex_reasoning"
+    TRUTHFULNESS = "sosyoloji_truthfulness"
     NLI = "nli"
+    MATH = "math"
+    TURKISH_VOCABULARY = "turkish_vocabulary"
+    METAPHORS_AND_IDIOMS = "metaphors_and_idioms"
+    TOPIC_DETECTION = "topic_detection"
+    STS = "sts"
+    MMLU = "mmlu"
+    BIAS_MC = "bias"
 
 
 class DeepEvalTaskManager:
@@ -49,9 +60,13 @@ class DeepEvalTaskManager:
         """Validate user tasks and store method references."""
         print(self.available_tasks.keys())
         print(user_tasks)
-        if not set(user_tasks).issubset(self.available_tasks.keys()):
-            invalid_tasks = set(user_tasks) - self.available_tasks.keys()
-            raise ValueError(f"Invalid task(s) requested: {invalid_tasks}")
+
+        try:
+            if not set(user_tasks).issubset(self.available_tasks.keys()):
+                invalid_tasks = set(user_tasks) - self.available_tasks.keys()
+                raise ValueError(f"Invalid task(s) requested: {invalid_tasks}")
+        except Exception as e:
+            print(f"Error: {e}")
 
         # Store actual method references instead of strings
         return {task : self.available_tasks[task] for task in user_tasks}
@@ -59,12 +74,22 @@ class DeepEvalTaskManager:
     def run_tasks(self):
         """Execute validated tasks in order."""
         results = {}
+        total_start_time = datetime.now()
         for task_name, task_method in self.tasks_to_run.items():
-            print("Running task: ", task_name)
-            task_enum = getattr(Task, task_name)
-            task_value = task_enum.value
-            results[task_value] = task_method()  # Call the stored method reference
-
+            try:
+                start_time = datetime.now()
+                print("Running task: ", task_name)
+                task_enum = getattr(Task, task_name)
+                task_value = task_enum.value
+                results[task_value] = task_method()  # Call the stored method reference
+                end_time = datetime.now()
+                print(f"Task {task_name} completed in {(end_time - start_time).seconds} seconds.")
+            except Exception as e:
+                print(f"Error At Task: {task_name} - {e}")
+                continue
+        total_end_time = datetime.now()
+        print(f"All tasks completed in {(total_end_time - total_start_time).seconds} seconds.")
+        print("All tasks completed.")
         return results
 
     def sentiment_analysis_tr(self):
@@ -78,32 +103,28 @@ class DeepEvalTaskManager:
         return res
 
     def summarization_tr(self):
-        task = SummarizationTask(self.model_name)
-        return task.evaluate()
+        summarization_task = SummarizationTask(self.model_name)
+        res = summarization_task.evaluate()
+        return res
 
-    def faithfulness_tr(self):
-        task = FaithfulnessTask(self.model_name)
-        return task.evaluate()
+    def sosyoloji_faithfulness(self):
+        faithfulness_task = FaithfulnessTask(self.model_name)
+        res = faithfulness_task.evaluate()
+        return res
 
-    def toxicity_tr(self):
-        task = ToxicityTask(self.model_name)
-        return task.evaluate()
+    def sosyoloji_toxicity(self):
+        toxicity_task = ToxicityTask(self.model_name)
+        res = toxicity_task.evaluate()
+        return res
 
-    def bias_tr(self):
-        task = BiasTask(self.model_name)
-        return task.evaluate()
+    def sosyoloji_bias(self):
+        bias_task = BiasTask(self.model_name)
+        res = bias_task.evaluate()
+        return res
 
     def instruction_following_tr(self):
-        task = InstructionFollowingTask(self.model_name)
-        return task.evaluate()
-
-    def reading_comprehension_tr(self):
-        task = ReadingComprehensionTask(self.model_name)
-        return task.evaluate()
-
-    def commonsense_reasoning(self):
-        commonsense_reasoning_task = CommonsenseReasoningTask(self.model_name)
-        res = commonsense_reasoning_task.evaluate()
+        instruction_following_task = InstructionFollowingTask(self.model_name)
+        res = instruction_following_task.evaluate()
         return res
 
     def reading_comprehension_mc(self):
@@ -111,17 +132,67 @@ class DeepEvalTaskManager:
         res = reading_comprehension_mc_task.evaluate()
         return res
 
+    def reading_comp_oe(self):
+        reading_comprehension_task = ReadingComprehensionTask(self.model_name)
+        res = reading_comprehension_task.evaluate()
+        return res
+
+    def commonsense_reasoning(self):
+        commonsense_reasoning_task = CommonsenseReasoningTask(self.model_name)
+        res = commonsense_reasoning_task.evaluate()
+        return res
+
     def complex_reasoning(self):
         complex_reasoning_task = ComplexReasoningTask(self.model_name)
         res = complex_reasoning_task.evaluate()
         return res
 
+    def sosyoloji_truthfulness(self):
+        truthfulness_task = TruthfulnessTask(self.model_name)
+        res = truthfulness_task.evaluate()
+        return res
+
     def nli(self):
         nli_task = NLITask(self.model_name)
         res = nli_task.evaluate()
         return res
 
+    def math(self):
+        math_task = MathTask(self.model_name)
+        res = math_task.evaluate()
+        return res
+
+    def turkish_vocabulary(self):
+        turkish_vocabulary_task = TurkishVocabularyTask(self.model_name)
+        res = turkish_vocabulary_task.evaluate()
+        return res
+
+    def metaphors_and_idioms(self):
+        metaphors_and_idioms_task = MetaphorsAndIdiomsTask(self.model_name)
+        res = metaphors_and_idioms_task.evaluate()
+        return res
+
+    def topic_detection(self):
+        topic_detection_task = TopicDetectionTask(self.model_name)
+        res = topic_detection_task.evaluate()
+        return res
+
+    def sts(self):
+        sts_task = STSTask(self.model_name)
+        res = sts_task.evaluate()
+        return res
+
+    def mmlu(self):
+        mmlu_task = MMLUTask(self.model_name)
+        res = mmlu_task.evaluate()
+        return res
+
+    def bias(self):
+        bias_task = BiasTask(self.model_name)
+        res = bias_task.evaluate()
+        return res
+
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("meta-llama/Llama-3.2-1B-Instruct", ["COMPLEX_REASONING","NLI"])
+    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["TOXICITY", "BIAS"])
     res = des.run_tasks()
     print(res)

src/deepeval/faithfulness_task.py

@@ -1,17 +1,15 @@
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import FaithfulnessMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
 
 class FaithfulnessTask(BaseTask):
-
     def __init__(self, model_name: str):
         super().__init__("metunlp/sosyoloji_faithfulness", model_name=model_name)
 
     def load_dataset_from_hf(self):
-
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
 
@@ -19,7 +17,7 @@ class FaithfulnessTask(BaseTask):
 
         for i, row in enumerate(self.dataset):
             context = row["context"]
-            question = row["soru"]
+            question = row["question"]
 
             prompt = (
                 f"Context: {context}\n"
@@ -36,7 +34,7 @@ class FaithfulnessTask(BaseTask):
             )
 
             metric = FaithfulnessMetric(
-                threshold=0.7,
+                threshold=0.0,
                 model="gpt-4o-mini",
                 include_reason=True
             )
@@ -52,18 +50,7 @@ class FaithfulnessTask(BaseTask):
                 "answer": generated_answer
             })
 
-        # Sonuçları ekrana bas (opsiyonel)
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Context ---")
-        #    print(res['context'])
-        #    print("--- Question ---")
-        #    print(res['question'])
-        #    print("--- Answer ---")
-        #    print(res['answer'])
-        #    print("\n---------------------------\n")
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
 
-        return {"results": results}
+        return {"results": overallScore}

src/deepeval/instruction_following_task.py

@@ -1,23 +1,19 @@
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import PromptAlignmentMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
 
 class InstructionFollowingTask(BaseTask):
 
-
     def __init__(self, model_name: str):
         super().__init__("metunlp/instruction_following_tr", model_name=model_name)
 
     def load_dataset_from_hf(self):
-
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
-
         results = []
-
         for i, row in enumerate(self.dataset):
             input_text = row.get("input", "")
             instruction_text = row.get("instruction", "")
@@ -51,18 +47,6 @@ class InstructionFollowingTask(BaseTask):
                 "instruction": instruction_text,
                 "output": output
             })
-
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Input ---")
-        #    print(res['input'])
-        #    print("--- Instruction ---")
-        #    print(res['instruction'])
-        #    print("--- Output ---")
-        #    print(res['output'])
-        #    print("\n---------------------------\n")
-
-        return {"results": results}
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        return {"results": overallScore}

src/deepeval/math.py

@@ -0,0 +1,128 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import re
+
+class MathTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/math_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_oeqa_multi_token(self, msg,max_new_tokens: int = 128):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user", "content": "You are a question-answering chatbot."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+        # Generate response with proper token limits
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            max_new_tokens=max_new_tokens,
+        )
+
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+
+        return generated_text
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            category = str(row["difficulty"])
+            answer = row["final_answer"]
+
+            # Prints for debugging
+            print(f"Answer: {answer}")
+            print("Type of answer:", type(answer))
+
+            # Construct the prompt/message
+            instruction = f"""Aşağıdaki matematik problemini verilen nihai cevap formatına uygun olacak şekilde çözün. Tüm adımları gösterdikten sonra, nihai cevabınızı sadece bir kez ve aşağıdaki kurallara uygun şekilde kutu (\\boxed{{}}) içinde verin.
+
+Nihai Cevap için Uyulması Gereken Format Kuralları:
+
+1. Kesirler her zaman en sade hallerinde verilmeli.
+    - Matris içi kesirler: x/y biçiminde.
+    - Diğer tüm kesirler: \\frac{{x}}{{y}} biçiminde.
+2. Çarpma işareti (*) kullanılmamalı. Örnek: 2x yazın, 2**x* değil.
+3. Birden çok değişken varsa alfabetik sıraya uyulmalı ve (x, y, z...), polinomları azalan derece sırasına göre yazılmalı.
+4. Her zaman aynı gösterim biçimi kullanılmalı. Ondalık yerine kesir kullanılmalı (ör. 0.5 yerine \\frac{{1}}{{2}} ).
+5. Faktörize polinomlar daima aynı faktör sırası ile verilsin; her sorguda aynı cevabı verecek şekilde tutarlılığı koruyun.
+6. Nihai cevabı kutu dışında tekrar etmeyin, biçimi değiştirmeyin. Aynı soru tekrarlandığında aynı formatı ve cevabı verin.
+7. Nihai cevap, tek seferde \\boxed{{...}} içinde verilmeli. Örnek: Cevap x ise, "\\boxed{{x}}".
+
+
+Görev: Problemi çözün, son adımda yukarıdaki kurallara tam uyan tek bir kutu içinde nihai cevabı verin.
+
+
+Çözüm:
+
+
+Nihai cevap:
+                        """
+            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_oeqa_multi_token(message)
+            responses.append(model_answer)
+            model_answer_cleaned = re.search(r"\\boxed{([^}]*)}", model_answer)
+
+            # Print answers
+            print(f"Correct Answer: {answer}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {answer == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if answer == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/metaphors_and_idioms.py

@@ -0,0 +1,87 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_split_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class MetaphorsAndIdiomsTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/metaphors_and_idioms", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset # dataset.select(range(min(10, len(dataset))))
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results =  defaultdict(lambda:  defaultdict(lambda: {'correct': 0, 'total': 0}))
+
+        total_count = 0
+        true = 0
+
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            category = "hard" if row["level"]== 1 else "easy" if row["level"] == 0 else None
+            answer_index = row["answer"]
+            correct_answer_letter = chr(65 + answer_index)
+            context = row["context"]
+            choices = ast.literal_eval(row["choices"])  # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+            subset = row["idiom_type"]
+
+            if subset == "atasözü":
+                question = "Aşağıda verilen durum hangi atasözü ile en iyi ifade edilebilir?"
+            elif subset == "deyim":
+                question = """Verilen bağlamda "[MASKED]" ile boş bırakılan yere hangi deyim getirilirse cümlenin akışı anlamlı olur?"""
+            else:
+                question = "Aşağıda verilen durum hangi atasözü ile en iyi ifade edilebilir?"
+
+            # Prints for debugging
+            print(f"Difficulty: {category}")
+            print("Type of difficulty:", type(category))
+            print(f"Answer: {correct_answer_letter}")
+            print("Type of answer:", type(answer_index))
+
+            # Construct the prompt/message
+            instruction = ""
+            prompt = f"Soru: {question}\nBağlam: {context}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[subset][category]['correct'] += 1
+
+            difficulty_results[subset][category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for subset in difficulty_results.keys():
+            subset_results = difficulty_results[subset]
+            for category, stats in subset_results.items():
+                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/mmlu.py

@@ -0,0 +1,87 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_config_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class MMLUTask(BaseTask):
+    def __init__(self, model_name):
+        self.subsets = get_dataset_config_names("metunlp/mmlu_tr")
+        print(self.subsets)
+        super().__init__("metunlp/mmlu_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        evaluate_count = 1
+        print("Loading dataset from Hugging Face.")
+        dataset_dict = {}
+        for subset in self.subsets:
+            subset_data = load_dataset(self.dataset_repo, subset, token=HF_TOKEN, split="train")
+            dataset_dict[subset] = subset_data.select(range(min(evaluate_count, len(subset_data))))
+        print("Dataset loaded.")
+        return dataset_dict
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+
+        total_count = 0
+        true = 0
+
+        for subset in self.subsets:
+            curr_dataset = self.dataset[subset]
+            print(curr_dataset[0])
+
+            for row in curr_dataset:
+                total_count += 1
+
+                # Get values from row
+                question = row["question"]
+                answer_index = row["answer"]
+                correct_answer_letter = chr(65 + answer_index)
+                choices = ast.literal_eval(row["choices"])  # Convert string to list
+                formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+
+                # Prints for debugging
+                print(f"Answer: {correct_answer_letter}")
+                print("Type of answer:", type(answer_index))
+
+                # Construct the prompt/message
+                instruction = f"Aşağıda {row["subject"]} konusunda çoktan seçmeli bir soru verilmiştir."
+                prompt = f"{instruction}\n\nSoru: {question}\nSeçenekler:\n{formatted_choices}\n\n"
+                message = prompt
+
+                # Get/format answer of the model
+                model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+                responses.append(model_answer)
+                model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+                # Print answers
+                print(f"Correct Answer: {correct_answer_letter}")
+                print(f"Model Answer: {model_answer}")
+                print(f"Model Answer Cleaned: {model_answer_cleaned}")
+                print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+                # Check if correct based on metric
+                if correct_answer_letter == model_answer_cleaned:
+                    true += 1
+                    difficulty_results[subset]['correct'] += 1
+
+                difficulty_results[subset]['total'] += 1
+
+        # Print results categorized by subset
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{subset.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/ner.py

@@ -0,0 +1,166 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import re
+
+class NERTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/tr_ner", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_oeqa_multi_token(self, msg,max_new_tokens: int = 128):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user", "content": "You are a question-answering chatbot."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+
+        # Generate response with proper token limits
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            max_new_tokens=max_new_tokens,
+        )
+
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+
+        return generated_text
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            category = str(row["difficulty"])
+            answer = row["final_answer"]
+
+            # Prints for debugging
+            print(f"Answer: {answer}")
+            print("Type of answer:", type(answer))
+
+            # Construct the prompt/message
+            instruction = ("Aşağıdaki Named Entity Recognition (NER) için etiketlenmesi gereken cümleler vardır. "
+                  "Cümlelerdeki varlıkları belirleyin ve şu kategorilere ayırın: CARDINAL, DATE, EVENT, FAC, GPE, LANGUAGE, LAW, LOC, MONEY, NORP, ORDINAL, ORG, PER, PERCENT, PERSON, PRODUCT, QUANTITY, TIME, TITLE, WORK_OF_ART. "
+                  ""
+                  "Varlıklar, anlamlı bilgiler içeren terimlerdir ve aşağıdaki şekilde tanımlanır: "
+                  "CARDINAL: Nicelik veya sıralama belirtmeyen sayısal ifadeler."
+                  "DATE: Belirli bir tarih veya zaman ifadeleri."
+                  "EVENT: Adlandırılmış olaylar veya durumlar."
+                  "FAC: Binalar veya önemli yerler gibi tesisler."
+                  "GPE: Ülke, şehir veya eyalet gibi coğrafi-politik varlıklar."
+                  "LANGUAGE: Adlandırılmış diller."
+                  "LAW: Yasal belgeler, düzenlemeler veya kanunlar."
+                  "LOC: Coğrafi veya fiziksel konumlar (GPE dışındaki)."
+                  "MONEY: Parasal değerler."
+                  "NORP: Milletler, dini veya siyasi gruplar."
+                  "ORDINAL: Sıralama veya dereceler."
+                  "ORG: Organizasyonlar veya kurumlar."
+                  "PER: Kişisel unvanlar veya sıfatlar."
+                  "PERSON: Bireylerin isimleri."
+                  "PRODUCT: Üretilen nesneler veya araçlar."
+                  "QUANTITY: Ölçülebilir miktarlar ve birimler."
+                  "TIME: Günün belirli saatleri."
+                  "TITLE: Kişi unvanları."
+                  "WORK_OF_ART: Sanat eserleri, kitaplar, müzik vb. Adlar, tarih ifadeleri, konumlar gibi belirgin bilgiler varlıktır."
+                  ""
+                  "Fiiller, sıfatlar, zarflar, soyut kavramlar gibi ifadeler varlık değildir. Çıktıyı aşağıdaki JSON formatında döndürün. "
+                  ""
+                  "Örnekler: "
+                  "Girdi: "
+                  "sentence: \"Üç yıl aradan sonra gerçekleştirilen ve Karadeniz, Ege ve Akdeniz’de düzenlenecek olan tatbikata ilişkin Yunanistan'ın Kathimerini gazetesi 'Türk-Yunan: Çetin donanma dengesinin gücü' başlığını kullandı.\""
+                  "Çıktı: "
+                  "Üç yıl,DATE"
+                  "Karadeniz,LOC"
+                  "Ege,LOC"
+                  "Akdeniz,LOC"
+                  "Yunanistan,GPE"
+                  "Kathimerini,ORG"
+                  "Türk,NORP"
+                  ""
+                  "Girdi:"
+                  "sentence: \"Evlendikten sonra oyunculuğu bırakan Makal, geçen yıl eşi ve oğluyla beraber İstanbul’dan Göcek’e taşınmıştı."
+                  "Çıktı: "
+                  "Makal,PERSON"
+                  "İstanbul,GPE"
+                  "Göcek,GPE"
+                  ""
+                  "Girdi:"
+                  "sentence: \"Yeşil-kırmızılılardan 2016’da ayrılıp 3 sezonluk aradan sonra 2019’da geri dönen Sarıca, takımına 2021 yılında Şampiyonlar Ligi’nde, 2023’te de Süper Lig’de iki final oynattı."
+                  "Çıktı:"
+                  "2016’da,DATE"
+                  "3,CARDINAL"
+                  "2019’da,DATE"
+                  "Sarıca,PERSON"
+                  "2021,DATE"
+                  "Şampiyonlar Ligi’nde,EVENT"
+                  "2023’te,DATE"
+                  "Süper Lig’de,EVENT"
+                  "iki,CARDINAL"
+                  ""
+                  "Verilen cümlelerdeki her varlığı csv formatında yukarıdaki örneklere benzer şekilde belirleyin. Çıktıdaki her satırı aşağıdaki gibi oluşturun: "
+                  "<Varlık metni>,<Varlık etiketi>"),
+            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_oeqa_multi_token(message)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer
+
+            # Print answers
+            print(f"Correct Answer: {answer}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {answer == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if answer == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/nli.py

@@ -10,7 +10,7 @@ class NLITask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -23,6 +23,9 @@ class NLITask(BaseTask):
             total_count += 1
 
             # Get values from row
+            text = row["text"]
+            premise = row["premise"]
+            hypothesis = row["hypothesis"]
             label = row["label"].lower().replace(' ','')
             choices=["entailment","contradiction","neutral"]
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
@@ -33,26 +36,26 @@ class NLITask(BaseTask):
 
 
             # Prints for debugging
-            print(f"Choices: {choices}")
-            print("Type of choices:", type(choices))
-            print("Label:", label)
+            # print(f"Choices: {choices}")
+            # print("Type of choices:", type(choices))
+            # print("Label:", label)
 
             # Construct the prompt/message
             instruction = ""
             question = "Yukarıdaki cümleler arasındaki ilişki “entailment” (bir cümle diğerini ima eder), “neutral (cümleler birbirini ima etmez ve çelişmez) veya “contradiction (cümleler birbirleriyle çelişir) olarak karakterize edilebilir. Bu ilişkilerden hangisi olduğunu söyleyin."
             context = f"Bağlam:\n{row["text"]}\n" # can add to prompt if needed
-            prompt = f"Cümle1:\n{row["premise"]}\nCümle2:{row["hypothesis"]}\nSoru:\n{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            prompt = f"Cümle1: {row["premise"]}\nCümle2: {row["hypothesis"]}\nSoru:\n{question}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
 
             # Print answers
-            print(f"Correct Answer: {correct_answer_letter}")
-            print(f"Model Answer: {model_answer}")
-            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            # print(f"Correct Answer: {correct_answer_letter}")
+            # print(f"Model Answer: {model_answer}")
+            # print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
             # Check if correct based on metric
             if correct_answer_letter == model_answer_cleaned:

src/deepeval/pos.py

@@ -0,0 +1,159 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import re
+
+class POSTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/tr_pos", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_oeqa_multi_token(self, msg,max_new_tokens: int = 128):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user", "content": "You are a question-answering chatbot."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+        prompt = ("Aşağıdaki Named Entity Recognition (NER) için etiketlenmesi gereken cümleler vardır. "
+                  "Cümlelerdeki varlıkları belirleyin ve şu kategorilere ayırın: CARDINAL, DATE, EVENT, FAC, GPE, LANGUAGE, LAW, LOC, MONEY, NORP, ORDINAL, ORG, PER, PERCENT, PERSON, PRODUCT, QUANTITY, TIME, TITLE, WORK_OF_ART. "
+                  ""
+                  "Varlıklar, anlamlı bilgiler içeren terimlerdir ve aşağıdaki şekilde tanımlanır: "
+                  "CARDINAL: Nicelik veya sıralama belirtmeyen sayısal ifadeler."
+                  "DATE: Belirli bir tarih veya zaman ifadeleri."
+                  "EVENT: Adlandırılmış olaylar veya durumlar."
+                  "FAC: Binalar veya önemli yerler gibi tesisler."
+                  "GPE: Ülke, şehir veya eyalet gibi coğrafi-politik varlıklar."
+                  "LANGUAGE: Adlandırılmış diller."
+                  "LAW: Yasal belgeler, düzenlemeler veya kanunlar."
+                  "LOC: Coğrafi veya fiziksel konumlar (GPE dışındaki)."
+                  "MONEY: Parasal değerler."
+                  "NORP: Milletler, dini veya siyasi gruplar."
+                  "ORDINAL: Sıralama veya dereceler."
+                  "ORG: Organizasyonlar veya kurumlar."
+                  "PER: Kişisel unvanlar veya sıfatlar."
+                  "PERSON: Bireylerin isimleri."
+                  "PRODUCT: Üretilen nesneler veya araçlar."
+                  "QUANTITY: Ölçülebilir miktarlar ve birimler."
+                  "TIME: Günün belirli saatleri."
+                  "TITLE: Kişi unvanları."
+                  "WORK_OF_ART: Sanat eserleri, kitaplar, müzik vb. Adlar, tarih ifadeleri, konumlar gibi belirgin bilgiler varlıktır."
+                  ""
+                  "Fiiller, sıfatlar, zarflar, soyut kavramlar gibi ifadeler varlık değildir. Çıktıyı aşağıdaki JSON formatında döndürün. "
+                  ""
+                  "Örnekler: "
+                  "Girdi: "
+                  "\"sentence\": \"Üç yıl aradan sonra gerçekleştirilen ve Karadeniz, Ege ve Akdeniz’de düzenlenecek olan tatbikata ilişkin Yunanistan'ın Kathimerini gazetesi 'Türk-Yunan: Çetin donanma dengesinin gücü' başlığını kullandı.\""
+                  "Çıktı: "
+                  "Üç yıl: DATE\" }, { \"text\": \"Karadeniz\", \"label\": \"LOC\" }, { \"text\": \"Ege\", \"label\": \"LOC\" }, { \"text\": \"Akdeniz\", \"label\": \"LOC\" }, { \"text\": \"Yunanistan\", \"label\": \"GPE\" }, { \"text\": \"Kathimerini\", \"label\": \"ORG\" }, { \"text\": \"Türk\", \"label\": \"NORP\" }]} Girdi: {\"sentence\": \"Evlendikten sonra oyunculuğu bırakan Makal, geçen yıl eşi ve oğluyla beraber İstanbul’dan Göcek’e taşınmıştı.\"} Çıktı: {\"entities\": [{ \"text\": \"Makal\", \"label\": \"PERSON\" }, { \"text\": \"İstanbul\", \"label\": \"GPE\" }, { \"text\": \"Göcek\", \"label\": \"GPE\" }]} Girdi: {\"sentence\": \"Yeşil-kırmızılılardan 2016’da ayrılıp 3 sezonluk aradan sonra 2019’da geri dönen Sarıca, takımına 2021 yılında Şampiyonlar Ligi’nde, 2023’te de Süper Lig’de iki final oynattı.\"} Çıktı: {\"entities\": [{ \"text\": \"2016’da\", \"label\": \"DATE\" }, { \"text\": \"3\", \"label\": \"CARDINAL\" }, { \"text\": \"2019’da\", \"label\": \"DATE\" }, { \"text\": \"Sarıca\", \"label\": \"PERSON\" }, { \"text\": \"2021\", \"label\": \"DATE\" }, { \"text\": \"Şampiyonlar Ligi’nde\", \"label\": \"EVENT\" }, { \"text\": \"2023’te\", \"label\": \"DATE\" }, { \"text\": \"Süper Lig’de\", \"label\": \"EVENT\" }, { \"text\": \"iki\", \"label\": \"CARDINAL\" }]}. Verilen cümlelerdeki varlıkları JSON formatında yukarıdaki örneklere benzer şekilde belirleyin. Çıktıyı aşağıdaki gibi oluşturun: Girdi Formatı: {\"sentence\": \"<CÜMLE>\"} Çıktı Formatı: {\"entities\": [{ \"text\": \"<Varlık metni>\", \"label\": \"<Varlık etiketi>\" }]}"),
+
+        # Generate response with proper token limits
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            max_new_tokens=max_new_tokens,
+        )
+
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+
+        return generated_text
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            category = str(row["difficulty"])
+            answer = row["final_answer"]
+
+            # Prints for debugging
+            print(f"Answer: {answer}")
+            print("Type of answer:", type(answer))
+
+            # Construct the prompt/message
+            instruction = f"""Aşağıdaki matematik problemini verilen nihai cevap formatına uygun olacak şekilde çözün. Tüm adımları gösterdikten sonra, nihai cevabınızı sadece bir kez ve aşağıdaki kurallara uygun şekilde kutu (\\boxed{{}}) içinde verin.
+
+Nihai Cevap için Uyulması Gereken Format Kuralları:
+
+1. Kesirler her zaman en sade hallerinde verilmeli.
+    - Matris içi kesirler: x/y biçiminde.
+    - Diğer tüm kesirler: \\frac{{x}}{{y}} biçiminde.
+2. Çarpma işareti (*) kullanılmamalı. Örnek: 2x yazın, 2**x* değil.
+3. Birden çok değişken varsa alfabetik sıraya uyulmalı ve (x, y, z...), polinomları azalan derece sırasına göre yazılmalı.
+4. Her zaman aynı gösterim biçimi kullanılmalı. Ondalık yerine kesir kullanılmalı (ör. 0.5 yerine \\frac{{1}}{{2}} ).
+5. Faktörize polinomlar daima aynı faktör sırası ile verilsin; her sorguda aynı cevabı verecek şekilde tutarlılığı koruyun.
+6. Nihai cevabı kutu dışında tekrar etmeyin, biçimi değiştirmeyin. Aynı soru tekrarlandığında aynı formatı ve cevabı verin.
+7. Nihai cevap, tek seferde \\boxed{{...}} içinde verilmeli. Örnek: Cevap x ise, "\\boxed{{x}}".
+
+
+Görev: Problemi çözün, son adımda yukarıdaki kurallara tam uyan tek bir kutu içinde nihai cevabı verin.
+
+
+Çözüm:
+
+
+Nihai cevap:
+                        """
+            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_oeqa_multi_token(message)
+            responses.append(model_answer)
+            model_answer_cleaned = re.search(r"\\boxed{([^}]*)}", model_answer)
+
+            # Print answers
+            print(f"Correct Answer: {answer}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {answer == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if answer == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/reading_comp_mc.py

@@ -11,7 +11,7 @@ class ReadingComprehensionMCTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -28,23 +28,27 @@ class ReadingComprehensionMCTask(BaseTask):
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
             category = row["difficulty"].lower().replace(' ','')
             answer = row["answer"]
+            text = row["text"]
+            question_about_the_text = row["question_about_the_text"]
 
             # Prints for debugging
-            print(f"Choices: {choices}")
-            print("Type of choices:", type(choices))
-            print("Type of answer:", type(answer))
+            # print(f"Choices: {choices}")
+            # print("Type of choices:", type(choices))
+            # print("Type of answer:", type(answer))
 
             # Get answer index (starting from 0)
             if type(answer) == int:
                 answer_index = answer
             else:
                 answer_index = int(answer)
+            
+            answer_index = answer_index - 1 # Because the answer is 1-indexed
             correct_answer_letter = chr(65 + answer_index)
 
 
             # Construct the prompt/message
             instruction = ""
-            prompt = f"Paragraf:\n{row["text"]}\nSoru:{row["question_about_the_text"]}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+            prompt = f"Paragraf:\n{text}\nSoru:{question_about_the_text}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
             message = prompt
 
             # Get/format answer of the model
@@ -53,9 +57,9 @@ class ReadingComprehensionMCTask(BaseTask):
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
 
             # Print answers
-            print(f"Correct Answer: {correct_answer_letter}")
-            print(f"Model Answer: {model_answer}")
-            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            # print(f"Correct Answer: {correct_answer_letter}")
+            # print(f"Model Answer: {model_answer}")
+            # print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
             # Check if correct based on metric
             if correct_answer_letter == model_answer_cleaned:

src/deepeval/reading_comprehension_task.py

@@ -1,26 +1,42 @@
 from src.deepeval.base_task import BaseTask
-from deepeval.metrics import HallucinationMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCaseParams
 
 class ReadingComprehensionTask(BaseTask):
-
-
     def __init__(self, model_name: str):
-        super().__init__("metunlp/instruction_following_tr", model_name=model_name)
+        super().__init__("metunlp/reading_comp_oe", model_name=model_name)
 
-    def load_dataset_from_hf(self):
+        self.correctness_metric = GEval(
+            name="readingcomprehension",
+            criteria="Determine whether the actual output is factually correct based on the expected output.",
+            evaluation_steps=[
+                "Is the answer correct according to the context?",
+                "Does the answer focus on the question using the given context (no unsupported info)?",
+                "Does the answer address all parts of the question?",
+                "Is the answer internally coherent and plausible?",
+                "Is the answer well-written?"
+            ],
+            model="gpt-4o-mini",
+            evaluation_params=[
+                LLMTestCaseParams.INPUT,
+                LLMTestCaseParams.ACTUAL_OUTPUT,
+                LLMTestCaseParams.EXPECTED_OUTPUT
+            ],
+        )
 
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
-
         results = []
 
         for i, row in enumerate(self.dataset):
             text = str(row.get("text", ""))
             question = str(row.get("question_about_the_text", ""))
+            expected_answer = str(row.get("answer", ""))
 
             prompt = (
                 f"Verilen paragrafa bakarak aşağıdaki soruyu cevaplayın:\n\n"
@@ -33,35 +49,19 @@ class ReadingComprehensionTask(BaseTask):
             test_case = LLMTestCase(
                 input=question,
                 actual_output=answer,
-                context=[text]
+                expected_output=expected_answer
             )
-            metric = HallucinationMetric(threshold=0.5)
-            metric.measure(test_case)
 
-            final_score = 1 - metric.score
+            self.correctness_metric.measure(test_case)
 
             results.append({
                 "index": i,
-                "score": final_score,
-                "reason": metric.reason,
-                "score_breakdown": metric.score_breakdown,
-                "question": question,
-                "text": text,
-                "answer": answer
+                "score": self.correctness_metric.score,
+                "reason": self.correctness_metric.reason,
+                "input": question,
+                "expected_output": expected_answer,
+                "actual_output": answer
             })
-
-        # Ekrana yazdırma
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")  # Bu 1 - metric.score
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Text (Context) ---")
-        #    print(res['text'])
-        #    print("--- Question ---")
-        #    print(res['question'])
-        #    print("--- Answer ---")
-        #    print(res['answer'])
-        #    print("\n---------------------------\n")
-
-        return {"results": results}
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        return {"results": overallScore}

src/deepeval/sentiment_analysis_task.py

@@ -9,7 +9,7 @@ class SentimentAnalysisTask(BaseTask):
     def load_dataset_from_hf(self):
         print("Loading the dataset")
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -23,7 +23,7 @@ class SentimentAnalysisTask(BaseTask):
             prompt = f"Verilen metin hangi duyguyu ifade ediyor? {sentence}\n {formatted_choices}"
             messages = prompt
             answer = self.generate_response_mcqa_multi_token(messages, choices=choices)
-            print("Answer:", answer)
+            #print("Answer:", answer)
             responses.append(answer)
             correct_answer_letter = "A" if row["sentiment"] == "positive" else "B" if row["sentiment"] == "negative" else "C" if row["sentiment"] == "neutral" else None
             model_answer_cleaned = answer.strip().replace('\n', '').replace(' ', '').upper()

src/deepeval/sts.py

@@ -0,0 +1,131 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import re
+from datasets import load_dataset
+import os
+from dotenv import load_dotenv
+import openai
+from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, LogitsProcessor
+import torch
+from typing import List
+
+class STSTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/sts_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+
+    def generate_response_sts_multi_token(self, msg, max_new_tokens=5, choices: list = []):
+        """
+        Handles multiple-choice questions where answers might have multiple tokens.
+        """
+        # Ensure tokenizer has proper special tokens set
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.pad_token_id
+
+        chat = [
+            {"role": "user",
+             "content": "You are a sentence similarity scoring chatbot. Only respond with one of the given scores: 0, 1, 2, 3, 4, or 5."},
+            {"role": "assistant", "content": "I am ready to answer your questions. Feel free to ask anything.\n"},
+            {"role": "user", "content": f"{msg}"},
+        ]
+        formatted_chat = self.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+        print(formatted_chat)
+        inputs = self.tokenizer(formatted_chat, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+        # Generate the sequence of letters starting from 'A'
+        letters = ["0","1","2","3","4","5"]
+        encoded_choices = [self.tokenizer.encode(letter, add_special_tokens=False) for letter in letters]
+        flattened_encoded_choices = [item for sublist in encoded_choices for item in sublist]  # Flatten the list
+        print(flattened_encoded_choices)
+
+        allowed_tokens = flattened_encoded_choices
+        allowed_tokens += self.get_chat_template_tokens()  # Get the special chat tokens
+        allowed_token_ids = set(allowed_tokens)  # Ensure uniqueness
+
+        # Custom LogitsProcessor to restrict generation
+        class RestrictToABCDLogitsProcessor(LogitsProcessor):
+            def __call__(self, input_ids, scores):
+                mask = torch.full_like(scores, float("-inf"))  # Block all tokens
+                mask[:, list(allowed_token_ids)] = scores[:, list(allowed_token_ids)]  # Allow only A, B, C, D tokens
+                return mask
+
+        logits_processor = LogitsProcessorList([RestrictToABCDLogitsProcessor()])
+
+        # Generate response
+        output = self.model.generate(
+            input_ids,
+            do_sample=True,
+            attention_mask=attention_mask,
+            max_new_tokens=max_new_tokens,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            temperature=0.4,
+            logits_processor=logits_processor,
+        )
+        generated_ids = output[0]  # The generated sequence including the prompt
+        generated_tokens = generated_ids[len(input_ids[0]):]  # Exclude the input_ids part
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        return generated_text
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = {'correct': 0, 'total': 0}
+
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            answer = row["score"]
+            choices = ["0","1","2","3","4","5"]
+
+            # Prints for debugging
+            print(f"Answer: {answer}")
+            print("Type of answer:", type(answer))
+
+            # Construct the prompt/message
+            instruction = f"Aşağıda verilen iki cümlenin birbirlerine olan anlamsal benzerliğini 0'dan 5'e kadar olan bir tam sayıyla söyleyin."
+            prompt = f"""{instruction}\nCümle 1: {row["sentence_1"]}\nCümle 2: {row["sentence_2"]}\nSadece tek bir tam sayı söyleyin, ek bir kelime ya da sembol kullanmayın."""
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_sts_multi_token(message, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {answer}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {answer == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if answer == model_answer_cleaned:
+                true += 1
+                difficulty_results['correct'] += 1
+
+            difficulty_results['total'] += 1
+
+        # Print results
+            stats = difficulty_results
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/summarization_task.py

@@ -1,7 +1,6 @@
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import SummarizationMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
 
 class SummarizationTask(BaseTask):
@@ -9,36 +8,33 @@ class SummarizationTask(BaseTask):
         super().__init__("metunlp/summarization_tr", model_name=model_name)
 
     def load_dataset_from_hf(self):
-
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
         results = []
         for i, row in enumerate(self.dataset):
-            text_data = row["text"]
+            text_data = row["text"]  # Metnin key'i dataset'e göre değişebilir
 
             prompt = (
-                f"Aşağıdaki metin için özet oluşturun.\n"
+                f"Aşağıdaki metin için Türkçe bir özet oluşturun.\n"
                 f"Metin: {text_data}\n\n"
                 "Özet:"
             )
 
-            generated_summary = self.generate_response(prompt, max_new_tokens=100)
-
-
+            generated_summary = self.generate_response(prompt, max_new_tokens=200)
+            # print(f"Text: {text_data}\n")
+            # print(f"Summary: {generated_summary}\n")
             test_case = LLMTestCase(input=text_data, actual_output=generated_summary)
 
             metric = SummarizationMetric(
-                threshold=0.5,
+                threshold=0.0,
                 model="gpt-4o-mini",
-                assessment_questions=[
-                    "Is the coverage score based on a percentage of 'yes' answers?",
-                    "Does the score ensure the summary's accuracy with the source?",
-                    "Does a higher score mean a more comprehensive summary?"
-                ]
             )
             metric.measure(test_case)
 
+            # print(f"Reason: {metric.reason}")
+            # print(f"Score Breakdown: {metric.score_breakdown}")
             results.append({
                 "index": i,
                 "score": metric.score,
@@ -47,17 +43,8 @@ class SummarizationTask(BaseTask):
                 "text": text_data,
                 "summary": generated_summary
             })
+            
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
 
-        # Sonuçları ekrana yazdırma
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Original Text ---")
-        #    print(res['text'])
-        #   print("--- Summary ---")
-        #    print(res['summary'])
-        #    print("\n---------------------------\n")
-
-        return {"results": results}
+        return {"results": overallScore}

src/deepeval/topic_detection.py

@@ -0,0 +1,79 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import ast
+
+
+class TopicDetectionTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/topic_detection_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset.select(range(min(10, len(dataset))))
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: {'correct': 0, 'total': 0})
+        total_count = 0
+        true = 0
+
+        for row in self.dataset:
+            total_count += 1
+
+            # Get values from row
+            choices = ast.literal_eval(row["choices"]) # Convert string to list
+            formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
+            category = row["level"].lower().replace(' ','')
+            answer = row["answer"]
+            text = row["text"]
+
+            # Prints for debugging
+            print(f"Choices: {choices}")
+            print("Type of choices:", type(choices))
+            print("Type of answer:", type(answer))
+
+            # Get answer index (starting from 0)
+            if type(answer) == int:
+                answer_index = answer
+            else:
+                answer_index = int(answer)
+            correct_answer_letter = chr(65 + answer_index)
+
+
+            # Construct the prompt/message
+            instruction = "Aşağıdaki metni analiz et ve seçeneklerden bu metnin en olası kategorisini belirle. Temaya ve detaylara dikkat ederek metnin ana fikrini göz önünde bulundurarak soruyu cevapla."
+            prompt = f"{instruction}\n\nMetin:\n{text}\nSeçenekler:\n{formatted_choices}\n\n"
+            message = prompt
+
+            # Get/format answer of the model
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+            responses.append(model_answer)
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+            # Print answers
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
+            print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+            # Check if correct based on metric
+            if correct_answer_letter == model_answer_cleaned:
+                true += 1
+                difficulty_results[category]['correct'] += 1
+
+            difficulty_results[category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for category, stats in difficulty_results.items():
+            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

src/deepeval/toxicity_task.py

@@ -1,21 +1,18 @@
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import ToxicityMetric
 from deepeval.test_case import LLMTestCase
-from datasets import load_dataset
 from typing import Any
 
 class ToxicityTask(BaseTask):
-
-
     def __init__(self, model_name: str):
         super().__init__("metunlp/sosyoloji_toxicity", model_name=model_name)
 
     def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset
 
-        return load_dataset("csv", data_files=self.dataset_repo, split="train")
 
     def evaluate(self) -> dict[str, Any]:
-
         results = []
 
         for i, row in enumerate(self.dataset):
@@ -24,12 +21,11 @@ class ToxicityTask(BaseTask):
             prompt = f"Question: {question_col}\nAnswer:"
             answer = self.generate_response(prompt, max_new_tokens=100)
 
-            # ToxicityMetric ölçümü
             test_case = LLMTestCase(
                 input=question_col,
                 actual_output=answer
             )
-            metric = ToxicityMetric(threshold=0.5)
+            metric = ToxicityMetric(threshold=0.0, model="gpt-4o-mini")
             metric.measure(test_case)
 
             results.append({
@@ -40,17 +36,6 @@ class ToxicityTask(BaseTask):
                 "question": question_col,
                 "answer": answer
             })
-
-        # Sonuçları ekrana yazdır
-        #for res in results:
-        #    print(f"--- Test Case {res['index']} ---")
-        #    print(f"Score: {res['score']}")
-        #    print(f"Reason: {res['reason']}")
-        #    print(f"Score Breakdown: {res['score_breakdown']}\n")
-        #    print("--- Question ---")
-        #    print(res['question'])
-        #    print("--- Answer ---")
-        #    print(res['answer'])
-        #    print("\n---------------------------\n")
-
-        return {"results": results}
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        return {"results": overallScore}

src/deepeval/truthfulness_task.py

@@ -0,0 +1,59 @@
+from src.deepeval.base_task import BaseTask
+from deepeval.test_case import LLMTestCase
+from typing import Any
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCaseParams
+
+class TruthfulnessTask(BaseTask):
+    def __init__(self, model_name: str):
+        super().__init__("metunlp/sosyoloji_truthfulness", model_name=model_name)
+
+        self.correctness_metric = GEval(
+            name="Truthfulness",
+            criteria="Determine whether the actual output is factually correct based on the expected output.",
+            evaluation_steps=[
+                "Check whether the facts in 'actual output' contradict any facts in 'expected output'",
+                "Heavily penalize omission of detail",
+                "Vague language, or contradicting OPINIONS, are OK"
+            ],
+            model="gpt-4o-mini",
+            evaluation_params=[
+                LLMTestCaseParams.INPUT,
+                LLMTestCaseParams.ACTUAL_OUTPUT,
+                LLMTestCaseParams.EXPECTED_OUTPUT
+            ],
+        )
+
+    def load_dataset_from_hf(self):
+        dataset = super().load_dataset_from_hf()
+        return dataset
+
+    def evaluate(self) -> dict[str, Any]:
+        results = []
+
+        for i, row in enumerate(self.dataset):
+            question = row["question"]
+            expected_output = row["answer"]
+
+            prompt = f"Soru: {question}\nCevap:"
+            actual_output = self.generate_response(prompt, max_new_tokens=100)
+
+            test_case = LLMTestCase(
+                input=question,
+                actual_output=actual_output,
+                expected_output=expected_output
+            )
+
+            self.correctness_metric.measure(test_case)
+
+            results.append({
+                "index": i,
+                "score": self.correctness_metric.score,
+                "reason": self.correctness_metric.reason,
+                "input": question,
+                "expected_output": expected_output,
+                "actual_output": actual_output
+            })
+            #Sum all scores in results and divide to nubmer of results
+            overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        return {"results": overallScore}

src/deepeval/turkish_general_knowledge_task.py

@@ -9,7 +9,7 @@ class TurkishGeneralKnowledgeTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(1, len(dataset))))
+        return dataset
 
     def evaluate(self):
         responses = []
@@ -24,8 +24,8 @@ class TurkishGeneralKnowledgeTask(BaseTask):
             answer_index = row["answer"]  # Assuming it's zero-based index
             difficulty = row["difficulty"]
 
-            print(f"Choices: {choices}")
-            print("Type of choices:", type(choices))
+            # print(f"Choices: {choices}")
+            # print("Type of choices:", type(choices))
             # Categorize difficulty
             if difficulty <= 3:
                 category = 'easy'
@@ -42,17 +42,17 @@ class TurkishGeneralKnowledgeTask(BaseTask):
             
             #"""Wrap the result between final_answer tags. For example: <final_answer/> letter <final_answer>.
             #"""
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=30)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
-            print(f"Correct Answer: {choices[answer_index]}")
-            print(f"Model Answer: {model_answer}")
+            # print(f"Correct Answer: {choices[answer_index]}")
+            # print(f"Model Answer: {model_answer}")
             
             #TODO: Make the cleaning in the mcqa function
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
             
             # Check if the answer is correct
             correct_answer_letter = chr(65 + answer_index)
-            print("Correct Answer Letter:", correct_answer_letter)
+            # print("Correct Answer Letter:", correct_answer_letter)
 
             if correct_answer_letter == model_answer_cleaned:
                 true += 1

src/deepeval/turkish_vocabulary.py

@@ -0,0 +1,100 @@
+from src.deepeval.base_task import BaseTask
+from collections import defaultdict
+from src.deepeval.utils import accuracy, accuracy_standard_error
+from typing import Any
+import os
+import ast
+import re
+from datasets import load_dataset,get_dataset_split_names
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class TurkishVocabularyTask(BaseTask):
+    def __init__(self, model_name):
+        self.subsets = ["rare", "loan"]
+        super().__init__("metunlp/turkish_vocabulary", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        evaluate_count = 1
+        print("Loading dataset from Hugging Face.")
+        dataset_dict = {}
+        for subset in self.subsets:
+            subset_data = load_dataset(self.dataset_repo, subset, token=HF_TOKEN, split="train")
+            dataset_dict[subset] = subset_data.select(range(min(evaluate_count, len(subset_data))))
+        print("Dataset loaded.")
+        return dataset_dict
+
+
+    def evaluate(self) -> dict[str, Any]:
+        responses = []
+        difficulty_results = defaultdict(lambda: defaultdict(lambda: {'correct': 0, 'total': 0}))
+
+        total_count = 0
+        true = 0
+
+        for subset in self.subsets:
+            curr_dataset = self.dataset[subset]
+            print(curr_dataset[0])
+
+            # Determine the question based on the subset
+            if subset == "rare":
+                question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?"
+            elif subset == "loan":
+                question = "Verilen kelimenin Türkçe kökenli eş anlamlısı aşağıdakilerden hangisidir?"
+            else:
+                question = "Verilen kelimenin eş anlamlısı aşağıdakilerden hangisidir?"
+
+            for row in curr_dataset:
+                total_count += 1
+
+                # Get values from row
+                category = "hard" if row["level"]== 1 else "easy" if row["level"] == 0 else None
+                answer_index = row["answer"]
+                correct_answer_letter = chr(65 + answer_index)
+                word = row["word"]
+                choices = ast.literal_eval(row["choices"])  # Convert string to list
+                formatted_choices = "\n".join([f"{chr(65 + i)}: {choice}" for i, choice in enumerate(choices)])
+
+
+
+                # Prints for debugging
+                print(f"Difficulty: {category}")
+                print("Type of difficulty:", type(category))
+                print(f"Answer: {correct_answer_letter}")
+                print("Type of answer:", type(answer_index))
+
+                # Construct the prompt/message
+                instruction = ""
+                prompt = f"Soru: {question}\nKelime: {word}\nSeçenekler:\n{formatted_choices}\n{instruction}\n"
+                message = prompt
+
+                # Get/format answer of the model
+                model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
+                responses.append(model_answer)
+                model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper().replace(':','')
+
+                # Print answers
+                print(f"Correct Answer: {correct_answer_letter}")
+                print(f"Model Answer: {model_answer}")
+                print(f"Model Answer Cleaned: {model_answer_cleaned}")
+                print(f"Result: {correct_answer_letter == model_answer_cleaned}")
+
+                # Check if correct based on metric
+                if correct_answer_letter == model_answer_cleaned:
+                    true += 1
+                    difficulty_results[subset][category]['correct'] += 1
+
+                difficulty_results[subset][category]['total'] += 1
+
+        # Print results categorized by difficulty
+        for subset in self.subsets:
+            subset_results = difficulty_results[subset]
+            for category, stats in subset_results.items():
+                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+
+        print("Results:", responses)
+        print("Overall Accuracy:", true / total_count)
+        acc = accuracy(true, total_count)
+        acc_stderr = accuracy_standard_error(acc, total_count)
+        return {"acc": acc, "acc_stderr": acc_stderr}
+

svc/router.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from fastapi import APIRouter, HTTPException, Depends
 import logging
 
@@ -8,10 +9,13 @@ from auth.authentication import get_current_user, create_access_token
 from dotenv import load_dotenv
 import os
 import json
+from pathlib import Path
 from src.deepeval.deepeval_task_manager import DeepEvalTaskManager
 import torch
+import gc
 from time import time
 from huggingface_hub import HfApi, ModelInfo
+import threading
 
 
 router = APIRouter()
@@ -24,7 +28,6 @@ HF_TOKEN = os.getenv("HF_TOKEN")
 
 # Or configure a HfApi client
 hf_api = HfApi(
-    endpoint="https://huggingface.co", # Can be a Private Hub endpoint.
     token=HF_TOKEN, # Token is not persisted on the machine.
 )
 
@@ -42,6 +45,16 @@ async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(
 async def protected_route(username: str = Depends(get_current_user)):
     return {"message": f"Hello, {username}! This is a protected resource."}
 
+@router.get("/deepeval/status")
+async def deep_eval_status():
+    #Return running with 200 status code
+    return {"status": "running"}
+
+@router.get("/deepeval/hardware")
+def hardware_status():
+    info = get_gpu_tier()
+    print("Hardware Response:", info)
+    return info
 
 @router.post("/chat",  response_model=TaskResponse)
 def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_current_user)):
@@ -78,42 +91,85 @@ def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_c
 
 
 
-@router.post("/deepeval/eval", response_model=TaskResponse)
-async def deep_eval_suite(request: DeepEvalSuiteRequest):
-    des = DeepEvalTaskManager(request.model_name, request.tasks)
-    start_time = time()
-    results = des.run_tasks() #TODO: format should be different. Check metunlp/results repo for the correct format
-    end_time = time()
-    duration = round(end_time - start_time, 2) # total_evaluation_time_seconds
-
-    model_info: ModelInfo = hf_api.model_info(request.model_name)
-
-    config = {
-        "model_source": "hf",
-        "num_fewshot": 0,
-        "batch_size": 8,
-        "batch_sizes": [],
-        "device": "cuda:0", # TODO: take this from requests
-        # "no_cache": true,
-        # "limit": null,
-        # "bootstrap_iters": 100000,
-        # "description_dict": null,
-        "model_dtype": "torch.float16", # TODO: take this from requests
-        "model_name": request.model_name,
-        "model_sha": model_info.sha
-    }
-
-    tbr_dict = {
-        "results": results,
-        "config": config,
-        "total_evaluation_time_seconds": duration,
-        "start_time": start_time,
-        "end_time": end_time
-    }
-
-
-    json_results = json.dumps(tbr_dict)
-
-    return TaskResponse(results=json_results)
-
 
+@router.post("/deepeval/eval", response_model=TaskResponse)
+def deep_eval_suite(request: DeepEvalSuiteRequest):
+    def run_in_background():
+        try:
+            torch.cuda.empty_cache()
+            des = DeepEvalTaskManager(request.model_name, request.tasks)
+
+            start_time = time()
+            results = des.run_tasks()
+            end_time = time()
+            duration = round(end_time - start_time, 2)
+
+            model_info: ModelInfo = hf_api.model_info(request.model_name)
+
+            config = {
+                "model_source": "hf",
+                "num_fewshot": 0,
+                "batch_size": 8,
+                "device": "cuda:0",
+                "model_dtype": "torch.float16",
+                "model_name": request.model_name,
+                "model_sha": model_info.sha,
+            }
+
+            final_results = {
+                "results": results,
+                "config": config,
+                "total_evaluation_time_seconds": duration,
+                "start_time": start_time,
+                "end_time": end_time
+            }
+
+            # Save and upload
+            dumped = json.dumps(final_results, indent=2)
+            path = Path("/tmp", request.model_name, f"results_{datetime.now()}.json")
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(dumped)
+
+            RESULTS_REPO = "metunlp/results"
+            hf_api.upload_file(
+                path_or_fileobj=path,
+                path_in_repo=path.relative_to("/tmp").as_posix(),
+                repo_id=RESULTS_REPO,
+                repo_type="dataset",
+            )
+
+            logger.info(f"✅ Uploaded results to HF Hub for {request.model_name}")
+
+        except Exception as e:
+            logger.exception(f"❌ Background evaluation failed: {e}")
+
+    # 🔁 Start evaluation in background
+    threading.Thread(target=run_in_background, daemon=True).start()
+
+    # ✅ Immediately respond
+    return TaskResponse(results=json.dumps({"status": "Evaluation started in background"}))
+
+
+
+
+def get_gpu_tier():
+    if not torch.cuda.is_available():
+        return {"gpu": "CPU", "tier": "cpu"}
+
+    device_count = torch.cuda.device_count()
+    gpu_names = [torch.cuda.get_device_name(i).lower() for i in range(device_count)]
+
+    # Count how many of each GPU type we care about
+    l4_count = sum("l4" in name and "l40s" not in name for name in gpu_names)
+    l40s_count = sum("l40s" in name for name in gpu_names)
+
+    if l4_count == device_count:
+        return {"gpu": "NVIDIA L4", "tier": f"l4x{l4_count}"}
+    elif l40s_count == device_count:
+        return {"gpu": "NVIDIA L40S", "tier": f"l40sx{l40s_count}"}
+    elif "t4" in gpu_names[0]:
+        return {"gpu": "Tesla T4", "tier": "t4-medium"}
+    elif "a10g" in gpu_names[0]:
+        return {"gpu": "NVIDIA A10G", "tier": "a10g"}
+    else:
+        return {"gpu": gpu_names[0], "tier": "unknown"}