merge

auth/authentication.py

@@ -1,33 +0,0 @@
-from fastapi.security import OAuth2PasswordBearer
-from fastapi import HTTPException, Depends
-from jose import JWTError, jwt
-from datetime import datetime, timedelta
-
-
-SECRET_KEY = "llmbenchmark_tr" # your secret key
-ALGORITHM = "HS256"
-ACCESS_TOKEN_EXPIRE_MINUTES = 30
-
-oauth2_scheme = OAuth2PasswordBearer(tokenUrl="api/token")
-
-def create_access_token(data: dict):
-    to_encode = data.copy()
-    expire = datetime.now() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
-    to_encode.update({"exp": expire})
-    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
-    return encoded_jwt
-
-def get_current_user(token: str = Depends(oauth2_scheme)):
-    credentials_exception = HTTPException(
-        status_code=401,
-        detail="Could not validate credentials",
-        headers={"WWW-Authenticate": "Bearer"},
-    )
-    try:
-        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
-        username: str = payload.get("sub")
-        if username is None:
-            raise credentials_exception
-        return username
-    except JWTError:
-        raise credentials_exception

src/deepeval/base_task.py

@@ -3,7 +3,7 @@ import itertools
 from datasets import load_dataset
 import os
 from dotenv import load_dotenv
-from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, LogitsProcessor, Gemma3ForCausalLM
+from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, LogitsProcessor#, Gemma3ForCausalLM
 import torch
 from typing import List
 load_dotenv()
@@ -29,7 +29,7 @@ class BaseTask(ABC):
     @staticmethod
     def load_model(model_name: str, device):
         """Loads model and tokenizer once and caches it."""
-        if "gemma-3" in model_name:
+        if False:#"gemma-3" in model_name:
             model = Gemma3ForCausalLM.from_pretrained(
                 model_name,
                 #device_map=device, #Gives Cannot copy out of meta tensor; no data! Please use torch.nn.Module.to_empty() instead of torch.nn.Module.to() when moving module from meta to a different device. error

src/deepeval/commonsense_reasoning_task.py

@@ -2,13 +2,14 @@ from src.deepeval.base_task import BaseTask
 from src.deepeval.utils import accuracy, accuracy_standard_error
 from typing import Any
 
-class SentimentAnalysisTask(BaseTask):
+class CommonsenseReasoningTask(BaseTask):
     def __init__(self, model_name):
         super().__init__("metunlp/commonsense", model_name=model_name)
 
     def load_dataset_from_hf(self):
+        print("Loading the dataset")
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset.select(range(min(1, len(dataset))))
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -16,7 +17,7 @@ class SentimentAnalysisTask(BaseTask):
         total_count = len(self.dataset)
         n_correct = 0
         for row in self.dataset:
-            sentence = row["sentence"]
+            print(row)
             label = row["label"]
             choices=[row["choice1"], row["choice2"]]
             formatted_choices = "\n".join([f"{chr(65+i)}: {choice}" for i, choice in enumerate(choices)])
@@ -28,17 +29,19 @@ class SentimentAnalysisTask(BaseTask):
             else:
                 question = "Seçeneklerden hangisi uygun?"  # Alternatif
 
-            prompt = f"Premise:\n{line["text"]}\nSoru:{question}\nSeçenekler:\n{formatted_choices}"
+            prompt = f"Bağlam:\n{row["text"]}\nÖnerme:\n{row["context"]}\nSoru:{question}\nSeçenekler:\n{formatted_choices}"
 
             messages = prompt
-            
-            answer = self.generate_response_mcqa_multi_token(messages, choices=choices)
-            print("Answer:", answer)
-            responses.append(answer)
-            correct_answer_letter = "A" if row["sentiment"] == "positive" else "B" if row["sentiment"] == "negative" else "C" if row["sentiment"] == "neutral" else None
-            model_answer_cleaned = answer.strip().replace('\n', '').replace(' ', '').upper()
+
+            model_answer = self.generate_response_mcqa_multi_token(messages, choices=choices)
+
+            correct_answer_letter = "A" if row["answer"] == 1 else "B" if row["answer"] == 2 else None
+            model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
             if correct_answer_letter == model_answer_cleaned:
                 n_correct += 1
+            print(f"Correct Answer: {correct_answer_letter}")
+            print(f"Model Answer: {model_answer}")
+            print(f"Model Answer Cleaned: {model_answer_cleaned}")
 
         acc = accuracy(n_correct, total_count)
         acc_stderr = accuracy_standard_error(acc, total_count)

src/deepeval/deepeval_task_manager.py

@@ -3,6 +3,7 @@ from dotenv import load_dotenv
 from enum import Enum
 from src.deepeval.turkish_general_knowledge_task import TurkishGeneralKnowledgeTask
 from src.deepeval.sentiment_analysis_task import SentimentAnalysisTask
+from src.deepeval.commonsense_reasoning_task import CommonsenseReasoningTask
 from typing import List
 load_dotenv()
 
@@ -12,6 +13,7 @@ class Task(Enum):
     # SUMMARIZATION = "summarization"
     SENTIMENT_ANALYSIS = "sentiment_analysis_tr"
     TURKISH_GENERAL_KNOWLEDGE = "turkish_general_knowledge"
+    COMMONSENSE_REASONING = "commonsense_reasoning"
 
 
 class DeepEvalTaskManager:
@@ -51,8 +53,13 @@ class DeepEvalTaskManager:
         res = turkish_general_knowledge_task.evaluate()
         return res
 
+    def commonsense_reasoning(self):
+        commonsense_reasoning_task = CommonsenseReasoningTask(self.model_name)
+        res = commonsense_reasoning_task.evaluate()
+        return res
+
 
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("google/gemma-3-4b-it", ["TURKISH_GENERAL_KNOWLEDGE"])
+    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["TURKISH_GENERAL_KNOWLEDGE","COMMONSENSE_REASONING"])
     res = des.run_tasks()
     print(res)

src/deepeval/sentiment_analysis_task.py

@@ -7,6 +7,7 @@ class SentimentAnalysisTask(BaseTask):
         super().__init__("metunlp/sentiment_analysis_tr", model_name=model_name)
 
     def load_dataset_from_hf(self):
+        print("Loading the dataset")
         dataset = super().load_dataset_from_hf()
         return dataset.select(range(min(10, len(dataset))))
 

src/deepeval/turkish_general_knowledge_task.py

@@ -8,7 +8,7 @@ class TurkishGeneralKnowledgeTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset.select(range(min(1, len(dataset))))
 
     def evaluate(self):
         responses = []