Create deep eval suite

src/deepeval/base_task.py

@@ -0,0 +1,85 @@
+from abc import ABC, abstractmethod
+from datasets import load_dataset
+import os
+from dotenv import load_dotenv
+from transformers import AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList
+import torch
+from typing import List
+load_dotenv()
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class BaseTask(ABC):
+    _model_cache = {}  # Class-level cache for models and tokenizers
+
+    def __init__(self, dataset_repo, model_name):
+        self.dataset_repo = dataset_repo
+        self.dataset = self.load_dataset_from_hf()
+        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        self.model, self.tokenizer = self.get_cached_model(model_name, self.device)
+
+
+    @classmethod
+    def get_cached_model(cls, model_name, device):
+        """Ensures the same model and tokenizer are used for every instance of subclasses."""
+        if model_name not in cls._model_cache:
+            cls._model_cache[model_name] = cls.load_model(model_name, device)
+        return cls._model_cache[model_name]
+
+    @staticmethod
+    def load_model(model_name: str, device):
+        """Loads model and tokenizer once and caches it."""
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype=torch.float16,
+            device_map=device,
+            token=HF_TOKEN,  # Replace with actual token
+        )
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        return model, tokenizer
+
+
+    def generate_response_mcqa(self, msg, max_new_tokens=1, choices: List[str]=[]):
+        # Ensure the tokenizer has a padding token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token  # Use EOS token as PAD token
+
+        inputs = self.tokenizer(msg, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs.input_ids.to(self.model.device)
+        attention_mask = inputs.attention_mask.to(self.model.device)
+
+        if self.model.config.pad_token_id is None:
+            self.model.config.pad_token_id = self.tokenizer.eos_token_id
+
+        # Get token IDs for answer choices
+        valid_answers = choices
+        valid_token_ids = [self.tokenizer.convert_tokens_to_ids(ans) for ans in valid_answers]
+
+        class MultipleChoiceLogitsProcessor:
+            def __call__(self, input_ids, scores):
+                mask = torch.full_like(scores, float("-inf"))
+                mask[:, valid_token_ids] = scores[:, valid_token_ids]  # Allow only valid tokens
+                return mask
+
+        logits_processor = LogitsProcessorList([MultipleChoiceLogitsProcessor()])
+
+        output = self.model.generate(
+            input_ids,
+            attention_mask=attention_mask,  # Fix: Pass attention_mask to avoid warning
+            max_new_tokens=max_new_tokens,
+            logits_processor=logits_processor
+        )
+        answer = self.tokenizer.decode(output[0][-1])
+
+        return answer
+
+    @abstractmethod
+    def load_dataset_from_hf(self):
+        """
+        Define your own loading method if needed.
+        :return: Dataset
+        """
+        return load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
+
+    @abstractmethod
+    def evaluate(self):
+        pass

src/deepeval/deepeval_task_manager.py

@@ -0,0 +1,48 @@
+import os
+from dotenv import load_dotenv
+from enum import Enum
+from src.deepeval.sentiment_analysis_task import SentimentAnalysisTask
+from typing import List
+load_dotenv()
+
+HF_TOKEN=os.getenv("HF_TOKEN")
+
+class Task(Enum):
+    # SUMMARIZATION = "summarization"
+    SENTIMENT_ANALYSIS = "sentiment_analysis"
+
+
+class DeepEvalTaskManager:
+    def __init__(self, model_name, tasks: List[str]):
+        self.model_name = model_name
+        self.available_tasks = {task.name: getattr(self, task.name.lower()) for task in Task}
+        self.tasks_to_run = self.validate_tasks(tasks)
+
+    def validate_tasks(self, user_tasks):
+        """Validate user tasks and store method references."""
+        print(self.available_tasks.keys())
+        if not set(user_tasks).issubset(self.available_tasks.keys()):
+            invalid_tasks = set(user_tasks) - self.available_tasks.keys()
+            raise ValueError(f"Invalid task(s) requested: {invalid_tasks}")
+
+        # Store actual method references instead of strings
+        return {task : self.available_tasks[task] for task in user_tasks}
+
+    def run_tasks(self):
+        """Execute validated tasks in order."""
+        results = {}
+        for task_name, task_method in self.tasks_to_run.items():
+            results[task_name] = task_method()  # Call the stored method reference
+
+        return results
+
+    def sentiment_analysis(self):
+        st_task = SentimentAnalysisTask(self.model_name)
+        res = st_task.evaluate()
+        return res
+
+
+if __name__ == "__main__":
+    des = DeepEvalTaskManager("meta-llama/Llama-3.2-1B-Instruct", ["SENTIMENT_ANALYSIS"])
+    res = des.run_tasks()
+    print(res)

src/deepeval/sentiment_analysis_task.py

@@ -0,0 +1,26 @@
+from src.deepeval.base_task import BaseTask
+
+class SentimentAnalysisTask(BaseTask):
+    def __init__(self, model_name):
+        super().__init__("metunlp/sentiment_analysis_tr", model_name=model_name)
+
+    def load_dataset_from_hf(self):
+        return super().load_dataset_from_hf()
+
+
+    def evaluate(self):
+        responses = []
+        total_count = len(self.dataset)
+        true = 0
+        for row in self.dataset:
+            sentence = row["sentence"]
+            prompt = f"Verilen metin hangi duyguyu ifade ediyor? {sentence}"
+            messages = prompt
+            answer = self.generate_response_mcqa(messages, choices=["positive", "negative", "neutral"])
+            responses.append(answer)
+            if row["sentiment"] == "positive":
+                true += 1
+
+        print(responses)
+        return true/total_count
+