Added new seperate logs for llm judges. Commented adapter loading for testing

src/deepeval/base_task.py

@@ -29,36 +29,53 @@ class BaseTask(ABC):
         if model_name not in cls._model_cache:
             cls._model_cache[model_name] = cls.load_model(model_name, device)
         return cls._model_cache[model_name]
-
+    
     @staticmethod
-    def load_model(model_name: str, device, weight, dtype, base_model):
+    def load_model(model_name: str, device):
         """Loads model and tokenizer once and caches it."""
         print(f"Loading model: {model_name}")
         start_time = datetime.now()
-        if weight == "Adapter":
-            base_model_1 = AutoModelForCausalLM.from_pretrained(
-                base_model,
-                torch_dtype=dtype,
-                device_map=device,
-                token=HF_TOKEN,  # Replace with actual token
-            )
-            model = PeftModel.from_pretrained(base_model_1, base_model)
-            tokenizer = AutoTokenizer.from_pretrained(base_model)
-            end_time = datetime.now()
-        else:
-            model = AutoModelForCausalLM.from_pretrained(
-                model_name,
-                torch_dtype=dtype,
-                device_map=device,
-                token=HF_TOKEN,  # Replace with actual token
-            )
-            tokenizer = AutoTokenizer.from_pretrained(model_name)
-            end_time = datetime.now()
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype=torch.float16,
+            device_map=device,
+            token=HF_TOKEN,  # Replace with actual token
+        )
+        end_time = datetime.now()
         print(f"Model loaded in {(end_time - start_time).seconds} seconds.")
         print("Model loaded.")
-        
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
         return model, tokenizer
 
+    # @staticmethod
+    # def load_model(model_name: str, device, weight, dtype, base_model):
+    #     """Loads model and tokenizer once and caches it."""
+    #     print(f"Loading model: {model_name}")
+    #     start_time = datetime.now()
+    #     if weight == "Adapter":
+    #         base_model_1 = AutoModelForCausalLM.from_pretrained(
+    #             base_model,
+    #             torch_dtype=dtype,
+    #             device_map=device,
+    #             token=HF_TOKEN,  # Replace with actual token
+    #         )
+    #         model = PeftModel.from_pretrained(base_model_1, base_model)
+    #         tokenizer = AutoTokenizer.from_pretrained(base_model)
+    #         end_time = datetime.now()
+    #     else:
+    #         model = AutoModelForCausalLM.from_pretrained(
+    #             model_name,
+    #             torch_dtype=dtype,
+    #             device_map=device,
+    #             token=HF_TOKEN,  # Replace with actual token
+    #         )
+    #         tokenizer = AutoTokenizer.from_pretrained(model_name)
+    #         end_time = datetime.now()
+    #     print(f"Model loaded in {(end_time - start_time).seconds} seconds.")
+    #     print("Model loaded.")
+        
+    #     return model, tokenizer
+
 
     def generate_response_mcqa(self, msg, max_new_tokens=1, choices: List[str]=[]):
         # Ensure the tokenizer has a padding token

src/deepeval/bias_task.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import BiasMetric
 from deepeval.test_case import LLMTestCase
@@ -13,10 +14,12 @@ class BiasTask(BaseTask):
         return dataset
 
     def evaluate(self) -> dict[str, Any]:
-
         results = []
+        total_model_time = 0
+        total_judge_time = 0
 
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             ambiguous_context = row.get("ambiguous_context", "")
             negative_question = row.get("question_ambiguous", "")
             disambiguated_context = row.get("disambiguated_context", "")
@@ -30,13 +33,18 @@ class BiasTask(BaseTask):
             )
 
             answer = self.generate_response(prompt, max_new_tokens=200)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=prompt,
                 actual_output=answer
             )
             metric = BiasMetric(threshold=0.0,model="gpt-4o-mini")
             metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -48,4 +56,7 @@ class BiasTask(BaseTask):
             })
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+            
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}

src/deepeval/faithfulness_task.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import FaithfulnessMetric
 from deepeval.test_case import LLMTestCase
@@ -14,8 +15,11 @@ class FaithfulnessTask(BaseTask):
     def evaluate(self) -> dict[str, Any]:
 
         results = []
+        total_model_time = 0
+        total_judge_time = 0
 
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             context = row["context"]
             question = row["question"]
 
@@ -26,7 +30,10 @@ class FaithfulnessTask(BaseTask):
             )
 
             generated_answer = self.generate_response(prompt, max_new_tokens=100)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=question,
                 actual_output=generated_answer,
@@ -39,6 +46,8 @@ class FaithfulnessTask(BaseTask):
                 include_reason=True
             )
             metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -53,4 +62,6 @@ class FaithfulnessTask(BaseTask):
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
 
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")  
         return {"results": overallScore}

src/deepeval/instruction_following_task.py

@@ -14,7 +14,11 @@ class InstructionFollowingTask(BaseTask):
 
     def evaluate(self) -> dict[str, Any]:
         results = []
+        total_model_time = 0
+        total_judge_time = 0
+        
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             input_text = row.get("input", "")
             instruction_text = row.get("instruction", "")
 
@@ -25,7 +29,10 @@ class InstructionFollowingTask(BaseTask):
             )
 
             output = self.generate_response(prompt, max_new_tokens=200)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=input_text,
                 actual_output=output
@@ -37,6 +44,8 @@ class InstructionFollowingTask(BaseTask):
                 include_reason=True
             )
             metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -49,4 +58,7 @@ class InstructionFollowingTask(BaseTask):
             })
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}

src/deepeval/reading_comprehension_task.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from src.deepeval.base_task import BaseTask
 from deepeval.test_case import LLMTestCase
 from typing import Any
@@ -32,8 +33,11 @@ class ReadingComprehensionTask(BaseTask):
 
     def evaluate(self) -> dict[str, Any]:
         results = []
+        total_model_time = 0
+        total_judge_time = 0
 
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             text = str(row.get("text", ""))
             question = str(row.get("question_about_the_text", ""))
             expected_answer = str(row.get("answer", ""))
@@ -45,7 +49,10 @@ class ReadingComprehensionTask(BaseTask):
             )
 
             answer = self.generate_response(prompt, max_new_tokens=150)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=question,
                 actual_output=answer,
@@ -53,6 +60,8 @@ class ReadingComprehensionTask(BaseTask):
             )
 
             self.correctness_metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -64,4 +73,7 @@ class ReadingComprehensionTask(BaseTask):
             })
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+            
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}

src/deepeval/summarization_task.py

@@ -1,3 +1,4 @@
+import datetime
 from src.deepeval.base_task import BaseTask
 from deepeval.metrics import SummarizationMetric
 from deepeval.test_case import LLMTestCase
@@ -13,7 +14,11 @@ class SummarizationTask(BaseTask):
 
     def evaluate(self) -> dict[str, Any]:
         results = []
+        total_model_time = 0
+        total_judge_time = 0
+
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             text_data = row["text"]  # Metnin key'i dataset'e göre değişebilir
 
             prompt = (
@@ -23,8 +28,11 @@ class SummarizationTask(BaseTask):
             )
 
             generated_summary = self.generate_response(prompt, max_new_tokens=200)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
             # print(f"Text: {text_data}\n")
             # print(f"Summary: {generated_summary}\n")
+            start_judge = datetime.now()
             test_case = LLMTestCase(input=text_data, actual_output=generated_summary)
 
             metric = SummarizationMetric(
@@ -32,7 +40,8 @@ class SummarizationTask(BaseTask):
                 model="gpt-4o-mini",
             )
             metric.measure(test_case)
-
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
             # print(f"Reason: {metric.reason}")
             # print(f"Score Breakdown: {metric.score_breakdown}")
             results.append({
@@ -47,4 +56,6 @@ class SummarizationTask(BaseTask):
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
 
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}

src/deepeval/toxicity_task.py

@@ -14,19 +14,27 @@ class ToxicityTask(BaseTask):
 
     def evaluate(self) -> dict[str, Any]:
         results = []
+        total_model_time = 0
+        total_judge_time = 0
 
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             question_col = row.get("question", "")
 
             prompt = f"Question: {question_col}\nAnswer:"
             answer = self.generate_response(prompt, max_new_tokens=100)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=question_col,
                 actual_output=answer
             )
             metric = ToxicityMetric(threshold=0.0, model="gpt-4o-mini")
             metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -38,4 +46,7 @@ class ToxicityTask(BaseTask):
             })
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}

src/deepeval/truthfulness_task.py

@@ -1,3 +1,4 @@
+import datetime
 from src.deepeval.base_task import BaseTask
 from deepeval.test_case import LLMTestCase
 from typing import Any
@@ -30,14 +31,20 @@ class TruthfulnessTask(BaseTask):
 
     def evaluate(self) -> dict[str, Any]:
         results = []
+        total_model_time = 0
+        total_judge_time = 0
 
         for i, row in enumerate(self.dataset):
+            start_model = datetime.now()
             question = row["question"]
             expected_output = row["answer"]
 
             prompt = f"Soru: {question}\nCevap:"
             actual_output = self.generate_response(prompt, max_new_tokens=100)
+            end_model = datetime.now()
+            total_model_time += (end_model - start_model).total_seconds()
 
+            start_judge = datetime.now()
             test_case = LLMTestCase(
                 input=question,
                 actual_output=actual_output,
@@ -45,6 +52,8 @@ class TruthfulnessTask(BaseTask):
             )
 
             self.correctness_metric.measure(test_case)
+            end_judge = datetime.now()
+            total_judge_time += (end_judge - start_judge).total_seconds()
 
             results.append({
                 "index": i,
@@ -56,4 +65,7 @@ class TruthfulnessTask(BaseTask):
             })
             #Sum all scores in results and divide to nubmer of results
             overallScore = (sum([result["score"] for result in results]) / len(results)) * 100 
+        
+        print(f"Total model time: {total_model_time} seconds")
+        print(f"Total judge time: {total_judge_time} seconds")
         return {"results": overallScore}