customize the data count

src/deepeval/base_task.py

@@ -246,8 +246,26 @@ class BaseTask(ABC):
         print("Dataset loaded.")
     
         # Load 50 from each dataset
-        if len(dataset) > 50:
-            dataset = dataset.shuffle(seed=42).select(range(50))
+        if len(dataset) > 100:
+            dataset = dataset.shuffle(seed=42).select(range(100))
+        end_time = datetime.now()
+        print(f"Dataset loaded in {(end_time - start_time).seconds} seconds.")
+        return dataset
+    
+    @abstractmethod
+    def load_dataset_lmjudge_from_hf(self):
+        """
+        Define your own loading method if needed.
+        :return: Dataset
+        """
+        print("Loading dataset from Hugging Face.")
+        start_time = datetime.now()
+        dataset= load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
+        print("Dataset loaded.")
+        
+        # Load 50 from each dataset
+        if len(dataset) > 10:
+            dataset = dataset.shuffle(seed=42).select(range(10))
         end_time = datetime.now()
         print(f"Dataset loaded in {(end_time - start_time).seconds} seconds.")
         return dataset

src/deepeval/bias.py

@@ -9,9 +9,15 @@ class BiasTask(BaseTask):
     def __init__(self, model_name):
         super().__init__("metunlp/sosyoloji_bias", model_name=model_name)
 
+<<<<<<< HEAD
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
         return dataset
+=======
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
+        return dataset.select(range(min(1, len(dataset))))
+>>>>>>> 2dca79c (customize the data count)
 
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/faithfulness_task.py

@@ -8,8 +8,8 @@ class FaithfulnessTask(BaseTask):
     def __init__(self, model_name: str):
         super().__init__("metunlp/sosyoloji_faithfulness", model_name=model_name)
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/instruction_following_task.py

@@ -9,8 +9,8 @@ class InstructionFollowingTask(BaseTask):
     def __init__(self, model_name: str):
         super().__init__("metunlp/instruction_following_tr", model_name=model_name)
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/reading_comprehension_task.py

@@ -27,8 +27,8 @@ class ReadingComprehensionTask(BaseTask):
             ],
         )
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/summarization_task.py

@@ -8,8 +8,8 @@ class SummarizationTask(BaseTask):
     def __init__(self, model_name: str):
         super().__init__("metunlp/summarization_tr", model_name=model_name)
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/toxicity_task.py

@@ -8,8 +8,8 @@ class ToxicityTask(BaseTask):
     def __init__(self, model_name: str):
         super().__init__("metunlp/sosyoloji_toxicity", model_name=model_name)
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
 

src/deepeval/truthfulness_task.py

@@ -25,8 +25,8 @@ class TruthfulnessTask(BaseTask):
             ],
         )
 
-    def load_dataset_from_hf(self):
-        dataset = super().load_dataset_from_hf()
+    def load_dataset_lmjudge_from_hf(self):
+        dataset = super().load_dataset_lmjudge_from_hf()
         return dataset
 
     def evaluate(self) -> dict[str, Any]: