Post merge fix

src/deepeval/base_task.py

@@ -206,7 +206,7 @@ class BaseTask(ABC):
         start_time = datetime.now()
         dataset= load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
         print("Dataset loaded.")
-        
+    
         # Load 50 from each dataset
         if len(dataset) > 50:
             dataset = dataset.shuffle(seed=42).select(range(50))

src/deepeval/bias.py

@@ -87,8 +87,10 @@ class BiasTask(BaseTask):
 
         # Print results categorized by difficulty
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/math.py

@@ -65,6 +65,7 @@ class MathTask(BaseTask):
             # Get values from row
             category = str(row["difficulty"])
             answer = row["final_answer"]
+            question = row["question"]
 
             # Prints for debugging
             print(f"Answer: {answer}")
@@ -94,7 +95,7 @@ Görev: Problemi çözün, son adımda yukarıdaki kurallara tam uyan tek bir ku
 
 Nihai cevap:
                         """
-            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            prompt = f"{instruction}\n\nSoru:\n{question}\n"
             message = prompt
 
             # Get/format answer of the model
@@ -117,8 +118,10 @@ Nihai cevap:
 
         # Print results categorized by difficulty
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/metaphors_and_idioms.py

@@ -76,8 +76,10 @@ class MetaphorsAndIdiomsTask(BaseTask):
         for subset in difficulty_results.keys():
             subset_results = difficulty_results[subset]
             for category, stats in subset_results.items():
-                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+                correct = stats['correct']
+                total = stats['total']
+                calculatedAccuracy = correct / total if total > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/mmlu.py

@@ -76,8 +76,10 @@ class MMLUTask(BaseTask):
 
         # Print results categorized by subset
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{subset.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{subset.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/ner.py

@@ -66,6 +66,7 @@ class NERTask(BaseTask):
             # Get values from row
             category = str(row["difficulty"])
             answer = row["final_answer"]
+            question = row["question"]
 
             # Prints for debugging
             print(f"Answer: {answer}")
@@ -132,7 +133,7 @@ class NERTask(BaseTask):
                   ""
                   "Verilen cümlelerdeki her varlığı csv formatında yukarıdaki örneklere benzer şekilde belirleyin. Çıktıdaki her satırı aşağıdaki gibi oluşturun: "
                   "<Varlık metni>,<Varlık etiketi>"),
-            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            prompt = f"{instruction}\n\nSoru:\n{question}\n"
             message = prompt
 
             # Get/format answer of the model
@@ -155,8 +156,10 @@ class NERTask(BaseTask):
 
         # Print results categorized by difficulty
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/pos.py

@@ -96,6 +96,7 @@ class POSTask(BaseTask):
             # Get values from row
             category = str(row["difficulty"])
             answer = row["final_answer"]
+            question = row["question"]
 
             # Prints for debugging
             print(f"Answer: {answer}")
@@ -125,7 +126,7 @@ Görev: Problemi çözün, son adımda yukarıdaki kurallara tam uyan tek bir ku
 
 Nihai cevap:
                         """
-            prompt = f"{instruction}\n\nSoru:\n{row["question"]}\n"
+            prompt = f"{instruction}\n\nSoru:\n{question}\n"
             message = prompt
 
             # Get/format answer of the model
@@ -148,8 +149,10 @@ Nihai cevap:
 
         # Print results categorized by difficulty
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/sts.py

@@ -90,6 +90,8 @@ class STSTask(BaseTask):
             # Get values from row
             answer = row["score"]
             choices = ["0","1","2","3","4","5"]
+            sentence_1 = row["sentence_1"]
+            sentence_2 = row["sentence_2"]
 
             # Prints for debugging
             print(f"Answer: {answer}")
@@ -97,7 +99,7 @@ class STSTask(BaseTask):
 
             # Construct the prompt/message
             instruction = f"Aşağıda verilen iki cümlenin birbirlerine olan anlamsal benzerliğini 0'dan 5'e kadar olan bir tam sayıyla söyleyin."
-            prompt = f"""{instruction}\nCümle 1: {row["sentence_1"]}\nCümle 2: {row["sentence_2"]}\nSadece tek bir tam sayı söyleyin, ek bir kelime ya da sembol kullanmayın."""
+            prompt = f"""{instruction}\nCümle 1: {sentence_1}\nCümle 2: {sentence_2}\nSadece tek bir tam sayı söyleyin, ek bir kelime ya da sembol kullanmayın."""
             message = prompt
 
             # Get/format answer of the model
@@ -119,9 +121,12 @@ class STSTask(BaseTask):
             difficulty_results['total'] += 1
 
         # Print results
-            stats = difficulty_results
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+        stats = difficulty_results
+        correct = stats['correct']
+        total = stats['total']
+
+        calculatedAccuracy = correct / total if total > 0 else 0
+        print(f"Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/topic_detection.py

@@ -68,8 +68,10 @@ class TopicDetectionTask(BaseTask):
 
         # Print results categorized by difficulty
         for category, stats in difficulty_results.items():
-            calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+            correct = stats['correct']
+            total = stats['total']
+            calculatedAccuracy = correct / total if total > 0 else 0
+            print(f"{category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

src/deepeval/turkish_vocabulary.py

@@ -89,8 +89,10 @@ class TurkishVocabularyTask(BaseTask):
         for subset in self.subsets:
             subset_results = difficulty_results[subset]
             for category, stats in subset_results.items():
-                calculatedAccuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
-                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({stats['correct']}/{stats['total']})")
+                correct = stats['correct']
+                total = stats['total']
+                calculatedAccuracy = correct / total if total > 0 else 0
+                print(f"{subset.capitalize()} {category.capitalize()} Accuracy: {calculatedAccuracy:.2%} ({correct}/{total})")
 
         print("Results:", responses)
         print("Overall Accuracy:", true / total_count)

svc/schemas.py

@@ -35,7 +35,6 @@ class DeepEvalSuiteRequest(BaseModel):
     tasks: Optional[List[str]] = None
 
 
-
 class TaskResponse(BaseModel):
     results: Json # dict[Any,Any]