Merge branch 'main' into aysu

requirements.txt

@@ -7,5 +7,6 @@ python-jose
 python-multipart
 deepeval
 --extra-index-url https://download.pytorch.org/whl/cu113
+huggingface-hub>=0.29.1
 torch
 sentencepiece

src/deepeval/base_task.py

@@ -76,8 +76,8 @@ class BaseTask(ABC):
         answer = self.tokenizer.decode(output[0][-1])
 
         return answer
-
-    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=5, choices: list = []):
+    
+    def generate_response_mcqa_multi_token(self, msg, max_new_tokens=2, choices: list = []):
         """
         Handles multiple-choice questions where answers might have multiple tokens.
         """
@@ -146,7 +146,7 @@ class BaseTask(ABC):
             {"role": "assistant", "content": "I am here to help you with any questions you may have."},
             {"role": "user", "content": prompt},
         ]
-
+        
         formatted_chat = self.tokenizer.apply_chat_template(
             chat,
             tokenize=False,
@@ -188,6 +188,11 @@ class BaseTask(ABC):
         print("Loading dataset from Hugging Face.")
         dataset= load_dataset(self.dataset_repo, token=HF_TOKEN, split="train")
         print("Dataset loaded.")
+        
+        # Load %25 of each dataset
+        print("Original dataset size: ", len(dataset))
+        dataset = dataset.shuffle(seed=42).select(range(int(len(dataset) * 0.25)))
+        print("Reduced dataset size: ", len(dataset))
         return dataset
 
     @abstractmethod

src/deepeval/bias_task.py

@@ -10,7 +10,7 @@ class BiasTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
 

src/deepeval/commonsense_reasoning_task.py

@@ -10,7 +10,7 @@ class CommonsenseReasoningTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -57,7 +57,7 @@ class CommonsenseReasoningTask(BaseTask):
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
 

src/deepeval/complex_reasoning.py

@@ -11,7 +11,7 @@ class ComplexReasoningTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/deepeval_task_manager.py

@@ -74,11 +74,15 @@ class DeepEvalTaskManager:
         """Execute validated tasks in order."""
         results = {}
         for task_name, task_method in self.tasks_to_run.items():
-            print("Running task: ", task_name)
-            task_enum = getattr(Task, task_name)
-            task_value = task_enum.value
-            results[task_value] = task_method()  # Call the stored method reference
-
+            try:
+                print("Running task: ", task_name)
+                task_enum = getattr(Task, task_name)
+                task_value = task_enum.value
+                results[task_value] = task_method()  # Call the stored method reference
+            except Exception as e:
+                print(f"Error At Task: {task_name} - {e}")
+                continue
+        print("All tasks completed.")
         return results
 
     def sentiment_analysis_tr(self):
@@ -182,6 +186,6 @@ class DeepEvalTaskManager:
         return res
 
 if __name__ == "__main__":
-    des = DeepEvalTaskManager("google/gemma-2b-it", ["BIAS_MC"])
+    des = DeepEvalTaskManager("google/gemma-2-2b-it", ["TOXICITY", "BIAS"])
     res = des.run_tasks()
     print(res)

src/deepeval/faithfulness_task.py

@@ -9,7 +9,7 @@ class FaithfulnessTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
 

src/deepeval/instruction_following_task.py

@@ -10,7 +10,7 @@ class InstructionFollowingTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
         results = []

src/deepeval/nli.py

@@ -10,7 +10,7 @@ class NLITask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -48,7 +48,7 @@ class NLITask(BaseTask):
             message = prompt
 
             # Get/format answer of the model
-            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=10)
+            model_answer = self.generate_response_mcqa_multi_token(message, choices=choices, max_new_tokens=2)
             responses.append(model_answer)
             model_answer_cleaned = model_answer.strip().replace('\n', '').replace(' ', '').upper()
 

src/deepeval/reading_comp_mc.py

@@ -11,7 +11,7 @@ class ReadingComprehensionMCTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:
@@ -41,8 +41,9 @@ class ReadingComprehensionMCTask(BaseTask):
                 answer_index = answer
             else:
                 answer_index = int(answer)
-            correct_answer_letter = chr(64 + answer_index) # 65 - 1 since we need the indexing to start from 0
-
+            
+            answer_index = answer_index - 1 # Because the answer is 1-indexed
+            correct_answer_letter = chr(65 + answer_index)
 
             # Construct the prompt/message
             instruction = ""

src/deepeval/reading_comprehension_task.py

@@ -28,7 +28,7 @@ class ReadingComprehensionTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
         results = []

src/deepeval/sentiment_analysis_task.py

@@ -9,7 +9,7 @@ class SentimentAnalysisTask(BaseTask):
     def load_dataset_from_hf(self):
         print("Loading the dataset")
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/summarization_task.py

@@ -9,7 +9,7 @@ class SummarizationTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
         results = []

src/deepeval/toxicity_task.py

@@ -9,7 +9,7 @@ class ToxicityTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
 
     def evaluate(self) -> dict[str, Any]:

src/deepeval/truthfulness_task.py

@@ -26,7 +26,7 @@ class TruthfulnessTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(3, len(dataset))))
+        return dataset
 
     def evaluate(self) -> dict[str, Any]:
         results = []

src/deepeval/turkish_general_knowledge_task.py

@@ -9,7 +9,7 @@ class TurkishGeneralKnowledgeTask(BaseTask):
 
     def load_dataset_from_hf(self):
         dataset = super().load_dataset_from_hf()
-        return dataset.select(range(min(10, len(dataset))))
+        return dataset
 
     def evaluate(self):
         responses = []

svc/router.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from fastapi import APIRouter, HTTPException, Depends
 import logging
 
@@ -8,11 +9,13 @@ from auth.authentication import get_current_user, create_access_token
 from dotenv import load_dotenv
 import os
 import json
+from pathlib import Path
 from src.deepeval.deepeval_task_manager import DeepEvalTaskManager
 import torch
 import gc
 from time import time
 from huggingface_hub import HfApi, ModelInfo
+import threading
 
 
 router = APIRouter()
@@ -25,7 +28,6 @@ HF_TOKEN = os.getenv("HF_TOKEN")
 
 # Or configure a HfApi client
 hf_api = HfApi(
-    endpoint="https://huggingface.co", # Can be a Private Hub endpoint.
     token=HF_TOKEN, # Token is not persisted on the machine.
 )
 
@@ -48,6 +50,11 @@ async def deep_eval_status():
     #Return running with 200 status code
     return {"status": "running"}
 
+@router.get("/deepeval/hardware")
+def hardware_status():
+    info = get_gpu_tier()
+    return info
+
 @router.post("/chat",  response_model=TaskResponse)
 def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_current_user)):
     logger.info(request)
@@ -82,48 +89,83 @@ def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_c
     return TaskResponse(results=dumped)
 
 
-@router.post("/deepeval/eval", response_model=TaskResponse)
-async def deep_eval_suite(request: DeepEvalSuiteRequest):
-    des = DeepEvalTaskManager(request.model_name, request.tasks)
-    start_time = time()
-    results = des.run_tasks() #TODO: format should be different. Check metunlp/results repo for the correct format
-    end_time = time()
-    duration = round(end_time - start_time, 2) # total_evaluation_time_seconds
-
-    model_info: ModelInfo = hf_api.model_info(request.model_name)
-
-    config = {
-        "model_source": "hf",
-        "num_fewshot": 0,
-        "batch_size": 8,
-        "batch_sizes": [],
-        "device": "cuda:0", # TODO: take this from requests
-        # "no_cache": true,
-        # "limit": null,
-        # "bootstrap_iters": 100000,
-        # "description_dict": null,
-        "model_dtype": "torch.float16", # TODO: take this from requests
-        "model_name": request.model_name,
-        "model_sha": model_info.sha
-    }
-
-    tbr_dict = {
-        "results": results,
-        "config": config,
-        "total_evaluation_time_seconds": duration,
-        "start_time": start_time,
-        "end_time": end_time
-    }
-
-    json_results = json.dumps(tbr_dict)
-
-    #Free up VRAM
-    torch.cuda.empty_cache()
-
-    #Free up RAM
-    des = None
-    gc.collect()
-
-    return TaskResponse(results=json_results)
 
 
+@router.post("/deepeval/eval", response_model=TaskResponse)
+def deep_eval_suite(request: DeepEvalSuiteRequest):
+    def run_in_background():
+        try:
+            torch.cuda.empty_cache()
+            des = DeepEvalTaskManager(request.model_name, request.tasks)
+
+            start_time = time()
+            results = des.run_tasks()
+            end_time = time()
+            duration = round(end_time - start_time, 2)
+
+            model_info: ModelInfo = hf_api.model_info(request.model_name)
+
+            config = {
+                "model_source": "hf",
+                "num_fewshot": 0,
+                "batch_size": 8,
+                "device": "cuda:0",
+                "model_dtype": "torch.float16",
+                "model_name": request.model_name,
+                "model_sha": model_info.sha,
+            }
+
+            final_results = {
+                "results": results,
+                "config": config,
+                "total_evaluation_time_seconds": duration,
+                "start_time": start_time,
+                "end_time": end_time
+            }
+
+            # Save and upload
+            dumped = json.dumps(final_results, indent=2)
+            path = Path("/tmp", request.model_name, f"results_{datetime.now()}.json")
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(dumped)
+
+            RESULTS_REPO = "metunlp/results"
+            hf_api.upload_file(
+                path_or_fileobj=path,
+                path_in_repo=path.relative_to("/tmp").as_posix(),
+                repo_id=RESULTS_REPO,
+                repo_type="dataset",
+            )
+
+            logger.info(f"✅ Uploaded results to HF Hub for {request.model_name}")
+
+        except Exception as e:
+            logger.exception(f"❌ Background evaluation failed: {e}")
+
+    # 🔁 Start evaluation in background
+    threading.Thread(target=run_in_background, daemon=True).start()
+
+    # ✅ Immediately respond
+    return TaskResponse(results=json.dumps({"status": "Evaluation started in background"}))
+
+
+
+
+def get_gpu_tier():
+    if not torch.cuda.is_available():
+        return {"gpu": "CPU", "tier": "cpu"}
+
+    gpu_name = torch.cuda.get_device_name(0).lower()
+
+    # Normalize GPU model to your custom tier system
+    if "t4" in gpu_name:
+        # You can improve this by checking memory or other context
+        return {"gpu": "Tesla T4", "tier": "t4-medium"}
+    elif "l4" in gpu_name:
+        return {"gpu": "NVIDIA L4", "tier": "l4x1"}
+    elif "l40s" in gpu_name:
+        return {"gpu": "NVIDIA L40S", "tier": "l40sx1"}
+    elif "a10g" in gpu_name:
+        return {"gpu": "NVIDIA A10G", "tier": "a10g"}
+    else:
+        return {"gpu": gpu_name, "tier": "unknown"}