Add deepeval endpoint

svc/router.py

@@ -2,14 +2,14 @@ from fastapi import APIRouter, HTTPException, Depends
 import logging
 
 from lm_eval import evaluator
-from svc.schemas import LMHarnessTaskRequest, LMHarnessTaskResponse, LoadModelRequest
+from svc.schemas import LMHarnessTaskRequest, TaskResponse, LoadModelRequest, DeepEvalSuiteRequest
 from fastapi.security import OAuth2PasswordRequestForm
 from auth.authentication import get_current_user, create_access_token
 from dotenv import load_dotenv
 import os
 import json
+from src.deepeval.deepeval_task_manager import DeepEvalTaskManager
 
-from transformers import AutoModelForCausalLM, AutoTokenizer
 
 router = APIRouter()
 
@@ -17,7 +17,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(
 logger = logging.getLogger(__name__)
 
 load_dotenv()
-
+HF_TOKEN = os.getenv("HF_TOKEN")
 @router.post("/token")
 async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
     auth_token = os.getenv("AUTH_UUID")
@@ -32,7 +32,7 @@ async def protected_route(username: str = Depends(get_current_user)):
     return {"message": f"Hello, {username}! This is a protected resource."}
 
 
-@router.post("/chat",  response_model=LMHarnessTaskResponse)
+@router.post("/chat",  response_model=TaskResponse)
 def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_current_user)):
     try:
         logger.info("Inside")
@@ -59,16 +59,14 @@ def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_c
     logger.info("-------------------results------------------\n")
     logger.info(dumped)
     logger.info("-------------------results end------------------\n")
-    return LMHarnessTaskResponse(results=dumped)
+    return TaskResponse(results=dumped)
+
+
 
+@router.post("/deepeval/eval", response_model=TaskResponse)
+async def deep_eval_suite(request: DeepEvalSuiteRequest):
+    des = DeepEvalTaskManager(request.model_name, request.tasks)
+    results = des.run_tasks() #TODO: format should be different. Check metunlp/results repo for the correct format
+    return TaskResponse(results=results)
 
 
-# @router.post("/load_model")
-# async def load_model(request: LoadModelRequest):
-#     model = AutoModelForCausalLM.from_pretrained(
-#         request.model_name,
-#         torch_dtype="auto",
-#         device_map="auto"
-#     )
-#     tokenizer = AutoTokenizer.from_pretrained(request.model_name)
-#     return model, tokenizer