Add other configs to result

svc/router.py

@@ -10,6 +10,9 @@ import os
 import json
 from src.deepeval.deepeval_task_manager import DeepEvalTaskManager
 import torch
+from time import time
+from huggingface_hub import HfApi, ModelInfo
+
 
 router = APIRouter()
 
@@ -18,6 +21,14 @@ logger = logging.getLogger(__name__)
 
 load_dotenv()
 HF_TOKEN = os.getenv("HF_TOKEN")
+
+# Or configure a HfApi client
+hf_api = HfApi(
+    endpoint="https://huggingface.co", # Can be a Private Hub endpoint.
+    token=HF_TOKEN, # Token is not persisted on the machine.
+)
+
+
 @router.post("/token")
 async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
     auth_token = os.getenv("AUTH_UUID")
@@ -70,8 +81,38 @@ def inference_model(request: LMHarnessTaskRequest, username: str = Depends(get_c
 @router.post("/deepeval/eval", response_model=TaskResponse)
 async def deep_eval_suite(request: DeepEvalSuiteRequest):
     des = DeepEvalTaskManager(request.model_name, request.tasks)
+    start_time = time()
     results = des.run_tasks() #TODO: format should be different. Check metunlp/results repo for the correct format
-    json_results = json.dumps(results)
+    end_time = time()
+    duration = round(end_time - start_time, 2) # total_evaluation_time_seconds
+
+    model_info: ModelInfo = hf_api.model_info(request.model_name)
+
+    config = {
+        "model_source": "hf",
+        "num_fewshot": 0,
+        "batch_size": 8,
+        "batch_sizes": [],
+        "device": "cuda:0", # TODO: take this from requests
+        # "no_cache": true,
+        # "limit": null,
+        # "bootstrap_iters": 100000,
+        # "description_dict": null,
+        "model_dtype": "torch.float16", # TODO: take this from requests
+        "model_name": request.model_name,
+        "model_sha": model_info.sha
+    }
+
+    tbr_dict = {
+        "results": results,
+        "config": config,
+        "total_evaluation_time_seconds": duration,
+        "start_time": start_time,
+        "end_time": end_time
+    }
+
+
+    json_results = json.dumps(tbr_dict)
 
     return TaskResponse(results=json_results)