import json import os from datetime import datetime, timezone import shutil from src.display.formatting import styled_error, styled_message, styled_warning from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO from src.submission.check_validity import ( already_submitted_models, check_model_card, get_model_size, is_model_on_hub, ) REQUESTED_MODELS = None USERS_TO_SUBMISSION_DATES = None def add_new_eval( model_name: str, output_format: str, revision_name: str, upload_file, version: str, ): global REQUESTED_MODELS global USERS_TO_SUBMISSION_DATES if not REQUESTED_MODELS: REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH) print(upload_file) print(version) # Does the model actually exist? if revision_name == "": revision_name = "main" folder_path = f"entry_{model_name}_{datetime.now()}" file_name = f"pred.json" current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") path = f"{EVAL_REQUESTS_PATH}/{version}/{folder_path}/{file_name}" shutil.copy(upload_file, file_name) print("Uploading eval file") API.upload_file( path_or_fileobj=file_name, path_in_repo=path.split("eval-queue/")[1], repo_id=QUEUE_REPO, repo_type="dataset", commit_message=f"Add {model_name} pred to eval queue", ) # Remove the local file os.remove(file_name) # Seems good, creating the eval print("Adding new eval") eval_entry = { "model": model_name, "revision": revision_name, "status": "PENDING", "submitted_time": current_time, "output_format": output_format, "submission_file": file_name, "private": False, } OUT_DIR = f"{EVAL_REQUESTS_PATH}/{version}/{folder_path}" os.makedirs(OUT_DIR, exist_ok=True) out_path = f"{OUT_DIR}/eval_request.json" with open(out_path, "w") as f: f.write(json.dumps(eval_entry)) print("Uploading eval file") API.upload_file( path_or_fileobj=out_path, path_in_repo=out_path.split("eval-queue/")[1], repo_id=QUEUE_REPO, repo_type="dataset", commit_message=f"Add {model_name} to eval queue", ) # Remove the local file os.remove(out_path) return styled_message( "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." )