Spaces:
Sleeping
Sleeping
File size: 2,527 Bytes
df66f6e 2a5f9fb 78a1bcb 2a5f9fb df66f6e 314f91a 2a5f9fb df66f6e 2a5f9fb 976f398 2a5f9fb 78a1bcb 336f5e2 2a5f9fb 976f398 78a1bcb 336f5e2 78a1bcb 2a5f9fb a3adbe2 2a5f9fb db0913f 36ff374 78a1bcb 2a5f9fb 93f34e1 2a5f9fb e6aadba 78a1bcb e6aadba 3d8e05d 78a1bcb 2a5f9fb 78a1bcb e6aadba 2a5f9fb 78a1bcb 2a5f9fb 78a1bcb c15e77e 2a5f9fb 36ff374 2a5f9fb a617daf 2a5f9fb 336f5e2 2a5f9fb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
import json
import os
from datetime import datetime, timezone
import shutil
from src.display.formatting import styled_error, styled_message, styled_warning
from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
from src.submission.check_validity import (
already_submitted_models,
check_model_card,
get_model_size,
is_model_on_hub,
)
REQUESTED_MODELS = None
USERS_TO_SUBMISSION_DATES = None
def add_new_eval(
model_name: str,
output_format: str,
revision_name: str,
upload_file,
version: str,
):
global REQUESTED_MODELS
global USERS_TO_SUBMISSION_DATES
if not REQUESTED_MODELS:
REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
print(upload_file)
print(version)
# Does the model actually exist?
if revision_name == "":
revision_name = "main"
folder_path = f"entry_{model_name}_{datetime.now()}"
file_name = f"pred.json"
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
path = f"{EVAL_REQUESTS_PATH}/{version}/{folder_path}/{file_name}"
shutil.copy(upload_file, file_name)
print("Uploading eval file")
API.upload_file(
path_or_fileobj=file_name,
path_in_repo=path.split("eval-queue/")[1],
repo_id=QUEUE_REPO,
repo_type="dataset",
commit_message=f"Add {model_name} pred to eval queue",
)
# Remove the local file
os.remove(file_name)
# Seems good, creating the eval
print("Adding new eval")
eval_entry = {
"model": model_name,
"revision": revision_name,
"status": "PENDING",
"submitted_time": current_time,
"output_format": output_format,
"submission_file": file_name,
"private": False,
}
OUT_DIR = f"{EVAL_REQUESTS_PATH}/{version}/{folder_path}"
os.makedirs(OUT_DIR, exist_ok=True)
out_path = f"{OUT_DIR}/eval_request.json"
with open(out_path, "w") as f:
f.write(json.dumps(eval_entry))
print("Uploading eval file")
API.upload_file(
path_or_fileobj=out_path,
path_in_repo=out_path.split("eval-queue/")[1],
repo_id=QUEUE_REPO,
repo_type="dataset",
commit_message=f"Add {model_name} to eval queue",
)
# Remove the local file
os.remove(out_path)
return styled_message(
"Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
)
|