Spaces:
Running
Running
File size: 3,097 Bytes
7628397 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
from email.utils import parseaddr
from huggingface_hub import HfApi
import os
import datetime
OWNER="locuslab"
SUBMISSION_DATASET = f"{OWNER}/submissions_internal"
RESULTS_DATASET = f"{OWNER}/results_public"
LEADERBOARD_PATH = f"{OWNER}/tofu_leaderboard"
api = HfApi()
TOKEN = os.environ.get("TOKEN", None)
YEAR_VERSION = "2024"
def format_error(msg):
return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
def format_warning(msg):
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
def format_log(msg):
return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
def model_hyperlink(link, model_name):
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def add_new_eval(
val_or_test: str,
model: str,
model_family: str,
system_prompt: str,
url: str,
path_to_file: str,
organisation: str,
mail: str,
):
# Very basic email parsing
_, parsed_mail = parseaddr(mail)
if not "@" in parsed_mail:
return format_warning("Please provide a valid email adress.")
print("Adding new eval")
# Check if the combination model/org already exists and prints a warning message if yes
# if model.lower() in set(eval_results[val_or_test]["model"]) and organisation.lower() in set(eval_results[val_or_test]["organisation"]):
# return format_warning("This model has been already submitted.")
if path_to_file is None:
return format_warning("Please attach a file.")
# Save submitted file
api.upload_file(
repo_id=SUBMISSION_DATASET,
path_or_fileobj=path_to_file.name,
path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_raw_{datetime.datetime.today()}.jsonl",
repo_type="dataset",
token=TOKEN
)
# Compute score
# Save scored file
api.upload_file(
repo_id=SUBMISSION_DATASET,
path_or_fileobj=f"scored/{organisation}_{model}.jsonl",
path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_scored_{datetime.datetime.today()}.jsonl",
repo_type="dataset",
token=TOKEN
)
# Actual submission
eval_entry = {
"model": model,
"model_family": model_family,
"system_prompt": system_prompt,
"url": url,
"organisation": organisation,
"mail": mail,
# "score": scores["all"]/num_questions["all"],
# "score_level1": scores[1]/num_questions[1],
# "score_level2": scores[2]/num_questions[2],
# "score_level3": scores[3]/num_questions[3],
}
# eval_results[val_or_test] = eval_results[val_or_test].add_item(eval_entry)
# print(eval_results)
# eval_results.push_to_hub(RESULTS_DATASET, config_name = YEAR_VERSION, token=TOKEN)
return format_log(f"Model {model} submitted by {organisation} successfully. \nPlease refresh the leaderboard, and wait a bit to see the score displayed")
|