File size: 3,411 Bytes
056e156
2b8f89d
e22e877
 
 
 
 
d1ca5fe
056e156
e1043c6
 
e22e877
741eae9
e1043c6
056e156
 
 
fd87b41
 
056e156
e22e877
 
 
 
 
056e156
e22e877
056e156
 
 
e22e877
056e156
 
 
e22e877
056e156
 
 
 
 
 
 
e22e877
 
 
 
 
 
 
 
056e156
 
 
8011593
056e156
 
 
 
d1ca5fe
056e156
 
 
 
 
e22e877
056e156
e22e877
8011593
 
 
 
 
 
 
 
 
 
 
 
 
 
 
056e156
e22e877
e1043c6
e22e877
e1043c6
 
 
 
 
e22e877
e1043c6
 
e22e877
e1043c6
 
056e156
e22e877
2b8f89d
 
 
9a20624
e22e877
 
741eae9
2b8f89d
e22e877
741eae9
7be8121
e22e877
741eae9
 
1eda6aa
741eae9
056e156
 
e22e877
8011593
056e156
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import os
import json
import datetime
from email.utils import parseaddr
from io import BytesIO

from huggingface_hub import HfApi
import gradio as gr

from eval_utils import get_evaluation_scores

# Constants
LEADERBOARD_PATH = "Exploration-Lab/IL-TUR-Leaderboard"
SUBMISSION_FORMAT = "predictions"
TOKEN = os.environ.get("TOKEN", None)
YEAR_VERSION = "2024"

api = HfApi(token=TOKEN)


# Helper functions for formatting messages
def format_message(msg, color):
    return f"<p style='color: {color}; font-size: 20px; text-align: center;'>{msg}</p>"


def format_error(msg):
    return format_message(msg, "red")


def format_warning(msg):
    return format_message(msg, "orange")


def format_log(msg):
    return format_message(msg, "green")


def model_hyperlink(link, model_name):
    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'


def input_verification(method_name, url, path_to_file, organisation, mail):
    """Verify the input fields for submission."""
    # Check if any field is empty
    if any(
        input == "" for input in [method_name, url, path_to_file, organisation, mail]
    ):
        return format_warning("Please fill all the fields.")

    # Check if file is attached
    if path_to_file is None:
        return format_warning("Please attach a file.")

    return


def add_new_eval(
    method_name: str,
    submitted_by: str,
    url: str,
    path_to_file: str,
    organisation: str,
    mail: str,
):
    """Add a new evaluation to the leaderboard."""

    # Verify input
    # Check if any field is empty
    if any(
        input == "" for input in [method_name, url, path_to_file, organisation, mail]
    ):
        return format_warning("Please fill all the fields.")

    # Check if file is attached
    if path_to_file is None:
        return format_warning("Please attach a file.")

    # Verify email format
    _, parsed_mail = parseaddr(mail)
    if "@" not in parsed_mail:
        print(parseaddr(mail))
        return format_warning("Please provide a valid email address.")

    # Process submission
    if SUBMISSION_FORMAT == "predictions":
        # Read submission and gold data
        with open(path_to_file, "r") as f:
            submission_data = json.load(f)
        with open("submissions/baseline/IL_TUR_eval_gold_small.json", "r") as f:
            gold_data = json.load(f)

        # Get evaluation scores
        submission = get_evaluation_scores(gold_data, submission_data)
    else:
        # Read submission directly if it's not in predictions format
        with open(path_to_file, "r") as f:
            submission = json.load(f)

    # Update results
    with open("submissions/baseline/results.json", "r") as f:
        results = json.load(f)
    results.append(submission[0])

    # Prepare buffer for upload
    leaderboard_buffer = BytesIO(json.dumps(results).encode())
    leaderboard_buffer.seek(0)

    # Upload to Hugging Face
    api.upload_file(
        repo_id=LEADERBOARD_PATH,
        path_in_repo="submissions/baseline/results.json",
        path_or_fileobj=leaderboard_buffer,
        token=TOKEN,
        repo_type="space",
    )

    return format_log(
        f"Method {method_name} submitted by {organisation} successfully. \n"
        "Please refresh the leaderboard, and wait for the evaluation results."
    )