jbnayahu commited on
Commit
3a4f28e
·
unverified ·
1 Parent(s): 57da4d5

Switch to read local results only.

Browse files

Signed-off-by: Jonathan Bnayahu <[email protected]>

app.py CHANGED
@@ -25,9 +25,8 @@ from src.display.utils import (
25
  # WeightType,
26
  # Precision
27
  )
28
- from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
29
  from src.populate import get_leaderboard_df
30
- from src.submission.submit import add_new_eval
31
 
32
 
33
  def restart_space():
@@ -41,13 +40,13 @@ def restart_space():
41
  # )
42
  # except Exception:
43
  # restart_space()
44
- try:
45
- print(EVAL_RESULTS_PATH)
46
- snapshot_download(
47
- repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
48
- )
49
- except Exception:
50
- restart_space()
51
 
52
  LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
53
 
 
25
  # WeightType,
26
  # Precision
27
  )
28
+ from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, TOKEN
29
  from src.populate import get_leaderboard_df
 
30
 
31
 
32
  def restart_space():
 
40
  # )
41
  # except Exception:
42
  # restart_space()
43
+ # try:
44
+ # print(EVAL_RESULTS_PATH)
45
+ # snapshot_download(
46
+ # repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
47
+ # )
48
+ # except Exception:
49
+ # restart_space()
50
 
51
  LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
52
 
src/envs.py CHANGED
@@ -10,16 +10,18 @@ OWNER = "jbnayahu" # Change to your org - don't forget to create a results and r
10
  # ----------------------------------
11
 
12
  REPO_ID = f"{OWNER}/bluebench"
13
- QUEUE_REPO = f"{OWNER}/bluebench-requests"
14
- RESULTS_REPO = f"{OWNER}/bluebench-results"
15
 
16
  # If you setup a cache later, just change HF_HOME
17
  CACHE_PATH=os.getenv("HF_HOME", ".")
18
 
 
 
 
19
  # Local caches
20
- EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
21
- EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
22
- EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
23
- EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
24
 
25
  API = HfApi(token=TOKEN)
 
10
  # ----------------------------------
11
 
12
  REPO_ID = f"{OWNER}/bluebench"
13
+ # QUEUE_REPO = f"{OWNER}/bluebench-requests"
 
14
 
15
  # If you setup a cache later, just change HF_HOME
16
  CACHE_PATH=os.getenv("HF_HOME", ".")
17
 
18
+ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "results/bluebench")
19
+
20
+
21
  # Local caches
22
+ # EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
23
+ # EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
24
+ # EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
25
+ # EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
26
 
27
  API = HfApi(token=TOKEN)
src/leaderboard/read_evals.py CHANGED
@@ -38,16 +38,16 @@ class EvalResult:
38
  with open(json_filepath) as fp:
39
  data = json.load(fp)
40
 
41
- env_info = data.get("environment_info")
42
-
43
- config = data.get("config")
44
 
 
 
45
  # Precision
46
  # precision = Precision.from_str(config.get("model_dtype"))
47
 
48
  # Get model and org
49
- org_and_model = config.get("model_name", config.get("model_args", None))
50
- org_and_model = org_and_model.split("/", 1)
51
 
52
  # if len(org_and_model) == 1:
53
  # org = None
@@ -57,7 +57,7 @@ class EvalResult:
57
  # org = org_and_model[0]
58
  # model = org_and_model[1]
59
  # result_key = f"{org}_{model}_{precision.value.name}"
60
- full_model = "/".join(org_and_model)
61
 
62
  # still_on_hub, _, model_config = is_model_on_hub(
63
  # full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
@@ -134,26 +134,26 @@ class EvalResult:
134
  return data_dict
135
 
136
 
137
- def get_request_file_for_model(requests_path, model_name, precision):
138
- """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
139
- request_files = os.path.join(
140
- requests_path,
141
- f"{model_name}_eval_request_*.json",
142
- )
143
- request_files = glob.glob(request_files)
144
-
145
- # Select correct request file (precision)
146
- request_file = ""
147
- request_files = sorted(request_files, reverse=True)
148
- for tmp_request_file in request_files:
149
- with open(tmp_request_file, "r") as f:
150
- req_content = json.load(f)
151
- if (
152
- req_content["status"] in ["FINISHED"]
153
- and req_content["precision"] == precision.split(".")[-1]
154
- ):
155
- request_file = tmp_request_file
156
- return request_file
157
 
158
 
159
  def get_raw_eval_results(results_path: str) -> list[EvalResult]:
 
38
  with open(json_filepath) as fp:
39
  data = json.load(fp)
40
 
41
+ env_info = data.get("environment_info").get("parsed_arguments")
 
 
42
 
43
+ full_model = env_info.get("model")
44
+
45
  # Precision
46
  # precision = Precision.from_str(config.get("model_dtype"))
47
 
48
  # Get model and org
49
+ # org_and_model = config.get("model_name", config.get("model_args", None))
50
+ # org_and_model = org_and_model.split("/", 1)
51
 
52
  # if len(org_and_model) == 1:
53
  # org = None
 
57
  # org = org_and_model[0]
58
  # model = org_and_model[1]
59
  # result_key = f"{org}_{model}_{precision.value.name}"
60
+ # full_model = "/".join(org_and_model)
61
 
62
  # still_on_hub, _, model_config = is_model_on_hub(
63
  # full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
 
134
  return data_dict
135
 
136
 
137
+ # def get_request_file_for_model(requests_path, model_name, precision):
138
+ # """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
139
+ # request_files = os.path.join(
140
+ # requests_path,
141
+ # f"{model_name}_eval_request_*.json",
142
+ # )
143
+ # request_files = glob.glob(request_files)
144
+
145
+ # # Select correct request file (precision)
146
+ # request_file = ""
147
+ # request_files = sorted(request_files, reverse=True)
148
+ # for tmp_request_file in request_files:
149
+ # with open(tmp_request_file, "r") as f:
150
+ # req_content = json.load(f)
151
+ # if (
152
+ # req_content["status"] in ["FINISHED"]
153
+ # and req_content["precision"] == precision.split(".")[-1]
154
+ # ):
155
+ # request_file = tmp_request_file
156
+ # return request_file
157
 
158
 
159
  def get_raw_eval_results(results_path: str) -> list[EvalResult]:
src/submission/check_validity.py DELETED
@@ -1,99 +0,0 @@
1
- import json
2
- import os
3
- import re
4
- from collections import defaultdict
5
- from datetime import datetime, timedelta, timezone
6
-
7
- import huggingface_hub
8
- from huggingface_hub import ModelCard
9
- from huggingface_hub.hf_api import ModelInfo
10
- from transformers import AutoConfig
11
- from transformers.models.auto.tokenization_auto import AutoTokenizer
12
-
13
- def check_model_card(repo_id: str) -> tuple[bool, str]:
14
- """Checks if the model card and license exist and have been filled"""
15
- try:
16
- card = ModelCard.load(repo_id)
17
- except huggingface_hub.utils.EntryNotFoundError:
18
- return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
19
-
20
- # Enforce license metadata
21
- if card.data.license is None:
22
- if not ("license_name" in card.data and "license_link" in card.data):
23
- return False, (
24
- "License not found. Please add a license to your model card using the `license` metadata or a"
25
- " `license_name`/`license_link` pair."
26
- )
27
-
28
- # Enforce card content
29
- if len(card.text) < 200:
30
- return False, "Please add a description to your model card, it is too short."
31
-
32
- return True, ""
33
-
34
- def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
35
- """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
36
- try:
37
- config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
38
- if test_tokenizer:
39
- try:
40
- tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
41
- except ValueError as e:
42
- return (
43
- False,
44
- f"uses a tokenizer which is not in a transformers release: {e}",
45
- None
46
- )
47
- except Exception as e:
48
- return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
49
- return True, None, config
50
-
51
- except ValueError:
52
- return (
53
- False,
54
- "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
55
- None
56
- )
57
-
58
- except Exception as e:
59
- return False, "was not found on hub!", None
60
-
61
-
62
- def get_model_size(model_info: ModelInfo, precision: str):
63
- """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
64
- try:
65
- model_size = round(model_info.safetensors["total"] / 1e9, 3)
66
- except (AttributeError, TypeError):
67
- return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
68
-
69
- size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
70
- model_size = size_factor * model_size
71
- return model_size
72
-
73
- def get_model_arch(model_info: ModelInfo):
74
- """Gets the model architecture from the configuration"""
75
- return model_info.config.get("architectures", "Unknown")
76
-
77
- def already_submitted_models(requested_models_dir: str) -> set[str]:
78
- """Gather a list of already submitted models to avoid duplicates"""
79
- depth = 1
80
- file_names = []
81
- users_to_submission_dates = defaultdict(list)
82
-
83
- for root, _, files in os.walk(requested_models_dir):
84
- current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
85
- if current_depth == depth:
86
- for file in files:
87
- if not file.endswith(".json"):
88
- continue
89
- with open(os.path.join(root, file), "r") as f:
90
- info = json.load(f)
91
- file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
92
-
93
- # Select organisation
94
- if info["model"].count("/") == 0 or "submitted_time" not in info:
95
- continue
96
- organisation, _ = info["model"].split("/")
97
- users_to_submission_dates[organisation].append(info["submitted_time"])
98
-
99
- return set(file_names), users_to_submission_dates
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/submit.py DELETED
@@ -1,119 +0,0 @@
1
- import json
2
- import os
3
- from datetime import datetime, timezone
4
-
5
- from src.display.formatting import styled_error, styled_message, styled_warning
6
- from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
7
- from src.submission.check_validity import (
8
- already_submitted_models,
9
- check_model_card,
10
- get_model_size,
11
- is_model_on_hub,
12
- )
13
-
14
- REQUESTED_MODELS = None
15
- USERS_TO_SUBMISSION_DATES = None
16
-
17
- def add_new_eval(
18
- model: str,
19
- base_model: str,
20
- revision: str,
21
- precision: str,
22
- weight_type: str,
23
- model_type: str,
24
- ):
25
- global REQUESTED_MODELS
26
- global USERS_TO_SUBMISSION_DATES
27
- if not REQUESTED_MODELS:
28
- REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
29
-
30
- user_name = ""
31
- model_path = model
32
- if "/" in model:
33
- user_name = model.split("/")[0]
34
- model_path = model.split("/")[1]
35
-
36
- precision = precision.split(" ")[0]
37
- current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
38
-
39
- if model_type is None or model_type == "":
40
- return styled_error("Please select a model type.")
41
-
42
- # Does the model actually exist?
43
- if revision == "":
44
- revision = "main"
45
-
46
- # Is the model on the hub?
47
- if weight_type in ["Delta", "Adapter"]:
48
- base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
49
- if not base_model_on_hub:
50
- return styled_error(f'Base model "{base_model}" {error}')
51
-
52
- if not weight_type == "Adapter":
53
- model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
54
- if not model_on_hub:
55
- return styled_error(f'Model "{model}" {error}')
56
-
57
- # Is the model info correctly filled?
58
- try:
59
- model_info = API.model_info(repo_id=model, revision=revision)
60
- except Exception:
61
- return styled_error("Could not get your model information. Please fill it up properly.")
62
-
63
- model_size = get_model_size(model_info=model_info, precision=precision)
64
-
65
- # Were the model card and license filled?
66
- try:
67
- license = model_info.cardData["license"]
68
- except Exception:
69
- return styled_error("Please select a license for your model")
70
-
71
- modelcard_OK, error_msg = check_model_card(model)
72
- if not modelcard_OK:
73
- return styled_error(error_msg)
74
-
75
- # Seems good, creating the eval
76
- print("Adding new eval")
77
-
78
- eval_entry = {
79
- "model": model,
80
- "base_model": base_model,
81
- "revision": revision,
82
- "precision": precision,
83
- "weight_type": weight_type,
84
- "status": "PENDING",
85
- "submitted_time": current_time,
86
- "model_type": model_type,
87
- "likes": model_info.likes,
88
- "params": model_size,
89
- "license": license,
90
- "private": False,
91
- }
92
-
93
- # Check for duplicate submission
94
- if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
95
- return styled_warning("This model has been already submitted.")
96
-
97
- print("Creating eval file")
98
- OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
99
- os.makedirs(OUT_DIR, exist_ok=True)
100
- out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
101
-
102
- with open(out_path, "w") as f:
103
- f.write(json.dumps(eval_entry))
104
-
105
- print("Uploading eval file")
106
- API.upload_file(
107
- path_or_fileobj=out_path,
108
- path_in_repo=out_path.split("eval-queue/")[1],
109
- repo_id=QUEUE_REPO,
110
- repo_type="dataset",
111
- commit_message=f"Add {model} to eval queue",
112
- )
113
-
114
- # Remove the local file
115
- os.remove(out_path)
116
-
117
- return styled_message(
118
- "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
119
- )