Spaces:
Running
Running
[email protected]
commited on
Commit
·
dc1fd7f
1
Parent(s):
d583100
read evals
Browse files
src/leaderboard/read_evals.py
CHANGED
@@ -157,7 +157,6 @@ def get_request_file_for_model(requests_path, model_name, precision):
|
|
157 |
def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
|
158 |
"""From the path of the results folder root, extract all needed info for results"""
|
159 |
model_result_filepaths = []
|
160 |
-
print(results_path)
|
161 |
for root, _, files in os.walk(results_path):
|
162 |
# We should only have json files in model results
|
163 |
if len(files) == 0 or any([not f.endswith(".json") for f in files]):
|
@@ -171,15 +170,11 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
|
|
171 |
|
172 |
for file in files:
|
173 |
model_result_filepaths.append(os.path.join(root, file))
|
174 |
-
print(model_result_filepaths)
|
175 |
eval_results = {}
|
176 |
for model_result_filepath in model_result_filepaths:
|
177 |
-
print(f'creation of result {model_result_filepath}')
|
178 |
# Creation of result
|
179 |
eval_result = EvalResult.init_from_json_file(model_result_filepath)
|
180 |
-
print(eval_result)
|
181 |
eval_result.update_with_request_file(requests_path)
|
182 |
-
print(eval_result)
|
183 |
|
184 |
# Store results of same eval together
|
185 |
eval_name = eval_result.eval_name
|
@@ -191,12 +186,10 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
|
|
191 |
results = []
|
192 |
for v in eval_results.values():
|
193 |
try:
|
194 |
-
print(str(v))
|
195 |
v.to_dict() # we test if the dict version is complete
|
196 |
-
print(f'to dict value {str(v)}')
|
197 |
results.append(v)
|
198 |
except KeyError: # not all eval values present
|
199 |
-
print('error key')
|
200 |
continue
|
201 |
|
202 |
return results
|
|
|
157 |
def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
|
158 |
"""From the path of the results folder root, extract all needed info for results"""
|
159 |
model_result_filepaths = []
|
|
|
160 |
for root, _, files in os.walk(results_path):
|
161 |
# We should only have json files in model results
|
162 |
if len(files) == 0 or any([not f.endswith(".json") for f in files]):
|
|
|
170 |
|
171 |
for file in files:
|
172 |
model_result_filepaths.append(os.path.join(root, file))
|
|
|
173 |
eval_results = {}
|
174 |
for model_result_filepath in model_result_filepaths:
|
|
|
175 |
# Creation of result
|
176 |
eval_result = EvalResult.init_from_json_file(model_result_filepath)
|
|
|
177 |
eval_result.update_with_request_file(requests_path)
|
|
|
178 |
|
179 |
# Store results of same eval together
|
180 |
eval_name = eval_result.eval_name
|
|
|
186 |
results = []
|
187 |
for v in eval_results.values():
|
188 |
try:
|
|
|
189 |
v.to_dict() # we test if the dict version is complete
|
|
|
190 |
results.append(v)
|
191 |
except KeyError: # not all eval values present
|
192 |
+
print(f'error key {str(v)}')
|
193 |
continue
|
194 |
|
195 |
return results
|