Spaces:
Sleeping
Sleeping
import os | |
import json | |
import argparse | |
def get_args(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument("--annotation-file", type=str) | |
parser.add_argument("--result-file", type=str) | |
parser.add_argument("--result-upload-file", type=str) | |
return parser.parse_args() | |
def eval_single(result_file, eval_only_type=None): | |
results = {} | |
for line in open(result_file): | |
row = json.loads(line) | |
results[row['question_id']] = row | |
type_counts = {} | |
correct_counts = {} | |
for question_data in data['questions']: | |
if eval_only_type is not None and question_data['data_type'] != eval_only_type: continue | |
data_type = question_data['question_type_id'] | |
type_counts[data_type] = type_counts.get(data_type, 0) + 1 | |
try: | |
question_id = int(question_data['question_id']) | |
except: | |
question_id = question_data['question_id'] | |
if question_id not in results: | |
correct_counts[data_type] = correct_counts.get(data_type, 0) | |
continue | |
row = results[question_id] | |
if row['text'] == question_data['answer']: | |
correct_counts[data_type] = correct_counts.get(data_type, 0) + 1 | |
total_count = 0 | |
total_correct = 0 | |
for data_type in sorted(type_counts.keys()): | |
accuracy = correct_counts[data_type] / type_counts[data_type] * 100 | |
if eval_only_type is None: | |
print(f"{ques_type_id_to_name[data_type]}: {accuracy:.2f}%") | |
total_count += type_counts[data_type] | |
total_correct += correct_counts[data_type] | |
total_accuracy = total_correct / total_count * 100 | |
if eval_only_type is None: | |
print(f"Total accuracy: {total_accuracy:.2f}%") | |
else: | |
print(f"{eval_only_type} accuracy: {total_accuracy:.2f}%") | |
return results | |
if __name__ == "__main__": | |
args = get_args() | |
data = json.load(open(args.annotation_file)) | |
ques_type_id_to_name = {id:n for n,id in data['question_type'].items()} | |
results = eval_single(args.result_file) | |
eval_single(args.result_file, eval_only_type='image') | |
eval_single(args.result_file, eval_only_type='video') | |
with open(args.result_upload_file, 'w') as fp: | |
for question in data['questions']: | |
qid = question['question_id'] | |
if qid in results: | |
result = results[qid] | |
else: | |
result = results[int(qid)] | |
fp.write(json.dumps({ | |
'question_id': qid, | |
'prediction': result['text'] | |
}) + '\n') | |