Spaces:
Sleeping
Sleeping
File size: 2,571 Bytes
b27b0a2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import os
import json
import argparse
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("--annotation-file", type=str)
parser.add_argument("--result-file", type=str)
parser.add_argument("--result-upload-file", type=str)
return parser.parse_args()
def eval_single(result_file, eval_only_type=None):
results = {}
for line in open(result_file):
row = json.loads(line)
results[row['question_id']] = row
type_counts = {}
correct_counts = {}
for question_data in data['questions']:
if eval_only_type is not None and question_data['data_type'] != eval_only_type: continue
data_type = question_data['question_type_id']
type_counts[data_type] = type_counts.get(data_type, 0) + 1
try:
question_id = int(question_data['question_id'])
except:
question_id = question_data['question_id']
if question_id not in results:
correct_counts[data_type] = correct_counts.get(data_type, 0)
continue
row = results[question_id]
if row['text'] == question_data['answer']:
correct_counts[data_type] = correct_counts.get(data_type, 0) + 1
total_count = 0
total_correct = 0
for data_type in sorted(type_counts.keys()):
accuracy = correct_counts[data_type] / type_counts[data_type] * 100
if eval_only_type is None:
print(f"{ques_type_id_to_name[data_type]}: {accuracy:.2f}%")
total_count += type_counts[data_type]
total_correct += correct_counts[data_type]
total_accuracy = total_correct / total_count * 100
if eval_only_type is None:
print(f"Total accuracy: {total_accuracy:.2f}%")
else:
print(f"{eval_only_type} accuracy: {total_accuracy:.2f}%")
return results
if __name__ == "__main__":
args = get_args()
data = json.load(open(args.annotation_file))
ques_type_id_to_name = {id:n for n,id in data['question_type'].items()}
results = eval_single(args.result_file)
eval_single(args.result_file, eval_only_type='image')
eval_single(args.result_file, eval_only_type='video')
with open(args.result_upload_file, 'w') as fp:
for question in data['questions']:
qid = question['question_id']
if qid in results:
result = results[qid]
else:
result = results[int(qid)]
fp.write(json.dumps({
'question_id': qid,
'prediction': result['text']
}) + '\n')
|