ignore old votes

get_results.py

@@ -15,6 +15,9 @@ load_dotenv()
 
 video_pairs = pd.read_csv('file_pairs.csv')[['file_name', 'vista_id', 'gem_id']].values.tolist()
 
+vista_ids = [x[1] for x in video_pairs]
+gem_ids = [x[2] for x in video_pairs]
+
 my_credentials = {
     "type": "service_account",
     "project_id": "human-eval-c4f83",
@@ -45,9 +48,15 @@ qids = [
     'q2',
 ]
 
+skips = 0
+
 for vote in tqdm(votes):
     vote_dict = vote.to_dict()
     for qid in qids:
+        if vote_dict['gem_id'] not in gem_ids or vote_dict['vista_id'] not in vista_ids:
+            skips += 1
+            continue
+
         category = qid
         if category not in gem_vote_count:
             gem_vote_count[category] = 0
@@ -55,18 +64,24 @@ for vote in tqdm(votes):
             no_preference_count[category] = 0
         
         if vote_dict[qid] == 'Video 1' and vote_dict['video1_id'] == vote_dict['gem_id']:
+            assert vote_dict['gem_id'] in gem_ids
             gem_vote_count[category] += 1
         elif vote_dict[qid] == 'Video 1' and vote_dict['video1_id'] == vote_dict['vista_id']:
+            assert vote_dict['vista_id'] in vista_ids
             vista_vote_count[category] += 1
         elif vote_dict[qid] == 'Video 2' and vote_dict['video2_id'] == vote_dict['gem_id']:
+            assert vote_dict['gem_id'] in gem_ids
             gem_vote_count[category] += 1
         elif vote_dict[qid] == 'Video 2' and vote_dict['video2_id'] == vote_dict['vista_id']:
+            assert vote_dict['vista_id'] in vista_ids
             vista_vote_count[category] += 1
         elif vote_dict[qid] == 'No preference':
             no_preference_count[category] += 1
         else:
             raise ValueError('Invalid vote: ' + str(vote_dict))
 
+print("Skipped", skips, "votes")
+
 # Create a table with the results. One row for GEM, one row for Vista, one row for no preference
 
 data = []

results.csv

@@ -1,3 +1,3 @@
 category,gem,vista,no_preference
-q1,59,8,11
-q2,52,12,14
+q1,12,24,54
+q2,43,27,20