final commit?

.gitignore

@@ -1,3 +1,2 @@
 *.json
 .env
-*vimeo*.py

get_results.py

@@ -0,0 +1,87 @@
+import gradio as gr
+import random
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+import uuid
+import json
+import os
+from dotenv import load_dotenv
+import re
+import pandas as pd
+from tqdm import tqdm
+
+load_dotenv()
+
+video_pairs = pd.read_csv('file_pairs.csv')[['file_name', 'vista_id', 'gem_id']].values.tolist()
+
+my_credentials = {
+    "type": "service_account",
+    "project_id": "human-eval-c4f83",
+    "private_key_id": os.environ.get("PRIVATE_KEY_ID"),
+    "private_key": os.environ.get("PRIVATE_KEY").replace(r'\n', '\n'),
+    "client_email": os.environ.get("CLIENT_EMAIL"),
+    "client_id": os.environ.get("CLIENT_ID"),
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": os.environ.get("AUTH_PROVIDER_X509_CERT_URL"),
+    "client_x509_cert_url": os.environ.get("CLIENT_X509_CERT_URL")
+}
+
+if not firebase_admin._apps:
+    cred = credentials.Certificate(my_credentials)
+    firebase_admin.initialize_app(cred)
+
+db = firestore.client()
+# get all votes
+votes = db.collection('votes').get()
+
+gem_vote_count = {}
+vista_vote_count = {}
+no_preference_count = {}
+
+qids = [
+    'realistic_dynamics',
+    'visual_quality',
+    'temporal_consistency',
+]
+
+for vote in tqdm(votes):
+    vote_dict = vote.to_dict()
+    is_long = 'LONG' in vote_dict['video_name']
+
+    for qid in qids:
+        category = qid + ('_long' if is_long else '_short')
+        if category not in gem_vote_count:
+            gem_vote_count[category] = 0
+            vista_vote_count[category] = 0
+            no_preference_count[category] = 0
+        
+        if vote_dict[qid] == 'Video 1' and vote_dict['video1_id'] == vote_dict['gem_id']:
+            gem_vote_count[category] += 1
+        elif vote_dict[qid] == 'Video 1' and vote_dict['video1_id'] == vote_dict['vista_id']:
+            vista_vote_count[category] += 1
+        elif vote_dict[qid] == 'Video 2' and vote_dict['video2_id'] == vote_dict['gem_id']:
+            gem_vote_count[category] += 1
+        elif vote_dict[qid] == 'Video 2' and vote_dict['video2_id'] == vote_dict['vista_id']:
+            vista_vote_count[category] += 1
+        elif vote_dict[qid] == 'No preference':
+            no_preference_count[category] += 1
+        else:
+            raise ValueError('Invalid vote: ' + str(vote_dict))
+
+# Create a table with the results. One row for GEM, one row for Vista, one row for no preference
+
+data = []
+for qid in qids:
+    for is_long in [True, False]:
+        category = qid + ('_long' if is_long else '_short')
+        data.append({
+            'category': category,
+            'gem': gem_vote_count[category],
+            'vista': vista_vote_count[category],
+            'no_preference': no_preference_count[category]
+        })
+
+df = pd.DataFrame(data)
+df.to_csv('results.csv', index=False)

get_vimeo.py

@@ -0,0 +1,43 @@
+import vimeo
+import pandas as pd
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+v = vimeo.VimeoClient(
+    token=os.environ.get("VIMEO_TOKEN"),
+    key=os.environ.get("VIMEO_KEY"),
+    secret=os.environ.get("VIMEO_SECRET")
+)
+
+user_id = 'pedrombr'  # Replace with the actual user ID
+endpoint = f'/users/{user_id}/videos'
+# Make the GET request
+response1 = v.get(f'{endpoint}?per_page=100').json()
+response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
+
+response = response1
+response['data'].extend(response2['data'])
+
+gem_videos = [
+    ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('GEM')
+]
+
+vista_videos = [
+    ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('Vista')
+]
+
+assert len(gem_videos) == len(vista_videos) and len(gem_videos) == 99
+
+# Match files by title and create a csv
+data = []
+
+for vista_file in vista_videos:
+    for gem_file in gem_videos:
+        if vista_file[0] == gem_file[0]:
+            data.append((vista_file[0], vista_file[1], gem_file[1]))
+
+# sort by file_name
+data.sort(key=lambda x: x[0])
+df = pd.DataFrame(data, columns=['file_name', 'vista_id', 'gem_id'])
+df.to_csv('file_pairs.csv', index=False)

rename_videos_vimeo.py

@@ -0,0 +1,72 @@
+import vimeo
+from tqdm import tqdm
+import time
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+v = vimeo.VimeoClient(
+    token=os.environ.get("VIMEO_TOKEN"),
+    key=os.environ.get("VIMEO_KEY"),
+    secret=os.environ.get("VIMEO_SECRET")
+)
+
+user_id = 'pedrombr'  # Replace with the actual user ID
+endpoint = f'/users/{user_id}/videos'
+# Make the GET request
+response1 = v.get(f'{endpoint}?per_page=100').json()
+response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
+
+response = response1
+response['data'].extend(response2['data'])
+
+def rename_video(video_id):
+    try:
+        # Get the current video details
+        video_response = v.get(f"/videos/{video_id}")
+        
+        if video_response.status_code != 200:
+            print(f"Failed to fetch video details: {video_response.json()}")
+            return
+        
+        video_data = video_response.json()
+        current_name = video_data.get("name", "")
+        
+        # Check if the name starts with 'GEM_'
+        if current_name.startswith("GEM_"):
+            new_name = current_name.replace("GEM_", f"{video_id}_", 1)  # Remove the 'GEM_' prefix
+            # Update the video's name
+            patch_data = {"name": new_name}
+            response = v.patch(f"/videos/{video_id}", data=patch_data)
+            
+            if response.status_code == 200:
+                print(f"Video renamed successfully to: {new_name}")
+            else:
+                print(f"Failed to rename video: {response.json()}")
+        elif current_name.startswith("Vista_"):
+            new_name = current_name.replace("Vista_", f"{video_id}_", 1)
+            patch_data = {"name": new_name}
+            response = v.patch(f"/videos/{video_id}", data=patch_data)
+            if response.status_code == 200:
+                print(f"Video renamed successfully to: {new_name}")
+            else:
+                print(f"Failed to rename video: {response.json()}")
+        else:
+            print("The video name does not start with 'GEM_' or 'Vista_'. No changes made.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        time.sleep(60)  # Wait for 60 seconds before retrying
+        rename_video(video_id)  # Retry the operation
+
+gem_videos = [
+    ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('GEM')
+]
+
+vista_videos = [
+    ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('Vista')
+]
+
+# assert len(gem_videos) == len(vista_videos) and len(gem_videos) == 99
+
+for video in gem_videos + vista_videos:
+    rename_video(video[1])  # Pass the video ID to the rename function

results.csv

@@ -0,0 +1,7 @@
+category,gem,vista,no_preference
+realistic_dynamics_long,75,23,18
+realistic_dynamics_short,35,26,76
+visual_quality_long,79,29,8
+visual_quality_short,38,51,48
+temporal_consistency_long,81,27,8
+temporal_consistency_short,41,36,60