controlability

QUALITY_file_pairs.csv

@@ -0,0 +1,100 @@
+file_name,vista_id,gem_id
+OpenDV_000002_LONG,1032061116,1032064937
+OpenDV_000005_LONG,1032061146,1032064968
+OpenDV_000007_LONG,1032061181,1032064996
+OpenDV_000011_LONG,1032061209,1032065029
+OpenDV_000039_LONG,1032061232,1032065073
+OpenDV_000047_LONG,1032061251,1032065122
+OpenDV_000054_LONG,1032061275,1032065174
+OpenDV_000083_LONG,1032061299,1032065217
+OpenDV_000087_LONG,1032061321,1032065259
+OpenDV_000099,1032061338,1032065317
+OpenDV_000117_LONG,1032061368,1032065379
+OpenDV_000120_LONG,1032061395,1032065446
+OpenDV_000123_LONG,1032061414,1032065510
+OpenDV_000127,1032061446,1032065569
+OpenDV_000139_LONG,1032061479,1032065608
+OpenDV_000146_LONG,1032061502,1032065647
+OpenDV_000150_LONG,1032061521,1032065698
+OpenDV_000162_LONG,1032061553,1032065759
+OpenDV_000173,1032061575,1032065799
+OpenDV_000178_LONG,1032061598,1032065842
+OpenDV_000184_LONG,1032061622,1032065912
+OpenDV_000192_LONG,1032061648,1032065968
+OpenDV_000200_LONG,1032061668,1032066005
+OpenDV_000217_LONG,1032061701,1032066052
+OpenDV_000219_LONG,1032061720,1032066105
+OpenDV_000220,1032061750,1032066125
+OpenDV_000234_LONG,1032061768,1032066153
+OpenDV_000239_LONG,1032061787,1032066192
+OpenDV_000246_LONG,1032061806,1032066218
+OpenDV_000253_LONG,1032061828,1032066244
+OpenDV_000258_LONG,1032061853,1032066264
+OpenDV_000276_LONG,1032061879,1032066296
+OpenDV_000279_LONG,1032061893,1032066317
+OpenDV_000290_LONG,1032061918,1032066350
+OpenDV_000291_LONG,1032061950,1032066369
+OpenDV_000294_LONG,1032061987,1032066392
+OpenDV_000295_LONG,1032062011,1032066420
+OpenDV_000297_LONG,1032062045,1032066458
+OpenDV_000299_LONG,1032062076,1032066493
+OpenDV_000304_LONG,1032062116,1032066529
+OpenDV_000322_LONG,1032062135,1032066573
+OpenDV_000323_LONG,1032062162,1032066640
+OpenDV_000345_LONG,1032062189,1032066699
+OpenDV_000350_LONG,1032062216,1032066742
+OpenDV_000369_LONG,1032062240,1032066787
+OpenDV_000370_LONG,1032062265,1032066855
+OpenDV_000382_LONG,1032062293,1032066897
+OpenDV_000394_LONG,1032062325,1032066970
+OpenDV_000395_LONG,1032062353,1032067033
+OpenDV_000396_LONG,1032062377,1032067079
+OpenDV_000399_LONG,1032062406,1032067140
+OpenDV_000411_LONG,1032062428,1032067205
+OpenDV_000414_LONG,1032062467,1032067271
+OpenDV_000628,1032062493,1032067319
+OpenDV_000644,1032062521,1032067348
+OpenDV_000693,1032062550,1032067386
+OpenDV_000748,1032062569,1032067449
+OpenDV_000782,1032062597,1032067481
+OpenDV_000868,1032062629,1032067531
+OpenDV_001015,1032062659,1032067559
+OpenDV_001151,1032062690,1032067584
+OpenDV_001237,1032062715,1032067601
+OpenDV_001268,1032062745,1032067628
+OpenDV_001369,1032062773,1032067652
+OpenDV_001482,1032062804,1032067680
+OpenDV_001540,1032062832,1032067724
+OpenDV_001661,1032062859,1032067759
+OpenDV_001714,1032062877,1032067812
+OpenDV_001755,1032062888,1032067860
+OpenDV_001907,1032062912,1032067921
+OpenDV_002142,1032062940,1032067955
+OpenDV_002240,1032062961,1032068002
+OpenDV_002251,1032062994,1032068029
+OpenDV_002276,1032063019,1032068074
+OpenDV_002319,1032063049,1032068121
+OpenDV_002574,1032063078,1032068169
+OpenDV_002596,1032063112,1032068230
+OpenDV_002706,1032063145,1032068268
+OpenDV_002847,1032063173,1032068325
+OpenDV_003243,1032063195,1032068347
+OpenDV_003414,1032063220,1032068369
+OpenDV_003423,1032063244,1032068411
+OpenDV_003521,1032063262,1032068445
+OpenDV_003563,1032063284,1032068478
+OpenDV_003681,1032063304,1032068507
+OpenDV_003758,1032063338,1032068551
+OpenDV_003783,1032063366,1032068577
+OpenDV_003792,1032063380,1032068608
+OpenDV_003919,1032063400,1032068645
+OpenDV_003959,1032063427,1032068701
+OpenDV_004275,1032063444,1032068744
+OpenDV_004393,1032063463,1032068780
+OpenDV_004418,1032063483,1032068822
+OpenDV_004570,1032063512,1032068861
+OpenDV_004623,1032063533,1032068910
+OpenDV_004717,1032063551,1032068950
+OpenDV_004848,1032063577,1032068998
+OpenDV_004881,1032063609,1032069043
+OpenDV_004934,1032063629,1032069091

QUALITY_results.csv

@@ -0,0 +1,7 @@
+category,gem,vista,no_preference
+realistic_dynamics_long,75,23,18
+realistic_dynamics_short,35,26,76
+visual_quality_long,79,29,8
+visual_quality_short,38,51,48
+temporal_consistency_long,81,27,8
+temporal_consistency_short,41,36,60

app.py

@@ -79,9 +79,8 @@ def save_vote(video_name, email, vista_id, gem_id, video1_id, video2_id, respons
         'gem_id': gem_id,
         'video1_id': video1_id,
         'video2_id': video2_id,
-        'visual_quality': responses['visual_quality'],
-        'temporal_consistency': responses['temporal_consistency'],
-        'realistic_dynamics': responses['realistic_dynamics'],
+        'q1': responses['q1'],
+        'q2': responses['q2'],
     }
     db.collection('votes').add(vote_data)
     # Update user's vote history
@@ -131,7 +130,6 @@ def update_interface(responses, state):
             output_message,            # output
             gr.update(value=None),     # q1
             gr.update(value=None),     # q2
-            gr.update(value=None),     # q3
             gr.update(interactive=False),  # next_btn
             state
         )
@@ -149,13 +147,12 @@ def update_interface(responses, state):
             "",                        # output
             gr.update(value=None),     # q1
             gr.update(value=None),     # q2
-            gr.update(value=None),     # q3
             gr.update(interactive=False),  # next_btn
             state
         )
 
-def check_all_answers(visual_quality, temporal_consistency, realistic_dynamics):
-    if visual_quality and temporal_consistency and realistic_dynamics:
+def check_all_answers(q1, q2):
+    if q1 and q2:
         return gr.update(interactive=True)
     else:
         return gr.update(interactive=False)
@@ -238,11 +235,11 @@ with gr.Blocks() as demo:
 
     gr.Markdown(
         """
-        You'll be seeing pairs of videos and asked to compare them based on visual quality, temporal consistency, and realistic dynamics.
+        You'll be seeing pairs of videos and will be asked to compare them.
+        The videos were generated by AI to move a car from one point to another (green boxes in the video).
 
-        **If you can complete at least 20-30 comparisons, that would be great! There are 99 pairs of videos in total.**
-
-        **Please try to reduce your “No Preference” choices as much as you can.**
+        **There are 10 videos in total.**
+        **If both videos are the same, select "No preference".**
         """
     )
 
@@ -266,17 +263,12 @@ with gr.Blocks() as demo:
         gr.Markdown("## Please answer the following questions:")
         q1 = gr.Radio(
             choices=["Video 1", "Video 2", "No preference"],
-            label="1. Which video has higher visual quality?",
+            label="1. Which video shows more natural or consistent motion?",
             type="value"
         )
         q2 = gr.Radio(
             choices=["Video 1", "Video 2", "No preference"],
-            label="2. Which video has more consistent motion over time?",
-            type="value"
-        )
-        q3 = gr.Radio(
-            choices=["Video 1", "Video 2", "No preference"],
-            label="3. Which video has more active agents?",
+            label="2. Which video follows the controls better?",
             type="value"
         )
 
@@ -287,17 +279,16 @@ with gr.Blocks() as demo:
 
     output = gr.Markdown()
 
-    def on_next(visual_quality, temporal_consistency, realistic_dynamics, state):
+    def on_next(q1, q2, state):
         responses = {
-            'visual_quality': visual_quality,
-            'temporal_consistency': temporal_consistency,
-            'realistic_dynamics': realistic_dynamics
+            'q1': q1,
+            'q2': q2,
         }
         return update_interface(responses, state)
 
     next_btn.click(
         fn=on_next,
-        inputs=[q1, q2, q3, state],
+        inputs=[q1, q2, state],
         outputs=[
             video1_column,        # Update video_column
             video2_column,        # Update video_column
@@ -306,28 +297,23 @@ with gr.Blocks() as demo:
             question_column,  # Update question_column
             button_row,       # Update button_row
             output,           # Update output message
-            q1, q2, q3,       # Reset questions
+            q1, q2,       # Reset questions
             next_btn,         # Update next_btn
             state
         ]
     )
 
-    def on_change(visual_quality, temporal_consistency, realistic_dynamics):
-        return check_all_answers(visual_quality, temporal_consistency, realistic_dynamics)
+    def on_change(q1, q2):
+        return check_all_answers(q1, q2)
 
     q1.change(
         fn=on_change,
-        inputs=[q1, q2, q3],
+        inputs=[q1, q2],
         outputs=next_btn
     )
     q2.change(
         fn=on_change,
-        inputs=[q1, q2, q3],
-        outputs=next_btn
-    )
-    q3.change(
-        fn=on_change,
-        inputs=[q1, q2, q3],
+        inputs=[q1, q2],
         outputs=next_btn
     )
 

file_pairs.csv

@@ -1,100 +1,11 @@
 file_name,vista_id,gem_id
-OpenDV_000002_LONG,1032061116,1032064937
-OpenDV_000005_LONG,1032061146,1032064968
-OpenDV_000007_LONG,1032061181,1032064996
-OpenDV_000011_LONG,1032061209,1032065029
-OpenDV_000039_LONG,1032061232,1032065073
-OpenDV_000047_LONG,1032061251,1032065122
-OpenDV_000054_LONG,1032061275,1032065174
-OpenDV_000083_LONG,1032061299,1032065217
-OpenDV_000087_LONG,1032061321,1032065259
-OpenDV_000099,1032061338,1032065317
-OpenDV_000117_LONG,1032061368,1032065379
-OpenDV_000120_LONG,1032061395,1032065446
-OpenDV_000123_LONG,1032061414,1032065510
-OpenDV_000127,1032061446,1032065569
-OpenDV_000139_LONG,1032061479,1032065608
-OpenDV_000146_LONG,1032061502,1032065647
-OpenDV_000150_LONG,1032061521,1032065698
-OpenDV_000162_LONG,1032061553,1032065759
-OpenDV_000173,1032061575,1032065799
-OpenDV_000178_LONG,1032061598,1032065842
-OpenDV_000184_LONG,1032061622,1032065912
-OpenDV_000192_LONG,1032061648,1032065968
-OpenDV_000200_LONG,1032061668,1032066005
-OpenDV_000217_LONG,1032061701,1032066052
-OpenDV_000219_LONG,1032061720,1032066105
-OpenDV_000220,1032061750,1032066125
-OpenDV_000234_LONG,1032061768,1032066153
-OpenDV_000239_LONG,1032061787,1032066192
-OpenDV_000246_LONG,1032061806,1032066218
-OpenDV_000253_LONG,1032061828,1032066244
-OpenDV_000258_LONG,1032061853,1032066264
-OpenDV_000276_LONG,1032061879,1032066296
-OpenDV_000279_LONG,1032061893,1032066317
-OpenDV_000290_LONG,1032061918,1032066350
-OpenDV_000291_LONG,1032061950,1032066369
-OpenDV_000294_LONG,1032061987,1032066392
-OpenDV_000295_LONG,1032062011,1032066420
-OpenDV_000297_LONG,1032062045,1032066458
-OpenDV_000299_LONG,1032062076,1032066493
-OpenDV_000304_LONG,1032062116,1032066529
-OpenDV_000322_LONG,1032062135,1032066573
-OpenDV_000323_LONG,1032062162,1032066640
-OpenDV_000345_LONG,1032062189,1032066699
-OpenDV_000350_LONG,1032062216,1032066742
-OpenDV_000369_LONG,1032062240,1032066787
-OpenDV_000370_LONG,1032062265,1032066855
-OpenDV_000382_LONG,1032062293,1032066897
-OpenDV_000394_LONG,1032062325,1032066970
-OpenDV_000395_LONG,1032062353,1032067033
-OpenDV_000396_LONG,1032062377,1032067079
-OpenDV_000399_LONG,1032062406,1032067140
-OpenDV_000411_LONG,1032062428,1032067205
-OpenDV_000414_LONG,1032062467,1032067271
-OpenDV_000628,1032062493,1032067319
-OpenDV_000644,1032062521,1032067348
-OpenDV_000693,1032062550,1032067386
-OpenDV_000748,1032062569,1032067449
-OpenDV_000782,1032062597,1032067481
-OpenDV_000868,1032062629,1032067531
-OpenDV_001015,1032062659,1032067559
-OpenDV_001151,1032062690,1032067584
-OpenDV_001237,1032062715,1032067601
-OpenDV_001268,1032062745,1032067628
-OpenDV_001369,1032062773,1032067652
-OpenDV_001482,1032062804,1032067680
-OpenDV_001540,1032062832,1032067724
-OpenDV_001661,1032062859,1032067759
-OpenDV_001714,1032062877,1032067812
-OpenDV_001755,1032062888,1032067860
-OpenDV_001907,1032062912,1032067921
-OpenDV_002142,1032062940,1032067955
-OpenDV_002240,1032062961,1032068002
-OpenDV_002251,1032062994,1032068029
-OpenDV_002276,1032063019,1032068074
-OpenDV_002319,1032063049,1032068121
-OpenDV_002574,1032063078,1032068169
-OpenDV_002596,1032063112,1032068230
-OpenDV_002706,1032063145,1032068268
-OpenDV_002847,1032063173,1032068325
-OpenDV_003243,1032063195,1032068347
-OpenDV_003414,1032063220,1032068369
-OpenDV_003423,1032063244,1032068411
-OpenDV_003521,1032063262,1032068445
-OpenDV_003563,1032063284,1032068478
-OpenDV_003681,1032063304,1032068507
-OpenDV_003758,1032063338,1032068551
-OpenDV_003783,1032063366,1032068577
-OpenDV_003792,1032063380,1032068608
-OpenDV_003919,1032063400,1032068645
-OpenDV_003959,1032063427,1032068701
-OpenDV_004275,1032063444,1032068744
-OpenDV_004393,1032063463,1032068780
-OpenDV_004418,1032063483,1032068822
-OpenDV_004570,1032063512,1032068861
-OpenDV_004623,1032063533,1032068910
-OpenDV_004717,1032063551,1032068950
-OpenDV_004848,1032063577,1032068998
-OpenDV_004881,1032063609,1032069043
-OpenDV_004934,1032063629,1032069091
+1,1051245161,1051244884
+11,1051245233,1051244979
+13,1051245253,1051245004
+17,1051245269,1051245040
+2,1051245188,1051244923
+21,1051245289,1051245077
+22,1051245311,1051245091
+25,1051245334,1051245112
+27,1051245352,1051245139
+3,1051245204,1051244950

get_results.py

@@ -41,17 +41,14 @@ vista_vote_count = {}
 no_preference_count = {}
 
 qids = [
-    'realistic_dynamics',
-    'visual_quality',
-    'temporal_consistency',
+    'q1',
+    'q2',
 ]
 
 for vote in tqdm(votes):
     vote_dict = vote.to_dict()
-    is_long = 'LONG' in vote_dict['video_name']
-
     for qid in qids:
-        category = qid + ('_long' if is_long else '_short')
+        category = qid
         if category not in gem_vote_count:
             gem_vote_count[category] = 0
             vista_vote_count[category] = 0
@@ -74,14 +71,13 @@ for vote in tqdm(votes):
 
 data = []
 for qid in qids:
-    for is_long in [True, False]:
-        category = qid + ('_long' if is_long else '_short')
-        data.append({
-            'category': category,
-            'gem': gem_vote_count[category],
-            'vista': vista_vote_count[category],
-            'no_preference': no_preference_count[category]
-        })
+    category = qid
+    data.append({
+        'category': category,
+        'gem': gem_vote_count[category],
+        'vista': vista_vote_count[category],
+        'no_preference': no_preference_count[category]
+    })
 
 df = pd.DataFrame(data)
 df.to_csv('results.csv', index=False)

get_vimeo.py

@@ -10,14 +10,11 @@ v = vimeo.VimeoClient(
     secret=os.environ.get("VIMEO_SECRET")
 )
 
-user_id = 'pedrombr'  # Replace with the actual user ID
+user_id = 'pedroooombr'  # Replace with the actual user ID
 endpoint = f'/users/{user_id}/videos'
 # Make the GET request
-response1 = v.get(f'{endpoint}?per_page=100').json()
-response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
-
+response1 = v.get(f'{endpoint}?per_page=50').json()
 response = response1
-response['data'].extend(response2['data'])
 
 gem_videos = [
     ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('GEM')
@@ -27,7 +24,8 @@ vista_videos = [
     ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('Vista')
 ]
 
-assert len(gem_videos) == len(vista_videos) and len(gem_videos) == 99
+print(len(gem_videos), len(vista_videos))
+assert len(gem_videos) == len(vista_videos)
 
 # Match files by title and create a csv
 data = []

rename_videos_vimeo.py

@@ -11,14 +11,16 @@ v = vimeo.VimeoClient(
     secret=os.environ.get("VIMEO_SECRET")
 )
 
-user_id = 'pedrombr'  # Replace with the actual user ID
+user_id = 'pedroooombr'  # Replace with the actual user ID
 endpoint = f'/users/{user_id}/videos'
 # Make the GET request
-response1 = v.get(f'{endpoint}?per_page=100').json()
-response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
+print("ENDPOINT: ", endpoint)
+response1 = v.get(f'{endpoint}?per_page=50')
+response1 = response1.json()
+# response2 = v.get(f'{endpoint}?per_page=100&page=2').json()
 
 response = response1
-response['data'].extend(response2['data'])
+# response['data'].extend(response2['data'])
 
 def rename_video(video_id):
     try:
@@ -65,6 +67,8 @@ gem_videos = [
 vista_videos = [
     ("_".join(v['name'].split("_")[1:]), v['uri'].split("/")[-1]) for v in response['data'] if v['name'].startswith('Vista')
 ]
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              
+print(len(gem_videos), len(vista_videos))
 
 # assert len(gem_videos) == len(vista_videos) and len(gem_videos) == 99
 

results.csv

@@ -1,7 +1,3 @@
 category,gem,vista,no_preference
-realistic_dynamics_long,75,23,18
-realistic_dynamics_short,35,26,76
-visual_quality_long,79,29,8
-visual_quality_short,38,51,48
-temporal_consistency_long,81,27,8
-temporal_consistency_short,41,36,60
+q1,10,0,0
+q2,8,0,2