kiwhansong commited on
Commit
a71f061
·
1 Parent(s): 2351462

fix all hf bugs

Browse files
Files changed (1) hide show
  1. app.py +91 -29
app.py CHANGED
@@ -243,7 +243,7 @@ def navigate_video(
243
  .cpu()
244
  )
245
  gen_video = torch.cat([video, next_video], dim=0)
246
- poses = conditions[0]
247
 
248
  images = (gen_video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
249
 
@@ -255,6 +255,7 @@ def navigate_video(
255
  [(image, f"t={i}") for i, image in enumerate(images)],
256
  )
257
 
 
258
  def undo_navigation(
259
  video: torch.Tensor,
260
  poses: torch.Tensor,
@@ -273,6 +274,7 @@ def undo_navigation(
273
  [(image, f"t={i}") for i, image in enumerate(images)],
274
  )
275
 
 
276
  def _interpolate_conditions(conditions, indices):
277
  """
278
  Interpolate conditions to fill out missing frames
@@ -307,9 +309,7 @@ def _interpolate_conditions(conditions, indices):
307
 
308
  return conditions
309
 
310
- @spaces.GPU(duration=300)
311
- @torch.autocast("cuda")
312
- @torch.no_grad()
313
  def _interpolate_between(
314
  xs: torch.Tensor,
315
  conditions: torch.Tensor,
@@ -356,6 +356,10 @@ def _interpolate_between(
356
  )
357
  return xs, long_conditions
358
 
 
 
 
 
359
  def smooth_navigation(
360
  video: torch.Tensor,
361
  poses: torch.Tensor,
@@ -371,7 +375,7 @@ def smooth_navigation(
371
  interpolation_factor,
372
  )
373
  video = dfot._unnormalize_x(video)[0].detach().cpu()
374
- poses = poses[0]
375
  images = (video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
376
  return (
377
  video,
@@ -382,8 +386,6 @@ def smooth_navigation(
382
  )
383
 
384
 
385
-
386
-
387
  # Create the Gradio Blocks
388
  with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
389
  gr.HTML(
@@ -487,7 +489,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
487
  def update_selection(selection: gr.SelectData):
488
  return selection.index
489
 
490
- demo1_scene_select_button = gr.Button("Select Scene", variant="primary")
 
 
491
 
492
  @demo1_scene_select_button.click(
493
  inputs=demo1_selected_scene_index, outputs=demo1_stage
@@ -519,7 +523,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
519
  choices=[(f"t={i}", i) for i in range(8)],
520
  value=[],
521
  )
522
- demo1_image_select_button = gr.Button("Select Input Images", variant="primary")
 
 
523
 
524
  @demo1_image_select_button.click(
525
  inputs=[demo1_selector],
@@ -606,7 +612,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
606
  def update_selection(selection: gr.SelectData):
607
  return selection.index
608
 
609
- demo2_select_button = gr.Button("Select Input Image", variant="primary")
 
 
610
 
611
  @demo2_select_button.click(
612
  inputs=demo2_selected_index, outputs=demo2_stage
@@ -707,7 +715,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
707
  def update_selection(selection: gr.SelectData):
708
  return selection.index
709
 
710
- demo3_select_button = gr.Button("Select Input Image", variant="primary")
 
 
711
 
712
  @demo3_select_button.click(
713
  inputs=demo3_selected_index,
@@ -763,14 +773,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
763
  with gr.Group():
764
  gr.Markdown("_**Select a direction to move:**_")
765
  with gr.Row(elem_id="basic-controls"):
766
- gr.Button("↰-60°\nTurn", size="sm", min_width=0, variant="primary").click(
 
 
 
 
 
767
  fn=partial(
768
  navigate_video,
769
  x_angle=0,
770
  y_angle=-60,
771
  distance=0,
772
  ),
773
- inputs=[demo3_current_video, demo3_current_poses],
 
 
 
774
  outputs=[
775
  demo3_current_video,
776
  demo3_current_poses,
@@ -780,14 +798,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
780
  ],
781
  )
782
 
783
- gr.Button("↖-30°\nVeer", size="sm", min_width=0, variant="primary").click(
 
 
 
 
 
784
  fn=partial(
785
  navigate_video,
786
  x_angle=0,
787
  y_angle=-30,
788
  distance=50,
789
  ),
790
- inputs=[demo3_current_video, demo3_current_poses],
 
 
 
791
  outputs=[
792
  demo3_current_video,
793
  demo3_current_poses,
@@ -797,14 +823,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
797
  ],
798
  )
799
 
800
- gr.Button("↑0°\nAhead", size="sm", min_width=0, variant="primary").click(
 
 
 
 
 
801
  fn=partial(
802
  navigate_video,
803
  x_angle=0,
804
  y_angle=0,
805
  distance=100,
806
  ),
807
- inputs=[demo3_current_video, demo3_current_poses],
 
 
 
808
  outputs=[
809
  demo3_current_video,
810
  demo3_current_poses,
@@ -813,14 +847,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
813
  demo3_generated_gallery,
814
  ],
815
  )
816
- gr.Button("↗30°\nVeer", size="sm", min_width=0, variant="primary").click(
 
 
 
 
 
817
  fn=partial(
818
  navigate_video,
819
  x_angle=0,
820
  y_angle=30,
821
  distance=50,
822
  ),
823
- inputs=[demo3_current_video, demo3_current_poses],
 
 
 
824
  outputs=[
825
  demo3_current_video,
826
  demo3_current_poses,
@@ -829,14 +871,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
829
  demo3_generated_gallery,
830
  ],
831
  )
832
- gr.Button("↱\n60° Turn", size="sm", min_width=0, variant="primary").click(
 
 
 
 
 
833
  fn=partial(
834
  navigate_video,
835
  x_angle=0,
836
  y_angle=60,
837
  distance=0,
838
  ),
839
- inputs=[demo3_current_video, demo3_current_poses],
 
 
 
840
  outputs=[
841
  demo3_current_video,
842
  demo3_current_poses,
@@ -874,11 +924,17 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
874
  interactive=True,
875
  )
876
 
877
- gr.Button("Generate Next Move", variant="primary").click(
878
- fn=partial(
879
- navigate_video,
880
- ),
881
- inputs=[demo3_current_video, demo3_current_poses, demo3_x_angle, demo3_y_angle, demo3_distance],
 
 
 
 
 
 
882
  outputs=[
883
  demo3_current_video,
884
  demo3_current_poses,
@@ -901,8 +957,10 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
901
  ],
902
  )
903
  with gr.Group():
904
- gr.Markdown("_At the end, apply temporal super-resolution to obtain a smoother video:_")
905
- demo3_interpolation_factor=gr.Slider(
 
 
906
  minimum=2,
907
  maximum=10,
908
  value=2,
@@ -912,7 +970,11 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
912
  )
913
  gr.Button("Smooth Out Video", variant="huggingface").click(
914
  fn=smooth_navigation,
915
- inputs=[demo3_current_video, demo3_current_poses, demo3_interpolation_factor],
 
 
 
 
916
  outputs=[
917
  demo3_current_video,
918
  demo3_current_poses,
 
243
  .cpu()
244
  )
245
  gen_video = torch.cat([video, next_video], dim=0)
246
+ poses = conditions[0].detach().cpu()
247
 
248
  images = (gen_video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
249
 
 
255
  [(image, f"t={i}") for i, image in enumerate(images)],
256
  )
257
 
258
+
259
  def undo_navigation(
260
  video: torch.Tensor,
261
  poses: torch.Tensor,
 
274
  [(image, f"t={i}") for i, image in enumerate(images)],
275
  )
276
 
277
+
278
  def _interpolate_conditions(conditions, indices):
279
  """
280
  Interpolate conditions to fill out missing frames
 
309
 
310
  return conditions
311
 
312
+
 
 
313
  def _interpolate_between(
314
  xs: torch.Tensor,
315
  conditions: torch.Tensor,
 
356
  )
357
  return xs, long_conditions
358
 
359
+
360
+ @spaces.GPU(duration=300)
361
+ @torch.autocast("cuda")
362
+ @torch.no_grad()
363
  def smooth_navigation(
364
  video: torch.Tensor,
365
  poses: torch.Tensor,
 
375
  interpolation_factor,
376
  )
377
  video = dfot._unnormalize_x(video)[0].detach().cpu()
378
+ poses = poses[0].detach().cpu()
379
  images = (video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
380
  return (
381
  video,
 
386
  )
387
 
388
 
 
 
389
  # Create the Gradio Blocks
390
  with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
391
  gr.HTML(
 
489
  def update_selection(selection: gr.SelectData):
490
  return selection.index
491
 
492
+ demo1_scene_select_button = gr.Button(
493
+ "Select Scene", variant="primary"
494
+ )
495
 
496
  @demo1_scene_select_button.click(
497
  inputs=demo1_selected_scene_index, outputs=demo1_stage
 
523
  choices=[(f"t={i}", i) for i in range(8)],
524
  value=[],
525
  )
526
+ demo1_image_select_button = gr.Button(
527
+ "Select Input Images", variant="primary"
528
+ )
529
 
530
  @demo1_image_select_button.click(
531
  inputs=[demo1_selector],
 
612
  def update_selection(selection: gr.SelectData):
613
  return selection.index
614
 
615
+ demo2_select_button = gr.Button(
616
+ "Select Input Image", variant="primary"
617
+ )
618
 
619
  @demo2_select_button.click(
620
  inputs=demo2_selected_index, outputs=demo2_stage
 
715
  def update_selection(selection: gr.SelectData):
716
  return selection.index
717
 
718
+ demo3_select_button = gr.Button(
719
+ "Select Input Image", variant="primary"
720
+ )
721
 
722
  @demo3_select_button.click(
723
  inputs=demo3_selected_index,
 
773
  with gr.Group():
774
  gr.Markdown("_**Select a direction to move:**_")
775
  with gr.Row(elem_id="basic-controls"):
776
+ gr.Button(
777
+ "↰-60°\nTurn",
778
+ size="sm",
779
+ min_width=0,
780
+ variant="primary",
781
+ ).click(
782
  fn=partial(
783
  navigate_video,
784
  x_angle=0,
785
  y_angle=-60,
786
  distance=0,
787
  ),
788
+ inputs=[
789
+ demo3_current_video,
790
+ demo3_current_poses,
791
+ ],
792
  outputs=[
793
  demo3_current_video,
794
  demo3_current_poses,
 
798
  ],
799
  )
800
 
801
+ gr.Button(
802
+ "↖-30°\nVeer",
803
+ size="sm",
804
+ min_width=0,
805
+ variant="primary",
806
+ ).click(
807
  fn=partial(
808
  navigate_video,
809
  x_angle=0,
810
  y_angle=-30,
811
  distance=50,
812
  ),
813
+ inputs=[
814
+ demo3_current_video,
815
+ demo3_current_poses,
816
+ ],
817
  outputs=[
818
  demo3_current_video,
819
  demo3_current_poses,
 
823
  ],
824
  )
825
 
826
+ gr.Button(
827
+ "↑0°\nAhead",
828
+ size="sm",
829
+ min_width=0,
830
+ variant="primary",
831
+ ).click(
832
  fn=partial(
833
  navigate_video,
834
  x_angle=0,
835
  y_angle=0,
836
  distance=100,
837
  ),
838
+ inputs=[
839
+ demo3_current_video,
840
+ demo3_current_poses,
841
+ ],
842
  outputs=[
843
  demo3_current_video,
844
  demo3_current_poses,
 
847
  demo3_generated_gallery,
848
  ],
849
  )
850
+ gr.Button(
851
+ "↗30°\nVeer",
852
+ size="sm",
853
+ min_width=0,
854
+ variant="primary",
855
+ ).click(
856
  fn=partial(
857
  navigate_video,
858
  x_angle=0,
859
  y_angle=30,
860
  distance=50,
861
  ),
862
+ inputs=[
863
+ demo3_current_video,
864
+ demo3_current_poses,
865
+ ],
866
  outputs=[
867
  demo3_current_video,
868
  demo3_current_poses,
 
871
  demo3_generated_gallery,
872
  ],
873
  )
874
+ gr.Button(
875
+ "↱\n60° Turn",
876
+ size="sm",
877
+ min_width=0,
878
+ variant="primary",
879
+ ).click(
880
  fn=partial(
881
  navigate_video,
882
  x_angle=0,
883
  y_angle=60,
884
  distance=0,
885
  ),
886
+ inputs=[
887
+ demo3_current_video,
888
+ demo3_current_poses,
889
+ ],
890
  outputs=[
891
  demo3_current_video,
892
  demo3_current_poses,
 
924
  interactive=True,
925
  )
926
 
927
+ gr.Button(
928
+ "Generate Next Move", variant="primary"
929
+ ).click(
930
+ fn=navigate_video,
931
+ inputs=[
932
+ demo3_current_video,
933
+ demo3_current_poses,
934
+ demo3_x_angle,
935
+ demo3_y_angle,
936
+ demo3_distance,
937
+ ],
938
  outputs=[
939
  demo3_current_video,
940
  demo3_current_poses,
 
957
  ],
958
  )
959
  with gr.Group():
960
+ gr.Markdown(
961
+ "_At the end, apply temporal super-resolution to obtain a smoother video:_"
962
+ )
963
+ demo3_interpolation_factor = gr.Slider(
964
  minimum=2,
965
  maximum=10,
966
  value=2,
 
970
  )
971
  gr.Button("Smooth Out Video", variant="huggingface").click(
972
  fn=smooth_navigation,
973
+ inputs=[
974
+ demo3_current_video,
975
+ demo3_current_poses,
976
+ demo3_interpolation_factor,
977
+ ],
978
  outputs=[
979
  demo3_current_video,
980
  demo3_current_poses,