Spaces:
Running
on
Zero
Running
on
Zero
Commit
·
a71f061
1
Parent(s):
2351462
fix all hf bugs
Browse files
app.py
CHANGED
@@ -243,7 +243,7 @@ def navigate_video(
|
|
243 |
.cpu()
|
244 |
)
|
245 |
gen_video = torch.cat([video, next_video], dim=0)
|
246 |
-
poses = conditions[0]
|
247 |
|
248 |
images = (gen_video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
|
249 |
|
@@ -255,6 +255,7 @@ def navigate_video(
|
|
255 |
[(image, f"t={i}") for i, image in enumerate(images)],
|
256 |
)
|
257 |
|
|
|
258 |
def undo_navigation(
|
259 |
video: torch.Tensor,
|
260 |
poses: torch.Tensor,
|
@@ -273,6 +274,7 @@ def undo_navigation(
|
|
273 |
[(image, f"t={i}") for i, image in enumerate(images)],
|
274 |
)
|
275 |
|
|
|
276 |
def _interpolate_conditions(conditions, indices):
|
277 |
"""
|
278 |
Interpolate conditions to fill out missing frames
|
@@ -307,9 +309,7 @@ def _interpolate_conditions(conditions, indices):
|
|
307 |
|
308 |
return conditions
|
309 |
|
310 |
-
|
311 |
-
@torch.autocast("cuda")
|
312 |
-
@torch.no_grad()
|
313 |
def _interpolate_between(
|
314 |
xs: torch.Tensor,
|
315 |
conditions: torch.Tensor,
|
@@ -356,6 +356,10 @@ def _interpolate_between(
|
|
356 |
)
|
357 |
return xs, long_conditions
|
358 |
|
|
|
|
|
|
|
|
|
359 |
def smooth_navigation(
|
360 |
video: torch.Tensor,
|
361 |
poses: torch.Tensor,
|
@@ -371,7 +375,7 @@ def smooth_navigation(
|
|
371 |
interpolation_factor,
|
372 |
)
|
373 |
video = dfot._unnormalize_x(video)[0].detach().cpu()
|
374 |
-
poses = poses[0]
|
375 |
images = (video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
|
376 |
return (
|
377 |
video,
|
@@ -382,8 +386,6 @@ def smooth_navigation(
|
|
382 |
)
|
383 |
|
384 |
|
385 |
-
|
386 |
-
|
387 |
# Create the Gradio Blocks
|
388 |
with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
389 |
gr.HTML(
|
@@ -487,7 +489,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
487 |
def update_selection(selection: gr.SelectData):
|
488 |
return selection.index
|
489 |
|
490 |
-
demo1_scene_select_button = gr.Button(
|
|
|
|
|
491 |
|
492 |
@demo1_scene_select_button.click(
|
493 |
inputs=demo1_selected_scene_index, outputs=demo1_stage
|
@@ -519,7 +523,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
519 |
choices=[(f"t={i}", i) for i in range(8)],
|
520 |
value=[],
|
521 |
)
|
522 |
-
demo1_image_select_button = gr.Button(
|
|
|
|
|
523 |
|
524 |
@demo1_image_select_button.click(
|
525 |
inputs=[demo1_selector],
|
@@ -606,7 +612,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
606 |
def update_selection(selection: gr.SelectData):
|
607 |
return selection.index
|
608 |
|
609 |
-
demo2_select_button = gr.Button(
|
|
|
|
|
610 |
|
611 |
@demo2_select_button.click(
|
612 |
inputs=demo2_selected_index, outputs=demo2_stage
|
@@ -707,7 +715,9 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
707 |
def update_selection(selection: gr.SelectData):
|
708 |
return selection.index
|
709 |
|
710 |
-
demo3_select_button = gr.Button(
|
|
|
|
|
711 |
|
712 |
@demo3_select_button.click(
|
713 |
inputs=demo3_selected_index,
|
@@ -763,14 +773,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
763 |
with gr.Group():
|
764 |
gr.Markdown("_**Select a direction to move:**_")
|
765 |
with gr.Row(elem_id="basic-controls"):
|
766 |
-
gr.Button(
|
|
|
|
|
|
|
|
|
|
|
767 |
fn=partial(
|
768 |
navigate_video,
|
769 |
x_angle=0,
|
770 |
y_angle=-60,
|
771 |
distance=0,
|
772 |
),
|
773 |
-
inputs=[
|
|
|
|
|
|
|
774 |
outputs=[
|
775 |
demo3_current_video,
|
776 |
demo3_current_poses,
|
@@ -780,14 +798,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
780 |
],
|
781 |
)
|
782 |
|
783 |
-
gr.Button(
|
|
|
|
|
|
|
|
|
|
|
784 |
fn=partial(
|
785 |
navigate_video,
|
786 |
x_angle=0,
|
787 |
y_angle=-30,
|
788 |
distance=50,
|
789 |
),
|
790 |
-
inputs=[
|
|
|
|
|
|
|
791 |
outputs=[
|
792 |
demo3_current_video,
|
793 |
demo3_current_poses,
|
@@ -797,14 +823,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
797 |
],
|
798 |
)
|
799 |
|
800 |
-
gr.Button(
|
|
|
|
|
|
|
|
|
|
|
801 |
fn=partial(
|
802 |
navigate_video,
|
803 |
x_angle=0,
|
804 |
y_angle=0,
|
805 |
distance=100,
|
806 |
),
|
807 |
-
inputs=[
|
|
|
|
|
|
|
808 |
outputs=[
|
809 |
demo3_current_video,
|
810 |
demo3_current_poses,
|
@@ -813,14 +847,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
813 |
demo3_generated_gallery,
|
814 |
],
|
815 |
)
|
816 |
-
gr.Button(
|
|
|
|
|
|
|
|
|
|
|
817 |
fn=partial(
|
818 |
navigate_video,
|
819 |
x_angle=0,
|
820 |
y_angle=30,
|
821 |
distance=50,
|
822 |
),
|
823 |
-
inputs=[
|
|
|
|
|
|
|
824 |
outputs=[
|
825 |
demo3_current_video,
|
826 |
demo3_current_poses,
|
@@ -829,14 +871,22 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
829 |
demo3_generated_gallery,
|
830 |
],
|
831 |
)
|
832 |
-
gr.Button(
|
|
|
|
|
|
|
|
|
|
|
833 |
fn=partial(
|
834 |
navigate_video,
|
835 |
x_angle=0,
|
836 |
y_angle=60,
|
837 |
distance=0,
|
838 |
),
|
839 |
-
inputs=[
|
|
|
|
|
|
|
840 |
outputs=[
|
841 |
demo3_current_video,
|
842 |
demo3_current_poses,
|
@@ -874,11 +924,17 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
874 |
interactive=True,
|
875 |
)
|
876 |
|
877 |
-
gr.Button(
|
878 |
-
|
879 |
-
|
880 |
-
|
881 |
-
inputs=[
|
|
|
|
|
|
|
|
|
|
|
|
|
882 |
outputs=[
|
883 |
demo3_current_video,
|
884 |
demo3_current_poses,
|
@@ -901,8 +957,10 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
901 |
],
|
902 |
)
|
903 |
with gr.Group():
|
904 |
-
gr.Markdown(
|
905 |
-
|
|
|
|
|
906 |
minimum=2,
|
907 |
maximum=10,
|
908 |
value=2,
|
@@ -912,7 +970,11 @@ with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
|
912 |
)
|
913 |
gr.Button("Smooth Out Video", variant="huggingface").click(
|
914 |
fn=smooth_navigation,
|
915 |
-
inputs=[
|
|
|
|
|
|
|
|
|
916 |
outputs=[
|
917 |
demo3_current_video,
|
918 |
demo3_current_poses,
|
|
|
243 |
.cpu()
|
244 |
)
|
245 |
gen_video = torch.cat([video, next_video], dim=0)
|
246 |
+
poses = conditions[0].detach().cpu()
|
247 |
|
248 |
images = (gen_video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
|
249 |
|
|
|
255 |
[(image, f"t={i}") for i, image in enumerate(images)],
|
256 |
)
|
257 |
|
258 |
+
|
259 |
def undo_navigation(
|
260 |
video: torch.Tensor,
|
261 |
poses: torch.Tensor,
|
|
|
274 |
[(image, f"t={i}") for i, image in enumerate(images)],
|
275 |
)
|
276 |
|
277 |
+
|
278 |
def _interpolate_conditions(conditions, indices):
|
279 |
"""
|
280 |
Interpolate conditions to fill out missing frames
|
|
|
309 |
|
310 |
return conditions
|
311 |
|
312 |
+
|
|
|
|
|
313 |
def _interpolate_between(
|
314 |
xs: torch.Tensor,
|
315 |
conditions: torch.Tensor,
|
|
|
356 |
)
|
357 |
return xs, long_conditions
|
358 |
|
359 |
+
|
360 |
+
@spaces.GPU(duration=300)
|
361 |
+
@torch.autocast("cuda")
|
362 |
+
@torch.no_grad()
|
363 |
def smooth_navigation(
|
364 |
video: torch.Tensor,
|
365 |
poses: torch.Tensor,
|
|
|
375 |
interpolation_factor,
|
376 |
)
|
377 |
video = dfot._unnormalize_x(video)[0].detach().cpu()
|
378 |
+
poses = poses[0].detach().cpu()
|
379 |
images = (video.permute(0, 2, 3, 1) * 255).clamp(0, 255).to(torch.uint8).numpy()
|
380 |
return (
|
381 |
video,
|
|
|
386 |
)
|
387 |
|
388 |
|
|
|
|
|
389 |
# Create the Gradio Blocks
|
390 |
with gr.Blocks(theme=gr.themes.Base(primary_hue="teal")) as demo:
|
391 |
gr.HTML(
|
|
|
489 |
def update_selection(selection: gr.SelectData):
|
490 |
return selection.index
|
491 |
|
492 |
+
demo1_scene_select_button = gr.Button(
|
493 |
+
"Select Scene", variant="primary"
|
494 |
+
)
|
495 |
|
496 |
@demo1_scene_select_button.click(
|
497 |
inputs=demo1_selected_scene_index, outputs=demo1_stage
|
|
|
523 |
choices=[(f"t={i}", i) for i in range(8)],
|
524 |
value=[],
|
525 |
)
|
526 |
+
demo1_image_select_button = gr.Button(
|
527 |
+
"Select Input Images", variant="primary"
|
528 |
+
)
|
529 |
|
530 |
@demo1_image_select_button.click(
|
531 |
inputs=[demo1_selector],
|
|
|
612 |
def update_selection(selection: gr.SelectData):
|
613 |
return selection.index
|
614 |
|
615 |
+
demo2_select_button = gr.Button(
|
616 |
+
"Select Input Image", variant="primary"
|
617 |
+
)
|
618 |
|
619 |
@demo2_select_button.click(
|
620 |
inputs=demo2_selected_index, outputs=demo2_stage
|
|
|
715 |
def update_selection(selection: gr.SelectData):
|
716 |
return selection.index
|
717 |
|
718 |
+
demo3_select_button = gr.Button(
|
719 |
+
"Select Input Image", variant="primary"
|
720 |
+
)
|
721 |
|
722 |
@demo3_select_button.click(
|
723 |
inputs=demo3_selected_index,
|
|
|
773 |
with gr.Group():
|
774 |
gr.Markdown("_**Select a direction to move:**_")
|
775 |
with gr.Row(elem_id="basic-controls"):
|
776 |
+
gr.Button(
|
777 |
+
"↰-60°\nTurn",
|
778 |
+
size="sm",
|
779 |
+
min_width=0,
|
780 |
+
variant="primary",
|
781 |
+
).click(
|
782 |
fn=partial(
|
783 |
navigate_video,
|
784 |
x_angle=0,
|
785 |
y_angle=-60,
|
786 |
distance=0,
|
787 |
),
|
788 |
+
inputs=[
|
789 |
+
demo3_current_video,
|
790 |
+
demo3_current_poses,
|
791 |
+
],
|
792 |
outputs=[
|
793 |
demo3_current_video,
|
794 |
demo3_current_poses,
|
|
|
798 |
],
|
799 |
)
|
800 |
|
801 |
+
gr.Button(
|
802 |
+
"↖-30°\nVeer",
|
803 |
+
size="sm",
|
804 |
+
min_width=0,
|
805 |
+
variant="primary",
|
806 |
+
).click(
|
807 |
fn=partial(
|
808 |
navigate_video,
|
809 |
x_angle=0,
|
810 |
y_angle=-30,
|
811 |
distance=50,
|
812 |
),
|
813 |
+
inputs=[
|
814 |
+
demo3_current_video,
|
815 |
+
demo3_current_poses,
|
816 |
+
],
|
817 |
outputs=[
|
818 |
demo3_current_video,
|
819 |
demo3_current_poses,
|
|
|
823 |
],
|
824 |
)
|
825 |
|
826 |
+
gr.Button(
|
827 |
+
"↑0°\nAhead",
|
828 |
+
size="sm",
|
829 |
+
min_width=0,
|
830 |
+
variant="primary",
|
831 |
+
).click(
|
832 |
fn=partial(
|
833 |
navigate_video,
|
834 |
x_angle=0,
|
835 |
y_angle=0,
|
836 |
distance=100,
|
837 |
),
|
838 |
+
inputs=[
|
839 |
+
demo3_current_video,
|
840 |
+
demo3_current_poses,
|
841 |
+
],
|
842 |
outputs=[
|
843 |
demo3_current_video,
|
844 |
demo3_current_poses,
|
|
|
847 |
demo3_generated_gallery,
|
848 |
],
|
849 |
)
|
850 |
+
gr.Button(
|
851 |
+
"↗30°\nVeer",
|
852 |
+
size="sm",
|
853 |
+
min_width=0,
|
854 |
+
variant="primary",
|
855 |
+
).click(
|
856 |
fn=partial(
|
857 |
navigate_video,
|
858 |
x_angle=0,
|
859 |
y_angle=30,
|
860 |
distance=50,
|
861 |
),
|
862 |
+
inputs=[
|
863 |
+
demo3_current_video,
|
864 |
+
demo3_current_poses,
|
865 |
+
],
|
866 |
outputs=[
|
867 |
demo3_current_video,
|
868 |
demo3_current_poses,
|
|
|
871 |
demo3_generated_gallery,
|
872 |
],
|
873 |
)
|
874 |
+
gr.Button(
|
875 |
+
"↱\n60° Turn",
|
876 |
+
size="sm",
|
877 |
+
min_width=0,
|
878 |
+
variant="primary",
|
879 |
+
).click(
|
880 |
fn=partial(
|
881 |
navigate_video,
|
882 |
x_angle=0,
|
883 |
y_angle=60,
|
884 |
distance=0,
|
885 |
),
|
886 |
+
inputs=[
|
887 |
+
demo3_current_video,
|
888 |
+
demo3_current_poses,
|
889 |
+
],
|
890 |
outputs=[
|
891 |
demo3_current_video,
|
892 |
demo3_current_poses,
|
|
|
924 |
interactive=True,
|
925 |
)
|
926 |
|
927 |
+
gr.Button(
|
928 |
+
"Generate Next Move", variant="primary"
|
929 |
+
).click(
|
930 |
+
fn=navigate_video,
|
931 |
+
inputs=[
|
932 |
+
demo3_current_video,
|
933 |
+
demo3_current_poses,
|
934 |
+
demo3_x_angle,
|
935 |
+
demo3_y_angle,
|
936 |
+
demo3_distance,
|
937 |
+
],
|
938 |
outputs=[
|
939 |
demo3_current_video,
|
940 |
demo3_current_poses,
|
|
|
957 |
],
|
958 |
)
|
959 |
with gr.Group():
|
960 |
+
gr.Markdown(
|
961 |
+
"_At the end, apply temporal super-resolution to obtain a smoother video:_"
|
962 |
+
)
|
963 |
+
demo3_interpolation_factor = gr.Slider(
|
964 |
minimum=2,
|
965 |
maximum=10,
|
966 |
value=2,
|
|
|
970 |
)
|
971 |
gr.Button("Smooth Out Video", variant="huggingface").click(
|
972 |
fn=smooth_navigation,
|
973 |
+
inputs=[
|
974 |
+
demo3_current_video,
|
975 |
+
demo3_current_poses,
|
976 |
+
demo3_interpolation_factor,
|
977 |
+
],
|
978 |
outputs=[
|
979 |
demo3_current_video,
|
980 |
demo3_current_poses,
|