Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -21,17 +21,6 @@ from funcs import (
|
|
21 |
save_videos
|
22 |
)
|
23 |
from transformers import pipeline
|
24 |
-
from diffusers import FluxPipeline
|
25 |
-
from PIL import Image
|
26 |
-
import numpy as np
|
27 |
-
from huggingface_hub import login
|
28 |
-
|
29 |
-
# Hugging Face ํ ํฐ ์ค์ ๋ฐ ๋ก๊ทธ์ธ
|
30 |
-
hf_token = os.getenv("HF_TOKEN")
|
31 |
-
if hf_token:
|
32 |
-
login(token=hf_token)
|
33 |
-
else:
|
34 |
-
print("Warning: HF_TOKEN not found in environment variables. You may encounter authentication issues.")
|
35 |
|
36 |
def download_model():
|
37 |
REPO_ID = 'Doubiiu/DynamiCrafter_1024'
|
@@ -44,11 +33,11 @@ def download_model():
|
|
44 |
hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_1024_v1/', force_download=True)
|
45 |
|
46 |
download_model()
|
47 |
-
ckpt_path
|
48 |
-
config_file
|
49 |
config = OmegaConf.load(config_file)
|
50 |
model_config = config.pop("model", OmegaConf.create())
|
51 |
-
model_config['params']['unet_config']['params']['use_checkpoint']
|
52 |
model = instantiate_from_config(model_config)
|
53 |
assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
|
54 |
model = load_model_checkpoint(model, ckpt_path)
|
@@ -56,75 +45,70 @@ model.eval()
|
|
56 |
model = model.cuda()
|
57 |
|
58 |
# ๋ฒ์ญ ๋ชจ๋ธ ์ด๊ธฐํ
|
59 |
-
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-ko-en"
|
60 |
-
|
61 |
-
# FLUX ํ์ดํ๋ผ์ธ ์ด๊ธฐํ ๋ถ๋ถ ์์
|
62 |
-
flux_pipe = FluxPipeline.from_pretrained(
|
63 |
-
"black-forest-labs/FLUX.1-dev",
|
64 |
-
torch_dtype=torch.bfloat16,
|
65 |
-
use_auth_token=hf_token # ํ ํฐ์ ์ฌ์ฉํ์ฌ ์ธ์ฆ
|
66 |
-
)
|
67 |
-
flux_pipe.enable_model_cpu_offload()
|
68 |
|
69 |
-
|
|
|
70 |
# ํ๊ธ ์
๋ ฅ ๊ฐ์ง ๋ฐ ๋ฒ์ญ
|
71 |
if any('\u3131' <= char <= '\u318E' or '\uAC00' <= char <= '\uD7A3' for char in prompt):
|
72 |
translated = translator(prompt, max_length=512)[0]['translation_text']
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
def generate_image_from_text(prompt, seed=0):
|
77 |
-
translated_prompt = translate_prompt(prompt)
|
78 |
-
generator = torch.Generator("cpu").manual_seed(seed)
|
79 |
-
image = flux_pipe(
|
80 |
-
translated_prompt,
|
81 |
-
height=576,
|
82 |
-
width=1024,
|
83 |
-
guidance_scale=3.5,
|
84 |
-
num_inference_steps=50,
|
85 |
-
max_sequence_length=512,
|
86 |
-
generator=generator
|
87 |
-
).images[0]
|
88 |
-
return image
|
89 |
|
90 |
-
import torch
|
91 |
-
|
92 |
-
def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, seed=123, video_length=2, fs=8):
|
93 |
-
translated_prompt = translate_prompt(prompt)
|
94 |
-
print(f"Translated prompt: {translated_prompt}")
|
95 |
resolution = (576, 1024)
|
96 |
-
|
97 |
seed_everything(seed)
|
98 |
transform = transforms.Compose([
|
99 |
-
transforms.Resize(min(resolution)
|
100 |
transforms.CenterCrop(resolution),
|
101 |
-
|
102 |
torch.cuda.empty_cache()
|
103 |
-
print('
|
104 |
start = time.time()
|
105 |
if steps > 60:
|
106 |
-
steps = 60
|
107 |
-
|
|
|
108 |
channels = model.model.diffusion_model.out_channels
|
109 |
-
frames = int(video_length *
|
110 |
h, w = resolution[0] // 8, resolution[1] // 8
|
111 |
noise_shape = [batch_size, channels, frames, h, w]
|
|
|
|
|
112 |
with torch.no_grad(), torch.cuda.amp.autocast():
|
113 |
-
text_emb = model.get_learned_conditioning([
|
|
|
|
|
114 |
img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
|
115 |
img_tensor = (img_tensor / 255. - 0.5) * 2
|
116 |
-
|
117 |
-
|
|
|
|
|
|
|
|
|
118 |
img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
|
119 |
-
|
|
|
120 |
img_emb = model.image_proj_model(cond_images)
|
|
|
121 |
imtext_cond = torch.cat([text_emb, img_emb], dim=1)
|
122 |
-
|
|
|
|
|
|
|
|
|
123 |
batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
|
|
|
|
|
124 |
video_path = './output.mp4'
|
125 |
-
save_videos(batch_samples, './', filenames=['output'], fps=
|
126 |
return video_path
|
127 |
|
|
|
|
|
|
|
|
|
128 |
|
129 |
css = """
|
130 |
.tab-nav {
|
@@ -163,68 +147,39 @@ css = """
|
|
163 |
.tab-nav button:nth-child(3) { border-top: 3px solid #f7b731; }
|
164 |
"""
|
165 |
|
166 |
-
def infer_t2v(prompt, seed=123, steps=50, cfg_scale=7.5, eta=1.0, fs=8, video_length=2):
|
167 |
-
# ๋จผ์ ํ
์คํธ๋ก๋ถํฐ ์ด๋ฏธ์ง๋ฅผ ์์ฑํฉ๋๋ค
|
168 |
-
initial_image = generate_image_from_text(prompt, seed)
|
169 |
-
|
170 |
-
# ๊ทธ ๋ค์ ์์ฑ๋ ์ด๋ฏธ์ง๋ฅผ ์ฌ์ฉํ์ฌ ๋น๋์ค๋ฅผ ์์ฑํฉ๋๋ค
|
171 |
-
return infer(initial_image, prompt, steps, cfg_scale, eta, seed, video_length, fs)
|
172 |
|
173 |
with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
|
174 |
-
gr.Markdown("
|
175 |
-
with gr.Tab(label='
|
176 |
-
with gr.Column():
|
177 |
-
with gr.Row():
|
178 |
-
img_input_text = gr.Text(label='Image Generation Prompt')
|
179 |
-
img_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
|
180 |
-
img_generate_btn = gr.Button("Generate Image")
|
181 |
-
with gr.Row():
|
182 |
-
img_output_image = gr.Image(label="Generated Image")
|
183 |
-
img_generate_btn.click(
|
184 |
-
inputs=[img_input_text, img_seed],
|
185 |
-
outputs=[img_output_image],
|
186 |
-
fn=generate_image_from_text
|
187 |
-
)
|
188 |
-
|
189 |
-
with gr.Tab(label='Image to Video Generation'):
|
190 |
-
with gr.Column():
|
191 |
-
with gr.Row():
|
192 |
-
video_input_image = gr.Image(label="Input Image for Video")
|
193 |
-
video_prompt = gr.Text(label='Video Generation Prompt')
|
194 |
-
video_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
|
195 |
-
video_steps = gr.Slider(label="Sampling steps", minimum=1, maximum=50, step=1, value=30)
|
196 |
-
video_cfg_scale = gr.Slider(label='CFG Scale', minimum=1.0, maximum=15.0, step=0.5, value=7.5)
|
197 |
-
video_eta = gr.Slider(label='ETA', minimum=0.0, maximum=1.0, step=0.1, value=1.0)
|
198 |
-
video_fs = gr.Slider(label='FS', minimum=1, maximum=60, step=1, value=10) # fps๋ฅผ fs๋ก ๋ณ๊ฒฝ
|
199 |
-
video_length = gr.Slider(label="Video Length (seconds)", minimum=2, maximum=8, step=1, value=2)
|
200 |
-
video_generate_btn = gr.Button("Generate Video")
|
201 |
-
with gr.Row():
|
202 |
-
video_output = gr.Video(label="Generated Video", autoplay=True, show_share_button=True)
|
203 |
-
video_generate_btn.click(
|
204 |
-
inputs=[video_input_image, video_prompt, video_seed, video_steps, video_cfg_scale, video_eta, video_fs, video_length],
|
205 |
-
outputs=[video_output],
|
206 |
-
fn=infer
|
207 |
-
)
|
208 |
-
|
209 |
-
with gr.Tab(label='Text to Video Generation'):
|
210 |
with gr.Column():
|
211 |
with gr.Row():
|
212 |
with gr.Column():
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
|
219 |
-
|
220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
221 |
with gr.Row():
|
222 |
-
|
223 |
|
224 |
-
|
225 |
-
|
226 |
-
|
227 |
-
|
|
|
228 |
)
|
|
|
|
|
|
|
|
|
229 |
|
230 |
-
dynamicrafter_iface.launch(show_api=True)
|
|
|
21 |
save_videos
|
22 |
)
|
23 |
from transformers import pipeline
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
|
25 |
def download_model():
|
26 |
REPO_ID = 'Doubiiu/DynamiCrafter_1024'
|
|
|
33 |
hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_1024_v1/', force_download=True)
|
34 |
|
35 |
download_model()
|
36 |
+
ckpt_path='checkpoints/dynamicrafter_1024_v1/model.ckpt'
|
37 |
+
config_file='configs/inference_1024_v1.0.yaml'
|
38 |
config = OmegaConf.load(config_file)
|
39 |
model_config = config.pop("model", OmegaConf.create())
|
40 |
+
model_config['params']['unet_config']['params']['use_checkpoint']=False
|
41 |
model = instantiate_from_config(model_config)
|
42 |
assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
|
43 |
model = load_model_checkpoint(model, ckpt_path)
|
|
|
45 |
model = model.cuda()
|
46 |
|
47 |
# ๋ฒ์ญ ๋ชจ๋ธ ์ด๊ธฐํ
|
48 |
+
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-ko-en")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
49 |
|
50 |
+
@spaces.GPU(duration=300)
|
51 |
+
def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123, video_length=2):
|
52 |
# ํ๊ธ ์
๋ ฅ ๊ฐ์ง ๋ฐ ๋ฒ์ญ
|
53 |
if any('\u3131' <= char <= '\u318E' or '\uAC00' <= char <= '\uD7A3' for char in prompt):
|
54 |
translated = translator(prompt, max_length=512)[0]['translation_text']
|
55 |
+
prompt = translated
|
56 |
+
print(f"Translated prompt: {prompt}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
|
|
|
|
|
|
|
|
|
|
|
58 |
resolution = (576, 1024)
|
59 |
+
save_fps = 8
|
60 |
seed_everything(seed)
|
61 |
transform = transforms.Compose([
|
62 |
+
transforms.Resize(min(resolution)),
|
63 |
transforms.CenterCrop(resolution),
|
64 |
+
])
|
65 |
torch.cuda.empty_cache()
|
66 |
+
print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
|
67 |
start = time.time()
|
68 |
if steps > 60:
|
69 |
+
steps = 60
|
70 |
+
|
71 |
+
batch_size=1
|
72 |
channels = model.model.diffusion_model.out_channels
|
73 |
+
frames = int(video_length * save_fps) # ๋น๋์ค ๊ธธ์ด์ ๋ฐ๋ฅธ ํ๋ ์ ์ ๊ณ์ฐ
|
74 |
h, w = resolution[0] // 8, resolution[1] // 8
|
75 |
noise_shape = [batch_size, channels, frames, h, w]
|
76 |
+
|
77 |
+
# text cond
|
78 |
with torch.no_grad(), torch.cuda.amp.autocast():
|
79 |
+
text_emb = model.get_learned_conditioning([prompt])
|
80 |
+
|
81 |
+
# img cond
|
82 |
img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
|
83 |
img_tensor = (img_tensor / 255. - 0.5) * 2
|
84 |
+
|
85 |
+
image_tensor_resized = transform(img_tensor) #3,256,256
|
86 |
+
videos = image_tensor_resized.unsqueeze(0) # bchw
|
87 |
+
|
88 |
+
z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
|
89 |
+
|
90 |
img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
|
91 |
+
|
92 |
+
cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
|
93 |
img_emb = model.image_proj_model(cond_images)
|
94 |
+
|
95 |
imtext_cond = torch.cat([text_emb, img_emb], dim=1)
|
96 |
+
|
97 |
+
fs = torch.tensor([fs], dtype=torch.long, device=model.device)
|
98 |
+
cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
|
99 |
+
|
100 |
+
## inference
|
101 |
batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
|
102 |
+
## b,samples,c,t,h,w
|
103 |
+
|
104 |
video_path = './output.mp4'
|
105 |
+
save_videos(batch_samples, './', filenames=['output'], fps=save_fps)
|
106 |
return video_path
|
107 |
|
108 |
+
i2v_examples = [
|
109 |
+
['prompts/1024/astronaut04.png', 'a man in an astronaut suit playing a guitar', 30, 7.5, 1.0, 6, 123, 2],
|
110 |
+
]
|
111 |
+
|
112 |
|
113 |
css = """
|
114 |
.tab-nav {
|
|
|
147 |
.tab-nav button:nth-child(3) { border-top: 3px solid #f7b731; }
|
148 |
"""
|
149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
150 |
|
151 |
with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
|
152 |
+
gr.Markdown("์ด๋ฏธ์ง๋ก ์์ ์์ฑ ํ
์คํธ (ํ๊ธ ํ๋กฌํํธ ์ง์)")
|
153 |
+
with gr.Tab(label='ImageAnimation_576x1024'):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
154 |
with gr.Column():
|
155 |
with gr.Row():
|
156 |
with gr.Column():
|
157 |
+
with gr.Row():
|
158 |
+
i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
|
159 |
+
with gr.Row():
|
160 |
+
i2v_input_text = gr.Text(label='Prompts')
|
161 |
+
with gr.Row():
|
162 |
+
i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
|
163 |
+
i2v_eta = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='ETA', value=1.0, elem_id="i2v_eta")
|
164 |
+
i2v_cfg_scale = gr.Slider(minimum=1.0, maximum=15.0, step=0.5, label='CFG Scale', value=7.5, elem_id="i2v_cfg_scale")
|
165 |
+
with gr.Row():
|
166 |
+
i2v_steps = gr.Slider(minimum=1, maximum=50, step=1, elem_id="i2v_steps", label="Sampling steps", value=30)
|
167 |
+
i2v_motion = gr.Slider(minimum=5, maximum=20, step=1, elem_id="i2v_motion", label="FPS", value=8)
|
168 |
+
with gr.Row():
|
169 |
+
i2v_video_length = gr.Slider(minimum=2, maximum=8, step=1, elem_id="i2v_video_length", label="Video Length (seconds)", value=2)
|
170 |
+
i2v_end_btn = gr.Button("Generate")
|
171 |
with gr.Row():
|
172 |
+
i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
|
173 |
|
174 |
+
gr.Examples(examples=i2v_examples,
|
175 |
+
inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed, i2v_video_length],
|
176 |
+
outputs=[i2v_output_video],
|
177 |
+
fn = infer,
|
178 |
+
cache_examples=True,
|
179 |
)
|
180 |
+
i2v_end_btn.click(inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed, i2v_video_length],
|
181 |
+
outputs=[i2v_output_video],
|
182 |
+
fn = infer
|
183 |
+
)
|
184 |
|
185 |
+
dynamicrafter_iface.queue(max_size=12).launch(show_api=True)
|