seawolf2357 commited on
Commit
192dfa7
ยท
verified ยท
1 Parent(s): 248f435

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +69 -114
app.py CHANGED
@@ -21,17 +21,6 @@ from funcs import (
21
  save_videos
22
  )
23
  from transformers import pipeline
24
- from diffusers import FluxPipeline
25
- from PIL import Image
26
- import numpy as np
27
- from huggingface_hub import login
28
-
29
- # Hugging Face ํ† ํฐ ์„ค์ • ๋ฐ ๋กœ๊ทธ์ธ
30
- hf_token = os.getenv("HF_TOKEN")
31
- if hf_token:
32
- login(token=hf_token)
33
- else:
34
- print("Warning: HF_TOKEN not found in environment variables. You may encounter authentication issues.")
35
 
36
  def download_model():
37
  REPO_ID = 'Doubiiu/DynamiCrafter_1024'
@@ -44,11 +33,11 @@ def download_model():
44
  hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_1024_v1/', force_download=True)
45
 
46
  download_model()
47
- ckpt_path = 'checkpoints/dynamicrafter_1024_v1/model.ckpt'
48
- config_file = 'configs/inference_1024_v1.0.yaml'
49
  config = OmegaConf.load(config_file)
50
  model_config = config.pop("model", OmegaConf.create())
51
- model_config['params']['unet_config']['params']['use_checkpoint'] = False
52
  model = instantiate_from_config(model_config)
53
  assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
54
  model = load_model_checkpoint(model, ckpt_path)
@@ -56,75 +45,70 @@ model.eval()
56
  model = model.cuda()
57
 
58
  # ๋ฒˆ์—ญ ๋ชจ๋ธ ์ดˆ๊ธฐํ™”
59
- translator = pipeline("translation", model="Helsinki-NLP/opus-mt-ko-en", device=0) # GPU ์‚ฌ์šฉ ์„ค์ •
60
-
61
- # FLUX ํŒŒ์ดํ”„๋ผ์ธ ์ดˆ๊ธฐํ™” ๋ถ€๋ถ„ ์ˆ˜์ •
62
- flux_pipe = FluxPipeline.from_pretrained(
63
- "black-forest-labs/FLUX.1-dev",
64
- torch_dtype=torch.bfloat16,
65
- use_auth_token=hf_token # ํ† ํฐ์„ ์‚ฌ์šฉํ•˜์—ฌ ์ธ์ฆ
66
- )
67
- flux_pipe.enable_model_cpu_offload()
68
 
69
- def translate_prompt(prompt):
 
70
  # ํ•œ๊ธ€ ์ž…๋ ฅ ๊ฐ์ง€ ๋ฐ ๋ฒˆ์—ญ
71
  if any('\u3131' <= char <= '\u318E' or '\uAC00' <= char <= '\uD7A3' for char in prompt):
72
  translated = translator(prompt, max_length=512)[0]['translation_text']
73
- return translated
74
- return prompt
75
-
76
- def generate_image_from_text(prompt, seed=0):
77
- translated_prompt = translate_prompt(prompt)
78
- generator = torch.Generator("cpu").manual_seed(seed)
79
- image = flux_pipe(
80
- translated_prompt,
81
- height=576,
82
- width=1024,
83
- guidance_scale=3.5,
84
- num_inference_steps=50,
85
- max_sequence_length=512,
86
- generator=generator
87
- ).images[0]
88
- return image
89
 
90
- import torch
91
-
92
- def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, seed=123, video_length=2, fs=8):
93
- translated_prompt = translate_prompt(prompt)
94
- print(f"Translated prompt: {translated_prompt}")
95
  resolution = (576, 1024)
96
- save_fs = torch.tensor(fs) # fs๋ฅผ tensor๋กœ ๋ณ€ํ™˜
97
  seed_everything(seed)
98
  transform = transforms.Compose([
99
- transforms.Resize(min(resolution), antialias=True),
100
  transforms.CenterCrop(resolution),
101
- ])
102
  torch.cuda.empty_cache()
103
- print('Start:', translated_prompt, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
104
  start = time.time()
105
  if steps > 60:
106
- steps = 60
107
- batch_size = 1
 
108
  channels = model.model.diffusion_model.out_channels
109
- frames = int(video_length * fs)
110
  h, w = resolution[0] // 8, resolution[1] // 8
111
  noise_shape = [batch_size, channels, frames, h, w]
 
 
112
  with torch.no_grad(), torch.cuda.amp.autocast():
113
- text_emb = model.get_learned_conditioning([translated_prompt])
 
 
114
  img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
115
  img_tensor = (img_tensor / 255. - 0.5) * 2
116
- image_tensor_resized = transform(img_tensor).unsqueeze(0) # bchw
117
- z = get_latent_z(model, image_tensor_resized.unsqueeze(2)) #bc,1,hw
 
 
 
 
118
  img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
119
- cond_images = model.embedder(img_tensor.unsqueeze(0)) # blc
 
120
  img_emb = model.image_proj_model(cond_images)
 
121
  imtext_cond = torch.cat([text_emb, img_emb], dim=1)
122
- cond = {"c_crossattn": [imtext_cond], "c_concat": [img_tensor_repeat], "fs": save_fs}
 
 
 
 
123
  batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
 
 
124
  video_path = './output.mp4'
125
- save_videos(batch_samples, './', filenames=['output'], fps=fs)
126
  return video_path
127
 
 
 
 
 
128
 
129
  css = """
130
  .tab-nav {
@@ -163,68 +147,39 @@ css = """
163
  .tab-nav button:nth-child(3) { border-top: 3px solid #f7b731; }
164
  """
165
 
166
- def infer_t2v(prompt, seed=123, steps=50, cfg_scale=7.5, eta=1.0, fs=8, video_length=2):
167
- # ๋จผ์ € ํ…์ŠคํŠธ๋กœ๋ถ€ํ„ฐ ์ด๋ฏธ์ง€๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค
168
- initial_image = generate_image_from_text(prompt, seed)
169
-
170
- # ๊ทธ ๋‹ค์Œ ์ƒ์„ฑ๋œ ์ด๋ฏธ์ง€๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ ๋น„๋””์˜ค๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค
171
- return infer(initial_image, prompt, steps, cfg_scale, eta, seed, video_length, fs)
172
 
173
  with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
174
- gr.Markdown("# ๋ฌด๋น„ ์ŠคํŠœ๋””์˜ค")
175
- with gr.Tab(label='Image Generation'):
176
- with gr.Column():
177
- with gr.Row():
178
- img_input_text = gr.Text(label='Image Generation Prompt')
179
- img_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
180
- img_generate_btn = gr.Button("Generate Image")
181
- with gr.Row():
182
- img_output_image = gr.Image(label="Generated Image")
183
- img_generate_btn.click(
184
- inputs=[img_input_text, img_seed],
185
- outputs=[img_output_image],
186
- fn=generate_image_from_text
187
- )
188
-
189
- with gr.Tab(label='Image to Video Generation'):
190
- with gr.Column():
191
- with gr.Row():
192
- video_input_image = gr.Image(label="Input Image for Video")
193
- video_prompt = gr.Text(label='Video Generation Prompt')
194
- video_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
195
- video_steps = gr.Slider(label="Sampling steps", minimum=1, maximum=50, step=1, value=30)
196
- video_cfg_scale = gr.Slider(label='CFG Scale', minimum=1.0, maximum=15.0, step=0.5, value=7.5)
197
- video_eta = gr.Slider(label='ETA', minimum=0.0, maximum=1.0, step=0.1, value=1.0)
198
- video_fs = gr.Slider(label='FS', minimum=1, maximum=60, step=1, value=10) # fps๋ฅผ fs๋กœ ๋ณ€๊ฒฝ
199
- video_length = gr.Slider(label="Video Length (seconds)", minimum=2, maximum=8, step=1, value=2)
200
- video_generate_btn = gr.Button("Generate Video")
201
- with gr.Row():
202
- video_output = gr.Video(label="Generated Video", autoplay=True, show_share_button=True)
203
- video_generate_btn.click(
204
- inputs=[video_input_image, video_prompt, video_seed, video_steps, video_cfg_scale, video_eta, video_fs, video_length],
205
- outputs=[video_output],
206
- fn=infer
207
- )
208
-
209
- with gr.Tab(label='Text to Video Generation'):
210
  with gr.Column():
211
  with gr.Row():
212
  with gr.Column():
213
- video_prompt = gr.Text(label='Video Generation Prompt')
214
- video_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
215
- video_steps = gr.Slider(label="Sampling steps", minimum=1, maximum=50, step=1, value=30)
216
- video_cfg_scale = gr.Slider(label='CFG Scale', minimum=1.0, maximum=15.0, step=0.5, value=7.5)
217
- video_eta = gr.Slider(label='ETA', minimum=0.0, maximum=1.0, step=0.1, value=1.0)
218
- video_fs = gr.Slider(label='FS', minimum=1, maximum=60, step=1, value=10) # fps๋ฅผ fs๋กœ ๋ณ€๊ฒฝ
219
- video_length = gr.Slider(label="Video Length (seconds)", minimum=2, maximum=8, step=1, value=2)
220
- video_generate_btn = gr.Button("Generate Video")
 
 
 
 
 
 
221
  with gr.Row():
222
- video_output = gr.Video(label="Generated Video", autoplay=True, show_share_button=True)
223
 
224
- video_generate_btn.click(
225
- inputs=[video_prompt, video_seed, video_steps, video_cfg_scale, video_eta, video_fs, video_length],
226
- outputs=[video_output],
227
- fn=infer_t2v
 
228
  )
 
 
 
 
229
 
230
- dynamicrafter_iface.launch(show_api=True)
 
21
  save_videos
22
  )
23
  from transformers import pipeline
 
 
 
 
 
 
 
 
 
 
 
24
 
25
  def download_model():
26
  REPO_ID = 'Doubiiu/DynamiCrafter_1024'
 
33
  hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_1024_v1/', force_download=True)
34
 
35
  download_model()
36
+ ckpt_path='checkpoints/dynamicrafter_1024_v1/model.ckpt'
37
+ config_file='configs/inference_1024_v1.0.yaml'
38
  config = OmegaConf.load(config_file)
39
  model_config = config.pop("model", OmegaConf.create())
40
+ model_config['params']['unet_config']['params']['use_checkpoint']=False
41
  model = instantiate_from_config(model_config)
42
  assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
43
  model = load_model_checkpoint(model, ckpt_path)
 
45
  model = model.cuda()
46
 
47
  # ๋ฒˆ์—ญ ๋ชจ๋ธ ์ดˆ๊ธฐํ™”
48
+ translator = pipeline("translation", model="Helsinki-NLP/opus-mt-ko-en")
 
 
 
 
 
 
 
 
49
 
50
+ @spaces.GPU(duration=300)
51
+ def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123, video_length=2):
52
  # ํ•œ๊ธ€ ์ž…๋ ฅ ๊ฐ์ง€ ๋ฐ ๋ฒˆ์—ญ
53
  if any('\u3131' <= char <= '\u318E' or '\uAC00' <= char <= '\uD7A3' for char in prompt):
54
  translated = translator(prompt, max_length=512)[0]['translation_text']
55
+ prompt = translated
56
+ print(f"Translated prompt: {prompt}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
 
 
 
 
 
58
  resolution = (576, 1024)
59
+ save_fps = 8
60
  seed_everything(seed)
61
  transform = transforms.Compose([
62
+ transforms.Resize(min(resolution)),
63
  transforms.CenterCrop(resolution),
64
+ ])
65
  torch.cuda.empty_cache()
66
+ print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
67
  start = time.time()
68
  if steps > 60:
69
+ steps = 60
70
+
71
+ batch_size=1
72
  channels = model.model.diffusion_model.out_channels
73
+ frames = int(video_length * save_fps) # ๋น„๋””์˜ค ๊ธธ์ด์— ๋”ฐ๋ฅธ ํ”„๋ ˆ์ž„ ์ˆ˜ ๊ณ„์‚ฐ
74
  h, w = resolution[0] // 8, resolution[1] // 8
75
  noise_shape = [batch_size, channels, frames, h, w]
76
+
77
+ # text cond
78
  with torch.no_grad(), torch.cuda.amp.autocast():
79
+ text_emb = model.get_learned_conditioning([prompt])
80
+
81
+ # img cond
82
  img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
83
  img_tensor = (img_tensor / 255. - 0.5) * 2
84
+
85
+ image_tensor_resized = transform(img_tensor) #3,256,256
86
+ videos = image_tensor_resized.unsqueeze(0) # bchw
87
+
88
+ z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
89
+
90
  img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
91
+
92
+ cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
93
  img_emb = model.image_proj_model(cond_images)
94
+
95
  imtext_cond = torch.cat([text_emb, img_emb], dim=1)
96
+
97
+ fs = torch.tensor([fs], dtype=torch.long, device=model.device)
98
+ cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
99
+
100
+ ## inference
101
  batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
102
+ ## b,samples,c,t,h,w
103
+
104
  video_path = './output.mp4'
105
+ save_videos(batch_samples, './', filenames=['output'], fps=save_fps)
106
  return video_path
107
 
108
+ i2v_examples = [
109
+ ['prompts/1024/astronaut04.png', 'a man in an astronaut suit playing a guitar', 30, 7.5, 1.0, 6, 123, 2],
110
+ ]
111
+
112
 
113
  css = """
114
  .tab-nav {
 
147
  .tab-nav button:nth-child(3) { border-top: 3px solid #f7b731; }
148
  """
149
 
 
 
 
 
 
 
150
 
151
  with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
152
+ gr.Markdown("์ด๋ฏธ์ง€๋กœ ์˜์ƒ ์ƒ์„ฑ ํ…Œ์ŠคํŠธ (ํ•œ๊ธ€ ํ”„๋กฌํ”„ํŠธ ์ง€์›)")
153
+ with gr.Tab(label='ImageAnimation_576x1024'):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
  with gr.Column():
155
  with gr.Row():
156
  with gr.Column():
157
+ with gr.Row():
158
+ i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
159
+ with gr.Row():
160
+ i2v_input_text = gr.Text(label='Prompts')
161
+ with gr.Row():
162
+ i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
163
+ i2v_eta = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='ETA', value=1.0, elem_id="i2v_eta")
164
+ i2v_cfg_scale = gr.Slider(minimum=1.0, maximum=15.0, step=0.5, label='CFG Scale', value=7.5, elem_id="i2v_cfg_scale")
165
+ with gr.Row():
166
+ i2v_steps = gr.Slider(minimum=1, maximum=50, step=1, elem_id="i2v_steps", label="Sampling steps", value=30)
167
+ i2v_motion = gr.Slider(minimum=5, maximum=20, step=1, elem_id="i2v_motion", label="FPS", value=8)
168
+ with gr.Row():
169
+ i2v_video_length = gr.Slider(minimum=2, maximum=8, step=1, elem_id="i2v_video_length", label="Video Length (seconds)", value=2)
170
+ i2v_end_btn = gr.Button("Generate")
171
  with gr.Row():
172
+ i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
173
 
174
+ gr.Examples(examples=i2v_examples,
175
+ inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed, i2v_video_length],
176
+ outputs=[i2v_output_video],
177
+ fn = infer,
178
+ cache_examples=True,
179
  )
180
+ i2v_end_btn.click(inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed, i2v_video_length],
181
+ outputs=[i2v_output_video],
182
+ fn = infer
183
+ )
184
 
185
+ dynamicrafter_iface.queue(max_size=12).launch(show_api=True)