init upload

README.md

@@ -3,53 +3,65 @@ license: apache-2.0
 ---
 
 ```
+from diffusers.utils import load_image, export_to_video
+from transformers import CLIPVisionModel, CLIPImageProcessor, UMT5EncoderModel
+from diffusers import WanI2VPipeline, WanTransformer3DModel
 import torch
-from transformers import AutoTokenizer, UMT5EncoderModel
-from diffusers import AutoencoderKLWan, WanPipeline, WanTransformer3DModel, FlowMatchEulerDiscreteScheduler
-from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
-from diffusers.utils import export_to_video
-from torchvision import transforms
-import os
-import cv2
-os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
-import numpy as np
-
-pretrained_model_name_or_path = "./wan_t2v"
-transformer_t2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer')
+
+pretrained_model_name_or_path = "./wan_i2v"  # TODO replace with our hf id
+image_encoder = CLIPVisionModel.from_pretrained(pretrained_model_name_or_path, subfolder='image_encoder',
+                                                torch_dtype=torch.float16)
+transformer_i2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer_i2v_480p')
+# for 720p
+# transformer_i2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer_i2v_720p',
+#                                                          torch_dtype=torch.bfloat16)
+
+image_processor = CLIPImageProcessor.from_pretrained(pretrained_model_name_or_path, subfolder='image_processor')
 
 text_encoder = UMT5EncoderModel.from_pretrained(pretrained_model_name_or_path, subfolder='text_encoder',
                                                 torch_dtype=torch.bfloat16)
 
-pipe = WanPipeline.from_pretrained(
+pipe = WanI2VPipeline.from_pretrained(
     pretrained_model_name_or_path,
-    transformer=transformer_t2v,
+    transformer=transformer_i2v,
     text_encoder=text_encoder,
+    image_encoder=image_encoder,
+    image_processor=image_processor,
 )
 
-negative_prompt = '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
-
+image = load_image(
+    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg"
+)
 device = "cuda"
 seed = 0
-
+prompt = ("An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in "
+          "the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot.")
 generator = torch.Generator(device=device).manual_seed(seed)
+
+# pipe.to(device)
+pipe.enable_model_cpu_offload()
+
+negative_prompt = '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+
 inputs = {
-    "prompt": "两只拟人化的猫咪身穿舒适的拳击装备，戴着鲜艳的手套，在聚光灯照射的舞台上激烈对战",
+    'image': image,
+    "prompt": prompt,
+    # 'max_area': 720 * 1280, # for 720p
     "negative_prompt": negative_prompt,
+    'max_area': 480 * 832,
     "generator": generator,
-    "num_inference_steps": 50,
-    "flow_shift": 5.0,
+    "num_inference_steps": 40,
     "guidance_scale": 5.0,
-    "height": 720,
-    "width": 1280,
     "num_frames": 81,
     "max_sequence_length": 512,
-    "output_type": "np"
+    "output_type": "np",
+    # 'flow_shift': 5.0, # for 720p
+    'flow_shift': 3.0
 }
 
-pipe.enable_model_cpu_offload()
+output = pipe(**inputs).frames[0]
 
-video = pipe(**inputs).frames[0]
+export_to_video(output, "output.mp4", fps=16)
 
-export_to_video(video, "output.mp4", fps=16)
 
 ```

wan_t2v_fp32_example.py → wan_i2v_example.py

@@ -1,48 +1,59 @@
+from diffusers.utils import load_image, export_to_video
+from transformers import CLIPVisionModel, CLIPImageProcessor, UMT5EncoderModel
+from diffusers import WanI2VPipeline, WanTransformer3DModel
 import torch
-from transformers import AutoTokenizer, UMT5EncoderModel
-from diffusers import AutoencoderKLWan, WanPipeline, WanTransformer3DModel, FlowMatchEulerDiscreteScheduler
-from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
-from diffusers.utils import export_to_video
-from torchvision import transforms
-import os
-import cv2
-os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
-import numpy as np
-
-pretrained_model_name_or_path = "./wan_t2v"
-transformer_t2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer')
+
+pretrained_model_name_or_path = "./wan_i2v"  # TODO replace with our hf id
+image_encoder = CLIPVisionModel.from_pretrained(pretrained_model_name_or_path, subfolder='image_encoder',
+                                                torch_dtype=torch.float16)
+transformer_i2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer_i2v_480p')
+# for 720p
+# transformer_i2v = WanTransformer3DModel.from_pretrained(pretrained_model_name_or_path, subfolder='transformer_i2v_720p',
+#                                                          torch_dtype=torch.bfloat16)
+
+image_processor = CLIPImageProcessor.from_pretrained(pretrained_model_name_or_path, subfolder='image_processor')
 
 text_encoder = UMT5EncoderModel.from_pretrained(pretrained_model_name_or_path, subfolder='text_encoder',
                                                 torch_dtype=torch.bfloat16)
 
-pipe = WanPipeline.from_pretrained(
+pipe = WanI2VPipeline.from_pretrained(
     pretrained_model_name_or_path,
-    transformer=transformer_t2v,
+    transformer=transformer_i2v,
     text_encoder=text_encoder,
+    image_encoder=image_encoder,
+    image_processor=image_processor,
 )
 
-negative_prompt = '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
-
+image = load_image(
+    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg"
+)
 device = "cuda"
 seed = 0
-
+prompt = ("An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in "
+          "the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot.")
 generator = torch.Generator(device=device).manual_seed(seed)
+
+# pipe.to(device)
+pipe.enable_model_cpu_offload()
+
+negative_prompt = '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+
 inputs = {
-    "prompt": "两只拟人化的猫咪身穿舒适的拳击装备，戴着鲜艳的手套，在聚光灯照射的舞台上激烈对战",
+    'image': image,
+    "prompt": prompt,
+    # 'max_area': 720 * 1280, # for 720p
     "negative_prompt": negative_prompt,
+    'max_area': 480 * 832,
     "generator": generator,
-    "num_inference_steps": 50,
-    "flow_shift": 5.0,
+    "num_inference_steps": 40,
     "guidance_scale": 5.0,
-    "height": 720,
-    "width": 1280,
     "num_frames": 81,
     "max_sequence_length": 512,
-    "output_type": "np"
+    "output_type": "np",
+    # 'flow_shift': 5.0, # for 720p
+    'flow_shift': 3.0
 }
 
-pipe.enable_model_cpu_offload()
-
-video = pipe(**inputs).frames[0]
+output = pipe(**inputs).frames[0]
 
-export_to_video(video, "output.mp4", fps=16)
+export_to_video(output, "output.mp4", fps=16)