Duplicate from a-r-r-o-w/animatediff-motion-adapter-sdxl-beta

Co-authored-by: Aryan V S <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,58 @@
+---
+license: apache-2.0
+library_name: diffusers
+---
+
+AnimateDiff original author checkpoints are available at: https://huggingface.co/guoyww
+
+This checkpoint was converted to Diffusers format by [a-r-r-o-w](https://github.com/a-r-r-o-w/). You can find results and more details adding AnimateDiff SDXL support (beta) to 🤗 Diffusers [here](https://github.com/huggingface/diffusers/pull/6721) The following description is copied from [here](https://huggingface.co/guoyww/animatediff-motion-adapter-v1-5-2).
+
+AnimateDiff is a method that allows you to create videos using pre-existing Stable Diffusion Text to Image models.
+
+It achieves this by inserting motion module layers into a frozen text to image model and training it on video clips to extract a motion prior. These motion modules are applied after the ResNet and Attention blocks in the Stable Diffusion UNet. Their purpose is to introduce coherent motion across image frames. To support these modules we introduce the concepts of a MotionAdapter and UNetMotionModel. These serve as a convenient way to use these motion modules with existing Stable Diffusion models.
+
+Note: The SDXL checkpoint for AnimateDiff is a beta version.
+
+### Usage
+
+```python
+import torch
+from diffusers import AnimateDiffSDXLPipeline
+from diffusers.schedulers import DDIMScheduler, EulerDiscreteScheduler, DEISMultistepScheduler
+from diffusers.models import MotionAdapter
+from diffusers.utils import export_to_gif
+
+model_id = "stabilityai/stable-diffusion-xl-base-1.0"
+adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-sdxl-beta", torch_dtype=torch.float16)
+scheduler = DDIMScheduler.from_pretrained(
+    model_id,
+    subfolder="scheduler",
+    clip_sample=False,
+    timestep_spacing="linspace",
+    beta_schedule="linear",
+    steps_offset=1,
+)
+pipe = AnimateDiffSDXLPipeline.from_pretrained(
+    model_id,
+    motion_adapter=adapter,
+    scheduler=scheduler,
+    torch_dtype=torch.float16,
+    variant="fp16",
+).to("cuda")
+
+# enable memory savings
+pipe.enable_vae_slicing()
+pipe.enable_vae_tiling()
+
+result = pipe(
+    prompt="a panda surfing in the ocean, realistic, hyperrealism, high quality",
+    negative_prompt="low quality, worst quality",
+    num_inference_steps=20,
+    guidance_scale=8,
+    width=1024,
+    height=1024,
+    num_frames=16,
+)
+
+export_to_gif(result.frames[0], "animation.gif")
+```

config.json

@@ -0,0 +1,15 @@
+{
+  "_class_name": "MotionAdapter",
+  "_diffusers_version": "0.26.0.dev0",
+  "block_out_channels": [
+    320,
+    640,
+    1280
+  ],
+  "motion_layers_per_block": 2,
+  "motion_max_seq_length": 32,
+  "motion_mid_block_layers_per_block": 1,
+  "motion_norm_num_groups": 32,
+  "motion_num_attention_heads": 8,
+  "use_motion_mid_block": false
+}

diffusion_pytorch_model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24c3c5f48006ce2ce7b06188622865c620b2d33db23b1af671cc1f21716b5826
+size 474328896

diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff83626d91ee729c4352dd29bffca4f2cc3fd2a52999998764c35736a2ca4695
+size 948604184