Update pipeline.py

pipeline.py

@@ -812,7 +812,19 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
         latents = latents.to(device)
         return latents, init_latents
 
- 
+    def prepare_control_latents(self, batch_size, contorl_frames, num_channels_latents, num_frames, height, width, dtype, device):
+            shape = (
+                num_frames,
+                num_channels_latents,
+                height // self.vae_scale_factor,
+                width // self.vae_scale_factor,
+            )
+            
+            # convert input control image array to latents tensor array
+            latents = torch.zeros(shape, dtype=dtype, device=device)
+
+            return latents
+        
     @torch.no_grad()
     # @replace_example_docstring(EXAMPLE_DOC_STRING)
     def __call__(
@@ -1005,16 +1017,25 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
 
         if self.controlnet != None:
             if isinstance(controlnet, ControlNetModel):
-                conditioning_frames = self.prepare_image(
-                    image=conditioning_frames,
-                    width=width,
-                    height=height,
-                    batch_size=batch_size * num_videos_per_prompt * num_frames,
-                    num_images_per_prompt=num_videos_per_prompt,
-                    device=device,
-                    dtype=controlnet.dtype,
-                    do_classifier_free_guidance=self.do_classifier_free_guidance,
-                    guess_mode=guess_mode,
+                # conditioning_frames = self.prepare_image(
+                #     image=conditioning_frames,
+                #     width=width,
+                #     height=height,
+                #     batch_size=batch_size * num_videos_per_prompt * num_frames,
+                #     num_images_per_prompt=num_videos_per_prompt,
+                #     device=device,
+                #     dtype=controlnet.dtype,
+                #     do_classifier_free_guidance=self.do_classifier_free_guidance,
+                #     guess_mode=guess_mode,
+                # )
+                conditioning_frames = self.prepare_control_latents(
+                    num_frames,
+                    conditioning_frames,
+                    num_channels_latents,
+                    height,
+                    width,
+                    prompt_embeds.dtype,
+                    device,
                 )
             elif isinstance(controlnet, MultiControlNetModel):
                 cond_prepared_frames = []