diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..c7d9f3332a950355d5a77d85000f05e6f45435ea --- /dev/null +++ b/.gitattributes @@ -0,0 +1,34 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..951c6c6d95daea0122b1c360ba8497512e23e1c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.mp4 +*.ipynb +*__pycache__* \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..72b18dcdc118c6f5be1d45e6641f6578faf18571 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +--- +title: EchoNet Video Diffusion +emoji: 🖤 +colorFrom: gray +colorTo: purple +sdk: gradio +sdk_version: 3.17.0 +app_file: app.py +pinned: false +license: mit +duplicated_from: anon-SGXT/echocardiogram-video-diffusion +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..354eda00797e8e58a694b4af37a03552c9e5f4f8 --- /dev/null +++ b/app.py @@ -0,0 +1,145 @@ +import gradio as gr +import os +from omegaconf import OmegaConf +from imagen_pytorch import Unet3D, ElucidatedImagen, ImagenTrainer, ElucidatedImagenConfig, NullUnet, Imagen +import torch +import numpy as np +import cv2 +from PIL import Image +import torchvision.transforms as T + + +device = "cuda" if torch.cuda.is_available() else "cpu" +exp_path = "model" + +class BetterCenterCrop(T.CenterCrop): + def __call__(self, img): + h = img.shape[-2] + w = img.shape[-1] + dim = min(h, w) + + return T.functional.center_crop(img, dim) + +class ImageLoader: + def __init__(self, path) -> None: + self.path = path + self.all_files = os.listdir(path) + self.transform = T.Compose([ + T.ToTensor(), + BetterCenterCrop((112, 112)), + T.Resize((112, 112)), + ]) + + def get_image(self): + idx = np.random.randint(0, len(self.all_files)) + img = Image.open(os.path.join(self.path, self.all_files[idx])) + return img + +class Context: + def __init__(self, path, device): + self.path = path + self.config_path = os.path.join(path, "config.yaml") + self.weight_path = os.path.join(path, "merged.pt") + + self.config = OmegaConf.load(self.config_path) + + self.config.dataset.num_frames = int(self.config.dataset.fps * self.config.dataset.duration) + + self.im_load = ImageLoader("echo_images") + + unets = [] + for i, (k, v) in enumerate(self.config.unets.items()): + unets.append(Unet3D(**v, lowres_cond=(i>0))) # type: ignore + + imagen_klass = ElucidatedImagen if self.config.imagen.elucidated == True else Imagen + del self.config.imagen.elucidated + imagen = imagen_klass( + unets = unets, + **OmegaConf.to_container(self.config.imagen), # type: ignore + ) + + self.trainer = ImagenTrainer( + imagen = imagen, + **self.config.trainer + ).to(device) + + print("Loading weights from", self.weight_path) + additional_data = self.trainer.load(self.weight_path) + print("Loaded weights from", self.weight_path) + + def reshape_image(self, image): + try: + image = self.im_load.transform(image).multiply(255).byte().permute(1,2,0).numpy() + return image + except: + return None + + def load_random_image(self): + print("Loading random image") + image = self.im_load.get_image() + return image + + def generate_video(self, image, lvef, cond_scale): + print("Generating video") + print(f"lvef: {lvef}, cond_scale: {cond_scale}") + + image = self.im_load.transform(image).unsqueeze(0) + + sample_kwargs = {} + sample_kwargs = { + "text_embeds": torch.tensor([[[lvef/100.0]]]), + "cond_scale": cond_scale, + "cond_images": image, + } + + self.trainer.eval() + with torch.no_grad(): + video = self.trainer.sample( + batch_size=1, + video_frames=self.config.dataset.num_frames, + **sample_kwargs, + use_tqdm = True, + ).detach().cpu() # C x F x H x W + if video.shape[-3:] != (64, 112, 112): + video = torch.nn.functional.interpolate(video, size=(64, 112, 112), mode='trilinear', align_corners=False) + video = video.repeat((1,1,5,1,1)) # make the video loop 5 times - easier to see + uid = np.random.randint(0, 10) # prevent overwriting if multiple users are using the app + path = f"tmp/{uid}.mp4" + video = video.multiply(255).byte().squeeze(0).permute(1, 2, 3, 0).numpy() + out = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*'mp4v'), 32, (112, 112)) + for i in video: + out.write(i) + out.release() + return path + +context = Context(exp_path, device) + +with gr.Blocks(css="style.css") as demo: + + with gr.Row(): + gr.Label("Cardiac Ultrasound Video Generation Demo (paper: 905)") + + with gr.Row(): + with gr.Column(): + with gr.Row(): + with gr.Column(scale=3, variant="panel"): + text = gr.Markdown(value="This is a live demo of our work on cardiac ultrasound video generation. The model is trained on 4-chamber cardiac ultrasound videos and can generate realistic 4-chamber videos given a target Left Ventricle Ejection Fraction. Please, start by sampling a random frame from the pool of 100 images taken from the EchoNet-Dynamic dataset, which will act as the conditional image, representing the anatomy of the video. Then, set the target LVEF, and click the button to generate a video. The process takes 30s to 60s. The model running here corresponds to the 1SCM from the paper. **Click on the video to play it.** ") + with gr.Column(scale=1, min_width="226"): + image = gr.Image(interactive=True) + with gr.Column(scale=1, min_width="226"): + video = gr.Video(interactive=False) + + slider_ef = gr.Slider(minimum=10, maximum=90, step=1, label="Target LVEF", value=60, interactive=True) + slider_cond = gr.Slider(minimum=0, maximum=20, step=1, label="Conditional scale (if set to more than 1, generation time is 60s)", value=1, interactive=True) + + with gr.Row(): + img_btn = gr.Button(value="❶ Get a random cardiac ultrasound image (4Ch)") + run_btn = gr.Button(value="❷ Generate a video (~30s) 🚀") + + image.change(context.reshape_image, inputs=[image], outputs=[image]) + img_btn.click(context.load_random_image, inputs=[], outputs=[image]) + run_btn.click(context.generate_video, inputs=[image, slider_ef, slider_cond], outputs=[video]) + +if __name__ == "__main__": + demo.queue() + demo.launch() \ No newline at end of file diff --git a/echo_images/0X10094BA0A028EAC3.png b/echo_images/0X10094BA0A028EAC3.png new file mode 100644 index 0000000000000000000000000000000000000000..b96d80aaf5695eb20cdbdd1d04af2c223e0c4360 Binary files /dev/null and b/echo_images/0X10094BA0A028EAC3.png differ diff --git a/echo_images/0X1013E8A4864781B.png b/echo_images/0X1013E8A4864781B.png new file mode 100644 index 0000000000000000000000000000000000000000..226738adfee09779739bb5b453de7409eaf7ae58 Binary files /dev/null and b/echo_images/0X1013E8A4864781B.png differ diff --git a/echo_images/0X12B890B1E2E14CC4.png b/echo_images/0X12B890B1E2E14CC4.png new file mode 100644 index 0000000000000000000000000000000000000000..45640d9586fd9d59df2eda03d2e107d75adcb09d Binary files /dev/null and b/echo_images/0X12B890B1E2E14CC4.png differ diff --git a/echo_images/0X13E043A35E3EB490.png b/echo_images/0X13E043A35E3EB490.png new file mode 100644 index 0000000000000000000000000000000000000000..2901aa2eeeff559dade652b66a8f4045b7f077fb Binary files /dev/null and b/echo_images/0X13E043A35E3EB490.png differ diff --git a/echo_images/0X159BDA520C61736A.png b/echo_images/0X159BDA520C61736A.png new file mode 100644 index 0000000000000000000000000000000000000000..9a94f16f3e0ad96ad2d7ecfdb02c470af49927f3 Binary files /dev/null and b/echo_images/0X159BDA520C61736A.png differ diff --git a/echo_images/0X15DA8D60960ABB2B.png b/echo_images/0X15DA8D60960ABB2B.png new file mode 100644 index 0000000000000000000000000000000000000000..48fea3336f50da18f54cc0a8d081dca119cf49f4 Binary files /dev/null and b/echo_images/0X15DA8D60960ABB2B.png differ diff --git a/echo_images/0X16AF26F9A372EEDE.png b/echo_images/0X16AF26F9A372EEDE.png new file mode 100644 index 0000000000000000000000000000000000000000..2f531735f5952ce038336c66fbc81b1cc4b038ad Binary files /dev/null and b/echo_images/0X16AF26F9A372EEDE.png differ diff --git a/echo_images/0X17BC4EF4BF83368B.png b/echo_images/0X17BC4EF4BF83368B.png new file mode 100644 index 0000000000000000000000000000000000000000..0fa2744b4a478c3d7fa43cb14261ed55ef4b9402 Binary files /dev/null and b/echo_images/0X17BC4EF4BF83368B.png differ diff --git a/echo_images/0X1B379931357428C0.png b/echo_images/0X1B379931357428C0.png new file mode 100644 index 0000000000000000000000000000000000000000..f43eab7fc719a369883d208cd1240684559ca360 Binary files /dev/null and b/echo_images/0X1B379931357428C0.png differ diff --git a/echo_images/0X1CDD9C054D8FB60D.png b/echo_images/0X1CDD9C054D8FB60D.png new file mode 100644 index 0000000000000000000000000000000000000000..57ea89b814a1ec74aa412ddf5c69297e1becde68 Binary files /dev/null and b/echo_images/0X1CDD9C054D8FB60D.png differ diff --git a/echo_images/0X1DF7163A74801695.png b/echo_images/0X1DF7163A74801695.png new file mode 100644 index 0000000000000000000000000000000000000000..e00310344930d33272ad6a0fc460ebe9d8dc78d8 Binary files /dev/null and b/echo_images/0X1DF7163A74801695.png differ diff --git a/echo_images/0X20C397F012441121.png b/echo_images/0X20C397F012441121.png new file mode 100644 index 0000000000000000000000000000000000000000..0942bc7ee570ffcb25b27c3912337e2212292610 Binary files /dev/null and b/echo_images/0X20C397F012441121.png differ diff --git a/echo_images/0X22A1A8A656653343.png b/echo_images/0X22A1A8A656653343.png new file mode 100644 index 0000000000000000000000000000000000000000..43ccf0ee34ed555dcfa5d32c21d83dadc36926b4 Binary files /dev/null and b/echo_images/0X22A1A8A656653343.png differ diff --git a/echo_images/0X22D7FDCF2827269E.png b/echo_images/0X22D7FDCF2827269E.png new file mode 100644 index 0000000000000000000000000000000000000000..63a42daaac6927276f3967d49a571589b428b5d2 Binary files /dev/null and b/echo_images/0X22D7FDCF2827269E.png differ diff --git a/echo_images/0X230F00FD0DF5D71C.png b/echo_images/0X230F00FD0DF5D71C.png new file mode 100644 index 0000000000000000000000000000000000000000..50624d296d504954c339b13fe0fbda45f21404a1 Binary files /dev/null and b/echo_images/0X230F00FD0DF5D71C.png differ diff --git a/echo_images/0X244CAB3550320216.png b/echo_images/0X244CAB3550320216.png new file mode 100644 index 0000000000000000000000000000000000000000..ac3d87ba449c702ca51c06f224d416b664e0cb15 Binary files /dev/null and b/echo_images/0X244CAB3550320216.png differ diff --git a/echo_images/0X24FEF7D294B35A5B.png b/echo_images/0X24FEF7D294B35A5B.png new file mode 100644 index 0000000000000000000000000000000000000000..7ef01105e7c728dbd9611432645dbb3d91fdf414 Binary files /dev/null and b/echo_images/0X24FEF7D294B35A5B.png differ diff --git a/echo_images/0X25D970C75A57B3F2.png b/echo_images/0X25D970C75A57B3F2.png new file mode 100644 index 0000000000000000000000000000000000000000..77fb87edb2004275ecb7e249516357d9f0d4bd64 Binary files /dev/null and b/echo_images/0X25D970C75A57B3F2.png differ diff --git a/echo_images/0X277FC348812C0E79.png b/echo_images/0X277FC348812C0E79.png new file mode 100644 index 0000000000000000000000000000000000000000..15ccc9739848361d3ebe27fd08e80c57807de8d7 Binary files /dev/null and b/echo_images/0X277FC348812C0E79.png differ diff --git a/echo_images/0X27836E538BD008A.png b/echo_images/0X27836E538BD008A.png new file mode 100644 index 0000000000000000000000000000000000000000..c81657d3d30d9cf73cab9305f10424e806490eb6 Binary files /dev/null and b/echo_images/0X27836E538BD008A.png differ diff --git a/echo_images/0X2840438B29E95F1F.png b/echo_images/0X2840438B29E95F1F.png new file mode 100644 index 0000000000000000000000000000000000000000..d8e82d5f9ae6de10e87b3e9bb5b834f7f920753c Binary files /dev/null and b/echo_images/0X2840438B29E95F1F.png differ diff --git a/echo_images/0X29A336DCE20541A0.png b/echo_images/0X29A336DCE20541A0.png new file mode 100644 index 0000000000000000000000000000000000000000..f2fe8634e947f2a3858a42a5a0429d3df258af69 Binary files /dev/null and b/echo_images/0X29A336DCE20541A0.png differ diff --git a/echo_images/0X29C81728B50A2E6C.png b/echo_images/0X29C81728B50A2E6C.png new file mode 100644 index 0000000000000000000000000000000000000000..c181f56714d888ea3c32ef35154205571f1e3940 Binary files /dev/null and b/echo_images/0X29C81728B50A2E6C.png differ diff --git a/echo_images/0X2A830BC4A3A36A93.png b/echo_images/0X2A830BC4A3A36A93.png new file mode 100644 index 0000000000000000000000000000000000000000..35a29fe5b0d83e782931d08f372c445ffcd0702f Binary files /dev/null and b/echo_images/0X2A830BC4A3A36A93.png differ diff --git a/echo_images/0X2AD994F98C491FA6.png b/echo_images/0X2AD994F98C491FA6.png new file mode 100644 index 0000000000000000000000000000000000000000..272e13c161a4f3e335b0fe9aafed725611698736 Binary files /dev/null and b/echo_images/0X2AD994F98C491FA6.png differ diff --git a/echo_images/0X2BB766EF1A13DECC.png b/echo_images/0X2BB766EF1A13DECC.png new file mode 100644 index 0000000000000000000000000000000000000000..793163b25737228b10f5b62d9d9dc756ef8d2f74 Binary files /dev/null and b/echo_images/0X2BB766EF1A13DECC.png differ diff --git a/echo_images/0X2DA99F9FC1DAD8A9.png b/echo_images/0X2DA99F9FC1DAD8A9.png new file mode 100644 index 0000000000000000000000000000000000000000..08e9442c67b8eee5493c0c0d813d34f16938bd9f Binary files /dev/null and b/echo_images/0X2DA99F9FC1DAD8A9.png differ diff --git a/echo_images/0X3545F8A008B34ED0.png b/echo_images/0X3545F8A008B34ED0.png new file mode 100644 index 0000000000000000000000000000000000000000..055e1b6fe02a45a704e8f50db3cf73da72f98f10 Binary files /dev/null and b/echo_images/0X3545F8A008B34ED0.png differ diff --git a/echo_images/0X36E4468C9E659B89.png b/echo_images/0X36E4468C9E659B89.png new file mode 100644 index 0000000000000000000000000000000000000000..c6d43863ffafc3897e9806259c43a0b41a1fdca6 Binary files /dev/null and b/echo_images/0X36E4468C9E659B89.png differ diff --git a/echo_images/0X39CA8CC96A5D5E8B.png b/echo_images/0X39CA8CC96A5D5E8B.png new file mode 100644 index 0000000000000000000000000000000000000000..e7e624990fd1fa8b13edc5658681db2d95955a7c Binary files /dev/null and b/echo_images/0X39CA8CC96A5D5E8B.png differ diff --git a/echo_images/0X3B01B7487E3D81EA.png b/echo_images/0X3B01B7487E3D81EA.png new file mode 100644 index 0000000000000000000000000000000000000000..66274d649291c6fd3c66433a47179c0a02d78a00 Binary files /dev/null and b/echo_images/0X3B01B7487E3D81EA.png differ diff --git a/echo_images/0X3B0D2D527C387A0E.png b/echo_images/0X3B0D2D527C387A0E.png new file mode 100644 index 0000000000000000000000000000000000000000..5473521b07c4327aca6bd42fa8a2a116d30957f0 Binary files /dev/null and b/echo_images/0X3B0D2D527C387A0E.png differ diff --git a/echo_images/0X3B54A5459841DCE8.png b/echo_images/0X3B54A5459841DCE8.png new file mode 100644 index 0000000000000000000000000000000000000000..0f535be457400d7b58327a8c6325e9c31f6a2180 Binary files /dev/null and b/echo_images/0X3B54A5459841DCE8.png differ diff --git a/echo_images/0X3B9FBD87EE113D62.png b/echo_images/0X3B9FBD87EE113D62.png new file mode 100644 index 0000000000000000000000000000000000000000..67359c9cf722bf2df099dd1b5ad025235cfb4fbe Binary files /dev/null and b/echo_images/0X3B9FBD87EE113D62.png differ diff --git a/echo_images/0X3BA9F7C9DB0CF55B.png b/echo_images/0X3BA9F7C9DB0CF55B.png new file mode 100644 index 0000000000000000000000000000000000000000..f9e55fc28bb31634903756230321520f3969dca0 Binary files /dev/null and b/echo_images/0X3BA9F7C9DB0CF55B.png differ diff --git a/echo_images/0X3DA2B290B58A6540.png b/echo_images/0X3DA2B290B58A6540.png new file mode 100644 index 0000000000000000000000000000000000000000..668772478567a517352d62c6f96e18ac1c8882c2 Binary files /dev/null and b/echo_images/0X3DA2B290B58A6540.png differ diff --git a/echo_images/0X3E2F182038897EA5.png b/echo_images/0X3E2F182038897EA5.png new file mode 100644 index 0000000000000000000000000000000000000000..7830b9ff3cd2b46bccbaab6113b9d46f8b02489f Binary files /dev/null and b/echo_images/0X3E2F182038897EA5.png differ diff --git a/echo_images/0X3F076329C702F768.png b/echo_images/0X3F076329C702F768.png new file mode 100644 index 0000000000000000000000000000000000000000..807fc1428fd52cf0775181774cb5e32a954e9be4 Binary files /dev/null and b/echo_images/0X3F076329C702F768.png differ diff --git a/echo_images/0X4130EB4CD7ED958B.png b/echo_images/0X4130EB4CD7ED958B.png new file mode 100644 index 0000000000000000000000000000000000000000..00fc8b339e1b4dc8663d930e14f7769f27f18d3c Binary files /dev/null and b/echo_images/0X4130EB4CD7ED958B.png differ diff --git a/echo_images/0X42E8226CA93B7BAC.png b/echo_images/0X42E8226CA93B7BAC.png new file mode 100644 index 0000000000000000000000000000000000000000..d0c6cdfb95c72885ce5d00aa12cfda588af464ed Binary files /dev/null and b/echo_images/0X42E8226CA93B7BAC.png differ diff --git a/echo_images/0X45418C574D97027A.png b/echo_images/0X45418C574D97027A.png new file mode 100644 index 0000000000000000000000000000000000000000..b254407d5242c31593879f756172750a13015792 Binary files /dev/null and b/echo_images/0X45418C574D97027A.png differ diff --git a/echo_images/0X45CE057EC2EB577F.png b/echo_images/0X45CE057EC2EB577F.png new file mode 100644 index 0000000000000000000000000000000000000000..03bf201638d6f89df79c7754976c17af0fb47f62 Binary files /dev/null and b/echo_images/0X45CE057EC2EB577F.png differ diff --git a/echo_images/0X463A7B7D46C6CA4.png b/echo_images/0X463A7B7D46C6CA4.png new file mode 100644 index 0000000000000000000000000000000000000000..86f1ed473ddca913be06523017ceac4ae0399fa1 Binary files /dev/null and b/echo_images/0X463A7B7D46C6CA4.png differ diff --git a/echo_images/0X463C296E8E65DA97.png b/echo_images/0X463C296E8E65DA97.png new file mode 100644 index 0000000000000000000000000000000000000000..848e4cfe8457324aedb3aaeb52667e1620be6543 Binary files /dev/null and b/echo_images/0X463C296E8E65DA97.png differ diff --git a/echo_images/0X46682D67FA3FE237.png b/echo_images/0X46682D67FA3FE237.png new file mode 100644 index 0000000000000000000000000000000000000000..a231082c4d6591c1f79463ff261b28f98ff18d06 Binary files /dev/null and b/echo_images/0X46682D67FA3FE237.png differ diff --git a/echo_images/0X487B52623BC14C25.png b/echo_images/0X487B52623BC14C25.png new file mode 100644 index 0000000000000000000000000000000000000000..fd30c8c9781d7f1e408bff9d70f68c42ea839e11 Binary files /dev/null and b/echo_images/0X487B52623BC14C25.png differ diff --git a/echo_images/0X49110DEED5E41E90.png b/echo_images/0X49110DEED5E41E90.png new file mode 100644 index 0000000000000000000000000000000000000000..7bfb7207a2f244df2a2a5964ae58b35a5349f611 Binary files /dev/null and b/echo_images/0X49110DEED5E41E90.png differ diff --git a/echo_images/0X4937EFDC69F9E448.png b/echo_images/0X4937EFDC69F9E448.png new file mode 100644 index 0000000000000000000000000000000000000000..22da6fccab2a48d7aec444b64d13be8c0b8f758b Binary files /dev/null and b/echo_images/0X4937EFDC69F9E448.png differ diff --git a/echo_images/0X498F356AD182EFAA.png b/echo_images/0X498F356AD182EFAA.png new file mode 100644 index 0000000000000000000000000000000000000000..5db84774a3fc19ed5da59e3ceadd36606c7f44b0 Binary files /dev/null and b/echo_images/0X498F356AD182EFAA.png differ diff --git a/echo_images/0X4E20A4C205622D65.png b/echo_images/0X4E20A4C205622D65.png new file mode 100644 index 0000000000000000000000000000000000000000..5db2aa7045e25e4d4da4dca7662f1efdebb6a575 Binary files /dev/null and b/echo_images/0X4E20A4C205622D65.png differ diff --git a/echo_images/0X4FFC94D07CF61F1D.png b/echo_images/0X4FFC94D07CF61F1D.png new file mode 100644 index 0000000000000000000000000000000000000000..d76e90d1c76f143d7ceba76f28a7db065a14c0f6 Binary files /dev/null and b/echo_images/0X4FFC94D07CF61F1D.png differ diff --git a/echo_images/0X5204CC14B02919D5.png b/echo_images/0X5204CC14B02919D5.png new file mode 100644 index 0000000000000000000000000000000000000000..67ae56d7de791572579640a88b09f12e395e7d04 Binary files /dev/null and b/echo_images/0X5204CC14B02919D5.png differ diff --git a/echo_images/0X5245782BC86D72CF.png b/echo_images/0X5245782BC86D72CF.png new file mode 100644 index 0000000000000000000000000000000000000000..e2f1956101569538250115d38c4577ea52172a80 Binary files /dev/null and b/echo_images/0X5245782BC86D72CF.png differ diff --git a/echo_images/0X52749A08918D145.png b/echo_images/0X52749A08918D145.png new file mode 100644 index 0000000000000000000000000000000000000000..8dc74c5992d106490abe7482969f2d011f8f133e Binary files /dev/null and b/echo_images/0X52749A08918D145.png differ diff --git a/echo_images/0X540FDA7545A0BE0D.png b/echo_images/0X540FDA7545A0BE0D.png new file mode 100644 index 0000000000000000000000000000000000000000..cf4d973f1737a76eee1af2423ec4bf8b91bab2f4 Binary files /dev/null and b/echo_images/0X540FDA7545A0BE0D.png differ diff --git a/echo_images/0X555625C9E3EC6CEC.png b/echo_images/0X555625C9E3EC6CEC.png new file mode 100644 index 0000000000000000000000000000000000000000..bb0ae114863ade9638f468149e0ed1f77785fb2a Binary files /dev/null and b/echo_images/0X555625C9E3EC6CEC.png differ diff --git a/echo_images/0X560279556FB69A44.png b/echo_images/0X560279556FB69A44.png new file mode 100644 index 0000000000000000000000000000000000000000..4df7e82a167829fe23f7dccdaabc7fef69d1638a Binary files /dev/null and b/echo_images/0X560279556FB69A44.png differ diff --git a/echo_images/0X56D0F1AB970BFE4D.png b/echo_images/0X56D0F1AB970BFE4D.png new file mode 100644 index 0000000000000000000000000000000000000000..c8e688c4c0b642cd16714c3b1b1e21400c8681e0 Binary files /dev/null and b/echo_images/0X56D0F1AB970BFE4D.png differ diff --git a/echo_images/0X58F8B32F2F4BB1FC.png b/echo_images/0X58F8B32F2F4BB1FC.png new file mode 100644 index 0000000000000000000000000000000000000000..151dc7cd1d3f8c64272fd6065a13cba8d82d8442 Binary files /dev/null and b/echo_images/0X58F8B32F2F4BB1FC.png differ diff --git a/echo_images/0X590E81D4160A19D3.png b/echo_images/0X590E81D4160A19D3.png new file mode 100644 index 0000000000000000000000000000000000000000..c59cf45845e476ebcd3d3a03358e939f393da38c Binary files /dev/null and b/echo_images/0X590E81D4160A19D3.png differ diff --git a/echo_images/0X5947CC66C2E5AB66.png b/echo_images/0X5947CC66C2E5AB66.png new file mode 100644 index 0000000000000000000000000000000000000000..d742e1dd7965223d3625ef513d8d7f9bce609af2 Binary files /dev/null and b/echo_images/0X5947CC66C2E5AB66.png differ diff --git a/echo_images/0X5C70DA5DA096EBF8.png b/echo_images/0X5C70DA5DA096EBF8.png new file mode 100644 index 0000000000000000000000000000000000000000..b52cea1d32a40b7b94785d33bb45dd93f71cc628 Binary files /dev/null and b/echo_images/0X5C70DA5DA096EBF8.png differ diff --git a/echo_images/0X5C7B3A4A12245C5E.png b/echo_images/0X5C7B3A4A12245C5E.png new file mode 100644 index 0000000000000000000000000000000000000000..011b389e89787561d9d4586a3be7fa93350069a0 Binary files /dev/null and b/echo_images/0X5C7B3A4A12245C5E.png differ diff --git a/echo_images/0X5D3D9A1B411AC0B6.png b/echo_images/0X5D3D9A1B411AC0B6.png new file mode 100644 index 0000000000000000000000000000000000000000..0698cb4c6b5e85cf248224959532711773c36796 Binary files /dev/null and b/echo_images/0X5D3D9A1B411AC0B6.png differ diff --git a/echo_images/0X5D7A013C97D80D53.png b/echo_images/0X5D7A013C97D80D53.png new file mode 100644 index 0000000000000000000000000000000000000000..250599364c0b0fde6766df7bd98d91121fdf2764 Binary files /dev/null and b/echo_images/0X5D7A013C97D80D53.png differ diff --git a/echo_images/0X5DAE4DC1DC55BB33.png b/echo_images/0X5DAE4DC1DC55BB33.png new file mode 100644 index 0000000000000000000000000000000000000000..5d3979c00d121ff20738fd21fcf1e896f79a978e Binary files /dev/null and b/echo_images/0X5DAE4DC1DC55BB33.png differ diff --git a/echo_images/0X5E002446A11D8489.png b/echo_images/0X5E002446A11D8489.png new file mode 100644 index 0000000000000000000000000000000000000000..7991b1ce2965a5126fdd33c20e1917aee5cf9a9a Binary files /dev/null and b/echo_images/0X5E002446A11D8489.png differ diff --git a/echo_images/0X5EDDEA1B2AB83813.png b/echo_images/0X5EDDEA1B2AB83813.png new file mode 100644 index 0000000000000000000000000000000000000000..0ae9b78db09aff854187658608afafe0b211487f Binary files /dev/null and b/echo_images/0X5EDDEA1B2AB83813.png differ diff --git a/echo_images/0X5FBBC76F7AD9FB4D.png b/echo_images/0X5FBBC76F7AD9FB4D.png new file mode 100644 index 0000000000000000000000000000000000000000..531ab75854fb918a2f0b2fae69ca5f169cdffc10 Binary files /dev/null and b/echo_images/0X5FBBC76F7AD9FB4D.png differ diff --git a/echo_images/0X5FD2001CD8BCA24A.png b/echo_images/0X5FD2001CD8BCA24A.png new file mode 100644 index 0000000000000000000000000000000000000000..9268ef80f4c7d67213812da0ef109dbc8c99ea54 Binary files /dev/null and b/echo_images/0X5FD2001CD8BCA24A.png differ diff --git a/echo_images/0X60D51EC3F6430CD.png b/echo_images/0X60D51EC3F6430CD.png new file mode 100644 index 0000000000000000000000000000000000000000..f821ec763aa16bea82cb02dbd9a1abd5e3327d51 Binary files /dev/null and b/echo_images/0X60D51EC3F6430CD.png differ diff --git a/echo_images/0X60EB9960DC4D3AAE.png b/echo_images/0X60EB9960DC4D3AAE.png new file mode 100644 index 0000000000000000000000000000000000000000..062baf947c3a444f8a4615ac6207b5c8c45a8c84 Binary files /dev/null and b/echo_images/0X60EB9960DC4D3AAE.png differ diff --git a/echo_images/0X61AF888022B3E17.png b/echo_images/0X61AF888022B3E17.png new file mode 100644 index 0000000000000000000000000000000000000000..fd64205c94526176cbad12c439d9e51555b99dde Binary files /dev/null and b/echo_images/0X61AF888022B3E17.png differ diff --git a/echo_images/0X61DBD51F03B76427.png b/echo_images/0X61DBD51F03B76427.png new file mode 100644 index 0000000000000000000000000000000000000000..176489982ed6ce8067705045b0f89811e4762949 Binary files /dev/null and b/echo_images/0X61DBD51F03B76427.png differ diff --git a/echo_images/0X638132579FB600C9.png b/echo_images/0X638132579FB600C9.png new file mode 100644 index 0000000000000000000000000000000000000000..0817315d20d49960f15a083fb90e3f3ebf89b9a2 Binary files /dev/null and b/echo_images/0X638132579FB600C9.png differ diff --git a/echo_images/0X65E85D2F82BA872E.png b/echo_images/0X65E85D2F82BA872E.png new file mode 100644 index 0000000000000000000000000000000000000000..f0a09a75ef00eae1615d94c7695ccc799721772c Binary files /dev/null and b/echo_images/0X65E85D2F82BA872E.png differ diff --git a/echo_images/0X6802C4DD3536B4A.png b/echo_images/0X6802C4DD3536B4A.png new file mode 100644 index 0000000000000000000000000000000000000000..d640015ad87eedf91827edc89908c893e2d52d72 Binary files /dev/null and b/echo_images/0X6802C4DD3536B4A.png differ diff --git a/echo_images/0X6930EF64D363CAD9.png b/echo_images/0X6930EF64D363CAD9.png new file mode 100644 index 0000000000000000000000000000000000000000..6007b02dce31f705efd5146c3d5c6e0418a5910f Binary files /dev/null and b/echo_images/0X6930EF64D363CAD9.png differ diff --git a/echo_images/0X6A38F525525AABEF.png b/echo_images/0X6A38F525525AABEF.png new file mode 100644 index 0000000000000000000000000000000000000000..d1c678effc367855e42ecd0961b3ebcf23fb09de Binary files /dev/null and b/echo_images/0X6A38F525525AABEF.png differ diff --git a/echo_images/0X6BD400324E98B517.png b/echo_images/0X6BD400324E98B517.png new file mode 100644 index 0000000000000000000000000000000000000000..2fa96e1d96fd5dfb7474ce52ada40ba9ade7609b Binary files /dev/null and b/echo_images/0X6BD400324E98B517.png differ diff --git a/echo_images/0X6EC16E49F1C9337C.png b/echo_images/0X6EC16E49F1C9337C.png new file mode 100644 index 0000000000000000000000000000000000000000..a9f97dc7571c6a934b86da56a58b0b0dfe661f63 Binary files /dev/null and b/echo_images/0X6EC16E49F1C9337C.png differ diff --git a/echo_images/0X71C8CFCE2B3483F6.png b/echo_images/0X71C8CFCE2B3483F6.png new file mode 100644 index 0000000000000000000000000000000000000000..aaaceca290e1e466d2e4c062eedec391ae67e058 Binary files /dev/null and b/echo_images/0X71C8CFCE2B3483F6.png differ diff --git a/echo_images/0X739DC030F18706BB.png b/echo_images/0X739DC030F18706BB.png new file mode 100644 index 0000000000000000000000000000000000000000..cd45dc15727124498462502aa0a47e693184c07a Binary files /dev/null and b/echo_images/0X739DC030F18706BB.png differ diff --git a/echo_images/0X746492156F4D1DBA.png b/echo_images/0X746492156F4D1DBA.png new file mode 100644 index 0000000000000000000000000000000000000000..7aca5cd509ce0799e9dfef385a08f9041a6109df Binary files /dev/null and b/echo_images/0X746492156F4D1DBA.png differ diff --git a/echo_images/0X7532E58FAD68C533.png b/echo_images/0X7532E58FAD68C533.png new file mode 100644 index 0000000000000000000000000000000000000000..4c9e5dce01ba1e40f627af4054f85f6249cbab57 Binary files /dev/null and b/echo_images/0X7532E58FAD68C533.png differ diff --git a/echo_images/0X77A68EB0325B1DFC.png b/echo_images/0X77A68EB0325B1DFC.png new file mode 100644 index 0000000000000000000000000000000000000000..f55f7518fc8dd4512d11f05b3ad1b83dd638228b Binary files /dev/null and b/echo_images/0X77A68EB0325B1DFC.png differ diff --git a/echo_images/0X77F14DF18E757A6A.png b/echo_images/0X77F14DF18E757A6A.png new file mode 100644 index 0000000000000000000000000000000000000000..b1883ea9cbd4197fe5de7044715127521a56ee5f Binary files /dev/null and b/echo_images/0X77F14DF18E757A6A.png differ diff --git a/echo_images/0X7889C9C82EAA329B.png b/echo_images/0X7889C9C82EAA329B.png new file mode 100644 index 0000000000000000000000000000000000000000..023d0dda75b52dbbbca518234c4a09e3bf9f0097 Binary files /dev/null and b/echo_images/0X7889C9C82EAA329B.png differ diff --git a/echo_images/0X78ED1868FE9D92E.png b/echo_images/0X78ED1868FE9D92E.png new file mode 100644 index 0000000000000000000000000000000000000000..c85acd86a1b30d6c63daf22c37f9c8c5da8ed3e2 Binary files /dev/null and b/echo_images/0X78ED1868FE9D92E.png differ diff --git a/echo_images/0X792F6CC3B5F51F92.png b/echo_images/0X792F6CC3B5F51F92.png new file mode 100644 index 0000000000000000000000000000000000000000..beea18c487c033607c4b25d3538668c3b76b35e0 Binary files /dev/null and b/echo_images/0X792F6CC3B5F51F92.png differ diff --git a/echo_images/0X7CB28DF1B556246B.png b/echo_images/0X7CB28DF1B556246B.png new file mode 100644 index 0000000000000000000000000000000000000000..b04bb450cef4ec7bab04113d590e879936fe6914 Binary files /dev/null and b/echo_images/0X7CB28DF1B556246B.png differ diff --git a/echo_images/0X7E8800EA8C40DF0C.png b/echo_images/0X7E8800EA8C40DF0C.png new file mode 100644 index 0000000000000000000000000000000000000000..3d6bab2a2fdc9b16070792e64ca14add368e10a2 Binary files /dev/null and b/echo_images/0X7E8800EA8C40DF0C.png differ diff --git a/echo_images/0X7ED073E0F80E43D3.png b/echo_images/0X7ED073E0F80E43D3.png new file mode 100644 index 0000000000000000000000000000000000000000..6bf4931597b577da0f14b4df7301d2ceb9be63f3 Binary files /dev/null and b/echo_images/0X7ED073E0F80E43D3.png differ diff --git a/echo_images/0X7F5FDD0D4BE306D8.png b/echo_images/0X7F5FDD0D4BE306D8.png new file mode 100644 index 0000000000000000000000000000000000000000..412df81bb508eb3a7a37103556427754ec003e17 Binary files /dev/null and b/echo_images/0X7F5FDD0D4BE306D8.png differ diff --git a/echo_images/0X7FBE933AD38C3009.png b/echo_images/0X7FBE933AD38C3009.png new file mode 100644 index 0000000000000000000000000000000000000000..3611d00108a2868d0bd960e2199078a03608e227 Binary files /dev/null and b/echo_images/0X7FBE933AD38C3009.png differ diff --git a/echo_images/0X8AE76DB8D580220.png b/echo_images/0X8AE76DB8D580220.png new file mode 100644 index 0000000000000000000000000000000000000000..98b2258ef6894466d4c50fceecc3cb41e51c64cc Binary files /dev/null and b/echo_images/0X8AE76DB8D580220.png differ diff --git a/echo_images/0X8E7B765866B0CA7.png b/echo_images/0X8E7B765866B0CA7.png new file mode 100644 index 0000000000000000000000000000000000000000..e21fa8d3cd8ed98a0204329460133af7c99fdd94 Binary files /dev/null and b/echo_images/0X8E7B765866B0CA7.png differ diff --git a/echo_images/0X93720259E693454.png b/echo_images/0X93720259E693454.png new file mode 100644 index 0000000000000000000000000000000000000000..11a620df6aadcab1b20d228c99a264629f1b1ebd Binary files /dev/null and b/echo_images/0X93720259E693454.png differ diff --git a/echo_images/0X9949F48CD62C825.png b/echo_images/0X9949F48CD62C825.png new file mode 100644 index 0000000000000000000000000000000000000000..def378fc5c6252ffb35f7217a1f5023edebb8ff2 Binary files /dev/null and b/echo_images/0X9949F48CD62C825.png differ diff --git a/echo_images/0XED8409F1DA4738A.png b/echo_images/0XED8409F1DA4738A.png new file mode 100644 index 0000000000000000000000000000000000000000..932bb7a760125511e4f8f943908bef04ab495618 Binary files /dev/null and b/echo_images/0XED8409F1DA4738A.png differ diff --git a/model/config.yaml b/model/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45ea63dc13a3f5dcaeca89a048ce190991e55d7 --- /dev/null +++ b/model/config.yaml @@ -0,0 +1,60 @@ +unets: + unet1: + dim: 64 + num_resnet_blocks: 2 + dim_mults: + - 1 + - 2 + - 4 + max_text_len: 1 + layer_attns: false + layer_cross_attns: + - false + - false + - true + cond_images_channels: 3 + attend_at_middle: false +imagen: + elucidated: True + condition_on_text: true + image_sizes: + - 112 + text_embed_dim: 1 + num_sample_steps: 64 + temporal_downsample_factor: + - 1 + lowres_sample_noise_level: 0.2 + sigma_min: 0.002 + sigma_max: 80 + sigma_data: 0.25 + rho: 7 + P_mean: -1.2 + P_std: 1.2 + S_churn: 160 + S_tmin: 0.05 + S_tmax: 50 + S_noise: 1.003 +trainer: + split_batches: false + lr: 0.0005 + dl_tuple_output_keywords_names: + - images + - text_embeds + - cond_images +dataset: + data_path: / + deactivate_cache: false + fps: 8 + duration: 2.0 + grayscale: false +dataloader: + batch_size: 8 + num_workers: 8 +wandb: + project: "" + entity: "" +checkpoint: + path: / + batch_size: 4 + cond_scale: 5.0 + save_every_x_it: 5000 diff --git a/model/merged.pt b/model/merged.pt new file mode 100644 index 0000000000000000000000000000000000000000..09ed91072ddb3db4cf0bc4d4a1f9c7d182dc1c88 --- /dev/null +++ b/model/merged.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724eb5f0580198979f174dd41ee3747ee8c8cc73ebcbdff973f7682ab00b1e74 +size 300968361 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..82a3cf53a798ec30f3a6c836cbe5c8379ec37269 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +imagen_pytorch==1.21.5 +omegaconf +opencv-python diff --git a/style.css b/style.css new file mode 100644 index 0000000000000000000000000000000000000000..671ceeb261c08db59cabbf628f997e7039ea9802 --- /dev/null +++ b/style.css @@ -0,0 +1,19 @@ +/* div.controls { + visibility: hidden; +} */ + +#component-0 { + width: 70%; + margin: 0 auto; +} + +#component-8, #component-10, div.wrap.svelte-3cbf20, img.svelte-1ku16gh{ + width: 224px!important; + margin: auto!important; + height: 224px!important; +} + +div.controls{ + visibility: hidden; + display: none; +} \ No newline at end of file diff --git a/tmp/.placeholder b/tmp/.placeholder new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391