diff --git a/.gitattributes b/.gitattributes index e6399ba316f4088c46378f54fc368976d4f01c75..19f846cfea6e8d0869d82423a206db6e52aaa6ff 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,10 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text -images filter=lfs diff=lfs merge=lfs -text -normal_images/normal_3.jpg filter=lfs diff=lfs merge=lfs -text -normal_images/normal_4.jpg filter=lfs diff=lfs merge=lfs -text -normal_images/normal_8.jpg filter=lfs diff=lfs merge=lfs -text -dis_images/dis_1.jpg filter=lfs diff=lfs merge=lfs -text -dis_images/dis_8.jpg filter=lfs diff=lfs merge=lfs -text -dis_images/dis_9.jpg filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/app.py b/app.py index 177f372b9d2f84207a2292af980513413f8a3109..b2a47945f8c23277d44e0373b64d4fb824ab74f8 100644 --- a/app.py +++ b/app.py @@ -378,9 +378,7 @@ def run_demo_server(pipe_depth, pipe_normal, pipe_dis, pipe_matting, pipe_seg, p ) filenames = [] - filenames.extend(["depth_anime_%d.jpg" %(i+1) for i in range(7)]) - filenames.extend(["depth_line_%d.jpg" %(i+1) for i in range(6)]) - filenames.extend(["depth_real_%d.jpg" %(i+1) for i in range(24)]) + filenames.extend(["depth_%d.jpg" %(i+1) for i in range(8)]) example_folder = os.path.join(os.path.dirname(__file__), "depth_images") Examples( @@ -426,7 +424,7 @@ def run_demo_server(pipe_depth, pipe_normal, pipe_dis, pipe_matting, pipe_seg, p ) filenames = [] - filenames.extend(["normal_%d.jpg" %(i+1) for i in range(10)]) + filenames.extend(["normal_%d.jpg" %(i+1) for i in range(8)]) # example_folder = "images" # print(os.path.join(example_folder, '1.jpg')) # example_folder = os.path.join(os.path.dirname(__file__), "images") @@ -572,7 +570,7 @@ def run_demo_server(pipe_depth, pipe_normal, pipe_dis, pipe_matting, pipe_seg, p ) filenames = [] - filenames.extend(["seg_%d.jpg" %(i+1) for i in range(5)]) + filenames.extend(["seg_%d.jpg" %(i+1) for i in range(6)]) example_folder = os.path.join(os.path.dirname(__file__), "seg_images") Examples( @@ -618,9 +616,7 @@ def run_demo_server(pipe_depth, pipe_normal, pipe_dis, pipe_matting, pipe_seg, p ) filenames = [] - filenames.extend(["depth_anime_%d.jpg" %(i+1) for i in range(7)]) - filenames.extend(["depth_line_%d.jpg" %(i+1) for i in range(6)]) - filenames.extend(["depth_real_%d.jpg" %(i+1) for i in range(24)]) + filenames.extend(["depth_%d.jpg" %(i+1) for i in range(8)]) example_folder = os.path.join(os.path.dirname(__file__), "depth_images") Examples( @@ -837,13 +833,22 @@ def main(): dtype = torch.float32 variant = None - - unet_depth_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_depth_v2", use_safetensors=True).to(dtype) - unet_normal_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_normal_v2", use_safetensors=True).to(dtype) - unet_dis_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_dis_v2", use_safetensors=True).to(dtype) - unet_matting_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_matting_v2", use_safetensors=True).to(dtype) - unet_disparity_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_disparity_v2", use_safetensors=True).to(dtype) - unet_seg_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/GenPercept-models', subfolder="unet_seg_v2", use_safetensors=True).to(dtype) + + sd21_repo = "stabilityai/stable-diffusion-2-1" + unet_depth_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-depth', subfolder="unet_depth_v2", use_safetensors=True).to(dtype) + unet_normal_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-normal', subfolder="unet_normal_v2", use_safetensors=True).to(dtype) + unet_dis_v1 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-dis', subfolder="unet_dis_v1", use_safetensors=True).to(dtype) + unet_matting_v1 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-matting', subfolder="unet_matting_v1", use_safetensors=True).to(dtype) + unet_disparity_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-disparity', subfolder="unet_disparity_v2", use_safetensors=True).to(dtype) + unet_seg_v2 = CustomUNet2DConditionModel.from_pretrained('guangkaixu/genpercept-seg', subfolder="unet_seg_v2", use_safetensors=True).to(dtype) + + # sd21_repo = "/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/pretrained_weights/stable-diffusion-2-1" + # unet_depth_v2 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_depth_bs8_per_accu_pixel_mse_ssi_grad_loss', subfolder="unet", use_safetensors=True).to(dtype) + # unet_normal_v2 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_normal_bs4_per_accu_768_angular_loss', subfolder="unet", use_safetensors=True).to(dtype) + # unet_dis_v1 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_dis_bs4_per_accu_768_pixel_mse_loss', subfolder="unet", use_safetensors=True).to(dtype) + # unet_matting_v1 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_matting_bs4_per_accu_768_pixel_mse_loss/unet', subfolder="unet", use_safetensors=True).to(dtype) + # unet_disparity_v2 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_disparity_bs4_per_accu_768_pixel_mse_ssi_grad_loss', subfolder="unet", use_safetensors=True).to(dtype) + # unet_seg_v2 = CustomUNet2DConditionModel.from_pretrained('/mnt/nas/share/home/xugk/code/repo_private/genpercept_from_marigold/weights/main_paper/sd21_train_genpercept_exr_1card_ensure_seg_bs4_per_accu_768_pixel_mse_loss', subfolder="unet", use_safetensors=True).to(dtype) scheduler = DDIMSchedulerCustomized.from_pretrained("hf_configs/scheduler_beta_1.0_1.0", subfolder='scheduler') genpercept_pipeline = True @@ -856,39 +861,39 @@ def main(): ) pipe_depth = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_depth_v2, **pre_loaded_dict, + sd21_repo, unet=unet_depth_v2, **pre_loaded_dict, ) pipe_normal = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_normal_v2, **pre_loaded_dict, + sd21_repo, unet=unet_normal_v2, **pre_loaded_dict, ) pipe_dis = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_dis_v2, **pre_loaded_dict, + sd21_repo, unet=unet_dis_v1, **pre_loaded_dict, ) pipe_matting = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_matting_v2, **pre_loaded_dict, + sd21_repo, unet=unet_matting_v1, **pre_loaded_dict, ) pipe_seg = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_seg_v2, **pre_loaded_dict, + sd21_repo, unet=unet_seg_v2, **pre_loaded_dict, ) pipe_disparity = GenPerceptPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1", unet=unet_disparity_v2, **pre_loaded_dict, + sd21_repo, unet=unet_disparity_v2, **pre_loaded_dict, ) - try: - import xformers - pipe_depth.enable_xformers_memory_efficient_attention() - pipe_normal.enable_xformers_memory_efficient_attention() - pipe_dis.enable_xformers_memory_efficient_attention() - pipe_matting.enable_xformers_memory_efficient_attention() - pipe_seg.enable_xformers_memory_efficient_attention() - pipe_disparity.enable_xformers_memory_efficient_attention() - except: - pass # run without xformers + # try: + # import xformers + # pipe_depth.enable_xformers_memory_efficient_attention() + # pipe_normal.enable_xformers_memory_efficient_attention() + # pipe_dis.enable_xformers_memory_efficient_attention() + # pipe_matting.enable_xformers_memory_efficient_attention() + # pipe_seg.enable_xformers_memory_efficient_attention() + # pipe_disparity.enable_xformers_memory_efficient_attention() + # except: + # pass # run without xformers pipe_depth = pipe_depth.to(device) pipe_normal = pipe_normal.to(device) diff --git a/depth_images/depth_1.jpg b/depth_images/depth_1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..52ca746ed5c6304eb8aa99e1d4503919c01098fc --- /dev/null +++ b/depth_images/depth_1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f527e7ad927546b28436a401ed99bdaa1433d98838d46963a490149f584c598 +size 434181 diff --git a/depth_images/depth_2.jpg b/depth_images/depth_2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dc5ff2eb1a060eb5dae14d8ac522863c6b26b9f8 --- /dev/null +++ b/depth_images/depth_2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f271391863d34dd1fc06d6490721ea5723b677cc1088aaf50584588353fd8128 +size 54150 diff --git a/depth_images/depth_3.jpg b/depth_images/depth_3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ee4d0a17a24ad8d78a8301326afd586618de349 --- /dev/null +++ b/depth_images/depth_3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d45db37ad4ec593c2b1d0aab347850785d9d065f2b72a777f412c42b32a81943 +size 417478 diff --git a/depth_images/depth_4.jpg b/depth_images/depth_4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..25238a80b557ba00a55beb6543947031d6a34c9c --- /dev/null +++ b/depth_images/depth_4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79fe38080cce958e20fd661dc29d69b115cbeffddccaf8a6a397f7b3e98f71e7 +size 79438 diff --git a/depth_images/depth_5.jpg b/depth_images/depth_5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..306e862863c657c07424f517a643fc69bf2394a1 --- /dev/null +++ b/depth_images/depth_5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3c52ac509ac4f32f83fb06b1d0f99f8e5a58e75ef5d0e6615416757befd8a23 +size 291864 diff --git a/depth_images/depth_6.jpg b/depth_images/depth_6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..05fc0520445f113b133cceb21e230e215e5b7e97 --- /dev/null +++ b/depth_images/depth_6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da86be40e88f33249ce3d7e31b8e725cdc7c8a7daaf45f2c9349860bb6e5deb +size 131006 diff --git a/depth_images/depth_7.jpg b/depth_images/depth_7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..40d81d6e65aac9ea6088a68d979e9ae900f105fe --- /dev/null +++ b/depth_images/depth_7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91672b78a0c7dd8ccd4f5ca91bceeb35a3420f4ee9b8f3d92800fb15d89978e2 +size 758998 diff --git a/depth_images/depth_8.jpg b/depth_images/depth_8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b6c153d549e11399daad9e0471de93f597d08702 --- /dev/null +++ b/depth_images/depth_8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f120dae6e73922c4fb24e9dc10c6ce8916a6cf9ca554a7ca4ba379d83f384e +size 665486 diff --git a/depth_images/depth_anime_1.jpg b/depth_images/depth_anime_1.jpg deleted file mode 100644 index 20d317be98ae8cc3037cb05a6690760e63fc7c69..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_1.jpg and /dev/null differ diff --git a/depth_images/depth_anime_2.jpg b/depth_images/depth_anime_2.jpg deleted file mode 100644 index a34c85f4f15c1f486cb7afb47c32e64be6e91d5e..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_2.jpg and /dev/null differ diff --git a/depth_images/depth_anime_3.jpg b/depth_images/depth_anime_3.jpg deleted file mode 100644 index 9aa39879efe5d3b33e6634aad4b94d9fb0d5fc0a..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_3.jpg and /dev/null differ diff --git a/depth_images/depth_anime_4.jpg b/depth_images/depth_anime_4.jpg deleted file mode 100644 index ec22082eb20f0737bb8d55fcddb993c801531049..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_4.jpg and /dev/null differ diff --git a/depth_images/depth_anime_5.jpg b/depth_images/depth_anime_5.jpg deleted file mode 100644 index 13c146a985457af60e849d6ce3280f5e1039e7bd..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_5.jpg and /dev/null differ diff --git a/depth_images/depth_anime_6.jpg b/depth_images/depth_anime_6.jpg deleted file mode 100644 index a8bb2f5dd6bc387d1160dc9fdcf9465485331b96..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_6.jpg and /dev/null differ diff --git a/depth_images/depth_anime_7.jpg b/depth_images/depth_anime_7.jpg deleted file mode 100644 index 0e9489f68c2398f19de604cf5e455d158acd2f47..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_anime_7.jpg and /dev/null differ diff --git a/depth_images/depth_line_1.jpg b/depth_images/depth_line_1.jpg deleted file mode 100644 index 7ed91adc133b04d3ac811dea27a674365bff3946..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_1.jpg and /dev/null differ diff --git a/depth_images/depth_line_2.jpg b/depth_images/depth_line_2.jpg deleted file mode 100644 index e0d9c682ec843bfd649ffcd7e3a281fcea4433c7..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_2.jpg and /dev/null differ diff --git a/depth_images/depth_line_3.jpg b/depth_images/depth_line_3.jpg deleted file mode 100644 index 36587cb570b004a31cb4e319cfdec596c90402af..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_3.jpg and /dev/null differ diff --git a/depth_images/depth_line_4.jpg b/depth_images/depth_line_4.jpg deleted file mode 100644 index 95c8b3d24b09ec0b8fca52013724b33ba7c8c7d2..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_4.jpg and /dev/null differ diff --git a/depth_images/depth_line_5.jpg b/depth_images/depth_line_5.jpg deleted file mode 100644 index d4682ad24b5691217e69ea47447a75855764eeb9..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_5.jpg and /dev/null differ diff --git a/depth_images/depth_line_6.jpg b/depth_images/depth_line_6.jpg deleted file mode 100644 index 1a1a470bef6978eb01518ef703617b29ba3c35f3..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_line_6.jpg and /dev/null differ diff --git a/depth_images/depth_real_1.jpg b/depth_images/depth_real_1.jpg deleted file mode 100644 index 5139756237e7807a2cb4826b6bfdcf76d45f1a5a..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_1.jpg and /dev/null differ diff --git a/depth_images/depth_real_10.jpg b/depth_images/depth_real_10.jpg deleted file mode 100644 index 58b81b7c0bac72203da75f8df9066281e7c26c0b..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_10.jpg and /dev/null differ diff --git a/depth_images/depth_real_11.jpg b/depth_images/depth_real_11.jpg deleted file mode 100644 index 1042a68d7ba79ed8948907742374a9c4011dd797..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_11.jpg and /dev/null differ diff --git a/depth_images/depth_real_12.jpg b/depth_images/depth_real_12.jpg deleted file mode 100644 index 749f2e0afdec03e319659c86839b80cc18985608..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_12.jpg and /dev/null differ diff --git a/depth_images/depth_real_13.jpg b/depth_images/depth_real_13.jpg deleted file mode 100644 index 47da423d894a24162d4a0b5293b5bf855474ef23..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_13.jpg and /dev/null differ diff --git a/depth_images/depth_real_14.jpg b/depth_images/depth_real_14.jpg deleted file mode 100644 index a650ef395102ff2560bb569e5cc6db3cd4bc9f60..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_14.jpg and /dev/null differ diff --git a/depth_images/depth_real_15.jpg b/depth_images/depth_real_15.jpg deleted file mode 100644 index 823578ae3485accb4e5474f9daead20e3a479083..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_15.jpg and /dev/null differ diff --git a/depth_images/depth_real_16.jpg b/depth_images/depth_real_16.jpg deleted file mode 100644 index 24fbe8dd8a70758a6e13921f5c5ecc28b88157b9..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_16.jpg and /dev/null differ diff --git a/depth_images/depth_real_17.jpg b/depth_images/depth_real_17.jpg deleted file mode 100644 index 27fa02de4c94d6c95fab432df7e1ed84334f368c..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_17.jpg and /dev/null differ diff --git a/depth_images/depth_real_18.jpg b/depth_images/depth_real_18.jpg deleted file mode 100644 index bbc9b07e9587ac10d12d9ab67b8c1beadc2402e9..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_18.jpg and /dev/null differ diff --git a/depth_images/depth_real_19.jpg b/depth_images/depth_real_19.jpg deleted file mode 100644 index 9f0df138dd3b63e9bb2b4cab27451a77a595ca9a..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_19.jpg and /dev/null differ diff --git a/depth_images/depth_real_2.jpg b/depth_images/depth_real_2.jpg deleted file mode 100644 index 0f8a61117968cb8fb6b826ce670df0d3cace1d9a..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_2.jpg and /dev/null differ diff --git a/depth_images/depth_real_20.jpg b/depth_images/depth_real_20.jpg deleted file mode 100644 index f6cd3d2b1bc787e7e6298929c3c03592c440b63c..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_20.jpg and /dev/null differ diff --git a/depth_images/depth_real_21.jpg b/depth_images/depth_real_21.jpg deleted file mode 100644 index fd409a3a9742d4baf0544bf626b793b7dd551b2b..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_21.jpg and /dev/null differ diff --git a/depth_images/depth_real_22.jpg b/depth_images/depth_real_22.jpg deleted file mode 100644 index d59aab6e414c58574554527f20781cc7688c5062..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_22.jpg and /dev/null differ diff --git a/depth_images/depth_real_23.jpg b/depth_images/depth_real_23.jpg deleted file mode 100644 index dd49e93316867ba0adc09e3387b15628acb3b798..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_23.jpg and /dev/null differ diff --git a/depth_images/depth_real_24.jpg b/depth_images/depth_real_24.jpg deleted file mode 100644 index 494eede22a4bad3316864b94c8cbd88acdf73461..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_24.jpg and /dev/null differ diff --git a/depth_images/depth_real_3.jpg b/depth_images/depth_real_3.jpg deleted file mode 100644 index 35049ef2a4828a836202b3846495989384bf74ab..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_3.jpg and /dev/null differ diff --git a/depth_images/depth_real_4.jpg b/depth_images/depth_real_4.jpg deleted file mode 100644 index 5fb98bd121313a885ddf6d096333bc61c71f8ed8..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_4.jpg and /dev/null differ diff --git a/depth_images/depth_real_5.jpg b/depth_images/depth_real_5.jpg deleted file mode 100644 index 8483a3e37e0e41b8dae3a5d57489d499b5a24293..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_5.jpg and /dev/null differ diff --git a/depth_images/depth_real_6.jpg b/depth_images/depth_real_6.jpg deleted file mode 100644 index d6273078fd906a52be8fb83848433806002547f2..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_6.jpg and /dev/null differ diff --git a/depth_images/depth_real_7.jpg b/depth_images/depth_real_7.jpg deleted file mode 100644 index c203f688a7e0a2b6e2ecaebad2dfaef8e45cd208..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_7.jpg and /dev/null differ diff --git a/depth_images/depth_real_8.jpg b/depth_images/depth_real_8.jpg deleted file mode 100644 index 7d6065b0b3d8921722eef782fe4583cd31f71120..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_8.jpg and /dev/null differ diff --git a/depth_images/depth_real_9.jpg b/depth_images/depth_real_9.jpg deleted file mode 100644 index 30608d73d6def859fdd8d2390431d7f91ca0b034..0000000000000000000000000000000000000000 Binary files a/depth_images/depth_real_9.jpg and /dev/null differ diff --git a/dis_images/dis_10.jpg b/dis_images/dis_10.jpg index a27da5e0a29334af1423ccc8fa4717bee8acf9cb..5746154ca9acc86af5cfc6e30fbe1c7eb6178f65 100644 Binary files a/dis_images/dis_10.jpg and b/dis_images/dis_10.jpg differ diff --git a/dis_images/dis_2.jpg b/dis_images/dis_2.jpg index 6bfb53b86277c737d4ab008a378575c27096f31f..b3f691baea456763d564e48785a367cf9a8818f4 100644 Binary files a/dis_images/dis_2.jpg and b/dis_images/dis_2.jpg differ diff --git a/dis_images/dis_3.jpg b/dis_images/dis_3.jpg index 45dfc52234a55567640a4305470afe5e29160293..4c2147ebc06067c9f3a5c0c73ef5ef22de03f65f 100644 Binary files a/dis_images/dis_3.jpg and b/dis_images/dis_3.jpg differ diff --git a/dis_images/dis_4.jpg b/dis_images/dis_4.jpg index e32865288ecafda1e8fc232c60d99102ba85ebbb..426df854b1f6886492410bff416c225078bea50d 100644 Binary files a/dis_images/dis_4.jpg and b/dis_images/dis_4.jpg differ diff --git a/dis_images/dis_5.jpg b/dis_images/dis_5.jpg index 25ef414622c4110f291887fcf7b4b827d0197ee9..b18a10ef6f1c548743cef85f37ca60937efe8f14 100644 Binary files a/dis_images/dis_5.jpg and b/dis_images/dis_5.jpg differ diff --git a/dis_images/dis_6.jpg b/dis_images/dis_6.jpg index dd8ff240cc04283be776b3e59b6de59d70f2838b..b4cc6edc11a4c037c84c1b0bee24d7318e0c68d4 100644 Binary files a/dis_images/dis_6.jpg and b/dis_images/dis_6.jpg differ diff --git a/dis_images/dis_7.jpg b/dis_images/dis_7.jpg index 1d1346fe8fe5fa37087a7e7e3e9771cf6208f10f..ebeb7b27dd0cb1f2494827045e6be85e7dc0ab39 100644 Binary files a/dis_images/dis_7.jpg and b/dis_images/dis_7.jpg differ diff --git a/genpercept/__pycache__/__init__.cpython-310.pyc b/genpercept/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dee910b0a8df3352b5737203dcc0744130d79ef Binary files /dev/null and b/genpercept/__pycache__/__init__.cpython-310.pyc differ diff --git a/genpercept/__pycache__/genpercept_pipeline.cpython-310.pyc b/genpercept/__pycache__/genpercept_pipeline.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b558a1a421ef3145d8fd450f9b17a42fe930d7f4 Binary files /dev/null and b/genpercept/__pycache__/genpercept_pipeline.cpython-310.pyc differ diff --git a/genpercept/customized_modules/__pycache__/ddim.cpython-310.pyc b/genpercept/customized_modules/__pycache__/ddim.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7665b307612506aedb200e208640ca0f3eab11bd Binary files /dev/null and b/genpercept/customized_modules/__pycache__/ddim.cpython-310.pyc differ diff --git a/genpercept/genpercept_pipeline.py b/genpercept/genpercept_pipeline.py index 8b353b2140ee3df7754851edad4d9c999e4f4c29..121ce0670abd288d99c8d239cb64ddc9f8c9c7a3 100644 --- a/genpercept/genpercept_pipeline.py +++ b/genpercept/genpercept_pipeline.py @@ -41,7 +41,6 @@ from .util.image_util import ( import matplotlib.pyplot as plt from genpercept.models.dpt_head import DPTNeckHeadForUnetAfterUpsampleIdentity -from genpercept.util.image_util import process_normals class GenPerceptOutput(BaseOutput): @@ -310,9 +309,6 @@ class GenPerceptPipeline(DiffusionPipeline): # Clip output range pipe_pred = pipe_pred.clip(0, 1) - if mode == 'normal': - pred_np = process_normals(torch.from_numpy(pred_np)[None]) - # Colorize if color_map is not None: assert self.mode in ['depth', 'disparity'] @@ -520,7 +516,7 @@ class GenPerceptPipeline(DiffusionPipeline): # decode z = self.vae.post_quant_conv(pred_latent) stacked = self.vae.decoder(z) - if self.mode in ['depth', 'matting', 'dis']: + if self.mode in ['depth', 'matting', 'dis', 'disparity']: # mean of output channels stacked = stacked.mean(dim=1, keepdim=True) return stacked diff --git a/genpercept/models/__pycache__/custom_unet.cpython-310.pyc b/genpercept/models/__pycache__/custom_unet.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..068d7fbab06e6f022c07d779e5e7985912403e82 Binary files /dev/null and b/genpercept/models/__pycache__/custom_unet.cpython-310.pyc differ diff --git a/genpercept/models/__pycache__/dpt_head.cpython-310.pyc b/genpercept/models/__pycache__/dpt_head.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff86347455355d64728ff3ceb9245f402b2ffd9e Binary files /dev/null and b/genpercept/models/__pycache__/dpt_head.cpython-310.pyc differ diff --git a/genpercept/util/__pycache__/batchsize.cpython-310.pyc b/genpercept/util/__pycache__/batchsize.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff6fa0583b77e68fd205f767dde024978d8f26da Binary files /dev/null and b/genpercept/util/__pycache__/batchsize.cpython-310.pyc differ diff --git a/genpercept/util/__pycache__/ensemble.cpython-310.pyc b/genpercept/util/__pycache__/ensemble.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a39ee8b0b7904d8fa8272283bd1c761882d6656 Binary files /dev/null and b/genpercept/util/__pycache__/ensemble.cpython-310.pyc differ diff --git a/genpercept/util/__pycache__/image_util.cpython-310.pyc b/genpercept/util/__pycache__/image_util.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13849997634700b2cbf5715d0fce5b7fa2aad139 Binary files /dev/null and b/genpercept/util/__pycache__/image_util.cpython-310.pyc differ diff --git a/genpercept/util/image_util.py b/genpercept/util/image_util.py index 96f0df276e03540ca0326dd0f63b526413c9a3b3..dd2c794b59b00aea9cc19b0461efa33764318092 100644 --- a/genpercept/util/image_util.py +++ b/genpercept/util/image_util.py @@ -25,19 +25,6 @@ import torch from torchvision.transforms import InterpolationMode from torchvision.transforms.functional import resize -def process_normals(input_images:torch.Tensor): - normal_preds = input_images - bsz, d, h, w = normal_preds.shape - normal_preds = normal_preds / (torch.norm(normal_preds, p=2, dim=1).unsqueeze(1)+1e-5) - phi = torch.atan2(normal_preds[:,1,:,:], normal_preds[:,0,:,:]).mean(dim=0) - theta = torch.atan2(torch.norm(normal_preds[:,:2,:,:], p=2, dim=1), normal_preds[:,2,:,:]).mean(dim=0) - normal_pred = torch.zeros((d,h,w)).to(normal_preds) - normal_pred[0,:,:] = torch.sin(theta) * torch.cos(phi) - normal_pred[1,:,:] = torch.sin(theta) * torch.sin(phi) - normal_pred[2,:,:] = torch.cos(theta) - angle_error = torch.acos(torch.clip(torch.cosine_similarity(normal_pred[None], normal_preds, dim=1),-0.999, 0.999)) - normal_idx = torch.argmin(angle_error.reshape(bsz,-1).sum(-1)) - return normal_preds[normal_idx] def colorize_depth_maps( depth_map, min_depth, max_depth, cmap="Spectral", valid_mask=None diff --git a/gradio_patches/__pycache__/examples.cpython-310.pyc b/gradio_patches/__pycache__/examples.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e6f497e39ea848df595f9c531a26c6c40ab9dde Binary files /dev/null and b/gradio_patches/__pycache__/examples.cpython-310.pyc differ diff --git a/matting_images/matting_0.jpg b/matting_images/matting_0.jpg index 9c447fe6013a15547bdd84bbc8cb9071d4e78a2c..3be5eb98b2672d648e58bcbfff7f585d598c3efd 100644 Binary files a/matting_images/matting_0.jpg and b/matting_images/matting_0.jpg differ diff --git a/matting_images/matting_1.jpg b/matting_images/matting_1.jpg index 9eaf5940a344aff8093e5b2a52eb5d4817f4222d..ddcfc022e0ff6e0ba03dbd6cf3d8ce8338aba650 100644 Binary files a/matting_images/matting_1.jpg and b/matting_images/matting_1.jpg differ diff --git a/matting_images/matting_2.jpg b/matting_images/matting_2.jpg index 92bfadffc099049dd1285016e56e64c37cf6d10a..9e4fa20334181700e9e66819e8e09f1666ad0ec3 100644 Binary files a/matting_images/matting_2.jpg and b/matting_images/matting_2.jpg differ diff --git a/matting_images/matting_3.jpg b/matting_images/matting_3.jpg index 1dae380adba0fa09d56c73c7f4814c454ee85912..e7ee57ecea2c3110f74ba020f401508c4fabd5ca 100644 Binary files a/matting_images/matting_3.jpg and b/matting_images/matting_3.jpg differ diff --git a/matting_images/matting_4.jpg b/matting_images/matting_4.jpg index a5ff2483786cc9eb4f75a793067924c8cbe1a802..aa534d1f036d5819d8b210523df1fd825589e1b2 100644 Binary files a/matting_images/matting_4.jpg and b/matting_images/matting_4.jpg differ diff --git a/normal_images/normal_1.jpg b/normal_images/normal_1.jpg index 20d317be98ae8cc3037cb05a6690760e63fc7c69..52ca746ed5c6304eb8aa99e1d4503919c01098fc 100644 Binary files a/normal_images/normal_1.jpg and b/normal_images/normal_1.jpg differ diff --git a/normal_images/normal_10.jpg b/normal_images/normal_10.jpg deleted file mode 100644 index f6cd3d2b1bc787e7e6298929c3c03592c440b63c..0000000000000000000000000000000000000000 Binary files a/normal_images/normal_10.jpg and /dev/null differ diff --git a/normal_images/normal_2.jpg b/normal_images/normal_2.jpg index a34c85f4f15c1f486cb7afb47c32e64be6e91d5e..dc5ff2eb1a060eb5dae14d8ac522863c6b26b9f8 100644 Binary files a/normal_images/normal_2.jpg and b/normal_images/normal_2.jpg differ diff --git a/normal_images/normal_3.jpg b/normal_images/normal_3.jpg index 013a1bfa9427b93674c43ac5c7e971dab9c7ac65..3ee4d0a17a24ad8d78a8301326afd586618de349 100644 --- a/normal_images/normal_3.jpg +++ b/normal_images/normal_3.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c13277ad6a3ff0a767f59f1ac6183607800ed0068cd1a1452ea1c9c4ad49ea08 -size 1215526 +oid sha256:d45db37ad4ec593c2b1d0aab347850785d9d065f2b72a777f412c42b32a81943 +size 417478 diff --git a/normal_images/normal_4.jpg b/normal_images/normal_4.jpg index 4b26168bfce0fb7138d9b55cb735825bb0897424..25238a80b557ba00a55beb6543947031d6a34c9c 100644 --- a/normal_images/normal_4.jpg +++ b/normal_images/normal_4.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1bf8317ae6d1ef6c412ff233bda2944568aa7e9169606ed5f694d9c2bc1032f9 -size 1208304 +oid sha256:79fe38080cce958e20fd661dc29d69b115cbeffddccaf8a6a397f7b3e98f71e7 +size 79438 diff --git a/normal_images/normal_5.jpg b/normal_images/normal_5.jpg index 9f0df138dd3b63e9bb2b4cab27451a77a595ca9a..306e862863c657c07424f517a643fc69bf2394a1 100644 Binary files a/normal_images/normal_5.jpg and b/normal_images/normal_5.jpg differ diff --git a/normal_images/normal_6.jpg b/normal_images/normal_6.jpg index 13c146a985457af60e849d6ce3280f5e1039e7bd..05fc0520445f113b133cceb21e230e215e5b7e97 100644 Binary files a/normal_images/normal_6.jpg and b/normal_images/normal_6.jpg differ diff --git a/normal_images/normal_7.jpg b/normal_images/normal_7.jpg index a8bb2f5dd6bc387d1160dc9fdcf9465485331b96..40d81d6e65aac9ea6088a68d979e9ae900f105fe 100644 Binary files a/normal_images/normal_7.jpg and b/normal_images/normal_7.jpg differ diff --git a/normal_images/normal_8.jpg b/normal_images/normal_8.jpg index 9b8e4774af2b85eb88e3ef0638a854a6d72e1690..b6c153d549e11399daad9e0471de93f597d08702 100644 --- a/normal_images/normal_8.jpg +++ b/normal_images/normal_8.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aa927a7f63005681bee068d913dd5327adf81e88229787ff9e0f06549955401e -size 1142609 +oid sha256:b9f120dae6e73922c4fb24e9dc10c6ce8916a6cf9ca554a7ca4ba379d83f384e +size 665486 diff --git a/normal_images/normal_9.jpg b/normal_images/normal_9.jpg deleted file mode 100644 index d4682ad24b5691217e69ea47447a75855764eeb9..0000000000000000000000000000000000000000 Binary files a/normal_images/normal_9.jpg and /dev/null differ diff --git a/seg_images/seg_1.jpg b/seg_images/seg_1.jpg index e9c2e3eaa7bf17caa4a90a380ed4661a9ae04242..9878a2ec138a8f94dda097cf730c8b472f33b33f 100644 Binary files a/seg_images/seg_1.jpg and b/seg_images/seg_1.jpg differ diff --git a/seg_images/seg_2.jpg b/seg_images/seg_2.jpg index 8eaf6749c3ae175592fbeaab124d5baa0b8564b1..403569abe66bec92b8611d09a6d9b78ed3f6a91e 100644 Binary files a/seg_images/seg_2.jpg and b/seg_images/seg_2.jpg differ diff --git a/seg_images/seg_3.jpg b/seg_images/seg_3.jpg index 7d123b61c16ecf03bb2f09a74a0f426183bf4996..d0ffd58d62d32a118cde63d8fe24bcb7b8f5286c 100644 Binary files a/seg_images/seg_3.jpg and b/seg_images/seg_3.jpg differ diff --git a/seg_images/seg_4.jpg b/seg_images/seg_4.jpg index 996c13935835ce4d7abbe0556a3d444b8c368dcc..bed09e69416e57813ece5dc94f6a4b6ecaf5674a 100644 Binary files a/seg_images/seg_4.jpg and b/seg_images/seg_4.jpg differ diff --git a/seg_images/seg_5.jpg b/seg_images/seg_5.jpg index 526971eb42a956d36e877880fad134beb8aae520..32729d3e5d49a5e0ce316879de8e6ef7c2473458 100644 Binary files a/seg_images/seg_5.jpg and b/seg_images/seg_5.jpg differ diff --git a/seg_images/seg_6.jpg b/seg_images/seg_6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..408669ea42837018d0998ed824450e8aefda1012 --- /dev/null +++ b/seg_images/seg_6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487b6dc05a6a85ac2be46e86f3c48d2cbf92a85fd68e1e36c15daa1f28e69243 +size 564209