Init Commits

README.md

@@ -1,3 +1,73 @@
----
-license: creativeml-openrail-m
----
+---
+language:
+- en
+license: creativeml-openrail-m
+library_name: diffusers
+tags:
+- art
+- diffusion
+- Interior
+---
+
+# KuJiaLe Layout ControlNet
+The models are not permitted for commercial usage. For inquiries regarding business, commercial licensing, custom models, and consultation, please contact [[email protected]](mailto:[email protected]).
+
+The model is trained on [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) for interior designs.
+
+### Layout ControlNet Example
+Keep the room layout consistent, re-furnish the room.
+<table style="table-layout: fixed;">
+  <tr>
+    <td style="text-align: center; vertical-align: middle; width: 50%"> <img src="https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_input.jpg"  alt="Input" width="100%" style="display: block; margin-left: auto; margin-right: auto;"></td>
+    <td style="text-align: center; vertical-align: middle; width: 50%"> <img src="https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_output.jpg" alt="Output" width="100%" style="display: block; margin-left: auto; margin-right: auto;"></td>
+   </tr> 
+   <tr>
+      <td style="text-align: center; vertical-align: middle; width: 50%">Input</td>
+      <td style="text-align: center; vertical-align: middle; width: 50%">Output</td>
+  </td>
+  </tr>
+</table>
+
+## News🔥🔥🔥
+
+* May.30, 2024. Our checkpoint Layout-ControlNet are publicly available on [HuggingFace Repo](https://huggingface.co/kujiale-ai/controlnet-layout).
+
+<!-- ## Try our Hugging Face demos:  -->
+
+## Checkpoints
+* `control_v1_sd15_layout_fp16`: Layout ControlNet checkpoint, for SD15 models.
+
+## Using in 🧨 diffusers
+### Layout ControlNet
+```python
+import torch
+from diffusers.utils import load_image
+import numpy as np
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline, UniPCMultistepScheduler
+
+controlnet_checkpoint = "kujiale-ai/controlnet-layout"
+# Load original image
+image = load_image("https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_input.jpg")
+depth_image = load_image("https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_depth.jpg").convert("L")
+normal_image = load_image("https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_normal.jpg")
+segm_image = load_image("https://huggingface.co/kujiale-ai/controlnet-layout/resolve/main/examples/layout_segm.jpg")
+W, H = image.size
+depth_image = depth_image.resize((W, H))
+normal_image = normal_image.resize((W, H))
+segm_image = segm_image.resize((W, H))
+# Prepare Layout Control Image
+depth_image = np.array(depth_image, dtype=np.float32) / 255.0
+depth_image = torch.from_numpy(depth_image[:, :, None])[None].permute(0, 3, 1, 2)
+normal_image = np.array(normal_image, dtype=np.float32)
+normal_image = normal_image / 127.5 - 1.0
+normal_image = torch.from_numpy(normal_image)[None].permute(0, 3, 1, 2)
+segm_image = np.array(segm_image, dtype=np.float32) / 255.0
+segm_image = torch.from_numpy(segm_image)[None].permute(0, 3, 1, 2)
+control_image = torch.cat([depth_image, normal_image, segm_image], dim=1)
+# Initialize pipeline
+controlnet = ControlNetModel.from_pretrained(controlnet_checkpoint, subfolder="control_v1_sd15_layout_fp16", torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16).to("cuda")
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+image = pipe("A modern bedroom,best quality", num_inference_steps=30, image=control_image, guidance_scale=7).images[0]
+image.save('layout_output.jpg')
+```

control_v1_sd15_layout_fp16/config.json

@@ -0,0 +1,51 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.27.2",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_channels": 7,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "global_pool_conditions": false,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "transformer_layers_per_block": 1,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}