Duplicate from philschmid/stable-diffusion-2-inpainting-endpoint

Co-authored-by: Philipp Schmid <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,87 @@
+---
+license: openrail++
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- text-guided-to-image-inpainting
+- endpoints-template
+thumbnail: >-
+  https://huggingface.co/philschmid/stable-diffusion-2-inpainting-endpoint/resolve/main/Stable%20Diffusion%20Inference%20endpoints%20-%20inpainting.png
+inference: true
+duplicated_from: philschmid/stable-diffusion-2-inpainting-endpoint
+---
+
+
+# Fork of [stabilityai/stable-diffusion-2-inpainting](https://huggingface.co/stabilityai/stable-diffusion-2-inpainting)
+
+> Stable Diffusion is a latent text-to-image diffusion model capable of generating photo-realistic images given any text input.
+> For more information about how Stable Diffusion functions, please have a look at [🤗's Stable Diffusion with 🧨Diffusers blog](https://huggingface.co/blog/stable_diffusion).
+
+For more information about the model, license and limitations check the original model card at [stabilityai/stable-diffusion-2-inpainting](https://huggingface.co/stabilityai/stable-diffusion-2-inpainting).
+
+---
+
+This repository implements a custom `handler` task for `text-guided-to-image-inpainting` for 🤗 Inference Endpoints. The code for the customized pipeline is in the [handler.py](https://huggingface.co/philschmid/stable-diffusion-2-inpainting-endpoint/blob/main/handler.py).
+
+There is also a [notebook](https://huggingface.co/philschmid/stable-diffusion-2-inpainting-endpoint/blob/main/create_handler.ipynb) included, on how to create the `handler.py`
+
+![thubmnail](Stable%20Diffusion%20Inference%20endpoints%20-%20inpainting.png)
+
+
+### expected Request payload
+
+```json
+{
+    "inputs": "A prompt used for image generation",
+    "image" : "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAABGdBTUEAALGPC",
+    "mask_image": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAABGdBTUEAALGPC",
+}
+```
+
+below is an example on how to run a request using Python and `requests`.
+
+## Run Request 
+```python
+import json
+from typing import List
+import requests as r
+import base64
+from PIL import Image
+from io import BytesIO
+
+ENDPOINT_URL = ""
+HF_TOKEN = ""
+
+# helper image utils
+def encode_image(image_path):
+  with open(image_path, "rb") as i:
+    b64 = base64.b64encode(i.read())
+  return b64.decode("utf-8")
+
+
+def predict(prompt, image, mask_image):
+    image = encode_image(image)
+    mask_image = encode_image(mask_image)
+
+    # prepare sample payload
+    request = {"inputs": prompt, "image": image, "mask_image": mask_image}
+    # headers
+    headers = {
+        "Authorization": f"Bearer {HF_TOKEN}",
+        "Content-Type": "application/json",
+        "Accept": "image/png" # important to get an image back
+    }
+
+    response = r.post(ENDPOINT_URL, headers=headers, json=payload)
+    img = Image.open(BytesIO(response.content))
+    return img
+
+prediction = predict(
+    prompt="Face of a bengal cat, high resolution, sitting on a park bench",
+    image="dog.png",
+    mask_image="mask_dog.png"
+)
+```
+expected output
+
+![sample](result.png)

handler.py

@@ -0,0 +1,69 @@
+from typing import  Dict, List, Any
+import torch
+from diffusers import DPMSolverMultistepScheduler, StableDiffusionInpaintPipeline
+from PIL import Image
+import base64
+from io import BytesIO
+
+
+# set device
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+if device.type != 'cuda':
+    raise ValueError("need to run on GPU")
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        # load StableDiffusionInpaintPipeline pipeline
+        self.pipe = StableDiffusionInpaintPipeline.from_pretrained(path, torch_dtype=torch.float16)
+        # use DPMSolverMultistepScheduler
+        self.pipe.scheduler = DPMSolverMultistepScheduler.from_config(self.pipe.scheduler.config)
+        # move to device        
+        self.pipe = self.pipe.to(device)
+
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+        """
+        :param data: A dictionary contains `inputs` and optional `image` field.
+        :return: A dictionary with `image` field contains image in base64.
+        """
+        inputs = data.pop("inputs", data)
+        encoded_image = data.pop("image", None)
+        encoded_mask_image = data.pop("mask_image", None)
+        
+        # hyperparamters
+        num_inference_steps = data.pop("num_inference_steps", 25)
+        guidance_scale = data.pop("guidance_scale", 7.5)
+        negative_prompt = data.pop("negative_prompt", None)
+        height = data.pop("height", None)
+        width = data.pop("width", None)
+        
+        # process image
+        if encoded_image is not None and encoded_mask_image is not None:
+            image = self.decode_base64_image(encoded_image)
+            mask_image = self.decode_base64_image(encoded_mask_image)
+        else:
+            image = None
+            mask_image = None 
+        
+        # run inference pipeline
+        out = self.pipe(inputs, 
+                        image=image,             
+                        mask_image=mask_image,             
+                        num_inference_steps=num_inference_steps,
+                        guidance_scale=guidance_scale,
+                        num_images_per_prompt=1,
+                        negative_prompt=negative_prompt,
+                        height=height,
+                        width=width
+        )
+            
+        # return first generate PIL image
+        return out.images[0]
+    
+    # helper to decode input image
+    def decode_base64_image(self, image_string):
+        base64_image = base64.b64decode(image_string)
+        buffer = BytesIO(base64_image)
+        image = Image.open(buffer)
+        return image

model_index.json

@@ -0,0 +1,33 @@
+{
+  "_class_name": "StableDiffusionInpaintPipeline",
+  "_diffusers_version": "0.10.2",
+  "feature_extractor": [
+    null,
+    null
+  ],
+  "requires_safety_checker": false,
+  "safety_checker": [
+    null,
+    null
+  ],
+  "scheduler": [
+    "diffusers",
+    "PNDMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

requirements.txt

@@ -0,0 +1 @@
+diffusers==0.10.2

scheduler/scheduler_config.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "PNDMScheduler",
+  "_diffusers_version": "0.10.2",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "trained_betas": null
+}

text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "/home/ubuntu/.cache/huggingface/diffusers/models--stabilityai--stable-diffusion-2-inpainting/snapshots/76b00d76134aca7fc5e7137a469498627ad6b4bf/text_encoder",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float16",
+  "transformers_version": "4.24.0",
+  "vocab_size": 49408
+}

text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f1fce5bf3a7d2f31cddebc1f67ec9b34c1786c5b5804fc9513a4231e8d1bf10
+size 680896215

tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "!",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "name_or_path": "/home/ubuntu/.cache/huggingface/diffusers/models--stabilityai--stable-diffusion-2-inpainting/snapshots/76b00d76134aca7fc5e7137a469498627ad6b4bf/tokenizer",
+  "pad_token": "<|endoftext|>",
+  "special_tokens_map_file": "./special_tokens_map.json",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

unet/config.json

@@ -0,0 +1,47 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.10.2",
+  "_name_or_path": "/home/ubuntu/.cache/huggingface/diffusers/models--stabilityai--stable-diffusion-2-inpainting/snapshots/76b00d76134aca7fc5e7137a469498627ad6b4bf/unet",
+  "act_fn": "silu",
+  "attention_head_dim": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 9,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "sample_size": 64,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": true
+}

unet/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20722649ed12ff926183129d7f2f7957388d03edc79888be6da8e3346ef9e873
+size 1732120805

vae/config.json

@@ -0,0 +1,30 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.10.2",
+  "_name_or_path": "/home/ubuntu/.cache/huggingface/diffusers/models--stabilityai--stable-diffusion-2-inpainting/snapshots/76b00d76134aca7fc5e7137a469498627ad6b4bf/vae",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 512,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

vae/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b0f1843b01fbf820827e257eec40db099cde863f07314ae6ab641298e11fc98
+size 167399505