Upload 41 files

.gitattributes

@@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/pipeline.png filter=lfs diff=lfs merge=lfs -text
+assets/poses/p1_1.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p1_2.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p2.png filter=lfs diff=lfs merge=lfs -text
+assets/poses/p3_1.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p3_2.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p4.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p5.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/poses/p6.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/0.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/12.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/13.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/21.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/27.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/40.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/43.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/44.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/49.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/51.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/man/52.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/0.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/16.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/26.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/35.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/38.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/53.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/59.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/66.jpg filter=lfs diff=lfs merge=lfs -text
+assets/subjects/woman/99.jpg filter=lfs diff=lfs merge=lfs -text
+assets/teaser.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,14 +1,18 @@
 ---
-title: ID Patch SDXL
-emoji: 👁
-colorFrom: yellow
+title: ID-Patch
+emoji: 📸
+colorFrom: red
 colorTo: indigo
 sdk: gradio
-sdk_version: 5.27.1
+sdk_version: 5.23.1
 app_file: app.py
-pinned: false
+pinned: true
 license: apache-2.0
 short_description: Robust ID Association for Group Photo Personalization.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+The images used in this demo are sourced from consented subjects or generated by the models. These pictures are intended solely to show the capabilities of our research. If you have any concerns, please contact us, and we will promptly remove any inappropriate content.
+
+The use of the released code, model, and demo must strictly adhere to the respective licenses. The code in this demo is licensed under the [Apache License 2.0](./LICENSE), and our model is released under the [Creative Commons Attribution-NonCommercial 4.0 International Public License](https://creativecommons.org/licenses/by-nc/4.0/legalcode) for academic research purposes only. Any manual or automatic downloading of the face models from [InsightFace](https://github.com/deepinsight/insightface), the [FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) base model, LoRAs ([Realism](https://civitai.com/models/631986?modelVersionId=706528) and [Anti-blur](https://civitai.com/models/675581/anti-blur-flux-lora)), *etc.*, must follow their original licenses and be used only for academic research purposes.
+
+This research aims to positively impact the field of Generative AI. Any usage of this method must be responsible and comply with local laws. The developers do not assume any responsibility for any potential misuse. We added the "AI Generated" watermark for enhanced safety.

app.py

@@ -0,0 +1,557 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import gradio as gr
+import huggingface_hub
+import pillow_avif
+import spaces
+import gc
+from huggingface_hub import snapshot_download
+from pillow_heif import register_heif_opener
+from PIL import Image, ImageDraw, ImageFont
+
+from pathlib import Path
+import numpy as np
+import cv2
+import tensorflow as tf
+from mtcnn import MTCNN
+from insightface.utils import face_align
+import facexlib
+import torch
+from modules.inferencer import IDPatchInferencer
+from rtmlib import Body
+from utils.draw_condition import draw_openpose_from_mmpose
+
+# Register HEIF support for Pillow
+register_heif_opener()
+
+loaded_pipeline_config = {
+    'pipeline': None,
+    'face_encoder': None,
+    'face_detector': None
+}
+
+body_estimator = Body(to_openpose=False, mode='balanced', backend='onnxruntime', device='cpu')
+
+def pil_to_cv2(pil_image):
+    """PIL.Image -> OpenCV BGR Image"""
+    cv2_image = np.array(pil_image)
+    cv2_image = cv2.cvtColor(cv2_image, cv2.COLOR_RGB2BGR)
+    return cv2_image
+
+def mtcnn_to_kps(mtcnn_results):
+    kps = np.array([mtcnn_results[0]['keypoints']['left_eye'], mtcnn_results[0]['keypoints']['right_eye'], mtcnn_results[0]['keypoints']['nose'], mtcnn_results[0]['keypoints']['mouth_left'], mtcnn_results[0]['keypoints']['mouth_right']])
+    return kps
+
+def extract_face_emb(arcface_encoder, cropped_face):
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    face_image = torch.from_numpy(cropped_face).unsqueeze(0).permute(0,3,1,2) / 255.
+    face_image = 2 * face_image - 1
+    face_image = face_image.to(device).contiguous()
+    face_emb = arcface_encoder(face_image)[0]
+    return face_emb
+
+
+def download_models():
+    snapshot_download(repo_id='ByteDance/ID-Patch', revision="5e5434dc43a8d1325aade8b0da65d96d7c4cf3d9", local_dir='./models/ID-Patch', local_dir_use_symlinks=False)
+    snapshot_download(repo_id='RunDiffusion/Juggernaut-X-v10', revision="main", local_dir='./models/Juggernaut-X-v10', local_dir_use_symlinks=False)
+
+
+def init_pipeline():
+    pipeline = loaded_pipeline_config['pipeline']
+    gc.collect()
+
+    model_path = f'./models/ID-Patch'
+    print(f'loading model from {model_path}')
+
+    pipeline = IDPatchInferencer(base_model_path='./models/Juggernaut-X-v10', idp_model_path='./models/ID-Patch')
+    
+    loaded_pipeline_config['pipeline'] = pipeline
+    return pipeline
+
+# Future works: Add more model variants 
+def prepare_pipeline():
+    pipeline = loaded_pipeline_config['pipeline']
+    return pipeline
+
+
+def add_safety_watermark(image, text='AI Generated: ID-Patch', font_path=None):
+    width, height = image.size
+    draw = ImageDraw.Draw(image)
+
+    font_size = int(height * 0.028)
+    if font_path:
+        font = ImageFont.truetype(font_path, font_size)
+    else:
+        font = ImageFont.load_default(size=font_size)
+    
+    text_bbox = draw.textbbox((0, 0), text, font=font)
+    text_width, text_height = text_bbox[2] - text_bbox[0], text_bbox[3] - text_bbox[1]
+    x = width - text_width - 10
+    y = height - text_height - 20
+
+    shadow_offset = 2
+    shadow_color = "black"
+    draw.text((x + shadow_offset, y + shadow_offset), text, font=font, fill=shadow_color)
+
+    draw.text((x, y), text, font=font, fill="white")
+
+    return image
+
+
+@spaces.GPU(duration=60)
+def generate_image(
+    id_images, 
+    id_order,
+    control_image, 
+    prompt, 
+    seed, 
+    guidance_scale, 
+    num_steps, 
+    controlnet_conditioning_scale,
+    id_injection_ratio,
+    negative_prompt
+):
+    try:
+        print("======= Start Generating =======")
+        print(f"ID Images uploaded: {len(id_images)}")
+
+        id_images_pil = []
+        for idx, img in enumerate(id_images):
+            img = img[0]
+            print(f"ID Image {idx} size: {img.size}")
+            id_images_pil.append(img)
+        id_images = id_images_pil
+
+        if id_order is not None and id_order.strip() != "":
+            id_order = id_order.split(',')
+            sorted_id_images = [id_images[int(i)] for i in id_order]
+            id_images = sorted_id_images
+        else:
+            id_order = [i for i in range(len(id_images))]
+
+        print(f"Control Image size: {control_image.size}")
+
+        pipeline = prepare_pipeline()
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        arcface_encoder = facexlib.recognition.init_recognition_model('arcface', device=device)
+        tf.config.set_visible_devices([], 'GPU')
+        mtcnn_inferencer = MTCNN() # MTCNN might be slow, could be replaced by other face detectors, as long as it provides 5 keypoints
+
+        if seed == 0:
+            seed = torch.seed() & 0xFFFFFFFF
+
+        face_embs = []
+        for subject in id_images:
+            image_subject = pil_to_cv2(subject)
+
+            mtcnn_subject = mtcnn_inferencer.detect_faces(image_subject[:,:,::-1])
+            if not mtcnn_subject:
+                print("Warning: No face detected in uploaded identity image.")
+                continue  # skip this image
+
+            try:
+                kps_subject = mtcnn_to_kps(mtcnn_subject)
+                cropped_face_subject = face_align.norm_crop(image_subject, landmark=kps_subject, image_size=112)
+                emb = extract_face_emb(arcface_encoder, cropped_face_subject)
+                face_embs.append(emb)
+            except Exception as e:
+                print(f"Error processing face: {e}")
+                continue
+
+        if len(face_embs) == 0:
+            raise ValueError("No valid face embeddings extracted. Please upload clear identity images.")
+        
+        face_embs = torch.stack(face_embs)
+
+        # load pose
+        image_reference = pil_to_cv2(control_image)
+
+        # estimate pose
+        keypoints, scores = body_estimator(image_reference)
+        
+        # Check 
+        print(f"Keypoints raw output: {keypoints}")
+        if keypoints is None:
+            raise ValueError("Keypoints is None.")
+        if not isinstance(keypoints, (list, np.ndarray)):
+            raise ValueError(f"Keypoints type wrong: {type(keypoints)}")
+        if len(keypoints) == 0:
+            raise ValueError("Keypoints length == 0.")
+
+        keypoints = np.array(keypoints)
+        print(f"Keypoints converted to np.array, shape = {keypoints.shape}")
+
+        if len(keypoints.shape) != 3:
+            raise ValueError(f"Keypoints wrong shape: {keypoints.shape}")
+        if keypoints.shape[0] == 0:
+            raise ValueError("No people detected in the pose control image.")
+
+        face_locations = keypoints[:, 0]
+        face_locations = sorted(face_locations, key=lambda x: x[0] if isinstance(x, (list, tuple, np.ndarray)) and len(x) > 0 else 0)
+        face_locations = torch.from_numpy(np.stack(face_locations))
+        
+        # Draw OpenPose image
+        control_image = Image.fromarray(draw_openpose_from_mmpose(image_reference * 0, keypoints, scores))
+
+        image = pipeline.generate(
+            face_embs,
+            face_locations,
+            control_image,
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_steps,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            id_injection_ratio=id_injection_ratio,
+            seed=seed
+        )
+        image = add_safety_watermark(image)
+    
+    except Exception as e:
+        print(e)
+        gr.Error(f"An error occurred: {e}")
+        return gr.update()
+
+    return gr.update(value=image, label=f"Generated Image, seed = {seed}"), gr.update(value=control_image, label=f"OpenPose")
+
+
+def generate_examples(id_image_paths, ui_id_order, control_image_path, prompt_text, seed):
+    id_images = [Image.open(p).convert('RGB') for p in id_image_paths]
+    control_image = Image.open(control_image_path).convert('RGB')
+    return generate_image(id_images, ui_id_order, control_image, prompt_text, seed, 5.5, 50, 0.8, 0.8, "nude, worst quality, low quality, normal quality, nsfw, abstract, glitch, deformed, mutated, ugly, disfigured, text, watermark, bad hands, error, jpeg artifacts, blurry, missing fingers")
+
+
+def load_example(selected_key):
+    if selected_key is None:
+        return None, None, None, None, None
+
+    example = example_choices[selected_key]
+    id_images = [Image.open(p).convert('RGB') for p in example['id_images']]
+    control_image = Image.open(example['pose_image']).convert('RGB')
+    return (
+        id_images,               # For ui_id_image (Gallery)
+        example['id_order'],      # For ui_id_order (Textbox)
+        control_image,            # For ui_control_image (Image)
+        example['prompt'],        # For ui_prompt_text (Textbox)
+        example['seed']           # For ui_seed (Number)
+    )
+
+# Get all available ID and pose images
+man_images = sorted(list(Path('./assets/subjects/man').glob('*.jpg')))
+woman_images = sorted(list(Path('./assets/subjects/woman').glob('*.jpg')))
+pose_images = sorted(list(Path('./assets/poses').glob('*.png')) + list(Path('./assets/poses').glob('*.jpeg')) + list(Path('./assets/poses').glob('*.jpg')))
+
+def random_select_id_images(num_men, num_women):
+    if int(num_men) > len(man_images) or int(num_women) > len(woman_images):
+        raise ValueError("Requested more images than available.")
+    selected_men = np.random.choice(man_images, size=int(num_men), replace=False)
+    selected_women = np.random.choice(woman_images, size=int(num_women), replace=False)
+    selected = list(selected_men) + list(selected_women)
+    images = [Image.open(p).convert('RGB') for p in selected]
+    id_order = ",".join(str(i) for i in range(len(images)))
+    return images, id_order
+
+
+
+
+with gr.Blocks() as demo:
+    session_state = gr.State({})
+    default_model_version = "v1.0"
+
+    gr.HTML("""
+    <div style="text-align: center; max-width: 900px; margin: 0 auto;">
+        <h1 style="font-size: 1.5rem; font-weight: 700; display: block;">ID-Patch-SDXL</h1>
+        <h2 style="font-size: 1.2rem; font-weight: 300; margin-bottom: 1rem; display: block;">Official Gradio Demo for Our CVPR 2025 Paper <br><br>
+            <b>"ID-Patch: Robust ID Association for Group Photo Personalization" </b>
+        </h2>
+        <a href="https://byteaigc.github.io/ID-Patch/">[Project Page]</a>&ensp;
+        <a href="https://arxiv.org/abs/2411.13632">[Paper]</a>&ensp;
+        <a href="https://damon-demon.github.io/links/ID_Patch_CVPR25_poster.pdf">[Poster]</a>&ensp;
+        <a href="https://github.com/bytedance/ID-Patch">[Code]</a>&ensp;
+        <a href="https://huggingface.co/ByteDance/ID-Patch">[Model]</a>
+    </div>
+    """)
+
+    # Add the pipeline image of ID-Patch: assets/pipeline.png and short description
+    # with gr.Column(elem_id="pipeline_block"):
+    #     gr.Image(
+    #         value="./assets/pipeline.png", 
+    #         interactive=False, 
+    #         show_label=False, 
+    #         height=300,
+    #         container=False
+    #     )
+    #     gr.HTML(
+    #         """
+    #         <div style="text-align:center; font-size:1.2rem; font-weight:300;">
+    #             Pipeline of ID-Patch: Build Identity-to-Position Association
+    #         </div>
+    #         """
+    #     )
+
+
+    gr.Markdown("""
+    ### 💡 How to Use This Demo?
+    1. **Upload ID images**:  
+        - Upload one or more ID images for each person you want to generate.  
+            *(The number of uploaded ID images should match the number of people in your pose reference image.)*
+    
+    2. **ID Order**:
+        - List the ID images separated by commas, following the **left-to-right order** of detected faces in the pose reference image.   
+            *(ID index starts from 0!)*
+        
+
+    3. **Upload a pose reference image**:  
+        - Choose an image that shows the desired pose(s) for the people you want to generate.
+            *(Tip: If the pose is too complicated, then the face detection and pose detection might fail.)*
+
+    4. **Enter a text prompt**:  
+        - Describe the scene you want to create.  
+            *(Tip: Try to match the interactions described in your text with the uploaded pose reference.)*
+
+    5. **[Optional] Adjust advanced settings**:  
+        Fine-tune generation details if needed.
+
+    6. **Click "Generate"**:  
+        Your personalized image will be created. Enjoy!
+                
+    We also offer example data that users can easily select and load by clicking the **“Load Example”** button for testing.
+                
+    Alternatively, you can randomly sample a specific number of male and female face images from our provided identity image dataset and choose a pose from the available options.
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+
+            example_choices = {
+                "Woman Playing Piano (1 People)": {
+                    "id_images": ['./assets/subjects/woman/66.jpg'],
+                    "id_order": "0",
+                    "pose_image": './assets/poses/p1_1.jpeg',
+                    "prompt": 'a woman is playing piano, (pianist:1.1), wearing an elegant metallic gold backless gown dress, silver earrings, on the stage, in the spotlight, bright and colorful lighting, LED screen background, vibrant fill light',
+                    "seed": 1111
+                },
+                "Man Playing Piano (1 People)": {
+                    "id_images": ['./assets/subjects/man/21.jpg'],
+                    "id_order": "0",
+                    "pose_image": './assets/poses/p1_2.jpeg',
+                    "prompt": 'a man is playing piano, (pianist:1.1), wearing an elegant black havana tuxedo, on the stage, in the spotlight, bright and colorful lighting, LED screen background',
+                    "seed": 1111
+                },
+                "Couple Cheers (2 People)": {
+                    "id_images": ['./assets/subjects/man/0.jpg', './assets/subjects/woman/0.jpg'],
+                    "id_order": "0,1",
+                    "pose_image": './assets/poses/p2.png',
+                    "prompt": 'a young couple in front of their burning home still managing to find a moment of joy amidst disaster. cheerfully raise glasses filled with a bright blue drink',
+                    "seed": 2222
+                },
+                "Friend Selfie (3 People)": {
+                    "id_images": ['./assets/subjects/woman/26.jpg', './assets/subjects/woman/53.jpg','./assets/subjects/man/52.jpg'],
+                    "id_order": "0,1,2",
+                    "pose_image": './assets/poses/p3_2.jpeg',
+                    "prompt": 'a joyful selfie of three friends, background of television studio setting.',
+                    "seed": 3333
+                },
+                "Happy Piano Moment (4 People)": {
+                    "id_images": ['./assets/subjects/man/40.jpg', './assets/subjects/woman/59.jpg','./assets/subjects/woman/79.jpg', './assets/subjects/man/43.jpg'],
+                    "id_order": "0,1,2,3",
+                    "pose_image": './assets/poses/p4.jpeg',
+                    "prompt": 'three adults watch one man playing the piano in a brightly lit, elegant room with vintage decor',
+                    "seed": 4444
+                },
+                "Outdoor Selfie (6 People)": {
+                    "id_images": ['./assets/subjects/man/51.jpg', './assets/subjects/man/52.jpg','./assets/subjects/woman/79.jpg', './assets/subjects/woman/66.jpg', './assets/subjects/woman/39.jpg', './assets/subjects/man/49.jpg'],
+                    "id_order": "0,1,2,3,4,5",
+                    "pose_image": './assets/poses/p6.jpeg',
+                    "prompt": 'A joyful group selfie of six adventurous people on a mountain at sunrise. Each person is dressed in outdoor apparel suitable for chilly weather',
+                    "seed": 6666
+                },
+            }
+
+            # Build pose_name_to_path mapping
+            pose_name_to_path = {
+                example_name: example_data["pose_image"]
+                for example_name, example_data in example_choices.items()
+            }
+
+
+            with gr.Column(scale=2):
+                selected_example = gr.Dropdown(
+                    choices=list(example_choices.keys()),
+                    label="Example Selection",
+                    interactive=True
+                )
+                load_example_btn = gr.Button("Load Example")
+                
+            with gr.Row():
+                with gr.Column(scale=3):
+                    ui_id_image = gr.Gallery(
+                            label="Identity Images", 
+                            type="pil", 
+                            scale=3, 
+                            height=370, 
+                            min_width=100, 
+                            columns=4, 
+                            rows=1, 
+                            allow_preview=True, 
+                            show_label=True, 
+                            interactive=True
+                        )
+                    
+                    # Random Identity Selection
+                    with gr.Row():
+                        num_men_dropdown = gr.Dropdown(
+                            choices=[str(i) for i in range(11)], 
+                            value="0", 
+                            label="Number of Men"
+                        )
+                        num_women_dropdown = gr.Dropdown(
+                            choices=[str(i) for i in range(11)], 
+                            value="0", 
+                            label="Number of Women"
+                        )
+                    random_select_button = gr.Button("Random Select IDs")
+
+
+                with gr.Column(scale=2, min_width=100):
+                    ui_control_image = gr.Image(label="Pose Reference Image", type="pil", height=370, min_width=100)
+
+                    # Pose Selection based on Example Choices
+                    pose_dropdown = gr.Dropdown(
+                        choices=list(pose_name_to_path.keys()),
+                        label="Select Pose Example",
+                        interactive=True
+                    )
+                    pose_select_button = gr.Button("Load Pose")
+
+            
+            ui_prompt_text = gr.Textbox(label="Text Prompt (Describe the image you would like to generate)", value="Portrait, 4K, high quality, cinematic")
+            ui_id_order = gr.Textbox(label="ID Order (If not specified, the images will follow the original upload order)", value = None)
+
+                
+            ui_btn_generate = gr.Button("Generate")
+
+            with gr.Accordion("Advanced Settings", open=True):
+                with gr.Row():
+                    ui_num_steps = gr.Number(label="Steps", value=50)
+                    ui_seed = gr.Number(label="Seed (0 for random seed)", value=0)
+                ui_guidance_scale = gr.Number(label="Guidance Scale", value=5.5, step=0.1)
+                ui_controlnet_conditioning_scale = gr.Slider(minimum=0.0, maximum=1.0, value=0.8, step=0.05, label="ControlNet Conditioning Scale")
+                ui_id_injection_ratio = gr.Slider(minimum=0.0, maximum=1.0, value=0.8, step=0.05, label="ID Injection Ratio")
+                ui_negative_prompt = gr.Textbox(label="Negative Prompt", value="nude, worst quality, low quality, normal quality, nsfw, abstract, glitch, deformed, mutated, ugly, disfigured, text, watermark, bad hands, error, jpeg artifacts, blurry, missing fingers")
+
+
+
+        with gr.Column(scale=2):
+            image_output = gr.Image(label="Generated Image", interactive=False, height=880, format='png')
+            openpose_control_image = gr.Image(label="OpenPose Image", interactive=False, height=549, format='png')
+            
+
+
+    ui_btn_generate.click(
+        generate_image, 
+        inputs=[
+            ui_id_image, 
+            ui_id_order,
+            ui_control_image, 
+            ui_prompt_text, 
+            ui_seed, 
+            ui_guidance_scale, 
+            ui_num_steps,
+            ui_controlnet_conditioning_scale,
+            ui_id_injection_ratio,
+            ui_negative_prompt 
+            
+        ], 
+        outputs=[image_output, openpose_control_image], 
+        concurrency_id="gpu"
+    )
+
+    load_example_btn.click(
+        load_example,
+        inputs=[selected_example],
+        outputs=[ui_id_image, ui_id_order, ui_control_image, ui_prompt_text, ui_seed]
+    )
+
+    random_select_button.click(
+        random_select_id_images,
+        inputs=[num_men_dropdown, num_women_dropdown],
+        outputs=[ui_id_image, ui_id_order]
+    )
+
+    def select_pose_image(pose_name):
+        if pose_name not in pose_name_to_path:
+            raise ValueError(f"Pose name {pose_name} not found.")
+        
+        pose_path = pose_name_to_path[pose_name]
+        pose_image = Image.open(pose_path).convert('RGB')
+
+        for example_name, example_data in example_choices.items():
+            if example_name == pose_name:
+                prompt = example_data['prompt']
+                break
+        else:
+            prompt = ""
+
+        return pose_image, prompt
+    
+    pose_select_button.click(
+        select_pose_image,
+        inputs=[pose_dropdown],
+        outputs=[ui_control_image, ui_prompt_text]
+    )
+
+    
+    gr.Markdown(
+        """
+        ---
+        ### 📜 Disclaimer and Licenses 
+        The images used in this demo are sourced from consented subjects or generated by the models. These pictures are intended solely to show the capabilities of our research. If you have any concerns, please contact us, and we will promptly remove any inappropriate content.
+        
+        The use of the released code, model, and demo must strictly adhere to the respective licenses. 
+        Our code is released under the [Apache License 2.0](https://github.com/bytedance/ID-Patch/blob/main/LICENSE), 
+        and our model is released under the [CreativeML Open RAIL++-M License](https://huggingface.co/ByteDance/ID-Patch/blob/main/LICENSE.md) 
+        for academic research purposes only. Any manual or automatic downloading of the face models from [InsightFace](https://github.com/deepinsight/insightface), 
+        the [Juggernaut-X-v10](https://huggingface.co/RunDiffusion/Juggernaut-X-v10) base model, *etc.*, must follow their original licenses and be used only for academic research purposes.
+
+        This research aims to positively impact the field of Generative AI. Any usage of this method must be responsible and comply with local laws. The developers do not assume any responsibility for any potential misuse. We added the "AI Generated: ID-Patch" watermark for enhanced safety.
+        """
+    )    
+
+    gr.Markdown(
+        """
+        ### 📖 Citation
+
+        If you find ID-Patch useful for your research or applications, please cite our paper:
+
+        ```bibtex
+        @InProceedings{zhang2025idpatch,
+            author    = {Zhang, Yimeng and Zhi, Tiancheng and Liu, Jing and Sang, Shen and Jiang, Liming and Yan, Qing and Liu, Sijia and Luo, Linjie},
+            title     = {ID-Patch: Robust ID Association for Group Photo Personalization},
+            booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
+            month     = {June},
+            year      = {2025}
+        }
+        ```
+
+        We also appreciate it if you could give a star ⭐ to our [Github repository](https://github.com/bytedance/ID-Patch). Thanks a lot!
+        """
+    )
+
+
+download_models()
+init_pipeline()
+demo.launch()

modules/inferencer.py

@@ -0,0 +1,110 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+from pathlib import Path
+import numpy as np
+from diffusers import ControlNetModel, EulerDiscreteScheduler
+from diffusers.loaders.unet import UNet2DConditionLoadersMixin
+
+from .pipeline_idpatch_sd_xl import StableDiffusionXLIDPatchPipeline
+
+class IDPatchInferencer:
+    def __init__(self, base_model_path, idp_model_path, patch_size=64, torch_device='cuda:0', torch_dtype=torch.bfloat16):
+        super().__init__()
+        self.patch_size = patch_size
+        self.torch_device = torch_device
+        self.torch_dtype = torch_dtype
+        idp_state_dict = torch.load(Path(idp_model_path) / 'id-patch.bin', map_location="cpu")
+        loader = UNet2DConditionLoadersMixin()
+        self.id_patch_projection = loader._convert_ip_adapter_image_proj_to_diffusers(idp_state_dict['patch_proj']).to(self.torch_device, dtype=self.torch_dtype).eval()
+        self.id_prompt_projection = loader._convert_ip_adapter_image_proj_to_diffusers(idp_state_dict['prompt_proj']).to(self.torch_device, dtype=self.torch_dtype).eval()
+        controlnet = ControlNetModel.from_pretrained(Path(idp_model_path) / 'ControlNetModel').to(self.torch_device, dtype=self.torch_dtype).eval()
+        scheduler = EulerDiscreteScheduler.from_pretrained(base_model_path, subfolder="scheduler")
+        self.pipe = StableDiffusionXLIDPatchPipeline.from_pretrained(
+            base_model_path,
+            controlnet=controlnet,
+            scheduler=scheduler,
+            torch_dtype=self.torch_dtype,
+        ).to(self.torch_device)
+        
+    def get_text_embeds_from_strings(self, text_strings):
+        pipe = self.pipe
+        device = pipe.device
+        tokenizer_1 = pipe.tokenizer
+        tokenizer_2 = pipe.tokenizer_2
+        text_encoder_1 = pipe.text_encoder
+        text_encoder_2 = pipe.text_encoder_2
+        
+        text_embeds = []
+        for tokenizer, text_encoder in [(tokenizer_1, text_encoder_1), (tokenizer_2, text_encoder_2)]:
+            input_ids = tokenizer(
+                text_strings,
+                max_length=tokenizer.model_max_length,
+                padding="max_length",
+                truncation=True,
+                return_tensors="pt"
+            ).input_ids.to(device)
+            text_embeds.append(text_encoder(input_ids, output_hidden_states=True))
+        pooled_embeds = text_embeds[1]['text_embeds']
+        text_embeds = torch.concat([text_embeds[0]['hidden_states'][-2], text_embeds[1]['hidden_states'][-2]], dim=2)
+        return text_embeds, pooled_embeds
+    
+    def generate(self, face_embeds, face_locations, control_image, prompt, negative_prompt="", guidance_scale=5.0, num_inference_steps=50, controlnet_conditioning_scale=0.8, id_injection_ratio=0.8, seed=-1):
+        """
+            face_embeds: n_faces x 512
+            face_locations: n_faces x 2[xy]
+            control_image: PIL image
+        """
+        
+        face_locations = face_locations.to(self.torch_device, self.torch_dtype)
+        control_image = torch.from_numpy(np.array(control_image)).to(self.torch_device, dtype=self.torch_dtype).permute(2,0,1)[None] / 255.0
+        height, width = control_image.shape[2:4]
+        
+        text_embeds, pooled_embeds = self.get_text_embeds_from_strings([negative_prompt, prompt]) # text_embeds: 2 x 77 x 2048, pooled_embeds: 2 x 1280
+        negative_pooled_embeds, pooled_embeds = pooled_embeds[:1], pooled_embeds[1:]
+        negative_text_embeds, text_embeds = text_embeds[:1], text_embeds[1:]
+        
+        n_faces = len(face_embeds)
+        negative_id_embeds = self.id_prompt_projection(torch.zeros(n_faces, 1, 512, device=self.torch_device, dtype=self.torch_dtype)) # (BxF) x 16 x 2048
+        negative_id_embeds = negative_id_embeds.reshape(1, -1, negative_id_embeds.shape[2]) # B x (Fx16) x 2048
+        negative_text_id_embeds = torch.concat([negative_text_embeds, negative_id_embeds], dim=1)
+        
+        face_embeds = face_embeds[None].to(self.torch_device, self.torch_dtype) # 1 x faces x 512
+        id_embeds = self.id_prompt_projection(face_embeds.reshape(-1, 1, 512)) # (BxF) x 16 x 2048
+        id_embeds = id_embeds.reshape(face_embeds.shape[0], -1, id_embeds.shape[2]) # B x (Fx16) x 2048
+        text_id_embeds = torch.concat([text_embeds, id_embeds], dim=1) # B x (77+Fx16) x 2048
+        
+        patch_prompt_embeds = self.id_patch_projection(face_embeds.reshape(-1, 1, 512)) # (Bxn_faces) x 3 x (64*64)
+        patch_prompt_embeds = patch_prompt_embeds.reshape(1, n_faces, 3, self.patch_size, self.patch_size)
+        pad = self.patch_size // 2
+        canvas = torch.zeros((1, 3, height + pad * 2, width + pad * 2), device=self.torch_device)
+        
+        xymin = torch.round(face_locations - self.patch_size // 2).int()
+        xymax =torch.round(face_locations + self.patch_size // 2).int()
+        for f in range(n_faces):
+            xmin, ymin = xymin[f,0], xymin[f,1]
+            xmax, ymax = xymax[f,0], xymax[f,1]
+            if xmin+pad < 0 or xmax-pad >= width or ymin+pad < 0 or ymax-pad >= height:
+                continue
+            canvas[0,:,ymin+pad:ymax+pad,xmin+pad:xmax+pad] += patch_prompt_embeds[0,f]
+        condition_image = control_image + canvas[:,:,pad:-pad,pad:-pad]
+        
+        if seed >= 0:
+            generator = torch.Generator(self.torch_device).manual_seed(seed)
+        else:
+            generator = None
+        output_image = self.pipe(
+            prompt_embeds=text_id_embeds,
+            pooled_prompt_embeds=pooled_embeds,
+            negative_prompt_embeds=negative_text_id_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_embeds,
+            image=condition_image,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            num_inference_steps=num_inference_steps,
+            id_injection_ratio=id_injection_ratio,
+            output_type='pil',
+            generator=generator,
+        ).images[0]
+        return output_image

modules/pipeline_idpatch_sd_xl.py

@@ -0,0 +1,463 @@
+# Copyright (c) 2024 The HuggingFace Team
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0 
+#
+# This file has been modified by Bytedance Ltd. and/or its affiliates on October 10, 2024.
+#
+# Original file was released under Apache License 2.0, with the full license text
+# available at https://github.com/huggingface/diffusers/blob/v0.30.3/LICENSE.
+#
+# This modified file is released under the same license.
+
+
+from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl import *
+
+class StableDiffusionXLIDPatchPipeline(StableDiffusionXLControlNetPipeline):
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]] = None,
+        prompt_2: Optional[Union[str, List[str]]] = None,
+        image: PipelineImageInput = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_inference_steps: int = 50,
+        timesteps: List[int] = None,
+        sigmas: List[float] = None,
+        denoising_end: Optional[float] = None,
+        guidance_scale: float = 5.0,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        negative_prompt_2: Optional[Union[str, List[str]]] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.Tensor] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_pooled_prompt_embeds: Optional[torch.Tensor] = None,
+        ip_adapter_image: Optional[PipelineImageInput] = None,
+        ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+        controlnet_conditioning_scale: Union[float, List[float]] = 1.0,
+        guess_mode: bool = False,
+        control_guidance_start: Union[float, List[float]] = 0.0,
+        control_guidance_end: Union[float, List[float]] = 1.0,
+        original_size: Tuple[int, int] = None,
+        crops_coords_top_left: Tuple[int, int] = (0, 0),
+        target_size: Tuple[int, int] = None,
+        negative_original_size: Optional[Tuple[int, int]] = None,
+        negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
+        negative_target_size: Optional[Tuple[int, int]] = None,
+        clip_skip: Optional[int] = None,
+        callback_on_step_end: Optional[
+            Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
+        ] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        id_injection_ratio: float = 1.0,
+        **kwargs,
+    ):
+
+        callback = kwargs.pop("callback", None)
+        callback_steps = kwargs.pop("callback_steps", None)
+
+        if callback is not None:
+            deprecate(
+                "callback",
+                "1.0.0",
+                "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
+            )
+        if callback_steps is not None:
+            deprecate(
+                "callback_steps",
+                "1.0.0",
+                "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
+            )
+
+        if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
+            callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
+
+        controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet
+
+        # align format for control guidance
+        if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list):
+            control_guidance_start = len(control_guidance_end) * [control_guidance_start]
+        elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list):
+            control_guidance_end = len(control_guidance_start) * [control_guidance_end]
+        elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):
+            mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1
+            control_guidance_start, control_guidance_end = (
+                mult * [control_guidance_start],
+                mult * [control_guidance_end],
+            )
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            prompt,
+            prompt_2,
+            image,
+            callback_steps,
+            negative_prompt,
+            negative_prompt_2,
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            ip_adapter_image,
+            ip_adapter_image_embeds,
+            negative_pooled_prompt_embeds,
+            controlnet_conditioning_scale,
+            control_guidance_start,
+            control_guidance_end,
+            callback_on_step_end_tensor_inputs,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._clip_skip = clip_skip
+        self._cross_attention_kwargs = cross_attention_kwargs
+        self._denoising_end = denoising_end
+
+        # 2. Define call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
+            controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)
+
+        global_pool_conditions = (
+            controlnet.config.global_pool_conditions
+            if isinstance(controlnet, ControlNetModel)
+            else controlnet.nets[0].config.global_pool_conditions
+        )
+        guess_mode = guess_mode or global_pool_conditions
+
+        # 3.1 Encode input prompt
+        text_encoder_lora_scale = (
+            self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
+        )
+        (
+            prompt_embeds,
+            negative_prompt_embeds,
+            pooled_prompt_embeds,
+            negative_pooled_prompt_embeds,
+        ) = self.encode_prompt(
+            prompt,
+            prompt_2,
+            device,
+            num_images_per_prompt,
+            self.do_classifier_free_guidance,
+            negative_prompt,
+            negative_prompt_2,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            pooled_prompt_embeds=pooled_prompt_embeds,
+            negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+            lora_scale=text_encoder_lora_scale,
+            clip_skip=self.clip_skip,
+        )
+
+        # 3.2 Encode ip_adapter_image
+        if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
+            image_embeds = self.prepare_ip_adapter_image_embeds(
+                ip_adapter_image,
+                ip_adapter_image_embeds,
+                device,
+                batch_size * num_images_per_prompt,
+                self.do_classifier_free_guidance,
+            )
+
+        # 4. Prepare image
+        if isinstance(controlnet, ControlNetModel):
+            image = self.prepare_image(
+                image=image,
+                width=width,
+                height=height,
+                batch_size=batch_size * num_images_per_prompt,
+                num_images_per_prompt=num_images_per_prompt,
+                device=device,
+                dtype=controlnet.dtype,
+                do_classifier_free_guidance=self.do_classifier_free_guidance,
+                guess_mode=guess_mode,
+            )
+            height, width = image.shape[-2:]
+        elif isinstance(controlnet, MultiControlNetModel):
+            images = []
+
+            for image_ in image:
+                image_ = self.prepare_image(
+                    image=image_,
+                    width=width,
+                    height=height,
+                    batch_size=batch_size * num_images_per_prompt,
+                    num_images_per_prompt=num_images_per_prompt,
+                    device=device,
+                    dtype=controlnet.dtype,
+                    do_classifier_free_guidance=self.do_classifier_free_guidance,
+                    guess_mode=guess_mode,
+                )
+
+                images.append(image_)
+
+            image = images
+            height, width = image[0].shape[-2:]
+        else:
+            assert False
+
+        # 5. Prepare timesteps
+        timesteps, num_inference_steps = retrieve_timesteps(
+            self.scheduler, num_inference_steps, device, timesteps, sigmas
+        )
+        self._num_timesteps = len(timesteps)
+
+        # 6. Prepare latent variables
+        num_channels_latents = self.unet.config.in_channels
+        latents = self.prepare_latents(
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 6.5 Optionally get Guidance Scale Embedding
+        timestep_cond = None
+        if self.unet.config.time_cond_proj_dim is not None:
+            guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
+            timestep_cond = self.get_guidance_scale_embedding(
+                guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
+            ).to(device=device, dtype=latents.dtype)
+
+        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 7.1 Create tensor stating which controlnets to keep
+        controlnet_keep = []
+        for i in range(len(timesteps)):
+            keeps = [
+                1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e)
+                for s, e in zip(control_guidance_start, control_guidance_end)
+            ]
+            controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps)
+
+        # 7.2 Prepare added time ids & embeddings
+        if isinstance(image, list):
+            original_size = original_size or image[0].shape[-2:]
+        else:
+            original_size = original_size or image.shape[-2:]
+        target_size = target_size or (height, width)
+
+        add_text_embeds = pooled_prompt_embeds
+        if self.text_encoder_2 is None:
+            text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
+        else:
+            text_encoder_projection_dim = self.text_encoder_2.config.projection_dim
+
+        add_time_ids = self._get_add_time_ids(
+            original_size,
+            crops_coords_top_left,
+            target_size,
+            dtype=prompt_embeds.dtype,
+            text_encoder_projection_dim=text_encoder_projection_dim,
+        )
+
+        if negative_original_size is not None and negative_target_size is not None:
+            negative_add_time_ids = self._get_add_time_ids(
+                negative_original_size,
+                negative_crops_coords_top_left,
+                negative_target_size,
+                dtype=prompt_embeds.dtype,
+                text_encoder_projection_dim=text_encoder_projection_dim,
+            )
+        else:
+            negative_add_time_ids = add_time_ids
+
+        if self.do_classifier_free_guidance:
+            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+            add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
+            add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
+
+        prompt_embeds = prompt_embeds.to(device)
+        add_text_embeds = add_text_embeds.to(device)
+        add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)
+
+        # 8. Denoising loop
+        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
+
+        # 8.1 Apply denoising_end
+        if (
+            self.denoising_end is not None
+            and isinstance(self.denoising_end, float)
+            and self.denoising_end > 0
+            and self.denoising_end < 1
+        ):
+            discrete_timestep_cutoff = int(
+                round(
+                    self.scheduler.config.num_train_timesteps
+                    - (self.denoising_end * self.scheduler.config.num_train_timesteps)
+                )
+            )
+            num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
+            timesteps = timesteps[:num_inference_steps]
+
+        is_unet_compiled = is_compiled_module(self.unet)
+        is_controlnet_compiled = is_compiled_module(self.controlnet)
+        is_torch_higher_equal_2_1 = is_torch_version(">=", "2.1")
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                # Relevant thread:
+                # https://dev-discuss.pytorch.org/t/cudagraphs-in-pytorch-2-0/1428
+                if (is_unet_compiled and is_controlnet_compiled) and is_torch_higher_equal_2_1:
+                    torch._inductor.cudagraph_mark_step_begin()
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
+
+                # controlnet(s) inference
+                if guess_mode and self.do_classifier_free_guidance:
+                    # Infer ControlNet only for the conditional batch.
+                    control_model_input = latents
+                    control_model_input = self.scheduler.scale_model_input(control_model_input, t)
+                    controlnet_prompt_embeds = prompt_embeds.chunk(2)[1]
+                    controlnet_added_cond_kwargs = {
+                        "text_embeds": add_text_embeds.chunk(2)[1],
+                        "time_ids": add_time_ids.chunk(2)[1],
+                    }
+                else:
+                    control_model_input = latent_model_input
+                    controlnet_prompt_embeds = prompt_embeds
+                    controlnet_added_cond_kwargs = added_cond_kwargs
+
+                if isinstance(controlnet_keep[i], list):
+                    cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])]
+                else:
+                    controlnet_cond_scale = controlnet_conditioning_scale
+                    if isinstance(controlnet_cond_scale, list):
+                        controlnet_cond_scale = controlnet_cond_scale[0]
+                    cond_scale = controlnet_cond_scale * controlnet_keep[i]
+                if i < len(timesteps) * (1 - id_injection_ratio):
+                    token_length = 77
+                else:
+                    token_length = 1000000
+                down_block_res_samples, mid_block_res_sample = self.controlnet(
+                    control_model_input,
+                    t,
+                    encoder_hidden_states=controlnet_prompt_embeds[:,:token_length],
+                    controlnet_cond=image,
+                    conditioning_scale=cond_scale,
+                    guess_mode=guess_mode,
+                    added_cond_kwargs=controlnet_added_cond_kwargs,
+                    return_dict=False,
+                )
+
+                if guess_mode and self.do_classifier_free_guidance:
+                    # Inferred ControlNet only for the conditional batch.
+                    # To apply the output of ControlNet to both the unconditional and conditional batches,
+                    # add 0 to the unconditional batch to keep it unchanged.
+                    down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples]
+                    mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample])
+
+                if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
+                    added_cond_kwargs["image_embeds"] = image_embeds
+
+                # predict the noise residual
+                noise_pred = self.unet(
+                    latent_model_input,
+                    t,
+                    encoder_hidden_states=prompt_embeds[:,:token_length],
+                    timestep_cond=timestep_cond,
+                    cross_attention_kwargs=self.cross_attention_kwargs,
+                    down_block_additional_residuals=down_block_res_samples,
+                    mid_block_additional_residual=mid_block_res_sample,
+                    added_cond_kwargs=added_cond_kwargs,
+                    return_dict=False,
+                )[0]
+
+                # perform guidance
+                if self.do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
+
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+                    add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
+                    negative_pooled_prompt_embeds = callback_outputs.pop(
+                        "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
+                    )
+                    add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
+                    negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        step_idx = i // getattr(self.scheduler, "order", 1)
+                        callback(step_idx, t, latents)
+
+        if not output_type == "latent":
+            # make sure the VAE is in float32 mode, as it overflows in float16
+            needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
+
+            if needs_upcasting:
+                self.upcast_vae()
+                latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
+
+            # unscale/denormalize the latents
+            # denormalize with the mean and std if available and not None
+            has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
+            has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
+            if has_latents_mean and has_latents_std:
+                latents_mean = (
+                    torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
+                )
+                latents_std = (
+                    torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
+                )
+                latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
+            else:
+                latents = latents / self.vae.config.scaling_factor
+
+            image = self.vae.decode(latents, return_dict=False)[0]
+
+            # cast back to fp16 if needed
+            if needs_upcasting:
+                self.vae.to(dtype=torch.float16)
+        else:
+            image = latents
+
+        if not output_type == "latent":
+            # apply watermark if available
+            if self.watermark is not None:
+                image = self.watermark.apply_watermark(image)
+
+            image = self.image_processor.postprocess(image, output_type=output_type)
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if not return_dict:
+            return (image,)
+
+        return StableDiffusionXLPipelineOutput(images=image)

requirements.txt

@@ -0,0 +1,21 @@
+accelerate==0.34.2
+diffusers==0.30.3
+facexlib==0.3.0
+gradio==5.23.1
+gradio_client==1.8.0
+httpcore==1.0.7
+httpx==0.28.1
+huggingface-hub==0.28.1
+insightface==0.7.3
+mtcnn==0.1.1
+numpy==1.26.4
+onnxruntime==1.19.2
+opencv-python==4.8.0.74
+pillow==10.4.0
+pillow-avif-plugin==1.5.0
+pillow-heif==0.21.0
+sentencepiece==0.2.0
+rtmlib==0.0.13
+tensorflow==2.17.0
+torch==2.1.2
+transformers==4.44.2

utils/draw_condition.py

@@ -0,0 +1,90 @@
+# Copyright (c) OpenMMLab
+# Copyright (c) 2024 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0 
+#
+# This file has been modified by Bytedance Ltd. and/or its affiliates on October 10, 2024.
+#
+# Original file was released under Apache License 2.0, with the full license text
+# available at https://github.com/open-mmlab/mmpose/blob/main/LICENSE.
+#
+# This modified file is released under the same license.
+
+
+import numpy as np
+import cv2
+from itertools import product
+import math
+
+def draw_openpose_from_mmpose(canvas, keypoints, keypoint_scores, kpt_thr=0.3, ignore_individual_points=False):
+    """
+        canvas: background image
+        keypoints: N x 17 x 2
+        keypoint_scores: N x 17
+        ret: RGB order (note: although we use cv2 to process image, result is in RGB order)
+    """
+    
+    # openpose format
+    limb_seq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10],
+        [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17],
+        [1, 16], [16, 18]]
+
+    colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
+        [85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255],
+        [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255], [255, 0, 255],
+        [255, 0, 170], [255, 0, 85]]
+
+    stickwidth = 4
+    num_openpose_kpt = 18
+    num_link = len(limb_seq)
+    
+    # concatenate scores and keypoints
+    keypoints = np.concatenate((keypoints, keypoint_scores.reshape(-1, 17, 1)), axis=-1)
+    
+    # compute neck joint
+    neck = (keypoints[:, 5] + keypoints[:, 6]) / 2
+    #if keypoints[:, 5, 2] < kpt_thr or keypoints[:, 6, 2] < kpt_thr:
+    #    neck[:, 2] = 0
+    neck[:, 2] = np.minimum(keypoints[:, 5, 2], keypoints[:, 6, 2])
+
+    # 17 keypoints to 18 keypoints
+    new_keypoints = np.insert(keypoints[:, ], 17, neck, axis=1)
+
+    # mmpose format to openpose format
+    openpose_idx = [15, 14, 17, 16, 2, 6, 3, 7, 4, 8, 12, 9, 13, 10, 1]
+    mmpose_idx = [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17]
+    new_keypoints[:, openpose_idx, :] = new_keypoints[:, mmpose_idx, :]
+    
+    black_img = canvas
+    num_instance = new_keypoints.shape[0]
+    
+    # drw keypoints
+    for i in range(num_instance):
+        valid = [False] * 18
+        for link_idx in range(num_link):
+            conf = new_keypoints[i][np.array(limb_seq[link_idx]) - 1, 2]
+            if np.sum(conf > kpt_thr) == 2:
+                valid[limb_seq[link_idx][0]-1] = True
+                valid[limb_seq[link_idx][1]-1] = True
+        for j in range(num_openpose_kpt):
+            x, y, conf = new_keypoints[i][j]
+            if conf > kpt_thr and valid[j]:
+                cv2.circle(black_img, (int(x), int(y)), 4, colors[j], thickness=-1)
+
+    # draw links
+    cur_black_img = black_img.copy()
+    for i, link_idx in product(range(num_instance), range(num_link)):
+        conf = new_keypoints[i][np.array(limb_seq[link_idx]) - 1, 2]
+        if np.sum(conf > kpt_thr) == 2:
+            Y = new_keypoints[i][np.array(limb_seq[link_idx]) - 1, 0]
+            X = new_keypoints[i][np.array(limb_seq[link_idx]) - 1, 1]
+            mX = np.mean(X)
+            mY = np.mean(Y)
+            length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
+            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
+            polygon = cv2.ellipse2Poly(
+                (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
+                0, 360, 1)
+            cv2.fillConvexPoly(cur_black_img, polygon, colors[link_idx])
+    black_img = cv2.addWeighted(black_img, 0.4, cur_black_img, 0.6, 0)
+    
+    return black_img