Upgrading from sd 1.5 to sdxl

app.py

@@ -1,24 +1,45 @@
+from diffusers import AutoencoderKL, StableDiffusionXLControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
 import torch
-from diffusers import StableDiffusionPipeline
+from controlnet_aux import OpenposeDetector
+from diffusers.utils import load_image
 import gradio as gr
 
+#sd1.5 bases
 #model_base = "SG161222/Realistic_Vision_V5.1_noVAE" #fantasy people
 #model_base = "Justin-Choo/epiCRealism-Natural_Sin_RC1_VAE" #cartoon people
 #model_base = "Lykon/DreamShaper" #unrealistic people
 #model_base = "runwayml/stable-diffusion-v1-5" #base
-model_base = "Krebzonide/LazyMixPlus" #nsfw people
+#model_base = "Krebzonide/LazyMixPlus" #nsfw people
 #model_base = "Krebzonide/Humans" #boring people
 #model_base = "aufahr/unofficial_aom3" #anime people
 
 #lora_model_path = "Krebzonide/LoRA-CH-0" #mecjh - Corey H, traind on epiCRealism
 #lora_model_path = "Krebzonide/LoRA-CH-1" #mecjh - Corey H, traind on epiCRealism
-lora_model_path = "Krebzonide/LoRA-EM1" #exgfem - Emily M, trained on LizyMixPlus
+#lora_model_path = "Krebzonide/LoRA-EM1" #exgfem - Emily M, trained on LizyMixPlus
 #lora_model_path = "Krebzonide/LoRA-EM-2-0" #exgfem - Emily M, trained on Humans
 #lora_model_path = "Krebzonide/LoRA-YX1" #uwspyx - Professor Xing, trained on Realistic_Vision
 
-pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16, use_safetensors=True, use_auth_token="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")
-pipe.unet.load_attn_procs(lora_model_path, use_auth_token="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")
-pipe.to("cuda")
+#pipe = StableDiffusionPipeline.from_pretrained(model_base, torch_dtype=torch.float16, use_safetensors=True, use_auth_token="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")
+#pipe.unet.load_attn_procs(lora_model_path, use_auth_token="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")
+#pipe.to("cuda")
+
+
+model_base = "stabilityai/stable-diffusion-xl-base-1.0"
+
+openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
+
+#image is a random guy. openpose_image is the pose of that guy.
+image = load_image(
+    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/person.png"
+)
+openpose_image = openpose(image)
+
+controlnet = ControlNetModel.from_pretrained("thibaud/controlnet-openpose-sdxl-1.0", torch_dtype=torch.float16)
+
+pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+    model_base, controlnet=controlnet, torch_dtype=torch.float16
+)
+pipe.enable_model_cpu_offload()
 
 css = """
 .btn-green {
@@ -37,8 +58,10 @@ def generate(prompt, neg_prompt, samp_steps, guide_scale, lora_scale, progress=g
         negative_prompt=neg_prompt,
         num_inference_steps=samp_steps,
         guidance_scale=guide_scale,
-        cross_attention_kwargs={"scale": lora_scale},
-        num_images_per_prompt=6
+        #cross_attention_kwargs={"scale": lora_scale},
+        num_images_per_prompt=4,
+        #generator=torch.manual_seed(97),
+        image=openpose_image.resize((1024, 1024)), #THIS IS THE OPENPOSE IMAGE
     ).images
     return [(img, f"Image {i+1}") for i, img in enumerate(images)]