import gradio as gr import numpy as np from diffusers import StableDiffusionXLControlNetInpaintPipeline from diffusers import StableDiffusionXLImg2ImgPipeline, DPMSolverMultistepScheduler, AutoencoderTiny, StableDiffusionXLControlNetPipeline, ControlNetModel from diffusers.utils import load_image from diffusers.image_processor import IPAdapterMaskProcessor import torch import os from transformers import CLIPVisionModelWithProjection, CLIPImageProcessor from diffusers.utils import make_image_grid from diffusers import DPMSolverSDEScheduler MAX_SEED = np.iinfo(np.int32).max MAX_IMAGE_SIZE = 1024 device = "cuda" if torch.cuda.is_available() else "cpu" if torch.cuda.is_available(): torch_dtype = torch.float16 else: torch_dtype = torch.float32 processor_mask = IPAdapterMaskProcessor() controlnets = [ ControlNetModel.from_pretrained( "diffusers/controlnet-depth-sdxl-1.0",variant="fp16",use_safetensors=True,torch_dtype=torch.float16 ), ControlNetModel.from_pretrained( "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16, use_safetensors=True,variant="fp16" ), ] pipe_CN = StableDiffusionXLControlNetPipeline.from_pretrained("SG161222/RealVisXL_V5.0", torch_dtype=torch.float16,controlnet=controlnets, use_safetensors=True, variant='fp16') pipe_CN.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float16) pipe_CN.scheduler=DPMSolverMultistepScheduler.from_pretrained("SG161222/RealVisXL_V5.0",subfolder="scheduler",use_karras_sigmas=True) pipe_CN.to("cuda") pipe_CN.load_lora_weights('CreativesCombined/hb8_cases_dreambooth_lora_test_1_14', weight_name='pytorch_lora_weights.safetensors',adapter_name='cases') refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0",text_encoder_2=pipe_CN.text_encoder_2,vae=pipe_CN.vae,torch_dtype=torch.float16,use_safetensors=True,variant="fp16") refiner.to("cuda") pipe_IN = StableDiffusionXLControlNetInpaintPipeline.from_pretrained("diffusers/stable-diffusion-xl-1.0-inpainting-0.1",controlnet=controlnets, torch_dtype=torch.float16, variant="fp16").to("cuda") pipe_IN.load_lora_weights('Tonioesparza/ourhood_training_dreambooth_lora_2_0', weight_name='pytorch_lora_weights.safetensors',adapter_name='ourhood') pipe_IN.to("cuda") def ourhood_inference(prompt1=str,num_inference_steps=int,scaffold=int,seed=int): ###pro_encode = pipe_cn.encode_text(prompt) ###pro_encode = pipe_CN.encode_text(prompt)[2] ### function has no formats defined scaff_dic={1:{'mask1':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/mask_in_square_2.png", 'depth_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/mask_depth_noroof_square.png", 'canny_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/mask_depth_solo_square.png"}, 2:{'mask1':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/mask_in_C.png", 'depth_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/depth_C.png", 'canny_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/canny_C_solo.png"}, 3:{'mask1':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/mask_in_B.png", 'depth_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/depth_B.png", 'canny_image':"https://huggingface.co/Tonioesparza/ourhood_training_dreambooth_lora_2_0/blob/main/canny_B_solo.png"}} ##############################load loras ###pipe_CN.fuse_lora() output_height = 1024 output_width = 1024 mask1 = load_image(scaff_dic[scaffold]['mask1']) masks = processor_mask.preprocess([mask1], height=output_height, width=output_width) masks = [masks.reshape(1, masks.shape[0], masks.shape[2], masks.shape[3])] ###ip_images init ###ip_img_1 = load_image(r"C:\Users\AntonioEsparzaGlisma\PycharmProjects\hB8\Cases\a-place-to_210930_HAY_A-PLACE-TO_091-768x1024.png") ###ip_images = [[ip_img_1]] ###pipe_CN.set_ip_adapter_scale([[0.7]]) n_steps = num_inference_steps ###precomputed depth image depth_image = load_image(scaff_dic[scaffold]['depth_image']) canny_image = load_image(scaff_dic[scaffold]['canny_image']) images_CN = [depth_image, canny_image] neg1 = 'text,watermark' prompt2 = 'Photorealistic rendering, of an OurHood privacy booth, with a silken oak frame, hickory stained melange polyester fabric, windows' neg2 = 'curtains, pillows' generator = torch.Generator(device="cuda").manual_seed(seed) results = pipe_CN( prompt=prompt1, ###ip_adapter_image=ip_images, negative_prompt=neg1, num_inference_steps=n_steps, num_images_per_prompt=1, generator=generator, denoising_end=0.8, image=images_CN, output_type="latent", control_guidance_start=[0.0,0.5], control_guidance_end=[0.5,1.0], controlnet_conditioning_scale=[0.5,1.0], ).images[0] image = refiner( prompt=prompt1, num_inference_steps=n_steps, denoising_start=0.8, image=results).images[0] image = pipe_IN( prompt=prompt2, negative_prompt=neg2, image=image, mask_image=mask1, num_inference_steps=n_steps, strength=1.0, control_guidance_end=[0.9,0.9], controlnet_conditioning_scale=[0.3, 0.45], control_image=images_CN, generator=generator, ).images[0] return image """ image = refiner( prompt=prompt, num_inference_steps=40, denoising_start=0.8, image=image, ).images[0] """ #@spaces.GPU #[uncomment to use ZeroGPU] examples = [ "A photograph, of an Ourhood privacy booth, front view, in a warehouse eventspace environment, in the style of event photography, silken oak frame, checkered warm grey exterior fabric, checkered warm grey interior fabric, curtains, diner seating, pillows", "A photograph, of an Ourhood privacy booth, side view, in a warehouse eventspace environment, in the style of event photography, silken oak frame, taupe exterior fabric", "A photograph, of an Ourhood privacy booth, close-up, in a HolmrisB8_HQ office environment, in the style of makeshift photoshoot, silken oak frame, taupe exterior fabric, taupe interior fabric, pillows", "A rendering, of an Ourhood privacy booth, front view, in a Nordic atrium environment, in the style of Keyshot, silken oak frame, taupe exterior fabric, taupe interior fabric, diner seating"] css=""" #col-container { margin: 0 auto; max-width: 640px; } """ with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.Markdown(f""" # HB8-Ourhood inference test """) with gr.Row(): prompt = gr.Text( label="Prompt", show_label=False, max_lines=1, placeholder="Enter your prompt", container=False, ) run_button = gr.Button("Run", scale=0) result = gr.Image(label="Result", show_label=False) with gr.Accordion("Advanced Settings", open=False): perspective = gr.Slider( label="perspective", minimum=1, maximum=3, step=1, value=1, ) seed = gr.Slider( label="tracking number (seed)", minimum=0, maximum=MAX_SEED, step=1, value=0, ) with gr.Row(): num_inference_steps = gr.Slider( label="Number of inference steps", minimum=35, maximum=50, step=1, value=35, #Replace with defaults that work for your model ) gr.Examples( examples = examples, inputs = [prompt] ) gr.on( triggers=[run_button.click, prompt.submit], fn = ourhood_inference, inputs = [prompt, num_inference_steps, perspective,seed], outputs = [result] ) demo.queue().launch()