ui_model.py

import torch
import cv2
import numpy as np

from torchvision.transforms.functional import to_tensor, center_crop, resize
from PIL import Image
from EngageEngine.pipeline import EngagePipeline

from diffusers import (
    EulerAncestralDiscreteScheduler,
    AutoencoderKL, ControlNetModel,
)


def process_sketch(x, im_size=(1024, 1024), sketch_detail=0.5, sketch_softness=0.5):
    x_b = Image.new("RGBA", x.size, "WHITE")
    x_b.paste(x, mask=x)
    x = to_tensor(x_b.convert('RGB')).unsqueeze(0)
    x = center_crop(x, x.shape[-1])
    x = resize(x, im_size)

    u_th = (1 - sketch_detail) * 190 + 10
    l_th = (1 - sketch_detail) ** (sketch_softness * 8 + 1) * 185 + 5

    edges = [cv2.Canny(x[i].mul(255).permute(1, 2, 0).numpy().astype(np.uint8),
                       u_th, l_th, L2gradient=True) for i in range(len(x))]
    edges = torch.stack([torch.tensor(e).div(255).unsqueeze(0) for e in edges], dim=0)
    edges = torch.concatenate([edges, edges, edges], dim=1)
    return edges


def process_mask(x, mask, im_size=(1024, 1024)):
    x = to_tensor(x.convert('RGB')).unsqueeze(0)
    x = center_crop(x, x.shape[-1])
    x = resize(x, im_size)

    mask = to_tensor(mask.convert('L')).unsqueeze(0)
    mask = center_crop(mask, mask.shape[-1])
    mask = resize(mask, im_size)

    return x, mask


def fetch_model():
    # Load VAE component
    vae = AutoencoderKL.from_pretrained(
        "madebyollin/sdxl-vae-fp16-fix",
        torch_dtype=torch.float16
    )

    controlnet = ControlNetModel.from_pretrained(
        "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16
    )

    # Configure the pipeline
    pipe = EngagePipeline.from_pretrained(
        "dataautogpt3/ProteusV0.4-Lightning",
        vae=vae,
        controlnet=controlnet,
        torch_dtype=torch.float16
    )
    pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
    pipe.load_lora_weights("EngageEngine/ENGAGE_LORA.safetensors", adapter_name="ENGAGE_LORA")
    pipe.load_lora_weights("EngageEngine/FILM_LORA.safetensors", adapter_name="FILM_LORA")
    pipe.load_lora_weights("EngageEngine/MJ_LORA.safetensors", adapter_name="MJ_LORA")
    pipe.load_lora_weights("EngageEngine/MORE_ART_LORA.safetensors", adapter_name="MORE_ART_LORA")
    pipe.set_adapters(["ENGAGE_LORA", "FILM_LORA", "MJ_LORA", "MORE_ART_LORA"], adapter_weights=[0.0, 0.0, 0.0, 0.0])
    pipe.to('cuda')

    return pipe