custom_nodes/ComfyUI-Frame-Interpolation/vfi_models/ops/cupy_ops/sepconv.py

import torch
from .utils import cuda_launch, cuda_kernel, cuda_int32

sepconv_vergrad = """
    extern "C" __global__ void __launch_bounds__(512) sepconv_vergrad(
        const int n,
        const {{type}}* __restrict__ tenIn,
        const {{type}}* __restrict__ tenVer,
        const {{type}}* __restrict__ tenHor,
        const {{type}}* __restrict__ tenOutgrad,
        {{type}}* __restrict__ tenIngrad,
        {{type}}* __restrict__ tenVergrad,
        {{type}}* __restrict__ tenHorgrad
    ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) {
        const int intN = ( intIndex / SIZE_3(tenVergrad) / SIZE_2(tenVergrad) / SIZE_1(tenVergrad) ) % SIZE_0(tenVergrad);
        const int intC = ( intIndex / SIZE_3(tenVergrad) / SIZE_2(tenVergrad)                      ) % SIZE_1(tenVergrad);
        const int intY = ( intIndex / SIZE_3(tenVergrad)                                           ) % SIZE_2(tenVergrad);
        const int intX = ( intIndex                                                                ) % SIZE_3(tenVergrad);

        {{type}} fltVergrad = 0.0;

        {{type}} fltKahanc = 0.0;
        {{type}} fltKahany = 0.0;
        {{type}} fltKahant = 0.0;

        for (int intI = 0; intI < SIZE_1(tenIn); intI += 1) {
            for (int intFx = 0; intFx < SIZE_1(tenHor); intFx += 1) {
                fltKahany = VALUE_4(tenHor, intN, intFx, intY, intX) * VALUE_4(tenIn, intN, intI, intY + intC, intX + intFx) * VALUE_4(tenOutgrad, intN, intI, intY, intX);
                fltKahany = fltKahany - fltKahanc;
                fltKahant = fltVergrad + fltKahany;
                fltKahanc = (fltKahant - fltVergrad) - fltKahany;
                fltVergrad = fltKahant;
            }
        }

        tenVergrad[intIndex] = fltVergrad;
    } }
"""

sepconv_ingrad = """
    extern "C" __global__ void __launch_bounds__(512) sepconv_ingrad(
        const int n,
        const {{type}}* __restrict__ tenIn,
        const {{type}}* __restrict__ tenVer,
        const {{type}}* __restrict__ tenHor,
        const {{type}}* __restrict__ tenOutgrad,
        {{type}}* __restrict__ tenIngrad,
        {{type}}* __restrict__ tenVergrad,
        {{type}}* __restrict__ tenHorgrad
    ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) {
        const int intN = ( intIndex / SIZE_3(tenIngrad) / SIZE_2(tenIngrad) / SIZE_1(tenIngrad) ) % SIZE_0(tenIngrad);
        const int intC = ( intIndex / SIZE_3(tenIngrad) / SIZE_2(tenIngrad)                     ) % SIZE_1(tenIngrad);
        const int intY = ( intIndex / SIZE_3(tenIngrad)                                         ) % SIZE_2(tenIngrad);
        const int intX = ( intIndex                                                             ) % SIZE_3(tenIngrad);

        {{type}} fltIngrad = 0.0;

        {{type}} fltKahanc = 0.0;
        {{type}} fltKahany = 0.0;
        {{type}} fltKahant = 0.0;

        for (int intFy = 0; intFy < SIZE_1(tenVer); intFy += 1) {
            int intKy = intY + intFy - (SIZE_1(tenVer) - 1);

            if (intKy < 0) { continue; }
            if (intKy >= SIZE_2(tenVer)) { continue; }

            for (int intFx = 0; intFx < SIZE_1(tenHor); intFx += 1) {
                int intKx = intX + intFx - (SIZE_1(tenHor) - 1);

                if (intKx < 0) { continue; }
                if (intKx >= SIZE_3(tenHor)) { continue; }

                fltKahany = VALUE_4(tenVer, intN, (SIZE_1(tenVer) - 1) - intFy, intKy, intKx) * VALUE_4(tenHor, intN, (SIZE_1(tenHor) - 1) - intFx, intKy, intKx) * VALUE_4(tenOutgrad, intN, intC, intKy, intKx);
                fltKahany = fltKahany - fltKahanc;
                fltKahant = fltIngrad + fltKahany;
                fltKahanc = (fltKahant - fltIngrad) - fltKahany;
                fltIngrad = fltKahant;
            }
        }

        tenIngrad[intIndex] = fltIngrad;
    } }
"""

sepconv_out = """
    extern "C" __global__ void __launch_bounds__(512) sepconv_out(
        const int n,
        const {{type}}* __restrict__ tenIn,
        const {{type}}* __restrict__ tenVer,
        const {{type}}* __restrict__ tenHor,
        {{type}}* __restrict__ tenOut
    ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) {
        const int intN = ( intIndex / SIZE_3(tenOut) / SIZE_2(tenOut) / SIZE_1(tenOut) ) % SIZE_0(tenOut);
        const int intC = ( intIndex / SIZE_3(tenOut) / SIZE_2(tenOut)                  ) % SIZE_1(tenOut);
        const int intY = ( intIndex / SIZE_3(tenOut)                                   ) % SIZE_2(tenOut);
        const int intX = ( intIndex                                                    ) % SIZE_3(tenOut);

        {{type}} fltOut = 0.0;

        {{type}} fltKahanc = 0.0;
        {{type}} fltKahany = 0.0;
        {{type}} fltKahant = 0.0;

        for (int intFy = 0; intFy < SIZE_1(tenVer); intFy += 1) {
            for (int intFx = 0; intFx < SIZE_1(tenHor); intFx += 1) {
                fltKahany = VALUE_4(tenIn, intN, intC, intY + intFy, intX + intFx) * VALUE_4(tenVer, intN, intFy, intY, intX) * VALUE_4(tenHor, intN, intFx, intY, intX);
                fltKahany = fltKahany - fltKahanc;
                fltKahant = fltOut + fltKahany;
                fltKahanc = (fltKahant - fltOut) - fltKahany;
                fltOut = fltKahant;
            }
        }

        tenOut[intIndex] = fltOut;
    } }
"""

sepconv_horgrad = """
    extern "C" __global__ void __launch_bounds__(512) sepconv_horgrad(
        const int n,
        const {{type}}* __restrict__ tenIn,
        const {{type}}* __restrict__ tenVer,
        const {{type}}* __restrict__ tenHor,
        const {{type}}* __restrict__ tenOutgrad,
        {{type}}* __restrict__ tenIngrad,
        {{type}}* __restrict__ tenVergrad,
        {{type}}* __restrict__ tenHorgrad
    ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) {
        const int intN = ( intIndex / SIZE_3(tenHorgrad) / SIZE_2(tenHorgrad) / SIZE_1(tenHorgrad) ) % SIZE_0(tenHorgrad);
        const int intC = ( intIndex / SIZE_3(tenHorgrad) / SIZE_2(tenHorgrad)                      ) % SIZE_1(tenHorgrad);
        const int intY = ( intIndex / SIZE_3(tenHorgrad)                                           ) % SIZE_2(tenHorgrad);
        const int intX = ( intIndex                                                                ) % SIZE_3(tenHorgrad);

        {{type}} fltHorgrad = 0.0;

        {{type}} fltKahanc = 0.0;
        {{type}} fltKahany = 0.0;
        {{type}} fltKahant = 0.0;

        for (int intI = 0; intI < SIZE_1(tenIn); intI += 1) {
            for (int intFy = 0; intFy < SIZE_1(tenVer); intFy += 1) {
                fltKahany = VALUE_4(tenVer, intN, intFy, intY, intX) * VALUE_4(tenIn, intN, intI, intY + intFy, intX + intC) * VALUE_4(tenOutgrad, intN, intI, intY, intX);
                fltKahany = fltKahany - fltKahanc;
                fltKahant = fltHorgrad + fltKahany;
                fltKahanc = (fltKahant - fltHorgrad) - fltKahany;
                fltHorgrad = fltKahant;
            }
        }

        tenHorgrad[intIndex] = fltHorgrad;
    } }
"""

class sepconv_func(torch.autograd.Function):
    @staticmethod
    @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32)
    def forward(self, tenIn, tenVer, tenHor):
        tenOut = tenIn.new_empty(
            [
                tenIn.shape[0],
                tenIn.shape[1],
                tenVer.shape[2] and tenHor.shape[2],
                tenVer.shape[3] and tenHor.shape[3],
            ]
        )

        if tenIn.is_cuda == True:
            cuda_launch(
                cuda_kernel(
                    "sepconv_out",
                    sepconv_out,
                    {
                        "tenIn": tenIn,
                        "tenVer": tenVer,
                        "tenHor": tenHor,
                        "tenOut": tenOut,
                    },
                )
            )(
                grid=tuple([int((tenOut.nelement() + 512 - 1) / 512), 1, 1]),
                block=tuple([512, 1, 1]),
                args=[
                    cuda_int32(tenOut.nelement()),
                    tenIn.data_ptr(),
                    tenVer.data_ptr(),
                    tenHor.data_ptr(),
                    tenOut.data_ptr(),
                ],
            )

        elif tenIn.is_cuda != True:
            assert False

        # end

        self.save_for_backward(tenIn, tenVer, tenHor)

        return tenOut

    # end

    @staticmethod
    @torch.cuda.amp.custom_bwd
    def backward(self, tenOutgrad):
        tenIn, tenVer, tenHor = self.saved_tensors

        tenOutgrad = tenOutgrad.contiguous()
        assert tenOutgrad.is_cuda == True

        tenIngrad = (
            tenIn.new_empty(
                [tenIn.shape[0], tenIn.shape[1], tenIn.shape[2], tenIn.shape[3]]
            )
            if self.needs_input_grad[0] == True
            else None
        )
        tenVergrad = (
            tenVer.new_empty(
                [tenVer.shape[0], tenVer.shape[1], tenVer.shape[2], tenVer.shape[3]]
            )
            if self.needs_input_grad[1] == True
            else None
        )
        tenHorgrad = (
            tenHor.new_empty(
                [tenHor.shape[0], tenHor.shape[1], tenHor.shape[2], tenHor.shape[3]]
            )
            if self.needs_input_grad[2] == True
            else None
        )

        if tenIngrad is not None:
            cuda_launch(
                cuda_kernel(
                    "sepconv_ingrad",
                    sepconv_ingrad,
                    {
                        "tenIn": tenIn,
                        "tenVer": tenVer,
                        "tenHor": tenHor,
                        "tenOutgrad": tenOutgrad,
                        "tenIngrad": tenIngrad,
                        "tenVergrad": tenVergrad,
                        "tenHorgrad": tenHorgrad,
                    },
                )
            )(
                grid=tuple([int((tenIngrad.nelement() + 512 - 1) / 512), 1, 1]),
                block=tuple([512, 1, 1]),
                args=[
                    cuda_int32(tenIngrad.nelement()),
                    tenIn.data_ptr(),
                    tenVer.data_ptr(),
                    tenHor.data_ptr(),
                    tenOutgrad.data_ptr(),
                    tenIngrad.data_ptr(),
                    None,
                    None,
                ],
            )
        # end

        if tenVergrad is not None:
            cuda_launch(
                cuda_kernel(
                    "sepconv_vergrad",
                    sepconv_vergrad,
                    {
                        "tenIn": tenIn,
                        "tenVer": tenVer,
                        "tenHor": tenHor,
                        "tenOutgrad": tenOutgrad,
                        "tenIngrad": tenIngrad,
                        "tenVergrad": tenVergrad,
                        "tenHorgrad": tenHorgrad,
                    },
                )
            )(
                grid=tuple([int((tenVergrad.nelement() + 512 - 1) / 512), 1, 1]),
                block=tuple([512, 1, 1]),
                args=[
                    cuda_int32(tenVergrad.nelement()),
                    tenIn.data_ptr(),
                    tenVer.data_ptr(),
                    tenHor.data_ptr(),
                    tenOutgrad.data_ptr(),
                    None,
                    tenVergrad.data_ptr(),
                    None,
                ],
            )
        # end

        if tenHorgrad is not None:
            cuda_launch(
                cuda_kernel(
                    "sepconv_horgrad",
                    sepconv_horgrad,
                    {
                        "tenIn": tenIn,
                        "tenVer": tenVer,
                        "tenHor": tenHor,
                        "tenOutgrad": tenOutgrad,
                        "tenIngrad": tenIngrad,
                        "tenVergrad": tenVergrad,
                        "tenHorgrad": tenHorgrad,
                    },
                )
            )(
                grid=tuple([int((tenHorgrad.nelement() + 512 - 1) / 512), 1, 1]),
                block=tuple([512, 1, 1]),
                args=[
                    cuda_int32(tenHorgrad.nelement()),
                    tenIn.data_ptr(),
                    tenVer.data_ptr(),
                    tenHor.data_ptr(),
                    tenOutgrad.data_ptr(),
                    None,
                    None,
                    tenHorgrad.data_ptr(),
                ],
            )
        # end

        return tenIngrad, tenVergrad, tenHorgrad

    # end


# end
__all__ = ["sepconv_func"]