models/PDNet.py

from pathlib import Path

import torch
from torch.func import vmap
from torch.utils.data import DataLoader
import deepinv as dinv
from deepinv.unfolded import unfolded_builder
from deepinv.utils.phantoms import RandomPhantomDataset, SheppLoganDataset
from deepinv.optim.optim_iterators import CPIteration, fStep, gStep
from deepinv.optim import Prior, DataFidelity
from deepinv.utils import TensorList

from physics.multiscale import MultiScaleLinearPhysics
from models.heads import Heads, Tails, InHead, OutTail, ConvChannels, SNRModule, EquivConvModule, EquivHeads


def get_PDNet_architecture(in_channels=[1, 2, 3], out_channels=[1, 2, 3], n_primal=3, n_dual=3, device='cuda'):
    class PDNetIteration(CPIteration):
        r"""Single iteration of learned primal dual.
        We only redefine the fStep and gStep classes.
        The forward method is inherited from the CPIteration class.
        """

        def __init__(self, **kwargs):
            super().__init__(**kwargs)
            self.g_step = gStepPDNet(**kwargs)
            self.f_step = fStepPDNet(**kwargs)

        def forward(
                self, X, cur_data_fidelity, cur_prior, cur_params, y, physics, *args, **kwargs
        ):
            r"""
            Single iteration of the Chambolle-Pock algorithm.

            :param dict X: Dictionary containing the current iterate and the estimated cost.
            :param deepinv.optim.DataFidelity cur_data_fidelity: Instance of the DataFidelity class defining the current data_fidelity.
            :param deepinv.optim.Prior cur_prior: Instance of the Prior class defining the current prior.
            :param dict cur_params: dictionary containing the current parameters of the algorithm.
            :param torch.Tensor y: Input data.
            :param deepinv.physics.Physics physics: Instance of the physics modeling the data-fidelity term.
            :return: Dictionary `{"est": (x, ), "cost": F}` containing the updated current iterate and the estimated current cost.
            """
            x_prev, z_prev, u_prev = X["est"]  # x : primal, z : relaxed primal, u : dual
            BS, C_primal, H_primal, W_primal = x_prev.shape
            _, C_dual, H_dual, W_dual = u_prev.shape
            n_channels = C_primal // n_primal
            K = lambda x: torch.cat(
                [physics.A(x[:, i * n_channels:(i + 1) * n_channels, :, :]) for i in range(n_primal)], dim=1)
            K_adjoint = lambda x: torch.cat(
                [physics.A_adjoint(x[:, i * n_channels:(i + 1) * n_channels, :, :]) for i in range(n_dual)], dim=1)
            u = self.f_step(u_prev, K(z_prev), cur_data_fidelity, y, physics, n_channels,
                            cur_params)  # dual update (data_fid)
            x = self.g_step(x_prev, K_adjoint(u), cur_prior, n_channels, cur_params)  # primal update (prior)
            z = x + cur_params["beta"] * (x - x_prev)
            F = (
                self.F_fn(x, cur_data_fidelity, cur_prior, cur_params, y, physics)
                if self.has_cost
                else None
            )
            return {"est": (x, z, u), "cost": F}

    class fStepPDNet(fStep):
        r"""
        Dual update of the PDNet algorithm.
        We write it as a proximal operator of the data fidelity term.
        This proximal mapping is to be replaced by a trainable model.
        """

        def __init__(self, **kwargs):
            super().__init__(**kwargs)

        def forward(self, x, w, cur_data_fidelity, y, physics, n_channels, *args):
            r"""
            :param torch.Tensor x: Current first variable :math:`u`.
            :param torch.Tensor w: Current second variable :math:`A z`.
            :param deepinv.optim.data_fidelity cur_data_fidelity: Instance of the DataFidelity class defining the current data fidelity term.
            :param torch.Tensor y: Input data.
            """
            return cur_data_fidelity.prox(x, w, y, n_channels)

    class gStepPDNet(gStep):
        r"""
        Primal update of the PDNet algorithm.
        We write it as a proximal operator of the prior term.
        This proximal mapping is to be replaced by a trainable model.
        """

        def __init__(self, **kwargs):
            super().__init__(**kwargs)

        def forward(self, x, w, cur_prior, n_channels, *args):
            r"""
            :param torch.Tensor x: Current first variable :math:`x`.
            :param torch.Tensor w: Current second variable :math:`A^\top u`.
            :param deepinv.optim.prior cur_prior: Instance of the Prior class defining the current prior.
            """
            return cur_prior.prox(x, w, n_channels)

    # %%
    # Define the trainable prior and data fidelity terms.
    # ---------------------------------------------------
    # Prior and data-fidelity are respectively defined as subclass of :class:`deepinv.optim.Prior` and :class:`deepinv.optim.DataFidelity`.
    # Their proximal operators are replaced by trainable models.

    class PDNetPrior(Prior):
        def __init__(self, model, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.model = model

        def prox(self, x, w, n_channels):
            # give to the model : full primal + premier de dual
            dual_cond = w[:, 0:n_channels, :, :]
            return self.model(x, dual_cond)

    class PDNetDataFid(DataFidelity):
        def __init__(self, model, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.model = model

        def prox(self, x, w, y, n_channels):
            # give to the model : full dual + deuxieme de primal + y = n_channel*n_dual + n_channel + n_channel
            if n_primal > 1:
                primal_cond = w[:, n_channels:(2 * n_channels), :, :]
            else:
                primal_cond = w[:, 0:n_channels, :, :]
            return self.model(x, primal_cond, y)

    # Unrolled optimization algorithm parameters
    max_iter = 10

    # Set up the data fidelity term. Each layer has its own data fidelity module.
    in_channels_dual = [in_channel * n_dual + in_channel + in_channel for in_channel in in_channels]
    out_channels_dual = [in_channel * n_dual for in_channel in in_channels]
    in_channels_primal = [in_channel * n_primal + in_channel for in_channel in in_channels]
    out_channels_primal = [in_channel * n_primal for in_channel in in_channels]

    data_fidelity = [
        PDNetDataFid(model=PDNet_DualBlock(in_channels=in_channels_dual, out_channels=out_channels_dual).to(device)) for
        i in range(max_iter)
    ]

    # Set up the trainable prior. Each layer has its own prior module.
    prior = [
        PDNetPrior(model=PDNet_PrimalBlock(in_channels=in_channels_primal, out_channels=out_channels_primal).to(device))
        for i in range(max_iter)]

    # %%
    # Define the model.
    # -------------------------------

    def custom_init(y, physics):
        x0 = physics.A_dagger(y).repeat(1, n_primal, 1, 1)
        u0 = (0 * y).repeat(1, n_dual, 1, 1)
        return {"est": (x0, x0, u0)}

    def custom_output(X):
        x = X["est"][0]
        n_channels = x.shape[1] // n_primal
        if n_primal > 1:
            return X["est"][0][:, n_channels:(2 * n_channels), :, :]
        else:
            return X["est"][0][:, 0:n_channels, :, :]

    # %%
    # Define the unfolded trainable model.
    # -------------------------------------
    # The original paper of the learned primal dual algorithm the authors used the adjoint operator
    # in the primal update. However, the same authors (among others) find in the paper
    #
    # A. Hauptmann, J. Adler, S. Arridge, O. Öktem,
    # Multi-scale learned iterative reconstruction,
    # IEEE Transactions on Computational Imaging 6, 843-856, 2020.
    #
    # that using a filtered gradient can improve both the training speed and reconstruction quality significantly.
    # Following this approach, we use the filtered backprojection instead of the adjoint operator in the primal step.

    model = unfolded_builder(
        iteration=PDNetIteration(),
        params_algo={"beta": 0.0},
        data_fidelity=data_fidelity,
        prior=prior,
        max_iter=max_iter,
        custom_init=custom_init,
        get_output=custom_output,
    )

    return model.to(device)


def init_weights(m):
    if isinstance(m, torch.nn.Linear):
        torch.torch.nn.init.xavier_uniform(m.weight)
        m.bias.data.fill_(0.0)


class PDNet_PrimalBlock(torch.nn.Module):
    r"""
    Primal block for the Primal-Dual unfolding model.

    From https://arxiv.org/abs/1707.06474.

    Primal variables are images of shape (batch_size, in_channels, height, width). The input of each
    primal block is the concatenation of the current primal variable and the backprojected dual variable along
    the channel dimension. The output of each primal block is the current primal variable.

    :param int in_channels: number of input channels. Default: 6.
    :param int out_channels: number of output channels. Default: 5.
    :param int depth: number of convolutional layers in the block. Default: 3.
    :param bool bias: whether to use bias in convolutional layers. Default: True.
    :param int nf: number of features in the convolutional layers. Default: 32.
    """

    def __init__(self, in_channels=[1, 2, 3], out_channels=[1, 2, 3], depth=3, bias=True, nf=32):
        super(PDNet_PrimalBlock, self).__init__()

        self.separate_head = isinstance(in_channels, list)
        self.depth = depth

        self.in_conv = InHead(in_channels, nf, bias=bias)
        # self.m_head.apply(init_weights)

        # self.in_conv = torch.nn.Conv2d(
        #     in_channels, nf, kernel_size=3, stride=1, padding=1, bias=bias
        # )

        self.in_conv.apply(init_weights)
        self.conv_list = torch.nn.ModuleList(
            [
                torch.nn.Conv2d(nf, nf, kernel_size=3, stride=1, padding=1, bias=bias)
                for _ in range(self.depth - 2)
            ]
        )
        self.conv_list.apply(init_weights)
        # self.out_conv = torch.nn.Conv2d(
        #     nf, out_channels, kernel_size=3, stride=1, padding=1, bias=bias
        # )
        self.out_conv = OutTail(nf, out_channels, bias=bias)
        self.out_conv.apply(init_weights)

        self.nl_list = torch.nn.ModuleList([torch.nn.PReLU() for _ in range(self.depth - 1)])

    def forward(self, x, Atu):
        r"""
        Forward pass of the primal block.

        :param torch.Tensor x: current primal variable.
        :param torch.Tensor Atu: backprojected dual variable.
        :return: (:class:`torch.Tensor`) the current primal variable.
        """
        primal_channels = x.shape[1]
        x_in = torch.cat((x, Atu), dim=1)

        x_ = self.in_conv(x_in)
        x_ = self.nl_list[0](x_)

        for i in range(self.depth - 2):
            x_l = self.conv_list[i](x_)
            x_ = self.nl_list[i + 1](x_l)

        return self.out_conv(x_, primal_channels) + x


class PDNet_DualBlock(torch.nn.Module):
    r"""
    Dual block for the Primal-Dual unfolding model.

    From https://arxiv.org/abs/1707.06474.

    Dual variables are images of shape (batch_size, in_channels, height, width). The input of each
    primal block is the concatenation of the current dual variable with the projected primal variable and
    the measurements. The output of each dual block is the current primal variable.

    :param int in_channels: number of input channels. Default: 7.
    :param int out_channels: number of output channels. Default: 5.
    :param int depth: number of convolutional layers in the block. Default: 3.
    :param bool bias: whether to use bias in convolutional layers. Default: True.
    :param int nf: number of features in the convolutional layers. Default: 32.
    """

    def __init__(self, in_channels=[1, 2, 3], out_channels=[6, 2, 3], depth=3, bias=True, nf=32):
        super(PDNet_DualBlock, self).__init__()

        self.depth = depth
        self.in_conv = InHead(in_channels, nf, bias=bias)
        # self.in_conv = torch.nn.Conv2d(
        #     in_channels, nf, kernel_size=3, stride=1, padding=1, bias=bias
        # )
        self.in_conv.apply(init_weights)
        self.conv_list = torch.nn.ModuleList(
            [
                torch.nn.Conv2d(nf, nf, kernel_size=3, stride=1, padding=1, bias=bias)
                for _ in range(self.depth - 2)
            ]
        )
        self.conv_list.apply(init_weights)
        self.out_conv = OutTail(nf, out_channels, bias=bias)
        # self.out_conv = torch.nn.Conv2d(
        #    nf, out_channels, kernel_size=3, stride=1, padding=1, bias=bias
        # )
        self.out_conv.apply(init_weights)

        self.nl_list = torch.nn.ModuleList([torch.nn.PReLU() for _ in range(self.depth - 1)])

    def forward(self, u, Ax_cur, y):
        r"""
        Forward pass of the dual block.

        :param torch.Tensor u: current dual variable.
        :param torch.Tensor Ax_cur: projection of the primal variable.
        :param torch.Tensor y: measurements.
        """
        dual_channels = u.shape[1]
        x_in = torch.cat((u, Ax_cur, y), dim=1)

        x_ = self.in_conv(x_in)
        x_ = self.nl_list[0](x_)

        for i in range(self.depth - 2):
            x_l = self.conv_list[i](x_)
            x_ = self.nl_list[i + 1](x_l)

        return self.out_conv(x_, dual_channels) + u