DL_models.py

import torch
import torch.nn as nn
import torch.nn.functional as F

import torch
import torch.nn as nn
import torchvision.models as models
from torchvision.models import ResNet50_Weights
import numpy as np
from torchvision import transforms

class CustomResNet(nn.Module):
    def __init__(self):
        super(CustomResNet, self).__init__()
        if torch.cuda.is_available():
            self.device = 'cuda'
        elif torch.backends.mps.is_available():
            self.device = 'mps'
        else:
            self.device = 'cpu'

        self.transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        ])

        self.resnet = models.resnet50(weights=ResNet50_Weights.DEFAULT)

        # Remove the final fully connected layer
        self.resnet = nn.Sequential(*list(self.resnet.children())[:-1])

        # Define MLP
        self.fc1 = nn.Linear(2 * (512 * 4), 256)
        self.fc2 = nn.Linear(256, 1)

        self.gradients = None

    def activations_hook(self, grad):
        self.gradients = grad

    def forward(self, x1, x2, Location):
        N = x1.shape[0]

        x1 = self.transform(x1.to(torch.float).to(self.device))
        x2 = self.transform(x2.to(torch.float).to(self.device))
        Location = Location.to(torch.float).to(self.device)

        # Process both images through the same ResNet
        f1 = self.resnet[:8](x1)
        h = f1.register_hook(self.activations_hook)
        f1 = self.resnet[8:](f1)
        f2 = self.resnet(x2)

        # Flatten the features
        f1 = f1.view(f1.size(0), -1)
        f2 = f2.view(f2.size(0), -1)

        f_ad = f1*Location[:,0].reshape(N,1) + f2*Location[:,1].reshape(N,1)
        f_context = f1*(1-Location[:,0].reshape(N,1)) + f2*(1-Location[:,1].reshape(N,1))

        # Concatenate the features
        combined = torch.cat((f_ad, f_context), dim=1)

        # Pass through MLP
        x = torch.relu(self.fc1(combined))
        x = self.fc2(x)

        return x
    
    # method for the gradient extraction
    def get_activations_gradient(self):
        return self.gradients

    # method for the activation exctraction
    def get_activations(self, x):
        x = self.transform(x.to(torch.float).to(self.device))
        return self.resnet[:8](x)