gui/gradcampp_explainer.py

from numpy.core.fromnumeric import size
from base_explainer import BaseExplainer
import tensorflow.keras as keras
import tensorflow as tf
import numpy as np
from PIL import Image
import matplotlib.cm as cm
import cv2

class GradCAMPPExplainer(BaseExplainer):

    #implementacao do metodo abstrato
    def get_explanation(self, img, model, img_size, props, preprocess_input = None, index=None):
        #clona o modelo e remove a softmax da ultima camada
        clone = tf.keras.models.clone_model(model)
        clone.layers[-1].activation = None
        
        #cria modelo grad-cam
        grad_model = tf.keras.models.Model([clone.inputs], [clone.get_layer(props["conv_layer"]).output, clone.output])
        #transforma a imagem em array
        img_array = self.get_img_array(img, size = img_size, expand=False)
        #pre processa a imagem
        if preprocess_input:
            img_procecessed_array = preprocess_input(img_array)
        else:
            img_procecessed_array = img_array
        #faz o heatmap
        heatmap = self.__grad_cam_plus(grad_model, img_procecessed_array, props["conv_layer"], index)
        #poe o heatmap na imagem
        heat, mask = self.__save_and_display_gradcam(img, heatmap)


        return keras.preprocessing.image.array_to_img(heat)


    #transforma a imagem em array
    def get_img_array(self, img_path, size, expand=True):
        # `img` is a PIL image of size 299x299
        img = keras.preprocessing.image.load_img(img_path, target_size=size)
        # `array` is a float32 Numpy array of shape (299, 299, 3)
        array = keras.preprocessing.image.img_to_array(img)
        # We add a dimension to transform our array into a "batch"
        # of size (1, 299, 299, 3)
        if expand:
            array = np.expand_dims(array, axis=0)
        return array

    #implementacao do grad-cam++
    def __grad_cam_plus(self, model, img, layer_name, category_id=None):
        img_tensor = np.expand_dims(img, axis=0)

        with tf.GradientTape() as gtape1:
            with tf.GradientTape() as gtape2:
                with tf.GradientTape() as gtape3:
                    conv_output, predictions = model(img_tensor)
                    if category_id==None:
                        category_id = tf.argmax(predictions[0])
                        print(category_id)
                    output = predictions[:, category_id]
                    conv_first_grad = gtape3.gradient(output, conv_output)
                conv_second_grad = gtape2.gradient(conv_first_grad, conv_output)
            conv_third_grad = gtape1.gradient(conv_second_grad, conv_output)

        global_sum = np.sum(conv_output, axis=(0, 1, 2))

        alpha_num = conv_second_grad[0]
        alpha_denom = conv_second_grad[0]*2.0 + conv_third_grad[0]*global_sum
        alpha_denom = np.where(alpha_denom != 0.0, alpha_denom, 1e-10)
        
        alphas = alpha_num/alpha_denom
        alpha_normalization_constant = np.sum(alphas, axis=(0,1))
        alphas /= alpha_normalization_constant

        weights = np.maximum(conv_first_grad[0], 0.0)

        deep_linearization_weights = np.sum(weights*alphas, axis=(0,1))
        grad_CAM_map = np.sum(deep_linearization_weights*conv_output[0], axis=2)

        heatmap = tf.maximum(grad_CAM_map, 0) / tf.math.reduce_max(grad_CAM_map)
        
        heatmap = heatmap.numpy()
        return heatmap

    def __save_and_display_gradcam(self, img_path, heatmap, cam_path="cam.jpg", alpha=0.4):
        # Load the original image
        img = keras.preprocessing.image.load_img(img_path)
        img = keras.preprocessing.image.img_to_array(img)

        # Rescale heatmap to a range 0-255
        heatmap = np.uint8(255 * heatmap)
        im = Image.fromarray(heatmap)
        im = im.resize((img.shape[1], img.shape[0]))
        
        im = np.asarray(im)
        im = np.where(im > 0, 1, im)

        # Use jet colormap to colorize heatmap
        jet = cm.get_cmap("jet")

        # Use RGB values of the colormap
        jet_colors = jet(np.arange(256))[:, :3]
        jet_heatmap = jet_colors[heatmap]

    

        # Create an image with RGB colorized heatmap
        jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)
        jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
        jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)

        # Superimpose the heatmap on original image
        superimposed_img = jet_heatmap * alpha + img
        #superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)

        # Save the superimposed image
        #superimposed_img.save(cam_path)

        # Display Grad CAM
        #display(Image(cam_path))
        return superimposed_img, im