Add show label on gradcam

app.py

@@ -14,15 +14,14 @@ import mediapy
 import numpy as np
 import pandas as pd
 import torch
-from deep_translator import GoogleTranslator
+import deep_translator
 from gradio_blocks import build_video_to_camvideo
 from Nets import CustomResNet18
 from PIL import Image, ImageDraw, ImageFont
 
-# from pytorch_grad_cam import GradCAM, HiResCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM, FullGrad
-from custom_code.custom_grad_cam import GradCAM, HiResCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM, FullGrad
-from custom_code.custom_grad_cam.utils.model_targets import ClassifierOutputTarget
-from custom_code.custom_grad_cam.utils.image import show_cam_on_image
+from pytorch_grad_cam import GradCAM, HiResCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM, FullGrad
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+from pytorch_grad_cam.utils.image import show_cam_on_image
 
 from tqdm import tqdm
 from util import transform
@@ -36,9 +35,9 @@ IMAGE_PATH = os.path.join(os.getcwd(), 'src/examples')
 IMAGES_PER_ROW = 5
 
 MAXIMAL_FRAMES = 700
-BATCHES_TO_PROCESS = 20
-OUTPUT_FPS = 10
-MAX_OUT_FRAMES = 70
+BATCHES_TO_PROCESS = 15
+OUTPUT_FPS = 15
+MAX_OUT_FRAMES = 90
 
 MODEL = CustomResNet18(111).eval()
 MODEL.load_state_dict(torch.load('src/results/models/best_model.pth', map_location=torch.device('cpu')))
@@ -51,8 +50,6 @@ LANGUAGES_TO_SELECT = {
     "Italian": "it",
     "Finnish": "fi",
     "Ukrainian": "uk",
-    "Japanese": "ja",
-    "Hebrew": "iw"
 }
 
 CAM_METHODS = {
@@ -106,8 +103,12 @@ def get_translated(to_translate, target_language="German"):
     target_language = LANGUAGES_TO_SELECT[target_language] if target_language in LANGUAGES_TO_SELECT else target_language
     if target_language == "en": return to_translate
     if target_language not in LANGUAGES_TO_SELECT.values(): raise gr.Error(f'Language {target_language} not found.')
-    return GoogleTranslator(source="en", target=target_language).translate(to_translate)
-# for idx in range(111): get_translated(get_class_name(idx))
+    try:
+        return deep_translator.GoogleTranslator(source="en", target=target_language).translate(to_translate)
+    except deep_translator.exceptions.TooManyRequests:
+        print(f'Too many requests for {to_translate} to {target_language}.')
+        return ("-/-")
+
 with ThreadPoolExecutor(max_workers=30) as executor:
     # give the executor the list of images and args (in this case, the target language)
     # and let the executor map the function to the list of images
@@ -156,8 +157,8 @@ def gradcam(image, colormap="Jet", use_eigen_smooth=False, use_aug_smooth=False,
 
     with CAM_METHODS[cam_method](model=MODEL, target_layers=layers) as cam:
         grayscale_cam = cam(input_tensor=image_tensor, targets=targets, aug_smooth=use_aug_smooth, eigen_smooth=use_eigen_smooth)
-        if label_image:
-            predicted_animal = get_class_name(np.argmax(cam.outputs.cpu().data.numpy(), axis=-1)[0])
+    if label_image:
+        predicted_animal = get_class_name(np.argmax(MODEL.output.cpu().data.numpy(), axis=-1)[0])
         
     grayscale_cam = grayscale_cam[0, :]
     grayscale_cam = cv2.resize(grayscale_cam, (image_width, image_height), interpolation=cv2.INTER_CUBIC)
@@ -185,7 +186,7 @@ def gradcam(image, colormap="Jet", use_eigen_smooth=False, use_aug_smooth=False,
     out_image = Image.fromarray(visualization)
     return out_image
 
-def gradcam_video(video, colormap="Jet", use_eigen_smooth=False, BWHighlight=False, alpha=0.5, cam_method=GradCAM, layer=None, specific_class="Predicted Class"):
+def gradcam_video(video, colormap="Jet", use_eigen_smooth=False, BWHighlight=False, alpha=0.5, cam_method=GradCAM, layer=None, specific_class="Predicted Class", label_image=True, target_lang="German"):
     global OUTPUT_FPS, MAXIMAL_FRAMES, BATCHES_TO_PROCESS, MAX_OUT_FRAMES
     if video is None: raise gr.Error("Please upload a video.")
     if colormap not in CV2_COLORMAPS.keys():
@@ -241,6 +242,18 @@ def gradcam_video(video, colormap="Jet", use_eigen_smooth=False, BWHighlight=Fal
                 else:
                     image = image / 255
                     visualization = show_cam_on_image(image, _grayscale_cam, use_rgb=True, image_weight=alpha, colormap=colormap)
+                    
+                if label_image:
+                    pass
+                    predicted_animal = get_class_name(np.argmax(MODEL.output.cpu().data.numpy(), axis=-1)[i])
+                    plt_image = Image.fromarray(visualization, mode="RGB")
+                    draw = ImageDraw.Draw(plt_image)
+                    draw.rectangle((5, 5, 150, 30), fill=(10, 10, 10, 100))
+                    animal = predicted_animal.capitalize()
+                    if target_lang is not None and target_lang != "None":
+                        animal += f' ({get_translated(animal, target_lang)})'
+                    draw.text((10, 7), animal, font=font, fill=(255, 125, 0, 255))
+                    visualization = np.array(plt_image)
                 results.append(visualization)
         
     # save video
@@ -479,7 +492,7 @@ with gr.Blocks(theme='freddyaboulton/dracula_revamped', css=css) as demo:
         #                Video CAM
         # -------------------------------------------
         with gr.Tab("Explain Video"):
-            build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gradcam_video)
+            build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gradcam_video, animal_translation_target_language)
         
         # -------------------------------------------
         #                EXAMPLES

requirements.txt

@@ -49,6 +49,7 @@ ipykernel==6.25.2
 ipython==8.16.0
 jedi==0.19.0
 Jinja2==3.1.2
+grad-cam==1.4.8
 joblib==1.3.2
 jsonschema==4.19.1
 jsonschema-specifications==2023.7.1

src/Nets.py

@@ -4,9 +4,11 @@ from torchvision import models
 class CustomResNet18(nn.Module):
     def __init__(self, num_classes=11):
         super(CustomResNet18, self).__init__()
+        self.output = None
         self.resnet = models.resnet18(pretrained=True)
         num_features = self.resnet.fc.in_features
         self.resnet.fc = nn.Linear(num_features, num_classes)
 
     def forward(self, x):
-        return self.resnet(x)
+        self.output = self.resnet(x)
+        return self.output

src/custom_code/custom_grad_cam/__init__.py

@@ -1,20 +0,0 @@
-from custom_grad_cam.grad_cam import GradCAM
-from custom_grad_cam.hirescam import HiResCAM
-from custom_grad_cam.grad_cam_elementwise import GradCAMElementWise
-from custom_grad_cam.ablation_layer import AblationLayer, AblationLayerVit, AblationLayerFasterRCNN
-from custom_grad_cam.ablation_cam import AblationCAM
-from custom_grad_cam.xgrad_cam import XGradCAM
-from custom_grad_cam.grad_cam_plusplus import GradCAMPlusPlus
-from custom_grad_cam.score_cam import ScoreCAM
-from custom_grad_cam.layer_cam import LayerCAM
-from custom_grad_cam.eigen_cam import EigenCAM
-from custom_grad_cam.eigen_grad_cam import EigenGradCAM
-from custom_grad_cam.random_cam import RandomCAM
-from custom_grad_cam.fullgrad_cam import FullGrad
-from custom_grad_cam.guided_backprop import GuidedBackpropReLUModel
-from custom_grad_cam.activations_and_gradients import ActivationsAndGradients
-from custom_grad_cam.feature_factorization.deep_feature_factorization import DeepFeatureFactorization, run_dff_on_image
-import custom_grad_cam.utils.model_targets
-import custom_grad_cam.utils.reshape_transforms
-import custom_grad_cam.metrics.cam_mult_image
-import custom_grad_cam.metrics.road

src/custom_code/custom_grad_cam/ablation_cam.py

@@ -1,148 +0,0 @@
-import numpy as np
-import torch
-import tqdm
-from typing import Callable, List
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.find_layers import replace_layer_recursive
-from custom_grad_cam.ablation_layer import AblationLayer
-
-
-""" Implementation of AblationCAM
-https://openaccess.thecvf.com/content_WACV_2020/papers/Desai_Ablation-CAM_Visual_Explanations_for_Deep_Convolutional_Network_via_Gradient-free_Localization_WACV_2020_paper.pdf
-
-Ablate individual activations, and then measure the drop in the target score.
-
-In the current implementation, the target layer activations is cached, so it won't be re-computed.
-However layers before it, if any, will not be cached.
-This means that if the target layer is a large block, for example model.featuers (in vgg), there will
-be a large save in run time.
-
-Since we have to go over many channels and ablate them, and every channel ablation requires a forward pass,
-it would be nice if we could avoid doing that for channels that won't contribute anwyay, making it much faster.
-The parameter ratio_channels_to_ablate controls how many channels should be ablated, using an experimental method
-(to be improved). The default 1.0 value means that all channels will be ablated.
-"""
-
-
-class AblationCAM(BaseCAM):
-    def __init__(self,
-                 model: torch.nn.Module,
-                 target_layers: List[torch.nn.Module],
-                 use_cuda: bool = False,
-                 reshape_transform: Callable = None,
-                 ablation_layer: torch.nn.Module = AblationLayer(),
-                 batch_size: int = 32,
-                 ratio_channels_to_ablate: float = 1.0) -> None:
-
-        super(AblationCAM, self).__init__(model,
-                                          target_layers,
-                                          use_cuda,
-                                          reshape_transform,
-                                          uses_gradients=False)
-        self.batch_size = batch_size
-        self.ablation_layer = ablation_layer
-        self.ratio_channels_to_ablate = ratio_channels_to_ablate
-
-    def save_activation(self, module, input, output) -> None:
-        """ Helper function to save the raw activations from the target layer """
-        self.activations = output
-
-    def assemble_ablation_scores(self,
-                                 new_scores: list,
-                                 original_score: float,
-                                 ablated_channels: np.ndarray,
-                                 number_of_channels: int) -> np.ndarray:
-        """ Take the value from the channels that were ablated,
-            and just set the original score for the channels that were skipped """
-
-        index = 0
-        result = []
-        sorted_indices = np.argsort(ablated_channels)
-        ablated_channels = ablated_channels[sorted_indices]
-        new_scores = np.float32(new_scores)[sorted_indices]
-
-        for i in range(number_of_channels):
-            if index < len(ablated_channels) and ablated_channels[index] == i:
-                weight = new_scores[index]
-                index = index + 1
-            else:
-                weight = original_score
-            result.append(weight)
-
-        return result
-
-    def get_cam_weights(self,
-                        input_tensor: torch.Tensor,
-                        target_layer: torch.nn.Module,
-                        targets: List[Callable],
-                        activations: torch.Tensor,
-                        grads: torch.Tensor) -> np.ndarray:
-
-        # Do a forward pass, compute the target scores, and cache the
-        # activations
-        handle = target_layer.register_forward_hook(self.save_activation)
-        with torch.no_grad():
-            outputs = self.model(input_tensor)
-            handle.remove()
-            original_scores = np.float32(
-                [target(output).cpu().item() for target, output in zip(targets, outputs)])
-
-        # Replace the layer with the ablation layer.
-        # When we finish, we will replace it back, so the original model is
-        # unchanged.
-        ablation_layer = self.ablation_layer
-        replace_layer_recursive(self.model, target_layer, ablation_layer)
-
-        number_of_channels = activations.shape[1]
-        weights = []
-        # This is a "gradient free" method, so we don't need gradients here.
-        with torch.no_grad():
-            # Loop over each of the batch images and ablate activations for it.
-            for batch_index, (target, tensor) in enumerate(
-                    zip(targets, input_tensor)):
-                new_scores = []
-                batch_tensor = tensor.repeat(self.batch_size, 1, 1, 1)
-
-                # Check which channels should be ablated. Normally this will be all channels,
-                # But we can also try to speed this up by using a low
-                # ratio_channels_to_ablate.
-                channels_to_ablate = ablation_layer.activations_to_be_ablated(
-                    activations[batch_index, :], self.ratio_channels_to_ablate)
-                number_channels_to_ablate = len(channels_to_ablate)
-
-                for i in tqdm.tqdm(
-                    range(
-                        0,
-                        number_channels_to_ablate,
-                        self.batch_size)):
-                    if i + self.batch_size > number_channels_to_ablate:
-                        batch_tensor = batch_tensor[:(
-                            number_channels_to_ablate - i)]
-
-                    # Change the state of the ablation layer so it ablates the next channels.
-                    # TBD: Move this into the ablation layer forward pass.
-                    ablation_layer.set_next_batch(
-                        input_batch_index=batch_index,
-                        activations=self.activations,
-                        num_channels_to_ablate=batch_tensor.size(0))
-                    score = [target(o).cpu().item()
-                             for o in self.model(batch_tensor)]
-                    new_scores.extend(score)
-                    ablation_layer.indices = ablation_layer.indices[batch_tensor.size(
-                        0):]
-
-                new_scores = self.assemble_ablation_scores(
-                    new_scores,
-                    original_scores[batch_index],
-                    channels_to_ablate,
-                    number_of_channels)
-                weights.extend(new_scores)
-
-        weights = np.float32(weights)
-        weights = weights.reshape(activations.shape[:2])
-        original_scores = original_scores[:, None]
-        weights = (original_scores - weights) / original_scores
-
-        # Replace the model back to the original state
-        replace_layer_recursive(self.model, ablation_layer, target_layer)
-        return weights

src/custom_code/custom_grad_cam/ablation_cam_multilayer.py

@@ -1,136 +0,0 @@
-import cv2
-import numpy as np
-import torch
-import tqdm
-from custom_grad_cam.base_cam import BaseCAM
-
-
-class AblationLayer(torch.nn.Module):
-    def __init__(self, layer, reshape_transform, indices):
-        super(AblationLayer, self).__init__()
-
-        self.layer = layer
-        self.reshape_transform = reshape_transform
-        # The channels to zero out:
-        self.indices = indices
-
-    def forward(self, x):
-        self.__call__(x)
-
-    def __call__(self, x):
-        output = self.layer(x)
-
-        # Hack to work with ViT,
-        # Since the activation channels are last and not first like in CNNs
-        # Probably should remove it?
-        if self.reshape_transform is not None:
-            output = output.transpose(1, 2)
-
-        for i in range(output.size(0)):
-
-            # Commonly the minimum activation will be 0,
-            # And then it makes sense to zero it out.
-            # However depending on the architecture,
-            # If the values can be negative, we use very negative values
-            # to perform the ablation, deviating from the paper.
-            if torch.min(output) == 0:
-                output[i, self.indices[i], :] = 0
-            else:
-                ABLATION_VALUE = 1e5
-                output[i, self.indices[i], :] = torch.min(
-                    output) - ABLATION_VALUE
-
-        if self.reshape_transform is not None:
-            output = output.transpose(2, 1)
-
-        return output
-
-
-def replace_layer_recursive(model, old_layer, new_layer):
-    for name, layer in model._modules.items():
-        if layer == old_layer:
-            model._modules[name] = new_layer
-            return True
-        elif replace_layer_recursive(layer, old_layer, new_layer):
-            return True
-    return False
-
-
-class AblationCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(AblationCAM, self).__init__(model, target_layers, use_cuda,
-                                          reshape_transform)
-
-        if len(target_layers) > 1:
-            print(
-                "Warning. You are usign Ablation CAM with more than 1 layers. "
-                "This is supported only if all layers have the same output shape")
-
-    def set_ablation_layers(self):
-        self.ablation_layers = []
-        for target_layer in self.target_layers:
-            ablation_layer = AblationLayer(target_layer,
-                                           self.reshape_transform, indices=[])
-            self.ablation_layers.append(ablation_layer)
-            replace_layer_recursive(self.model, target_layer, ablation_layer)
-
-    def unset_ablation_layers(self):
-        # replace the model back to the original state
-        for ablation_layer, target_layer in zip(
-                self.ablation_layers, self.target_layers):
-            replace_layer_recursive(self.model, ablation_layer, target_layer)
-
-    def set_ablation_layer_batch_indices(self, indices):
-        for ablation_layer in self.ablation_layers:
-            ablation_layer.indices = indices
-
-    def trim_ablation_layer_batch_indices(self, keep):
-        for ablation_layer in self.ablation_layers:
-            ablation_layer.indices = ablation_layer.indices[:keep]
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_category,
-                        activations,
-                        grads):
-        with torch.no_grad():
-            outputs = self.model(input_tensor).cpu().numpy()
-            original_scores = []
-            for i in range(input_tensor.size(0)):
-                original_scores.append(outputs[i, target_category[i]])
-        original_scores = np.float32(original_scores)
-
-        self.set_ablation_layers()
-
-        if hasattr(self, "batch_size"):
-            BATCH_SIZE = self.batch_size
-        else:
-            BATCH_SIZE = 32
-
-        number_of_channels = activations.shape[1]
-        weights = []
-
-        with torch.no_grad():
-            # Iterate over the input batch
-            for tensor, category in zip(input_tensor, target_category):
-                batch_tensor = tensor.repeat(BATCH_SIZE, 1, 1, 1)
-                for i in tqdm.tqdm(range(0, number_of_channels, BATCH_SIZE)):
-                    self.set_ablation_layer_batch_indices(
-                        list(range(i, i + BATCH_SIZE)))
-
-                    if i + BATCH_SIZE > number_of_channels:
-                        keep = number_of_channels - i
-                        batch_tensor = batch_tensor[:keep]
-                        self.trim_ablation_layer_batch_indices(self, keep)
-                    score = self.model(batch_tensor)[:, category].cpu().numpy()
-                    weights.extend(score)
-
-        weights = np.float32(weights)
-        weights = weights.reshape(activations.shape[:2])
-        original_scores = original_scores[:, None]
-        weights = (original_scores - weights) / original_scores
-
-        # replace the model back to the original state
-        self.unset_ablation_layers()
-        return weights

src/custom_code/custom_grad_cam/ablation_layer.py

@@ -1,155 +0,0 @@
-import torch
-from collections import OrderedDict
-import numpy as np
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-
-class AblationLayer(torch.nn.Module):
-    def __init__(self):
-        super(AblationLayer, self).__init__()
-
-    def objectiveness_mask_from_svd(self, activations, threshold=0.01):
-        """ Experimental method to get a binary mask to compare if the activation is worth ablating.
-            The idea is to apply the EigenCAM method by doing PCA on the activations.
-            Then we create a binary mask by comparing to a low threshold.
-            Areas that are masked out, are probably not interesting anyway.
-        """
-
-        projection = get_2d_projection(activations[None, :])[0, :]
-        projection = np.abs(projection)
-        projection = projection - projection.min()
-        projection = projection / projection.max()
-        projection = projection > threshold
-        return projection
-
-    def activations_to_be_ablated(
-            self,
-            activations,
-            ratio_channels_to_ablate=1.0):
-        """ Experimental method to get a binary mask to compare if the activation is worth ablating.
-            Create a binary CAM mask with objectiveness_mask_from_svd.
-            Score each Activation channel, by seeing how much of its values are inside the mask.
-            Then keep the top channels.
-
-        """
-        if ratio_channels_to_ablate == 1.0:
-            self.indices = np.int32(range(activations.shape[0]))
-            return self.indices
-
-        projection = self.objectiveness_mask_from_svd(activations)
-
-        scores = []
-        for channel in activations:
-            normalized = np.abs(channel)
-            normalized = normalized - normalized.min()
-            normalized = normalized / np.max(normalized)
-            score = (projection * normalized).sum() / normalized.sum()
-            scores.append(score)
-        scores = np.float32(scores)
-
-        indices = list(np.argsort(scores))
-        high_score_indices = indices[::-
-                                     1][: int(len(indices) *
-                                              ratio_channels_to_ablate)]
-        low_score_indices = indices[: int(
-            len(indices) * ratio_channels_to_ablate)]
-        self.indices = np.int32(high_score_indices + low_score_indices)
-        return self.indices
-
-    def set_next_batch(
-            self,
-            input_batch_index,
-            activations,
-            num_channels_to_ablate):
-        """ This creates the next batch of activations from the layer.
-            Just take corresponding batch member from activations, and repeat it num_channels_to_ablate times.
-        """
-        self.activations = activations[input_batch_index, :, :, :].clone(
-        ).unsqueeze(0).repeat(num_channels_to_ablate, 1, 1, 1)
-
-    def __call__(self, x):
-        output = self.activations
-        for i in range(output.size(0)):
-            # Commonly the minimum activation will be 0,
-            # And then it makes sense to zero it out.
-            # However depending on the architecture,
-            # If the values can be negative, we use very negative values
-            # to perform the ablation, deviating from the paper.
-            if torch.min(output) == 0:
-                output[i, self.indices[i], :] = 0
-            else:
-                ABLATION_VALUE = 1e7
-                output[i, self.indices[i], :] = torch.min(
-                    output) - ABLATION_VALUE
-
-        return output
-
-
-class AblationLayerVit(AblationLayer):
-    def __init__(self):
-        super(AblationLayerVit, self).__init__()
-
-    def __call__(self, x):
-        output = self.activations
-        output = output.transpose(1, len(output.shape) - 1)
-        for i in range(output.size(0)):
-
-            # Commonly the minimum activation will be 0,
-            # And then it makes sense to zero it out.
-            # However depending on the architecture,
-            # If the values can be negative, we use very negative values
-            # to perform the ablation, deviating from the paper.
-            if torch.min(output) == 0:
-                output[i, self.indices[i], :] = 0
-            else:
-                ABLATION_VALUE = 1e7
-                output[i, self.indices[i], :] = torch.min(
-                    output) - ABLATION_VALUE
-
-        output = output.transpose(len(output.shape) - 1, 1)
-
-        return output
-
-    def set_next_batch(
-            self,
-            input_batch_index,
-            activations,
-            num_channels_to_ablate):
-        """ This creates the next batch of activations from the layer.
-            Just take corresponding batch member from activations, and repeat it num_channels_to_ablate times.
-        """
-        repeat_params = [num_channels_to_ablate] + \
-            len(activations.shape[:-1]) * [1]
-        self.activations = activations[input_batch_index, :, :].clone(
-        ).unsqueeze(0).repeat(*repeat_params)
-
-
-class AblationLayerFasterRCNN(AblationLayer):
-    def __init__(self):
-        super(AblationLayerFasterRCNN, self).__init__()
-
-    def set_next_batch(
-            self,
-            input_batch_index,
-            activations,
-            num_channels_to_ablate):
-        """ Extract the next batch member from activations,
-            and repeat it num_channels_to_ablate times.
-        """
-        self.activations = OrderedDict()
-        for key, value in activations.items():
-            fpn_activation = value[input_batch_index,
-                                   :, :, :].clone().unsqueeze(0)
-            self.activations[key] = fpn_activation.repeat(
-                num_channels_to_ablate, 1, 1, 1)
-
-    def __call__(self, x):
-        result = self.activations
-        layers = {0: '0', 1: '1', 2: '2', 3: '3', 4: 'pool'}
-        num_channels_to_ablate = result['pool'].size(0)
-        for i in range(num_channels_to_ablate):
-            pyramid_layer = int(self.indices[i] / 256)
-            index_in_pyramid_layer = int(self.indices[i] % 256)
-            result[layers[pyramid_layer]][i,
-                                          index_in_pyramid_layer, :, :] = -1000
-        return result

src/custom_code/custom_grad_cam/activations_and_gradients.py

@@ -1,46 +0,0 @@
-class ActivationsAndGradients:
-    """ Class for extracting activations and
-    registering gradients from targetted intermediate layers """
-
-    def __init__(self, model, target_layers, reshape_transform):
-        self.model = model
-        self.gradients = []
-        self.activations = []
-        self.reshape_transform = reshape_transform
-        self.handles = []
-        for target_layer in target_layers:
-            self.handles.append(
-                target_layer.register_forward_hook(self.save_activation))
-            # Because of https://github.com/pytorch/pytorch/issues/61519,
-            # we don't use backward hook to record gradients.
-            self.handles.append(
-                target_layer.register_forward_hook(self.save_gradient))
-
-    def save_activation(self, module, input, output):
-        activation = output
-
-        if self.reshape_transform is not None:
-            activation = self.reshape_transform(activation)
-        self.activations.append(activation.cpu().detach())
-
-    def save_gradient(self, module, input, output):
-        if not hasattr(output, "requires_grad") or not output.requires_grad:
-            # You can only register hooks on tensor requires grad.
-            return
-
-        # Gradients are computed in reverse order
-        def _store_grad(grad):
-            if self.reshape_transform is not None:
-                grad = self.reshape_transform(grad)
-            self.gradients = [grad.cpu().detach()] + self.gradients
-
-        output.register_hook(_store_grad)
-
-    def __call__(self, x):
-        self.gradients = []
-        self.activations = []
-        return self.model(x)
-
-    def release(self):
-        for handle in self.handles:
-            handle.remove()

src/custom_code/custom_grad_cam/base_cam.py

@@ -1,205 +0,0 @@
-import numpy as np
-import torch
-import ttach as tta
-from typing import Callable, List, Tuple
-from custom_grad_cam.activations_and_gradients import ActivationsAndGradients
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-from custom_grad_cam.utils.image import scale_cam_image
-from custom_grad_cam.utils.model_targets import ClassifierOutputTarget
-
-
-class BaseCAM:
-    def __init__(self,
-                 model: torch.nn.Module,
-                 target_layers: List[torch.nn.Module],
-                 use_cuda: bool = False,
-                 reshape_transform: Callable = None,
-                 compute_input_gradient: bool = False,
-                 uses_gradients: bool = True) -> None:
-        self.model = model.eval()
-        self.target_layers = target_layers
-        self.cuda = use_cuda
-        if self.cuda:
-            self.model = model.cuda()
-        self.reshape_transform = reshape_transform
-        self.compute_input_gradient = compute_input_gradient
-        self.uses_gradients = uses_gradients
-        self.activations_and_grads = ActivationsAndGradients(
-            self.model, target_layers, reshape_transform)
-        self.outputs = None
-
-    """ Get a vector of weights for every channel in the target layer.
-        Methods that return weights channels,
-        will typically need to only implement this function. """
-
-    def get_cam_weights(self,
-                        input_tensor: torch.Tensor,
-                        target_layers: List[torch.nn.Module],
-                        targets: List[torch.nn.Module],
-                        activations: torch.Tensor,
-                        grads: torch.Tensor) -> np.ndarray:
-        raise Exception("Not Implemented")
-
-    def get_cam_image(self,
-                      input_tensor: torch.Tensor,
-                      target_layer: torch.nn.Module,
-                      targets: List[torch.nn.Module],
-                      activations: torch.Tensor,
-                      grads: torch.Tensor,
-                      eigen_smooth: bool = False) -> np.ndarray:
-
-        weights = self.get_cam_weights(input_tensor,
-                                       target_layer,
-                                       targets,
-                                       activations,
-                                       grads)
-        weighted_activations = weights[:, :, None, None] * activations
-        if eigen_smooth:
-            cam = get_2d_projection(weighted_activations)
-        else:
-            cam = weighted_activations.sum(axis=1)
-        return cam
-
-    def forward(self,
-                input_tensor: torch.Tensor,
-                targets: List[torch.nn.Module],
-                eigen_smooth: bool = False) -> np.ndarray:
-
-        if self.cuda:
-            input_tensor = input_tensor.cuda()
-
-        if self.compute_input_gradient:
-            input_tensor = torch.autograd.Variable(input_tensor,
-                                                   requires_grad=True)
-
-        outputs = self.activations_and_grads(input_tensor)
-        self.outputs = outputs
-        if targets is None:
-            target_categories = np.argmax(outputs.cpu().data.numpy(), axis=-1)
-            targets = [ClassifierOutputTarget(
-                category) for category in target_categories]
-
-        if self.uses_gradients:
-            self.model.zero_grad()
-            loss = sum([target(output)
-                       for target, output in zip(targets, outputs)])
-            loss.backward(retain_graph=True)
-
-        # In most of the saliency attribution papers, the saliency is
-        # computed with a single target layer.
-        # Commonly it is the last convolutional layer.
-        # Here we support passing a list with multiple target layers.
-        # It will compute the saliency image for every image,
-        # and then aggregate them (with a default mean aggregation).
-        # This gives you more flexibility in case you just want to
-        # use all conv layers for example, all Batchnorm layers,
-        # or something else.
-        cam_per_layer = self.compute_cam_per_layer(input_tensor,
-                                                   targets,
-                                                   eigen_smooth)
-        return self.aggregate_multi_layers(cam_per_layer)
-
-    def get_target_width_height(self,
-                                input_tensor: torch.Tensor) -> Tuple[int, int]:
-        width, height = input_tensor.size(-1), input_tensor.size(-2)
-        return width, height
-
-    def compute_cam_per_layer(
-            self,
-            input_tensor: torch.Tensor,
-            targets: List[torch.nn.Module],
-            eigen_smooth: bool) -> np.ndarray:
-        activations_list = [a.cpu().data.numpy()
-                            for a in self.activations_and_grads.activations]
-        grads_list = [g.cpu().data.numpy()
-                      for g in self.activations_and_grads.gradients]
-        target_size = self.get_target_width_height(input_tensor)
-
-        cam_per_target_layer = []
-        # Loop over the saliency image from every layer
-        for i in range(len(self.target_layers)):
-            target_layer = self.target_layers[i]
-            layer_activations = None
-            layer_grads = None
-            if i < len(activations_list):
-                layer_activations = activations_list[i]
-            if i < len(grads_list):
-                layer_grads = grads_list[i]
-
-            cam = self.get_cam_image(input_tensor,
-                                     target_layer,
-                                     targets,
-                                     layer_activations,
-                                     layer_grads,
-                                     eigen_smooth)
-            cam = np.maximum(cam, 0)
-            scaled = scale_cam_image(cam, target_size)
-            cam_per_target_layer.append(scaled[:, None, :])
-
-        return cam_per_target_layer
-
-    def aggregate_multi_layers(
-            self,
-            cam_per_target_layer: np.ndarray) -> np.ndarray:
-        cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1)
-        cam_per_target_layer = np.maximum(cam_per_target_layer, 0)
-        result = np.mean(cam_per_target_layer, axis=1)
-        return scale_cam_image(result)
-
-    def forward_augmentation_smoothing(self,
-                                       input_tensor: torch.Tensor,
-                                       targets: List[torch.nn.Module],
-                                       eigen_smooth: bool = False) -> np.ndarray:
-        transforms = tta.Compose(
-            [
-                tta.HorizontalFlip(),
-                tta.Multiply(factors=[0.9, 1, 1.1]),
-            ]
-        )
-        cams = []
-        for transform in transforms:
-            augmented_tensor = transform.augment_image(input_tensor)
-            cam = self.forward(augmented_tensor,
-                               targets,
-                               eigen_smooth)
-
-            # The ttach library expects a tensor of size BxCxHxW
-            cam = cam[:, None, :, :]
-            cam = torch.from_numpy(cam)
-            cam = transform.deaugment_mask(cam)
-
-            # Back to numpy float32, HxW
-            cam = cam.numpy()
-            cam = cam[:, 0, :, :]
-            cams.append(cam)
-
-        cam = np.mean(np.float32(cams), axis=0)
-        return cam
-
-    def __call__(self,
-                 input_tensor: torch.Tensor,
-                 targets: List[torch.nn.Module] = None,
-                 aug_smooth: bool = False,
-                 eigen_smooth: bool = False) -> np.ndarray:
-
-        # Smooth the CAM result with test time augmentation
-        if aug_smooth is True:
-            return self.forward_augmentation_smoothing(
-                input_tensor, targets, eigen_smooth)
-
-        return self.forward(input_tensor,
-                            targets, eigen_smooth)
-
-    def __del__(self):
-        self.activations_and_grads.release()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self.activations_and_grads.release()
-        if isinstance(exc_value, IndexError):
-            # Handle IndexError here...
-            print(
-                f"An exception occurred in CAM with block: {exc_type}. Message: {exc_value}")
-            return True

src/custom_code/custom_grad_cam/eigen_cam.py

@@ -1,23 +0,0 @@
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-# https://arxiv.org/abs/2008.00299
-
-
-class EigenCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(EigenCAM, self).__init__(model,
-                                       target_layers,
-                                       use_cuda,
-                                       reshape_transform,
-                                       uses_gradients=False)
-
-    def get_cam_image(self,
-                      input_tensor,
-                      target_layer,
-                      target_category,
-                      activations,
-                      grads,
-                      eigen_smooth):
-        return get_2d_projection(activations)

src/custom_code/custom_grad_cam/eigen_grad_cam.py

@@ -1,21 +0,0 @@
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-# Like Eigen CAM: https://arxiv.org/abs/2008.00299
-# But multiply the activations x gradients
-
-
-class EigenGradCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(EigenGradCAM, self).__init__(model, target_layers, use_cuda,
-                                           reshape_transform)
-
-    def get_cam_image(self,
-                      input_tensor,
-                      target_layer,
-                      target_category,
-                      activations,
-                      grads,
-                      eigen_smooth):
-        return get_2d_projection(grads * activations)

src/custom_code/custom_grad_cam/feature_factorization/deep_feature_factorization.py

@@ -1,131 +0,0 @@
-import numpy as np
-from PIL import Image
-import torch
-from typing import Callable, List, Tuple, Optional
-from sklearn.decomposition import NMF
-from custom_grad_cam.activations_and_gradients import ActivationsAndGradients
-from custom_grad_cam.utils.image import scale_cam_image, create_labels_legend, show_factorization_on_image
-
-
-def dff(activations: np.ndarray, n_components: int = 5):
-    """ Compute Deep Feature Factorization on a 2d Activations tensor.
-
-    :param activations: A numpy array of shape batch x channels x height x width
-    :param n_components: The number of components for the non negative matrix factorization
-    :returns: A tuple of the concepts (a numpy array with shape channels x components),
-              and the explanation heatmaps (a numpy arary with shape batch x height x width)
-    """
-
-    batch_size, channels, h, w = activations.shape
-    reshaped_activations = activations.transpose((1, 0, 2, 3))
-    reshaped_activations[np.isnan(reshaped_activations)] = 0
-    reshaped_activations = reshaped_activations.reshape(
-        reshaped_activations.shape[0], -1)
-    offset = reshaped_activations.min(axis=-1)
-    reshaped_activations = reshaped_activations - offset[:, None]
-
-    model = NMF(n_components=n_components, init='random', random_state=0)
-    W = model.fit_transform(reshaped_activations)
-    H = model.components_
-    concepts = W + offset[:, None]
-    explanations = H.reshape(n_components, batch_size, h, w)
-    explanations = explanations.transpose((1, 0, 2, 3))
-    return concepts, explanations
-
-
-class DeepFeatureFactorization:
-    """ Deep Feature Factorization: https://arxiv.org/abs/1806.10206
-        This gets a model andcomputes the 2D activations for a target layer,
-        and computes Non Negative Matrix Factorization on the activations.
-
-        Optionally it runs a computation on the concept embeddings,
-        like running a classifier on them.
-
-        The explanation heatmaps are scalled to the range [0, 1]
-        and to the input tensor width and height.
-     """
-
-    def __init__(self,
-                 model: torch.nn.Module,
-                 target_layer: torch.nn.Module,
-                 reshape_transform: Callable = None,
-                 computation_on_concepts=None
-                 ):
-        self.model = model
-        self.computation_on_concepts = computation_on_concepts
-        self.activations_and_grads = ActivationsAndGradients(
-            self.model, [target_layer], reshape_transform)
-
-    def __call__(self,
-                 input_tensor: torch.Tensor,
-                 n_components: int = 16):
-        batch_size, channels, h, w = input_tensor.size()
-        _ = self.activations_and_grads(input_tensor)
-
-        with torch.no_grad():
-            activations = self.activations_and_grads.activations[0].cpu(
-            ).numpy()
-
-        concepts, explanations = dff(activations, n_components=n_components)
-
-        processed_explanations = []
-
-        for batch in explanations:
-            processed_explanations.append(scale_cam_image(batch, (w, h)))
-
-        if self.computation_on_concepts:
-            with torch.no_grad():
-                concept_tensors = torch.from_numpy(
-                    np.float32(concepts).transpose((1, 0)))
-                concept_outputs = self.computation_on_concepts(
-                    concept_tensors).cpu().numpy()
-            return concepts, processed_explanations, concept_outputs
-        else:
-            return concepts, processed_explanations
-
-    def __del__(self):
-        self.activations_and_grads.release()
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self.activations_and_grads.release()
-        if isinstance(exc_value, IndexError):
-            # Handle IndexError here...
-            print(
-                f"An exception occurred in ActivationSummary with block: {exc_type}. Message: {exc_value}")
-            return True
-
-
-def run_dff_on_image(model: torch.nn.Module,
-                     target_layer: torch.nn.Module,
-                     classifier: torch.nn.Module,
-                     img_pil: Image,
-                     img_tensor: torch.Tensor,
-                     reshape_transform=Optional[Callable],
-                     n_components: int = 5,
-                     top_k: int = 2) -> np.ndarray:
-    """ Helper function to create a Deep Feature Factorization visualization for a single image.
-        TBD: Run this on a batch with several images.
-    """
-    rgb_img_float = np.array(img_pil) / 255
-    dff = DeepFeatureFactorization(model=model,
-                                   reshape_transform=reshape_transform,
-                                   target_layer=target_layer,
-                                   computation_on_concepts=classifier)
-
-    concepts, batch_explanations, concept_outputs = dff(
-        img_tensor[None, :], n_components)
-
-    concept_outputs = torch.softmax(
-        torch.from_numpy(concept_outputs),
-        axis=-1).numpy()
-    concept_label_strings = create_labels_legend(concept_outputs,
-                                                 labels=model.config.id2label,
-                                                 top_k=top_k)
-    visualization = show_factorization_on_image(
-        rgb_img_float,
-        batch_explanations[0],
-        image_weight=0.3,
-        concept_labels=concept_label_strings)
-
-    result = np.hstack((np.array(img_pil), visualization))
-    return result

src/custom_code/custom_grad_cam/feature_factorization/utils.py

@@ -1,19 +0,0 @@
-import requests
-import numpy as np
-from typing import Dict
-
-
-def create_labels_legend(concept_scores: np.ndarray,
-                         labels: Dict[int, str],
-                         top_k=2):
-    concept_categories = np.argsort(concept_scores, axis=1)[:, ::-1][:, :top_k]
-    concept_labels_topk = []
-    for concept_index in range(concept_categories.shape[0]):
-        categories = concept_categories[concept_index, :]
-        concept_labels = []
-        for category in categories:
-            score = concept_scores[concept_index, category]
-            label = f"{labels[category].split(',')[0]}:{score:.2f}"
-            concept_labels.append(label)
-        concept_labels_topk.append("\n".join(concept_labels))
-    return concept_labels_topk

src/custom_code/custom_grad_cam/fullgrad_cam.py

@@ -1,95 +0,0 @@
-import numpy as np
-import torch
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.find_layers import find_layer_predicate_recursive
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-from custom_grad_cam.utils.image import scale_accross_batch_and_channels, scale_cam_image
-
-# https://arxiv.org/abs/1905.00780
-
-
-class FullGrad(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        if len(target_layers) > 0:
-            print(
-                "Warning: target_layers is ignored in FullGrad. All bias layers will be used instead")
-
-        def layer_with_2D_bias(layer):
-            bias_target_layers = [torch.nn.Conv2d, torch.nn.BatchNorm2d]
-            if type(layer) in bias_target_layers and layer.bias is not None:
-                return True
-            return False
-        target_layers = find_layer_predicate_recursive(
-            model, layer_with_2D_bias)
-        super(
-            FullGrad,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform,
-            compute_input_gradient=True)
-        self.bias_data = [self.get_bias_data(
-            layer).cpu().numpy() for layer in target_layers]
-
-    def get_bias_data(self, layer):
-        # Borrowed from official paper impl:
-        # https://github.com/idiap/fullgrad-saliency/blob/master/saliency/tensor_extractor.py#L47
-        if isinstance(layer, torch.nn.BatchNorm2d):
-            bias = - (layer.running_mean * layer.weight
-                      / torch.sqrt(layer.running_var + layer.eps)) + layer.bias
-            return bias.data
-        else:
-            return layer.bias.data
-
-    def compute_cam_per_layer(
-            self,
-            input_tensor,
-            target_category,
-            eigen_smooth):
-        input_grad = input_tensor.grad.data.cpu().numpy()
-        grads_list = [g.cpu().data.numpy() for g in
-                      self.activations_and_grads.gradients]
-        cam_per_target_layer = []
-        target_size = self.get_target_width_height(input_tensor)
-
-        gradient_multiplied_input = input_grad * input_tensor.data.cpu().numpy()
-        gradient_multiplied_input = np.abs(gradient_multiplied_input)
-        gradient_multiplied_input = scale_accross_batch_and_channels(
-            gradient_multiplied_input,
-            target_size)
-        cam_per_target_layer.append(gradient_multiplied_input)
-
-        # Loop over the saliency image from every layer
-        assert(len(self.bias_data) == len(grads_list))
-        for bias, grads in zip(self.bias_data, grads_list):
-            bias = bias[None, :, None, None]
-            # In the paper they take the absolute value,
-            # but possibily taking only the positive gradients will work
-            # better.
-            bias_grad = np.abs(bias * grads)
-            result = scale_accross_batch_and_channels(
-                bias_grad, target_size)
-            result = np.sum(result, axis=1)
-            cam_per_target_layer.append(result[:, None, :])
-        cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1)
-        if eigen_smooth:
-            # Resize to a smaller image, since this method typically has a very large number of channels,
-            # and then consumes a lot of memory
-            cam_per_target_layer = scale_accross_batch_and_channels(
-                cam_per_target_layer, (target_size[0] // 8, target_size[1] // 8))
-            cam_per_target_layer = get_2d_projection(cam_per_target_layer)
-            cam_per_target_layer = cam_per_target_layer[:, None, :, :]
-            cam_per_target_layer = scale_accross_batch_and_channels(
-                cam_per_target_layer,
-                target_size)
-        else:
-            cam_per_target_layer = np.sum(
-                cam_per_target_layer, axis=1)[:, None, :]
-
-        return cam_per_target_layer
-
-    def aggregate_multi_layers(self, cam_per_target_layer):
-        result = np.sum(cam_per_target_layer, axis=1)
-        return scale_cam_image(result)

src/custom_code/custom_grad_cam/grad_cam.py

@@ -1,22 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-
-
-class GradCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(
-            GradCAM,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_layer,
-                        target_category,
-                        activations,
-                        grads):
-        return np.mean(grads, axis=(2, 3))

src/custom_code/custom_grad_cam/grad_cam_elementwise.py

@@ -1,30 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-
-class GradCAMElementWise(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(
-            GradCAMElementWise,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_image(self,
-                      input_tensor,
-                      target_layer,
-                      target_category,
-                      activations,
-                      grads,
-                      eigen_smooth):
-        elementwise_activations = np.maximum(grads * activations, 0)
-
-        if eigen_smooth:
-            cam = get_2d_projection(elementwise_activations)
-        else:
-            cam = elementwise_activations.sum(axis=1)
-        return cam

src/custom_code/custom_grad_cam/grad_cam_plusplus.py

@@ -1,32 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-
-# https://arxiv.org/abs/1710.11063
-
-
-class GradCAMPlusPlus(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(GradCAMPlusPlus, self).__init__(model, target_layers, use_cuda,
-                                              reshape_transform)
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_layers,
-                        target_category,
-                        activations,
-                        grads):
-        grads_power_2 = grads**2
-        grads_power_3 = grads_power_2 * grads
-        # Equation 19 in https://arxiv.org/abs/1710.11063
-        sum_activations = np.sum(activations, axis=(2, 3))
-        eps = 0.000001
-        aij = grads_power_2 / (2 * grads_power_2 +
-                               sum_activations[:, :, None, None] * grads_power_3 + eps)
-        # Now bring back the ReLU from eq.7 in the paper,
-        # And zero out aijs where the activations are 0
-        aij = np.where(grads != 0, aij, 0)
-
-        weights = np.maximum(grads, 0) * aij
-        weights = np.sum(weights, axis=(2, 3))
-        return weights

src/custom_code/custom_grad_cam/guided_backprop.py

@@ -1,100 +0,0 @@
-import numpy as np
-import torch
-from torch.autograd import Function
-from custom_grad_cam.utils.find_layers import replace_all_layer_type_recursive
-
-
-class GuidedBackpropReLU(Function):
-    @staticmethod
-    def forward(self, input_img):
-        positive_mask = (input_img > 0).type_as(input_img)
-        output = torch.addcmul(
-            torch.zeros(
-                input_img.size()).type_as(input_img),
-            input_img,
-            positive_mask)
-        self.save_for_backward(input_img, output)
-        return output
-
-    @staticmethod
-    def backward(self, grad_output):
-        input_img, output = self.saved_tensors
-        grad_input = None
-
-        positive_mask_1 = (input_img > 0).type_as(grad_output)
-        positive_mask_2 = (grad_output > 0).type_as(grad_output)
-        grad_input = torch.addcmul(
-            torch.zeros(
-                input_img.size()).type_as(input_img),
-            torch.addcmul(
-                torch.zeros(
-                    input_img.size()).type_as(input_img),
-                grad_output,
-                positive_mask_1),
-            positive_mask_2)
-        return grad_input
-
-
-class GuidedBackpropReLUasModule(torch.nn.Module):
-    def __init__(self):
-        super(GuidedBackpropReLUasModule, self).__init__()
-
-    def forward(self, input_img):
-        return GuidedBackpropReLU.apply(input_img)
-
-
-class GuidedBackpropReLUModel:
-    def __init__(self, model, use_cuda):
-        self.model = model
-        self.model.eval()
-        self.cuda = use_cuda
-        if self.cuda:
-            self.model = self.model.cuda()
-
-    def forward(self, input_img):
-        return self.model(input_img)
-
-    def recursive_replace_relu_with_guidedrelu(self, module_top):
-
-        for idx, module in module_top._modules.items():
-            self.recursive_replace_relu_with_guidedrelu(module)
-            if module.__class__.__name__ == 'ReLU':
-                module_top._modules[idx] = GuidedBackpropReLU.apply
-        print("b")
-
-    def recursive_replace_guidedrelu_with_relu(self, module_top):
-        try:
-            for idx, module in module_top._modules.items():
-                self.recursive_replace_guidedrelu_with_relu(module)
-                if module == GuidedBackpropReLU.apply:
-                    module_top._modules[idx] = torch.nn.ReLU()
-        except BaseException:
-            pass
-
-    def __call__(self, input_img, target_category=None):
-        replace_all_layer_type_recursive(self.model,
-                                         torch.nn.ReLU,
-                                         GuidedBackpropReLUasModule())
-
-        if self.cuda:
-            input_img = input_img.cuda()
-
-        input_img = input_img.requires_grad_(True)
-
-        output = self.forward(input_img)
-
-        if target_category is None:
-            target_category = np.argmax(output.cpu().data.numpy())
-
-        loss = output[0, target_category]
-        loss.backward(retain_graph=True)
-
-        output = input_img.grad.cpu().data.numpy()
-        output = output[0, :, :, :]
-        output = output.transpose((1, 2, 0))
-
-        replace_all_layer_type_recursive(self.model,
-                                         GuidedBackpropReLUasModule,
-                                         torch.nn.ReLU())
-
-        return output

src/custom_code/custom_grad_cam/hirescam.py

@@ -1,32 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-
-class HiResCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(
-            HiResCAM,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_image(self,
-                      input_tensor,
-                      target_layer,
-                      target_category,
-                      activations,
-                      grads,
-                      eigen_smooth):
-        elementwise_activations = grads * activations
-
-        if eigen_smooth:
-            print(
-                "Warning: HiResCAM's faithfulness guarantees do not hold if smoothing is applied")
-            cam = get_2d_projection(elementwise_activations)
-        else:
-            cam = elementwise_activations.sum(axis=1)
-        return cam

src/custom_code/custom_grad_cam/layer_cam.py

@@ -1,36 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-
-# https://ieeexplore.ieee.org/document/9462463
-
-
-class LayerCAM(BaseCAM):
-    def __init__(
-            self,
-            model,
-            target_layers,
-            use_cuda=False,
-            reshape_transform=None):
-        super(
-            LayerCAM,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_image(self,
-                      input_tensor,
-                      target_layer,
-                      target_category,
-                      activations,
-                      grads,
-                      eigen_smooth):
-        spatial_weighted_activations = np.maximum(grads, 0) * activations
-
-        if eigen_smooth:
-            cam = get_2d_projection(spatial_weighted_activations)
-        else:
-            cam = spatial_weighted_activations.sum(axis=1)
-        return cam

src/custom_code/custom_grad_cam/metrics/cam_mult_image.py

@@ -1,37 +0,0 @@
-import torch
-import numpy as np
-from typing import List, Callable
-from custom_grad_cam.metrics.perturbation_confidence import PerturbationConfidenceMetric
-
-
-def multiply_tensor_with_cam(input_tensor: torch.Tensor,
-                             cam: torch.Tensor):
-    """ Multiply an input tensor (after normalization)
-        with a pixel attribution map
-    """
-    return input_tensor * cam
-
-
-class CamMultImageConfidenceChange(PerturbationConfidenceMetric):
-    def __init__(self):
-        super(CamMultImageConfidenceChange,
-              self).__init__(multiply_tensor_with_cam)
-
-
-class DropInConfidence(CamMultImageConfidenceChange):
-    def __init__(self):
-        super(DropInConfidence, self).__init__()
-
-    def __call__(self, *args, **kwargs):
-        scores = super(DropInConfidence, self).__call__(*args, **kwargs)
-        scores = -scores
-        return np.maximum(scores, 0)
-
-
-class IncreaseInConfidence(CamMultImageConfidenceChange):
-    def __init__(self):
-        super(IncreaseInConfidence, self).__init__()
-
-    def __call__(self, *args, **kwargs):
-        scores = super(IncreaseInConfidence, self).__call__(*args, **kwargs)
-        return np.float32(scores > 0)

src/custom_code/custom_grad_cam/metrics/perturbation_confidence.py

@@ -1,109 +0,0 @@
-import torch
-import numpy as np
-from typing import List, Callable
-
-import numpy as np
-import cv2
-
-
-class PerturbationConfidenceMetric:
-    def __init__(self, perturbation):
-        self.perturbation = perturbation
-
-    def __call__(self, input_tensor: torch.Tensor,
-                 cams: np.ndarray,
-                 targets: List[Callable],
-                 model: torch.nn.Module,
-                 return_visualization=False,
-                 return_diff=True):
-
-        if return_diff:
-            with torch.no_grad():
-                outputs = model(input_tensor)
-                scores = [target(output).cpu().numpy()
-                          for target, output in zip(targets, outputs)]
-                scores = np.float32(scores)
-
-        batch_size = input_tensor.size(0)
-        perturbated_tensors = []
-        for i in range(batch_size):
-            cam = cams[i]
-            tensor = self.perturbation(input_tensor[i, ...].cpu(),
-                                       torch.from_numpy(cam))
-            tensor = tensor.to(input_tensor.device)
-            perturbated_tensors.append(tensor.unsqueeze(0))
-        perturbated_tensors = torch.cat(perturbated_tensors)
-
-        with torch.no_grad():
-            outputs_after_imputation = model(perturbated_tensors)
-        scores_after_imputation = [
-            target(output).cpu().numpy() for target, output in zip(
-                targets, outputs_after_imputation)]
-        scores_after_imputation = np.float32(scores_after_imputation)
-
-        if return_diff:
-            result = scores_after_imputation - scores
-        else:
-            result = scores_after_imputation
-
-        if return_visualization:
-            return result, perturbated_tensors
-        else:
-            return result
-
-
-class RemoveMostRelevantFirst:
-    def __init__(self, percentile, imputer):
-        self.percentile = percentile
-        self.imputer = imputer
-
-    def __call__(self, input_tensor, mask):
-        imputer = self.imputer
-        if self.percentile != 'auto':
-            threshold = np.percentile(mask.cpu().numpy(), self.percentile)
-            binary_mask = np.float32(mask < threshold)
-        else:
-            _, binary_mask = cv2.threshold(
-                np.uint8(mask * 255), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
-
-        binary_mask = torch.from_numpy(binary_mask)
-        binary_mask = binary_mask.to(mask.device)
-        return imputer(input_tensor, binary_mask)
-
-
-class RemoveLeastRelevantFirst(RemoveMostRelevantFirst):
-    def __init__(self, percentile, imputer):
-        super(RemoveLeastRelevantFirst, self).__init__(percentile, imputer)
-
-    def __call__(self, input_tensor, mask):
-        return super(RemoveLeastRelevantFirst, self).__call__(
-            input_tensor, 1 - mask)
-
-
-class AveragerAcrossThresholds:
-    def __init__(
-        self,
-        imputer,
-        percentiles=[
-            10,
-            20,
-            30,
-            40,
-            50,
-            60,
-            70,
-            80,
-            90]):
-        self.imputer = imputer
-        self.percentiles = percentiles
-
-    def __call__(self,
-                 input_tensor: torch.Tensor,
-                 cams: np.ndarray,
-                 targets: List[Callable],
-                 model: torch.nn.Module):
-        scores = []
-        for percentile in self.percentiles:
-            imputer = self.imputer(percentile)
-            scores.append(imputer(input_tensor, cams, targets, model))
-        return np.mean(np.float32(scores), axis=0)

src/custom_code/custom_grad_cam/metrics/road.py

@@ -1,181 +0,0 @@
-# A Consistent and Efficient Evaluation Strategy for Attribution Methods
-# https://arxiv.org/abs/2202.00449
-# Taken from https://raw.githubusercontent.com/tleemann/road_evaluation/main/imputations.py
-# MIT License
-
-# Copyright (c) 2022 Tobias Leemann
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-
-# Implementations of our imputation models.
-import torch
-import numpy as np
-from scipy.sparse import lil_matrix, csc_matrix
-from scipy.sparse.linalg import spsolve
-from typing import List, Callable
-from custom_grad_cam.metrics.perturbation_confidence import PerturbationConfidenceMetric, \
-    AveragerAcrossThresholds, \
-    RemoveMostRelevantFirst, \
-    RemoveLeastRelevantFirst
-
-# The weights of the surrounding pixels
-neighbors_weights = [((1, 1), 1 / 12),
-                     ((0, 1), 1 / 6),
-                     ((-1, 1), 1 / 12),
-                     ((1, -1), 1 / 12),
-                     ((0, -1), 1 / 6),
-                     ((-1, -1), 1 / 12),
-                     ((1, 0), 1 / 6),
-                     ((-1, 0), 1 / 6)]
-
-
-class NoisyLinearImputer:
-    def __init__(self,
-                 noise: float = 0.01,
-                 weighting: List[float] = neighbors_weights):
-        """
-                Noisy linear imputation.
-                noise: magnitude of noise to add (absolute, set to 0 for no noise)
-                weighting: Weights of the neighboring pixels in the computation.
-                List of tuples of (offset, weight)
-        """
-        self.noise = noise
-        self.weighting = neighbors_weights
-
-    @staticmethod
-    def add_offset_to_indices(indices, offset, mask_shape):
-        """ Add the corresponding offset to the indices.
-    Return new indices plus a valid bit-vector. """
-        cord1 = indices % mask_shape[1]
-        cord0 = indices // mask_shape[1]
-        cord0 += offset[0]
-        cord1 += offset[1]
-        valid = ((cord0 < 0) | (cord1 < 0) |
-                 (cord0 >= mask_shape[0]) |
-                 (cord1 >= mask_shape[1]))
-        return ~valid, indices + offset[0] * mask_shape[1] + offset[1]
-
-    @staticmethod
-    def setup_sparse_system(mask, img, neighbors_weights):
-        """ Vectorized version to set up the equation system.
-                mask: (H, W)-tensor of missing pixels.
-                Image: (H, W, C)-tensor of all values.
-                Return (N,N)-System matrix, (N,C)-Right hand side for each of the C channels.
-        """
-        maskflt = mask.flatten()
-        imgflat = img.reshape((img.shape[0], -1))
-    # Indices that are imputed in the flattened mask:
-        indices = np.argwhere(maskflt == 0).flatten()
-        coords_to_vidx = np.zeros(len(maskflt), dtype=int)
-        coords_to_vidx[indices] = np.arange(len(indices))
-        numEquations = len(indices)
-    # System matrix:
-        A = lil_matrix((numEquations, numEquations))
-        b = np.zeros((numEquations, img.shape[0]))
-    # Sum of weights assigned:
-        sum_neighbors = np.ones(numEquations)
-        for n in neighbors_weights:
-            offset, weight = n[0], n[1]
-            # Take out outliers
-            valid, new_coords = NoisyLinearImputer.add_offset_to_indices(
-                indices, offset, mask.shape)
-            valid_coords = new_coords[valid]
-            valid_ids = np.argwhere(valid == 1).flatten()
-            # Add values to the right hand-side
-            has_values_coords = valid_coords[maskflt[valid_coords] > 0.5]
-            has_values_ids = valid_ids[maskflt[valid_coords] > 0.5]
-            b[has_values_ids, :] -= weight * imgflat[:, has_values_coords].T
-            # Add weights to the system (left hand side)
-# Find coordinates in the system.
-            has_no_values = valid_coords[maskflt[valid_coords] < 0.5]
-            variable_ids = coords_to_vidx[has_no_values]
-            has_no_values_ids = valid_ids[maskflt[valid_coords] < 0.5]
-            A[has_no_values_ids, variable_ids] = weight
-            # Reduce weight for invalid
-            sum_neighbors[np.argwhere(valid == 0).flatten()] = \
-                sum_neighbors[np.argwhere(valid == 0).flatten()] - weight
-
-        A[np.arange(numEquations), np.arange(numEquations)] = -sum_neighbors
-        return A, b
-
-    def __call__(self, img: torch.Tensor, mask: torch.Tensor):
-        """ Our linear inputation scheme. """
-        """
-		This is the function to do the linear infilling
-		img: original image (C,H,W)-tensor;
-		mask: mask; (H,W)-tensor
-
-		"""
-        imgflt = img.reshape(img.shape[0], -1)
-        maskflt = mask.reshape(-1)
-    # Indices that need to be imputed.
-        indices_linear = np.argwhere(maskflt == 0).flatten()
-        # Set up sparse equation system, solve system.
-        A, b = NoisyLinearImputer.setup_sparse_system(
-            mask.numpy(), img.numpy(), neighbors_weights)
-        res = torch.tensor(spsolve(csc_matrix(A), b), dtype=torch.float)
-
-        # Fill the values with the solution of the system.
-        img_infill = imgflt.clone()
-        img_infill[:, indices_linear] = res.t() + self.noise * \
-            torch.randn_like(res.t())
-
-        return img_infill.reshape_as(img)
-
-
-class ROADMostRelevantFirst(PerturbationConfidenceMetric):
-    def __init__(self, percentile=80):
-        super(ROADMostRelevantFirst, self).__init__(
-            RemoveMostRelevantFirst(percentile, NoisyLinearImputer()))
-
-
-class ROADLeastRelevantFirst(PerturbationConfidenceMetric):
-    def __init__(self, percentile=20):
-        super(ROADLeastRelevantFirst, self).__init__(
-            RemoveLeastRelevantFirst(percentile, NoisyLinearImputer()))
-
-
-class ROADMostRelevantFirstAverage(AveragerAcrossThresholds):
-    def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
-        super(ROADMostRelevantFirstAverage, self).__init__(
-            ROADMostRelevantFirst, percentiles)
-
-
-class ROADLeastRelevantFirstAverage(AveragerAcrossThresholds):
-    def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
-        super(ROADLeastRelevantFirstAverage, self).__init__(
-            ROADLeastRelevantFirst, percentiles)
-
-
-class ROADCombined:
-    def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
-        self.percentiles = percentiles
-        self.morf_averager = ROADMostRelevantFirstAverage(percentiles)
-        self.lerf_averager = ROADLeastRelevantFirstAverage(percentiles)
-
-    def __call__(self,
-                 input_tensor: torch.Tensor,
-                 cams: np.ndarray,
-                 targets: List[Callable],
-                 model: torch.nn.Module):
-
-        scores_lerf = self.lerf_averager(input_tensor, cams, targets, model)
-        scores_morf = self.morf_averager(input_tensor, cams, targets, model)
-        return (scores_lerf - scores_morf) / 2

src/custom_code/custom_grad_cam/random_cam.py

@@ -1,22 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-
-
-class RandomCAM(BaseCAM):
-    def __init__(self, model, target_layers, use_cuda=False,
-                 reshape_transform=None):
-        super(
-            RandomCAM,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_layer,
-                        target_category,
-                        activations,
-                        grads):
-        return np.random.uniform(-1, 1, size=(grads.shape[0], grads.shape[1]))

src/custom_code/custom_grad_cam/score_cam.py

@@ -1,60 +0,0 @@
-import torch
-import tqdm
-from custom_grad_cam.base_cam import BaseCAM
-
-
-class ScoreCAM(BaseCAM):
-    def __init__(
-            self,
-            model,
-            target_layers,
-            use_cuda=False,
-            reshape_transform=None):
-        super(ScoreCAM, self).__init__(model,
-                                       target_layers,
-                                       use_cuda,
-                                       reshape_transform=reshape_transform,
-                                       uses_gradients=False)
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_layer,
-                        targets,
-                        activations,
-                        grads):
-        with torch.no_grad():
-            upsample = torch.nn.UpsamplingBilinear2d(
-                size=input_tensor.shape[-2:])
-            activation_tensor = torch.from_numpy(activations)
-            if self.cuda:
-                activation_tensor = activation_tensor.cuda()
-
-            upsampled = upsample(activation_tensor)
-
-            maxs = upsampled.view(upsampled.size(0),
-                                  upsampled.size(1), -1).max(dim=-1)[0]
-            mins = upsampled.view(upsampled.size(0),
-                                  upsampled.size(1), -1).min(dim=-1)[0]
-
-            maxs, mins = maxs[:, :, None, None], mins[:, :, None, None]
-            upsampled = (upsampled - mins) / (maxs - mins)
-
-            input_tensors = input_tensor[:, None,
-                                         :, :] * upsampled[:, :, None, :, :]
-
-            if hasattr(self, "batch_size"):
-                BATCH_SIZE = self.batch_size
-            else:
-                BATCH_SIZE = 16
-
-            scores = []
-            for target, tensor in zip(targets, input_tensors):
-                for i in tqdm.tqdm(range(0, tensor.size(0), BATCH_SIZE)):
-                    batch = tensor[i: i + BATCH_SIZE, :]
-                    outputs = [target(o).cpu().item()
-                               for o in self.model(batch)]
-                    scores.extend(outputs)
-            scores = torch.Tensor(scores)
-            scores = scores.view(activations.shape[0], activations.shape[1])
-            weights = torch.nn.Softmax(dim=-1)(scores).numpy()
-            return weights

src/custom_code/custom_grad_cam/sobel_cam.py

@@ -1,11 +0,0 @@
-import cv2
-
-
-def sobel_cam(img):
-    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
-    grad_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
-    grad_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)
-    abs_grad_x = cv2.convertScaleAbs(grad_x)
-    abs_grad_y = cv2.convertScaleAbs(grad_y)
-    grad = cv2.addWeighted(abs_grad_x, 0.5, abs_grad_y, 0.5, 0)
-    return grad

src/custom_code/custom_grad_cam/utils/__init__.py

@@ -1,4 +0,0 @@
-from custom_grad_cam.utils.image import deprocess_image
-from custom_grad_cam.utils.svd_on_activations import get_2d_projection
-from custom_grad_cam.utils import model_targets
-from custom_grad_cam.utils import reshape_transforms

src/custom_code/custom_grad_cam/utils/find_layers.py

@@ -1,30 +0,0 @@
-def replace_layer_recursive(model, old_layer, new_layer):
-    for name, layer in model._modules.items():
-        if layer == old_layer:
-            model._modules[name] = new_layer
-            return True
-        elif replace_layer_recursive(layer, old_layer, new_layer):
-            return True
-    return False
-
-
-def replace_all_layer_type_recursive(model, old_layer_type, new_layer):
-    for name, layer in model._modules.items():
-        if isinstance(layer, old_layer_type):
-            model._modules[name] = new_layer
-        replace_all_layer_type_recursive(layer, old_layer_type, new_layer)
-
-
-def find_layer_types_recursive(model, layer_types):
-    def predicate(layer):
-        return type(layer) in layer_types
-    return find_layer_predicate_recursive(model, predicate)
-
-
-def find_layer_predicate_recursive(model, predicate):
-    result = []
-    for name, layer in model._modules.items():
-        if predicate(layer):
-            result.append(layer)
-        result.extend(find_layer_predicate_recursive(layer, predicate))
-    return result

src/custom_code/custom_grad_cam/utils/image.py

@@ -1,183 +0,0 @@
-import matplotlib
-from matplotlib import pyplot as plt
-from matplotlib.lines import Line2D
-import cv2
-import numpy as np
-import torch
-from torchvision.transforms import Compose, Normalize, ToTensor
-from typing import List, Dict
-import math
-
-
-def preprocess_image(
-    img: np.ndarray, mean=[
-        0.5, 0.5, 0.5], std=[
-            0.5, 0.5, 0.5]) -> torch.Tensor:
-    preprocessing = Compose([
-        ToTensor(),
-        Normalize(mean=mean, std=std)
-    ])
-    return preprocessing(img.copy()).unsqueeze(0)
-
-
-def deprocess_image(img):
-    """ see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """
-    img = img - np.mean(img)
-    img = img / (np.std(img) + 1e-5)
-    img = img * 0.1
-    img = img + 0.5
-    img = np.clip(img, 0, 1)
-    return np.uint8(img * 255)
-
-
-def show_cam_on_image(img: np.ndarray,
-                      mask: np.ndarray,
-                      use_rgb: bool = False,
-                      colormap: int = cv2.COLORMAP_JET,
-                      image_weight: float = 0.5) -> np.ndarray:
-    """ This function overlays the cam mask on the image as an heatmap.
-    By default the heatmap is in BGR format.
-
-    :param img: The base image in RGB or BGR format.
-    :param mask: The cam mask.
-    :param use_rgb: Whether to use an RGB or BGR heatmap, this should be set to True if 'img' is in RGB format.
-    :param colormap: The OpenCV colormap to be used.
-    :param image_weight: The final result is image_weight * img + (1-image_weight) * mask.
-    :returns: The default image with the cam overlay.
-    """
-    heatmap = cv2.applyColorMap(np.uint8(255 * mask), colormap)
-    if use_rgb:
-        heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
-    heatmap = np.float32(heatmap) / 255
-
-    if np.max(img) > 1:
-        raise Exception(
-            "The input image should np.float32 in the range [0, 1]")
-
-    if image_weight < 0 or image_weight > 1:
-        raise Exception(
-            f"image_weight should be in the range [0, 1].\
-                Got: {image_weight}")
-
-    cam = (1 - image_weight) * heatmap + image_weight * img
-    cam = cam / np.max(cam)
-    return np.uint8(255 * cam)
-
-
-def create_labels_legend(concept_scores: np.ndarray,
-                         labels: Dict[int, str],
-                         top_k=2):
-    concept_categories = np.argsort(concept_scores, axis=1)[:, ::-1][:, :top_k]
-    concept_labels_topk = []
-    for concept_index in range(concept_categories.shape[0]):
-        categories = concept_categories[concept_index, :]
-        concept_labels = []
-        for category in categories:
-            score = concept_scores[concept_index, category]
-            label = f"{','.join(labels[category].split(',')[:3])}:{score:.2f}"
-            concept_labels.append(label)
-        concept_labels_topk.append("\n".join(concept_labels))
-    return concept_labels_topk
-
-
-def show_factorization_on_image(img: np.ndarray,
-                                explanations: np.ndarray,
-                                colors: List[np.ndarray] = None,
-                                image_weight: float = 0.5,
-                                concept_labels: List = None) -> np.ndarray:
-    """ Color code the different component heatmaps on top of the image.
-        Every component color code will be magnified according to the heatmap itensity
-        (by modifying the V channel in the HSV color space),
-        and optionally create a lagend that shows the labels.
-
-        Since different factorization component heatmaps can overlap in principle,
-        we need a strategy to decide how to deal with the overlaps.
-        This keeps the component that has a higher value in it's heatmap.
-
-    :param img: The base image RGB format.
-    :param explanations: A tensor of shape num_componetns x height x width, with the component visualizations.
-    :param colors: List of R, G, B colors to be used for the components.
-                   If None, will use the gist_rainbow cmap as a default.
-    :param image_weight: The final result is image_weight * img + (1-image_weight) * visualization.
-    :concept_labels: A list of strings for every component. If this is paseed, a legend that shows
-                     the labels and their colors will be added to the image.
-    :returns: The visualized image.
-    """
-    n_components = explanations.shape[0]
-    if colors is None:
-        # taken from https://github.com/edocollins/DFF/blob/master/utils.py
-        _cmap = plt.cm.get_cmap('gist_rainbow')
-        colors = [
-            np.array(
-                _cmap(i)) for i in np.arange(
-                0,
-                1,
-                1.0 /
-                n_components)]
-    concept_per_pixel = explanations.argmax(axis=0)
-    masks = []
-    for i in range(n_components):
-        mask = np.zeros(shape=(img.shape[0], img.shape[1], 3))
-        mask[:, :, :] = colors[i][:3]
-        explanation = explanations[i]
-        explanation[concept_per_pixel != i] = 0
-        mask = np.uint8(mask * 255)
-        mask = cv2.cvtColor(mask, cv2.COLOR_RGB2HSV)
-        mask[:, :, 2] = np.uint8(255 * explanation)
-        mask = cv2.cvtColor(mask, cv2.COLOR_HSV2RGB)
-        mask = np.float32(mask) / 255
-        masks.append(mask)
-
-    mask = np.sum(np.float32(masks), axis=0)
-    result = img * image_weight + mask * (1 - image_weight)
-    result = np.uint8(result * 255)
-
-    if concept_labels is not None:
-        px = 1 / plt.rcParams['figure.dpi']  # pixel in inches
-        fig = plt.figure(figsize=(result.shape[1] * px, result.shape[0] * px))
-        plt.rcParams['legend.fontsize'] = int(
-            14 * result.shape[0] / 256 / max(1, n_components / 6))
-        lw = 5 * result.shape[0] / 256
-        lines = [Line2D([0], [0], color=colors[i], lw=lw)
-                 for i in range(n_components)]
-        plt.legend(lines,
-                   concept_labels,
-                   mode="expand",
-                   fancybox=True,
-                   shadow=True)
-
-        plt.tight_layout(pad=0, w_pad=0, h_pad=0)
-        plt.axis('off')
-        fig.canvas.draw()
-        data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
-        plt.close(fig=fig)
-        data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-        data = cv2.resize(data, (result.shape[1], result.shape[0]))
-        result = np.hstack((result, data))
-    return result
-
-
-def scale_cam_image(cam, target_size=None):
-    result = []
-    for img in cam:
-        img = img - np.min(img)
-        img = img / (1e-7 + np.max(img))
-        if target_size is not None:
-            img = cv2.resize(img, target_size)
-        result.append(img)
-    result = np.float32(result)
-
-    return result
-
-
-def scale_accross_batch_and_channels(tensor, target_size):
-    batch_size, channel_size = tensor.shape[:2]
-    reshaped_tensor = tensor.reshape(
-        batch_size * channel_size, *tensor.shape[2:])
-    result = scale_cam_image(reshaped_tensor, target_size)
-    result = result.reshape(
-        batch_size,
-        channel_size,
-        target_size[1],
-        target_size[0])
-    return result

src/custom_code/custom_grad_cam/utils/model_targets.py

@@ -1,103 +0,0 @@
-import numpy as np
-import torch
-import torchvision
-
-
-class ClassifierOutputTarget:
-    def __init__(self, category):
-        self.category = category
-
-    def __call__(self, model_output):
-        if len(model_output.shape) == 1:
-            return model_output[self.category]
-        return model_output[:, self.category]
-
-
-class ClassifierOutputSoftmaxTarget:
-    def __init__(self, category):
-        self.category = category
-
-    def __call__(self, model_output):
-        if len(model_output.shape) == 1:
-            return torch.softmax(model_output, dim=-1)[self.category]
-        return torch.softmax(model_output, dim=-1)[:, self.category]
-
-
-class BinaryClassifierOutputTarget:
-    def __init__(self, category):
-        self.category = category
-
-    def __call__(self, model_output):
-        if self.category == 1:
-            sign = 1
-        else:
-            sign = -1
-        return torch.abs(model_output) * sign
-
-
-class SoftmaxOutputTarget:
-    def __init__(self):
-        pass
-
-    def __call__(self, model_output):
-        return torch.softmax(model_output, dim=-1)
-
-
-class RawScoresOutputTarget:
-    def __init__(self):
-        pass
-
-    def __call__(self, model_output):
-        return model_output
-
-
-class SemanticSegmentationTarget:
-    """ Gets a binary spatial mask and a category,
-        And return the sum of the category scores,
-        of the pixels in the mask. """
-
-    def __init__(self, category, mask):
-        self.category = category
-        self.mask = torch.from_numpy(mask)
-        if torch.cuda.is_available():
-            self.mask = self.mask.cuda()
-
-    def __call__(self, model_output):
-        return (model_output[self.category, :, :] * self.mask).sum()
-
-
-class FasterRCNNBoxScoreTarget:
-    """ For every original detected bounding box specified in "bounding boxes",
-        assign a score on how the current bounding boxes match it,
-            1. In IOU
-            2. In the classification score.
-        If there is not a large enough overlap, or the category changed,
-        assign a score of 0.
-
-        The total score is the sum of all the box scores.
-    """
-
-    def __init__(self, labels, bounding_boxes, iou_threshold=0.5):
-        self.labels = labels
-        self.bounding_boxes = bounding_boxes
-        self.iou_threshold = iou_threshold
-
-    def __call__(self, model_outputs):
-        output = torch.Tensor([0])
-        if torch.cuda.is_available():
-            output = output.cuda()
-
-        if len(model_outputs["boxes"]) == 0:
-            return output
-
-        for box, label in zip(self.bounding_boxes, self.labels):
-            box = torch.Tensor(box[None, :])
-            if torch.cuda.is_available():
-                box = box.cuda()
-
-            ious = torchvision.ops.box_iou(box, model_outputs["boxes"])
-            index = ious.argmax()
-            if ious[0, index] > self.iou_threshold and model_outputs["labels"][index] == label:
-                score = ious[0, index] + model_outputs["scores"][index]
-                output = output + score
-        return output

src/custom_code/custom_grad_cam/utils/reshape_transforms.py

@@ -1,34 +0,0 @@
-import torch
-
-
-def fasterrcnn_reshape_transform(x):
-    target_size = x['pool'].size()[-2:]
-    activations = []
-    for key, value in x.items():
-        activations.append(
-            torch.nn.functional.interpolate(
-                torch.abs(value),
-                target_size,
-                mode='bilinear'))
-    activations = torch.cat(activations, axis=1)
-    return activations
-
-
-def swinT_reshape_transform(tensor, height=7, width=7):
-    result = tensor.reshape(tensor.size(0),
-                            height, width, tensor.size(2))
-
-    # Bring the channels to the first dimension,
-    # like in CNNs.
-    result = result.transpose(2, 3).transpose(1, 2)
-    return result
-
-
-def vit_reshape_transform(tensor, height=14, width=14):
-    result = tensor[:, 1:, :].reshape(tensor.size(0),
-                                      height, width, tensor.size(2))
-
-    # Bring the channels to the first dimension,
-    # like in CNNs.
-    result = result.transpose(2, 3).transpose(1, 2)
-    return result

src/custom_code/custom_grad_cam/utils/svd_on_activations.py

@@ -1,19 +0,0 @@
-import numpy as np
-
-
-def get_2d_projection(activation_batch):
-    # TBD: use pytorch batch svd implementation
-    activation_batch[np.isnan(activation_batch)] = 0
-    projections = []
-    for activations in activation_batch:
-        reshaped_activations = (activations).reshape(
-            activations.shape[0], -1).transpose()
-        # Centering before the SVD seems to be important here,
-        # Otherwise the image returned is negative
-        reshaped_activations = reshaped_activations - \
-            reshaped_activations.mean(axis=0)
-        U, S, VT = np.linalg.svd(reshaped_activations, full_matrices=True)
-        projection = reshaped_activations @ VT[0, :]
-        projection = projection.reshape(activations.shape[1:])
-        projections.append(projection)
-    return np.float32(projections)

src/custom_code/custom_grad_cam/xgrad_cam.py

@@ -1,31 +0,0 @@
-import numpy as np
-from custom_grad_cam.base_cam import BaseCAM
-
-
-class XGradCAM(BaseCAM):
-    def __init__(
-            self,
-            model,
-            target_layers,
-            use_cuda=False,
-            reshape_transform=None):
-        super(
-            XGradCAM,
-            self).__init__(
-            model,
-            target_layers,
-            use_cuda,
-            reshape_transform)
-
-    def get_cam_weights(self,
-                        input_tensor,
-                        target_layer,
-                        target_category,
-                        activations,
-                        grads):
-        sum_activations = np.sum(activations, axis=(2, 3))
-        eps = 1e-7
-        weights = grads * activations / \
-            (sum_activations[:, :, None, None] + eps)
-        weights = weights.sum(axis=(2, 3))
-        return weights

src/gradio_blocks.py

@@ -3,7 +3,7 @@ import os
 
 VIDEOS_PER_ROW = 3
 VIDEO_EXAMPLES_PATH = "src/example_videos"
-def build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gradcam_video):
+def build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gradcam_video, language):
     with gr.Row():
         with gr.Column(scale=2):
             gr.Markdown("### Video to GradCAM-Video")
@@ -36,13 +36,21 @@ def build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gra
                         scale=2,
                     )
             
-            video_animal_to_explain = gr.Dropdown(
-                choices=["Predicted Class"] + ALL_CLASSES,
-                label="Animal",
-                value="Predicted Class",
-                interactive=True,
-                scale=2,
-            )
+            with gr.Row():
+                video_animal_to_explain = gr.Dropdown(
+                    choices=["Predicted Class"] + ALL_CLASSES,
+                    label="Animal",
+                    value="Predicted Class",
+                    interactive=True,
+                    scale=4,
+                )
+                            
+                show_predicted_class = gr.Checkbox(
+                    label="Show Predicted Class",
+                    value=False,
+                    interactive=True,
+                    scale=1,
+                )
                 
             with gr.Row():
                 colormap = gr.Dropdown(
@@ -70,11 +78,11 @@ def build_video_to_camvideo(CAM_METHODS, CV2_COLORMAPS, LAYERS, ALL_CLASSES, gra
                 
         with gr.Column(scale=1):
             with gr.Column():
-                video_in = gr.Video(autoplay=False, include_audio=False)
-                video_out = gr.Video(autoplay=False, include_audio=False)
+                video_in = gr.Video(autoplay=False, include_audio=False, label="Input Video")
+                video_out = gr.Video(autoplay=False, include_audio=False, show_label=False)
                 
             gif_cam_mode_button = gr.Button(value="Show GradCAM-Video", label="GradCAM", scale=1)
-            gif_cam_mode_button.click(fn=gradcam_video, inputs=[video_in, colormap, use_eigen_smooth, bw_highlight, video_alpha, video_cam_method, video_layer, video_animal_to_explain], outputs=[video_out], queue=True)
+            gif_cam_mode_button.click(fn=gradcam_video, inputs=[video_in, colormap, use_eigen_smooth, bw_highlight, video_alpha, video_cam_method, video_layer, video_animal_to_explain, show_predicted_class, language], outputs=[video_out], queue=True)
     
     with gr.Row():
         with gr.Column():

src/results/gradcam_video.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d88ec14ff35116bf5d8bd65454616aba242d8f79bde4dcbd717aabbcc910670a
-size 917687
+oid sha256:f0bdd085324b0f6cf522ac7fbf79b31c0eca97ae780cd6d0093bb87fc71ad142
+size 734566