import streamlit as st from PIL import Image import settings import captum import numpy as np import torch import torch.nn.functional as F import torch.backends.cudnn as cudnn from utils import get_args from utils import CTCLabelConverter, AttnLabelConverter, Averager, TokenLabelConverter import string import time import sys from dataset import hierarchical_dataset, AlignCollate import validators from model import Model, STRScore from PIL import Image from lime.wrappers.scikit_image import SegmentationAlgorithm from captum._utils.models.linear_model import SkLearnLinearModel, SkLearnRidge import random import os from skimage.color import gray2rgb import pickle from train_shap_corr import getPredAndConf import re from captum_test import acquire_average_auc, saveAttrData import copy from skimage.color import gray2rgb from matplotlib import pyplot as plt from torchvision import transforms from captum.attr import ( GradientShap, DeepLift, DeepLiftShap, IntegratedGradients, LayerConductance, NeuronConductance, NoiseTunnel, Saliency, InputXGradient, GuidedBackprop, Deconvolution, GuidedGradCam, FeatureAblation, ShapleyValueSampling, Lime, KernelShap ) from captum.metrics import ( infidelity, sensitivity_max ) from captum.attr._utils.visualization import visualize_image_attr device = torch.device('cpu') opt = get_args(is_train=False) if opt.sensitive: opt.character = string.printable[:-6] # same with ASTER setting (use 94 char). cudnn.benchmark = True cudnn.deterministic = True # opt.num_gpu = torch.cuda.device_count() # combineBestDataXAI(opt) # acquire_average_auc(opt) # acquireSingleCharAttrAve(opt) modelName = "parseq" opt.modelName = modelName # opt.eval_data = "datasets/data_lmdb_release/evaluation" if modelName=="vitstr": opt.benchmark_all_eval = True opt.Transformation = "None" opt.FeatureExtraction = "None" opt.SequenceModeling = "None" opt.Prediction = "None" opt.Transformer = True opt.sensitive = True opt.imgH = 224 opt.imgW = 224 opt.data_filtering_off = True opt.TransformerModel= "vitstr_base_patch16_224" opt.saved_model = "pretrained/vitstr_base_patch16_224_aug.pth" opt.batch_size = 1 opt.workers = 0 opt.scorer = "mean" opt.blackbg = True elif modelName=="parseq": opt.benchmark_all_eval = True opt.Transformation = "None" opt.FeatureExtraction = "None" opt.SequenceModeling = "None" opt.Prediction = "None" opt.Transformer = True opt.sensitive = True opt.imgH = 32 opt.imgW = 128 opt.data_filtering_off = True opt.batch_size = 1 opt.workers = 0 opt.scorer = "mean" opt.blackbg = True segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=4, max_dist=200, ratio=0.2, random_seed=random.randint(0, 1000)) if modelName=="vitstr": if opt.Transformer: converter = TokenLabelConverter(opt) elif 'CTC' in opt.Prediction: converter = CTCLabelConverter(opt.character) else: converter = AttnLabelConverter(opt.character) opt.num_class = len(converter.character) if opt.rgb: opt.input_channel = 3 model_obj = Model(opt) model = torch.nn.DataParallel(model_obj).to(device) modelCopy = copy.deepcopy(model) scoring_singlechar = STRScore(opt=opt, converter=converter, device=device, enableSingleCharAttrAve=True) super_pixel_model_singlechar = torch.nn.Sequential( # super_pixler, # numpy2torch_converter, modelCopy, scoring_singlechar ).to(device) modelCopy.eval() scoring_singlechar.eval() super_pixel_model_singlechar.eval() # Single Char Attribution Averaging # enableSingleCharAttrAve - set to True scoring = STRScore(opt=opt, converter=converter, device=device) super_pixel_model = torch.nn.Sequential( # super_pixler, # numpy2torch_converter, model, scoring ).to(device) model.eval() scoring.eval() super_pixel_model.eval() elif modelName=="parseq": model = torch.hub.load('baudm/parseq', 'parseq', pretrained=True) # checkpoint = torch.hub.load_state_dict_from_url('https://github.com/baudm/parseq/releases/download/v1.0.0/parseq-bb5792a6.pt', map_location="cpu") # # state_dict = {key.replace("module.", ""): value for key, value in checkpoint["state_dict"].items()} # model.load_state_dict(checkpoint) model = model.to(device) model_obj = model converter = TokenLabelConverter(opt) modelCopy = copy.deepcopy(model) scoring_singlechar = STRScore(opt=opt, converter=converter, device=device, enableSingleCharAttrAve=True, model=modelCopy) super_pixel_model_singlechar = torch.nn.Sequential( # super_pixler, # numpy2torch_converter, modelCopy, scoring_singlechar ).to(device) modelCopy.eval() scoring_singlechar.eval() super_pixel_model_singlechar.eval() # Single Char Attribution Averaging # enableSingleCharAttrAve - set to True scoring = STRScore(opt=opt, converter=converter, device=device, model=model) super_pixel_model = torch.nn.Sequential( # super_pixler, # numpy2torch_converter, model, scoring ).to(device) model.eval() scoring.eval() super_pixel_model.eval() if opt.blackbg: shapImgLs = np.zeros(shape=(1, 1, 224, 224)).astype(np.float32) trainList = np.array(shapImgLs) background = torch.from_numpy(trainList).to(device) # x = st.slider('Select a value') # st.write(x, 'squared is', x * x) ### Returns the mean for each segmentation having shape as the same as the input ### This function can only one attribution image at a time def averageSegmentsOut(attr, segments): averagedInput = torch.clone(attr) sortedDict = {} for x in np.unique(segments): segmentMean = torch.mean(attr[segments == x][:]) sortedDict[x] = float(segmentMean.detach().cpu().numpy()) averagedInput[segments == x] = segmentMean return averagedInput, sortedDict ### Acquire pixelwise attributions and replace them with ranked numbers averaged ### across segmentation with the largest contribution having the largest number ### and the smallest set to 1, which is the minimum number. ### attr - original attribution ### segm - image segmentations def rankedAttributionsBySegm(attr, segm): aveSegmentations, sortedDict = averageSegmentsOut(attr[0,0], segm) totalSegm = len(sortedDict.keys()) # total segmentations sortedKeys = [k for k, v in sorted(sortedDict.items(), key=lambda item: item[1])] sortedKeys = sortedKeys[::-1] ### A list that should contain largest to smallest score currentRank = totalSegm rankedSegmImg = torch.clone(attr) for totalSegToHide in range(0, len(sortedKeys)): currentSegmentToHide = sortedKeys[totalSegToHide] rankedSegmImg[0,0][segm == currentSegmentToHide] = currentRank currentRank -= 1 return rankedSegmImg labels = st.text_input('Drag one of the images from the right towards the box below (or you can choose your own image). ' 'You need to put the text of the image in the textbox below first (e.g. GAS) before dragging the image.') image = Image.open('demo_image/demo_gas.jpg') #Brand logo image (optional) image2 = Image.open('demo_image/demo_shakeshack.jpg') #Brand logo image (optional) image3 = Image.open('demo_image/demo_ronaldo.jpg') #Brand logo image (optional) image4 = Image.open('demo_image/demo_car.jpg') #Brand logo image (optional) #Create two columns with different width col1, col2 = st.columns( [0.8, 0.2]) with col1: # To display the header text using css style st.markdown(""" """, unsafe_allow_html=True) st.markdown('
STRExp (Explaining PARSeq STR Model)...
', unsafe_allow_html=True) with col2: # To display brand logo st.image(image, width=150) st.image(image2, width=150) st.image(image3, width=150) st.image(image4, width=150) uploaded_file = st.file_uploader("Choose a file", type=["png", "jpg"]) if uploaded_file is not None: # To read file as bytes: bytes_data = uploaded_file.getvalue() pilImg = Image.open(uploaded_file) pilImg = pilImg.resize((opt.imgW, opt.imgH)) orig_img_tensors = transforms.ToTensor()(pilImg).unsqueeze(0) img1 = orig_img_tensors.to(device) # image_tensors = ((torch.clone(orig_img_tensors) + 1.0) / 2.0) * 255.0 image_tensors = torch.mean(orig_img_tensors, dim=1).unsqueeze(0).unsqueeze(0) imgDataDict = {} img_numpy = image_tensors.cpu().detach().numpy()[0] ### Need to set batch size to 1 only if img_numpy.shape[0] == 1: img_numpy = gray2rgb(img_numpy[0]) # print("img_numpy shape: ", img_numpy.shape) # (1, 32, 128, 3) segmOutput = segmentation_fn(img_numpy[0]) results_dict = {} aveAttr = [] aveAttr_charContrib = [] # labels: RONALDO segmDataNP = segmOutput img1.requires_grad = True bgImg = torch.zeros(img1.shape).to(device) # preds = model(img1, seqlen=converter.batch_max_length) input = img1 origImgNP = torch.clone(orig_img_tensors).detach().cpu().numpy()[0][0] # (1, 1, 224, 224) origImgNP = gray2rgb(origImgNP) charOffset = 0 img1 = transforms.Normalize(0.5, 0.5)(img1) # Between -1 to 1 labels = labels.lower() target = converter.encode([labels]) ### Local explanations only collectedAttributions = [] for charIdx in range(0, len(labels)): scoring_singlechar.setSingleCharOutput(charIdx + charOffset) gtClassNum = target[0][charIdx + charOffset] gs = GradientShap(super_pixel_model_singlechar) baseline_dist = torch.zeros((1, 3, opt.imgH, opt.imgW)) baseline_dist = baseline_dist.to(device) attributions = gs.attribute(input, baselines=baseline_dist, target=0) collectedAttributions.append(attributions) aveAttributions = torch.mean(torch.cat(collectedAttributions,dim=0), dim=0).unsqueeze(0) # if not torch.isnan(aveAttributions).any(): # rankedAttr = rankedAttributionsBySegm(aveAttributions, segmDataNP) # rankedAttr = rankedAttr.detach().cpu().numpy()[0][0] # rankedAttr = gray2rgb(rankedAttr) # mplotfig, _ = visualize_image_attr(rankedAttr, origImgNP, method='blended_heat_map', cmap='RdYlGn') # mplotfig.savefig(outputDir + '{}_shapley_l.png'.format(nameNoExt)) # mplotfig.clear() # plt.close(mplotfig) ### Local Sampling gs = GradientShap(super_pixel_model) baseline_dist = torch.zeros((1, 3, opt.imgH, opt.imgW)) baseline_dist = baseline_dist.to(device) attributions = gs.attribute(input, baselines=baseline_dist, target=0) # if not torch.isnan(attributions).any(): # collectedAttributions.append(attributions) # rankedAttr = rankedAttributionsBySegm(attributions, segmDataNP) # rankedAttr = rankedAttr.detach().cpu().numpy()[0][0] # rankedAttr = gray2rgb(rankedAttr) # mplotfig, _ = visualize_image_attr(rankedAttr, origImgNP, method='blended_heat_map', cmap='RdYlGn') # mplotfig.savefig(outputDir + '{}_shapley.png'.format(nameNoExt)) # mplotfig.clear() # plt.close(mplotfig) ### Global + Local context aveAttributions = torch.mean(torch.cat(collectedAttributions,dim=0), dim=0).unsqueeze(0) if not torch.isnan(aveAttributions).any(): rankedAttr = rankedAttributionsBySegm(aveAttributions, segmDataNP) rankedAttr = rankedAttr.detach().cpu().numpy()[0][0] rankedAttr = gray2rgb(rankedAttr) mplotfig, _ = visualize_image_attr(rankedAttr, origImgNP, method='blended_heat_map', cmap='RdYlGn') # fig = mplotfig.figure(figsize=(8,8)) st.pyplot(mplotfig) # mplotfig.savefig(outputDir + '{}_shapley_gl.png'.format(nameNoExt)) # mplotfig.clear() # plt.close(mplotfig)