Spaces:
Sleeping
Sleeping
| import torch | |
| from transformers import ViTForImageClassification, ViTImageProcessor | |
| import matplotlib.pyplot as plt | |
| import gradio as gr | |
| import plotly.graph_objects as go | |
| import torch | |
| import numpy as np | |
| from PIL import Image | |
| model_name = "./best_model" | |
| processor = ViTImageProcessor.from_pretrained(model_name) | |
| labels = ['Acne or Rosacea', 'Actinic Keratosis Basal Cell Carcinoma and other Malignant Lesions', 'Atopic Dermatitis', 'Bullous Disease', 'Cellulitis Impetigo and other Bacterial Infections', 'Contact Dermatitis', 'Eczema', 'Exanthems and Drug Eruptions', 'Hair Loss Photos Alopecia and other Hair Diseases', 'Herpes HPV and other STDs', 'Light Diseases and Disorders of Pigmentation', 'Lupus and other Connective Tissue diseases', 'Melanoma Skin Cancer Nevi and Moles', 'Nail Fungus and other Nail Disease', 'Psoriasis pictures Lichen Planus and related diseases', 'Scabies Lyme Disease and other Infestations and Bites', 'Seborrheic Keratoses and other Benign Tumors', 'Systemic Disease', 'Tinea Ringworm Candidiasis and other Fungal Infections', 'Urticaria Hives', 'Vascular Tumors', 'Vasculitis', 'Warts Molluscum and other Viral Infections'] | |
| class ViTForImageClassificationWithAttention(ViTForImageClassification): | |
| def forward(self, pixel_values, output_attentions=True): | |
| outputs = super().forward(pixel_values, output_attentions=output_attentions) | |
| attention = outputs.attentions | |
| return outputs, attention | |
| model = ViTForImageClassificationWithAttention.from_pretrained(model_name,attn_implementation="eager") | |
| def classify_image(image): | |
| img = Image.open(image) | |
| inputs = processor(images=img, return_tensors="pt") | |
| outputs, attention = model(**inputs, output_attentions=True) | |
| logits = outputs.logits | |
| probs = torch.nn.functional.softmax(logits, dim=1) | |
| top_k_probs, top_k_indices = torch.topk(probs, k=5) # show top 5 predicted labels | |
| predicted_class_idx = torch.argmax(logits) | |
| predicted_class_label = labels[predicted_class_idx] | |
| top_k_labels = [labels[idx] for idx in top_k_indices[0]] | |
| top_k_label_probs = [(label, prob.item()) for label, prob in zip(top_k_labels, top_k_probs[0])] | |
| # Create a bar chart | |
| fig_bar = go.Figure( | |
| data=[go.Bar(x=[label for label, prob in top_k_label_probs], y=[prob for label, prob in top_k_label_probs])]) | |
| fig_bar.update_layout(title="Top 5 Predicted Labels with Probabilities", xaxis_title="Label", | |
| yaxis_title="Probability") | |
| # Create a heatmap | |
| if attention is not None: | |
| fig_heatmap = go.Figure( | |
| data=[go.Heatmap(z=attention[0][0, 0, :, :].detach().numpy(), colorscale='Viridis', showscale=False)]) | |
| fig_heatmap.update_layout(title="Attention Heatmap") | |
| else: | |
| fig_heatmap = None | |
| # Overlay the attention heatmap on the input image | |
| # Overlay the attention heatmap on the input image | |
| # Overlay the attention heatmap on the input image | |
| if attention is not None: | |
| img_array = np.array(img) | |
| heatmap = np.array(attention[0][0, 0, :, :].detach().numpy()) | |
| heatmap = np.resize(heatmap, (img_array.shape[0], img_array.shape[1])) | |
| heatmap = heatmap / heatmap.max() * 255 # Normalize heatmap to [0, 255] | |
| heatmap = heatmap.astype(np.uint8) | |
| heatmap_color = np.zeros((img_array.shape[0], img_array.shape[1], 3), dtype=np.uint8) | |
| heatmap_color[:, :, 0] = heatmap # Red channel | |
| heatmap_color[:, :, 1] = heatmap # Green channel | |
| heatmap_color[:, :, 2] = 0 # Blue channel | |
| attention_overlay = (img_array * 0.5 + heatmap_color * 0.5).astype(np.uint8) | |
| img = Image.fromarray(attention_overlay) | |
| img.save("attention_overlay.png") | |
| attention_overlay = gr.Image("attention_overlay.png") | |
| else: | |
| attention_overlay = None | |
| # Return the predicted label, the bar chart, and the heatmap | |
| return predicted_class_label, fig_bar, attention_overlay | |
| demo = gr.Interface( | |
| fn=classify_image, | |
| inputs=gr.Image(type="filepath"), | |
| outputs=[ | |
| gr.Label(label="Predicted Class Label"), | |
| gr.Plot(label="Top 5 Predicted Labels with Probabilities"), | |
| gr.Image(label="Attention Overlay") | |
| ], | |
| title="Dermatological Image Classification Demo", | |
| description="Upload an image to see the predicted class label, top 5 predicted labels with probabilities, and attention overlay", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() |