Update

app.py

@@ -45,15 +45,15 @@ demo = gr.Interface(
     theme="gradio/soft",
     fn=process_file,
     title="HANDETECT",
-    description=generate_description,
+    # description=generate_description,
     inputs=[
         gr.components.Image(type="filepath", label="Choose Image", source="upload"),
     ],
     outputs=[
-        gr.outputs.Textbox(label="Prediction 1"),
-        gr.outputs.Textbox(label="Prediction 2"),
-        gr.outputs.Textbox(label="Prediction 3"),
+        gr.outputs.Textbox(label="Probability 1"),
+        gr.outputs.Textbox(label="Probability 2"),
+        gr.outputs.Textbox(label="Probability 3"),
     ],
 )
 
-demo.launch()
+demo.launch(inbrowser=True)

augment.py

@@ -5,8 +5,13 @@ from configs import *
 import uuid
 
 tasks = ["1", "2", "3", "4", "5", "6"]
+num_of_images = 100
+
+shutil.rmtree(TEMP_DATA_DIR + "1/", ignore_errors=True)
+
 
 for task in ["1"]:
+    shutil.rmtree(AUG_DATA_DIR + task, ignore_errors=True)
     # Loop through all folders in Task 1 and generate augmented images for each class
     for class_label in [
         "Alzheimer Disease",
@@ -46,8 +51,7 @@ for task in ["1"]:
             p.random_contrast(probability=0.8, min_factor=0.5, max_factor=1.5)
             p.random_color(probability=0.8, min_factor=0.5, max_factor=1.5)
             p.rotate_random_90(probability=0.8)
-            # Generate 100 - total of original images so that the total number of images in each class is 100
-            p.sample(100 - len(p.augmentor_images))
+            p.sample(num_of_images - len(p.augmentor_images))
             # Move the folder to data/train/Task 1/augmented
             # Create the folder if it does not exist
             if not os.path.exists(f"{AUG_DATA_DIR}{task}/"):
@@ -67,3 +71,5 @@ for task in ["1"]:
                     f"{AUG_DATA_DIR}{task}/{class_label}/{file}",
                     f"{AUG_DATA_DIR}{task}/{class_label}/{number}.png",
                 )
+
+shutil.rmtree(TEMP_DATA_DIR + task, ignore_errors=True)

calculate.py

@@ -0,0 +1,18 @@
+from scipy.optimize import linprog
+
+# Coefficients for the objective function (negative because linprog does minimization)
+c = [-0.88, -0.88, -0.85]
+
+# Coefficients for the inequality constraint (sum of weights = 1)
+A = [[1, 1, 1]]
+b = [1]
+
+# Bounds for each weight (between 0 and 1)
+bounds = [(0, 1), (0, 1), (0, 1)]
+
+# Solve the linear programming problem
+result = linprog(c, A_eq=A, b_eq=b, bounds=bounds)
+
+# The optimal weights
+optimal_weights = result.x
+print("Optimal weights:", optimal_weights)

combine_dats.py → combine_data.py

@@ -5,38 +5,39 @@ import uuid
 
 from configs import *
 
+shutil.rmtree(COMBINED_DATA_DIR + "1/", ignore_errors=True)
+
 for disease in CLASSES:
     # check if the original folder exists
-    if os.path.exists(RAW_DATA_DIR + '1/' + disease):
+    if os.path.exists(RAW_DATA_DIR + "1/" + disease):
         print("Copying raw data for disease: ", disease)
-        if not os.path.exists(COMBINED_DATA_DIR + '1/' + disease):
-            os.makedirs(COMBINED_DATA_DIR + '1/' + disease)
-        for file in os.listdir(RAW_DATA_DIR + '1/' + disease):
+        if not os.path.exists(COMBINED_DATA_DIR + "1/" + disease):
+            os.makedirs(COMBINED_DATA_DIR + "1/" + disease)
+        for file in os.listdir(RAW_DATA_DIR + "1/" + disease):
             random_name = str(uuid.uuid4()) + ".png"
             shutil.copy(
-                RAW_DATA_DIR + '1/' + disease + '/' + file,
-                COMBINED_DATA_DIR + '1/' + disease + '/' + random_name,
+                RAW_DATA_DIR + "1/" + disease + "/" + file,
+                COMBINED_DATA_DIR + "1/" + disease + "/" + random_name,
             )
-        
-    if os.path.exists(EXTERNAL_DATA_DIR + '1/' + disease):
+
+    if os.path.exists(EXTERNAL_DATA_DIR + "1/" + disease):
         print("Copying external data for disease: ", disease)
-        if not os.path.exists(COMBINED_DATA_DIR + '1/' + disease):
-            os.makedirs(COMBINED_DATA_DIR + '1/' + disease)
-        for file in os.listdir(EXTERNAL_DATA_DIR + '1/' + disease):
+        if not os.path.exists(COMBINED_DATA_DIR + "1/" + disease):
+            os.makedirs(COMBINED_DATA_DIR + "1/" + disease)
+        for file in os.listdir(EXTERNAL_DATA_DIR + "1/" + disease):
             random_name = str(uuid.uuid4()) + ".png"
             shutil.copy(
-                EXTERNAL_DATA_DIR + '1/' + disease + '/' + file,
-                COMBINED_DATA_DIR + '1/' + disease + '/' + random_name,
+                EXTERNAL_DATA_DIR + "1/" + disease + "/" + file,
+                COMBINED_DATA_DIR + "1/" + disease + "/" + random_name,
             )
-        
-    if os.path.exists(AUG_DATA_DIR + '1/' + disease):
+
+    if os.path.exists(AUG_DATA_DIR + "1/" + disease):
         print("Copying augmented data for disease: ", disease)
-        if not os.path.exists(COMBINED_DATA_DIR + '1/' + disease):
-            os.makedirs(COMBINED_DATA_DIR + '1/' + disease)
-        for file in os.listdir(AUG_DATA_DIR + '1/' + disease):
+        if not os.path.exists(COMBINED_DATA_DIR + "1/" + disease):
+            os.makedirs(COMBINED_DATA_DIR + "1/" + disease)
+        for file in os.listdir(AUG_DATA_DIR + "1/" + disease):
             random_name = str(uuid.uuid4()) + ".png"
             shutil.copy(
-                AUG_DATA_DIR + '1/' + disease + '/' + file,
-                COMBINED_DATA_DIR + '1/' + disease + '/' + random_name,
+                AUG_DATA_DIR + "1/" + disease + "/" + file,
+                COMBINED_DATA_DIR + "1/" + disease + "/" + random_name,
             )
-    

configs.py

@@ -13,29 +13,50 @@ from torchvision.models import (
     ShuffleNet_V2_X2_0_Weights,
     mobilenet_v3_small,
     MobileNet_V3_Small_Weights,
+    efficientnet_v2_s,
+    EfficientNet_V2_S_Weights,
+    efficientnet_b0,
+    EfficientNet_B0_Weights,
+    efficientnet_b1,
+    EfficientNet_B1_Weights,
+    efficientnet_b2,
+    EfficientNet_B2_Weights,
+    efficientnet_b3,
+    EfficientNet_B3_Weights,
+    mobilenet_v3_small,
+    MobileNet_V3_Small_Weights,
+    mobilenet_v3_large,
+    MobileNet_V3_Large_Weights,
+    googlenet,
+    GoogLeNet_Weights,
+    MobileNet_V2_Weights,
+    mobilenet_v2,
 )
 
 import torch.nn.functional as F
-from pytorchcv.model_provider import get_model as ptcv_get_model
 
 # Constants
 RANDOM_SEED = 123
-BATCH_SIZE = 16
-NUM_EPOCHS = 40
-LEARNING_RATE = 0.00016662575248025378
+BATCH_SIZE = 8
+NUM_EPOCHS = 150
+WARMUP_EPOCHS = 5
+LEARNING_RATE = 0.0001
 STEP_SIZE = 10
-GAMMA = 0.9
-DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+GAMMA = 0.3
+CUTMIX_ALPHA = 0.3
+# DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+DEVICE = torch.device("cpu")
 NUM_PRINT = 100
 TASK = 1
+WARMUP_EPOCHS = 5
 RAW_DATA_DIR = r"data/train/raw/Task "
 AUG_DATA_DIR = r"data/train/augmented/Task "
 EXTERNAL_DATA_DIR = r"data/train/external/Task "
 COMBINED_DATA_DIR = r"data/train/combined/Task "
-TEMP_DATA_DIR = "data/temp/"
+TEST_DATA_DIR = r"data/test/Task "
+TEMP_DATA_DIR = "data/temp/Task "
 NUM_CLASSES = 7
 LABEL_SMOOTHING_EPSILON = 0.1
-MIXUP_ALPHA = 0.2
 EARLY_STOPPING_PATIENCE = 20
 CLASSES = [
     "Alzheimer Disease",
@@ -46,12 +67,29 @@ CLASSES = [
     "Huntington Disease",
     "Parkinson Disease",
 ]
-MODEL_SAVE_PATH = r"output/checkpoints/model.pth"
 
 
-class SqueezeNet1_0WithDropout(nn.Module):
+class SE_Block(nn.Module):
+    def __init__(self, channel, reduction=16):
+        super(SE_Block, self).__init__()
+        self.avg_pool = nn.AdaptiveAvgPool2d(1)
+        self.fc = nn.Sequential(
+            nn.Linear(channel, channel // reduction, bias=False),
+            nn.ReLU(inplace=True),
+            nn.Linear(channel // reduction, channel, bias=False),
+            nn.Sigmoid(),  # Sigmoid activation to produce attention scores
+        )
+
+    def forward(self, x):
+        b, c, _, _ = x.size()
+        y = self.avg_pool(x).view(b, c)
+        y = self.fc(y).view(b, c, 1, 1)
+        return x * y.expand_as(x)
+
+
+class SqueezeNet1_0WithSE(nn.Module):
     def __init__(self, num_classes, dropout_prob=0.5):
-        super(SqueezeNet1_0WithDropout, self).__init__()
+        super(SqueezeNet1_0WithSE, self).__init__()
         squeezenet = squeezenet1_0(weights=SqueezeNet1_0_Weights.DEFAULT)
         self.features = squeezenet.features
         self.classifier = nn.Sequential(
@@ -64,17 +102,26 @@ class SqueezeNet1_0WithDropout(nn.Module):
             dropout_prob
         )  # Add dropout layer with the specified probability
 
+        # Adjust channel for SqueezeNet1.0 (original SqueezeNet1.0 has 1000 classes)
+        num_classes_squeezenet1_0 = 7
+
+        # Add Squeeze-and-Excitation block
+        self.se_block = SE_Block(
+            channel=num_classes_squeezenet1_0
+        )  # Adjust channel as needed
+
     def forward(self, x):
         x = self.features(x)
         x = self.classifier(x)
+        # x = self.se_block(x)  # Apply the SE block
         x = F.dropout(x, training=self.training)  # Apply dropout during training
         x = torch.flatten(x, 1)
         return x
 
 
-class SqueezeNet1_1WithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.5):
-        super(SqueezeNet1_1WithDropout, self).__init__()
+class SqueezeNet1_1WithSE(nn.Module):
+    def __init__(self, num_classes, dropout_prob=0.2):
+        super(SqueezeNet1_1WithSE, self).__init__()
         squeezenet = squeezenet1_1(weights=SqueezeNet1_1_Weights.DEFAULT)
         self.features = squeezenet.features
         self.classifier = nn.Sequential(
@@ -87,21 +134,26 @@ class SqueezeNet1_1WithDropout(nn.Module):
             dropout_prob
         )  # Add dropout layer with the specified probability
 
+        # Add Squeeze-and-Excitation block
+        self.se_block = SE_Block(channel=num_classes)  # Adjust channel as needed
+
     def forward(self, x):
         x = self.features(x)
         x = self.classifier(x)
+        x = self.se_block(x)  # Apply the SE block
         x = F.dropout(x, training=self.training)  # Apply dropout during training
         x = torch.flatten(x, 1)
         return x
 
 
-class ShuffleNetV2WithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.5):
-        super(ShuffleNetV2WithDropout, self).__init__()
-        shufflenet = shufflenet_v2_x2_0(weights=ShuffleNet_V2_X2_0_Weights.DEFAULT)
-        self.features = shufflenet.features
+class EfficientNetB2WithDropout(nn.Module):
+    #  0.00022015769999619205
+    def __init__(self, num_classes, dropout_prob=0.2):
+        super(EfficientNetB2WithDropout, self).__init__()
+        efficientnet = efficientnet_b2(weights=EfficientNet_B2_Weights.DEFAULT)
+        self.features = efficientnet.features
         self.classifier = nn.Sequential(
-            nn.Conv2d(1024, num_classes, kernel_size=1),
+            nn.Conv2d(1408, num_classes, kernel_size=1),
             nn.BatchNorm2d(num_classes),  # add batch normalization
             nn.ReLU(inplace=True),
             nn.AdaptiveAvgPool2d((1, 1)),
@@ -118,13 +170,62 @@ class ShuffleNetV2WithDropout(nn.Module):
         return x
 
 
-class MobileNetV3SmallWithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.5):
-        super(MobileNetV3SmallWithDropout, self).__init__()
-        mobilenet = mobilenet_v3_small(weights=MobileNet_V3_Small_Weights.DEFAULT)
+class EfficientNetB3WithDropout(nn.Module):
+    def __init__(self, num_classes, dropout_prob=0.2):
+        super(EfficientNetB3WithDropout, self).__init__()
+        efficientnet = efficientnet_b3(weights=EfficientNet_B3_Weights.DEFAULT)
+        self.features = efficientnet.features
+        self.classifier = nn.Sequential(
+            nn.Conv2d(1536, num_classes, kernel_size=1),
+            nn.BatchNorm2d(num_classes),  # add batch normalization
+            nn.ReLU(inplace=True),
+            nn.AdaptiveAvgPool2d((1, 1)),
+        )
+        self.dropout = nn.Dropout(
+            dropout_prob
+        )  # Add dropout layer with the specified probability
+
+    def forward(self, x):
+        x = self.features(x)
+        x = self.classifier(x)
+        x = F.dropout(x, training=self.training)  # Apply dropout during training
+        x = torch.flatten(x, 1)
+        return x
+
+
+class ResNet18WithNorm(nn.Module):
+    def __init__(self, num_classes=1000):
+        super(ResNet18WithNorm, self).__init__()
+        resnet = resnet18(pretrained=False)
+
+        # Remove the last block (Block 4)
+        self.features = nn.Sequential(
+            *list(resnet.children())[:-1]  # Exclude the last block
+        )
+
+        self.classifier = nn.Sequential(
+            nn.AdaptiveAvgPool2d((1, 1)),
+            nn.Flatten(),
+            nn.Linear(
+                512, num_classes
+            ),  # Adjust input size for the fully connected layer
+            nn.BatchNorm1d(num_classes),  # Add batch normalization
+        )
+
+    def forward(self, x):
+        x = self.features(x)
+        x = self.classifier(x)
+        x = torch.flatten(x, 1)
+        return x
+
+
+class MobileNetV3LargeWithDropout(nn.Module):
+    def __init__(self, num_classes, dropout_prob=0.2):
+        super(MobileNetV3LargeWithDropout, self).__init__()
+        mobilenet = mobilenet_v3_large(weights=MobileNet_V3_Large_Weights.DEFAULT)
         self.features = mobilenet.features
         self.classifier = nn.Sequential(
-            nn.Conv2d(576, num_classes, kernel_size=1),
+            nn.Conv2d(960, num_classes, kernel_size=1),
             nn.BatchNorm2d(num_classes),  # add batch normalization
             nn.ReLU(inplace=True),
             nn.AdaptiveAvgPool2d((1, 1)),
@@ -141,15 +242,60 @@ class MobileNetV3SmallWithDropout(nn.Module):
         return x
 
 
-MODEL = SqueezeNet1_0WithDropout(num_classes=7)
-# MODEL = ptcv_get_model("sqnxt23v5_w2", pretrained=False, num_classes=7)
-print(CLASSES)
+class GoogLeNetWithSE(nn.Module):
+    def __init__(self, num_classes):
+        super(GoogLeNetWithSE, self).__init__()
+        googlenet = googlenet(weights=GoogLeNet_Weights.DEFAULT)
+        # self.features = googlenet.features
+        self.classifier = nn.Sequential(
+            nn.Conv2d(1024, num_classes, kernel_size=1),
+            nn.BatchNorm2d(num_classes),  # add batch normalization
+            nn.ReLU(inplace=True),
+            nn.AdaptiveAvgPool2d((1, 1)),
+        )
+
+        # Add Squeeze-and-Excitation block
+        self.se_block = SE_Block(channel=num_classes)  # Adjust channel as needed
+
+    def forward(self, x):
+        # x = self.features(x)
+        x = self.classifier(x)
+        x = self.se_block(x)  # Apply the SE block
+        x = torch.flatten(x, 1)
+        return x
+
+
+class MobileNetV2WithDropout(nn.Module):
+    def __init__(self, num_classes, dropout_prob=0.2):
+        super(MobileNetV2WithDropout, self).__init__()
+        mobilenet = mobilenet_v2(weights=MobileNet_V2_Weights.DEFAULT)
+        self.features = mobilenet.features
+        self.classifier = nn.Sequential(
+            nn.Conv2d(1280, num_classes, kernel_size=1),
+            nn.BatchNorm2d(num_classes),  # add batch normalization
+            nn.ReLU(inplace=True),
+            nn.AdaptiveAvgPool2d((1, 1)),
+        )
+        self.dropout = nn.Dropout(
+            dropout_prob
+        )  # Add dropout layer with the specified probability
+
+    def forward(self, x):
+        x = self.features(x)
+        x = self.classifier(x)
+        x = F.dropout(x, training=self.training)  # Apply dropout during training
+        x = torch.flatten(x, 1)
+        return x
+
 
+MODEL = EfficientNetB3WithDropout(num_classes=NUM_CLASSES)
+MODEL_SAVE_PATH = r"output/checkpoints/" + MODEL.__class__.__name__ + ".pth"
+# MODEL_SAVE_PATH = r"C:\Users\User\Downloads\bestsqueezenetSE.pth"
 preprocess = transforms.Compose(
     [
-        transforms.Resize((274, 274)),  # Resize to 112x112
+        transforms.Resize((224, 224)),
         transforms.ToTensor(),  # Convert to tensor
-        transforms.Grayscale(num_output_channels=3),  # Convert to 3 channels
+        # transforms.Grayscale(num_output_channels=3),  # Convert to 3 channels
         # Normalize 3 channels
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
     ]
@@ -167,3 +313,14 @@ class CustomDataset(Dataset):
     def __getitem__(self, idx):
         img, label = self.data[idx]
         return img, label
+
+
+def ensemble_predictions(models, image):
+    all_predictions = []
+
+    with torch.no_grad():
+        for model in models:
+            output = model(image)
+            all_predictions.append(output)
+
+    return torch.stack(all_predictions, dim=0).mean(dim=0)

convert.py

@@ -3,7 +3,7 @@ import onnx2tf
 from configs import *
 
 torch.onnx.export(
-    model=MODEL,
+    model=model,
     args=torch.randn(1, 3, 64, 64),
     f="output/checkpoints/model.onnx",
     verbose=True,

data-splitting.py

@@ -0,0 +1,23 @@
+# Take 10% of the data in data\train\combined\Task 1\<class> and move it to data\test\Task 1\<class>
+
+import os
+import shutil
+import uuid
+
+from configs import *
+
+shutil.rmtree(TEST_DATA_DIR + "1/", ignore_errors=True)
+
+for disease in CLASSES:
+    # check if the original folder exists
+    if os.path.exists(COMBINED_DATA_DIR + "1/" + disease):
+        print("Splitting data for disease: ", disease)
+        if not os.path.exists(TEST_DATA_DIR + "1/" + disease):
+            os.makedirs(TEST_DATA_DIR + "1/" + disease)
+        files = os.listdir(COMBINED_DATA_DIR + "1/" + disease)
+        files.sort()
+        for file in files[: int(len(files) * 0.1)]:
+            shutil.move(
+                COMBINED_DATA_DIR + "1/" + disease + "/" + file,
+                TEST_DATA_DIR + "1/" + disease + "/" + file,
+            )

data_loader.py

@@ -1,22 +1,20 @@
 from configs import *
 from torchvision.datasets import ImageFolder
 from torch.utils.data import random_split, DataLoader, Dataset
+import torch
 
+torch.manual_seed(RANDOM_SEED)
 
-def load_data(raw_dir, augmented_dir, external_dir, preprocess, batch_size=BATCH_SIZE):
-    # Load the dataset using ImageFolder
-    raw_dataset = ImageFolder(root=raw_dir, transform=preprocess)
-    external_dataset = ImageFolder(root=external_dir, transform=preprocess)
-    augmented_dataset = ImageFolder(root=augmented_dir, transform=preprocess)
-    dataset = raw_dataset + external_dataset + augmented_dataset
+# Set seed
+torch.manual_seed(RANDOM_SEED)
+
+def load_data(combined_dir, preprocess, batch_size=BATCH_SIZE):
+    dataset = ImageFolder(combined_dir, transform=preprocess)
 
     # Classes
-    classes = augmented_dataset.classes
+    classes = dataset.classes
 
     print("Classes: ", *classes, sep=", ")
-    print("Length of raw dataset: ", len(raw_dataset))
-    print("Length of external dataset: ", len(external_dataset))
-    print("Length of augmented dataset: ", len(augmented_dataset))
     print("Length of total dataset: ", len(dataset))
 
     # Split the dataset into train and validation sets
@@ -29,8 +27,18 @@ def load_data(raw_dir, augmented_dir, external_dir, preprocess, batch_size=BATCH
         CustomDataset(train_dataset), batch_size=batch_size, shuffle=True, num_workers=0
     )
     valid_loader = DataLoader(
-        CustomDataset(val_dataset), batch_size=batch_size, num_workers=0
+        CustomDataset(val_dataset), batch_size=batch_size, num_workers=0, shuffle=False
     )
 
     return train_loader, valid_loader
 
+
+def load_test_data(test_dir, preprocess, batch_size=BATCH_SIZE):
+    test_dataset = ImageFolder(test_dir, transform=preprocess)
+
+    # Create a DataLoader for the test data
+    test_dataloader = DataLoader(
+        CustomDataset(test_dataset), batch_size=batch_size, shuffle=False, num_workers=0
+    )
+
+    return test_dataloader

ensemble.py

@@ -0,0 +1,249 @@
+import matplotlib.pyplot as plt
+from torch.optim.lr_scheduler import CosineAnnealingLR
+import torch
+import torch.nn as nn
+from torchvision.datasets import ImageFolder
+from torch.utils.data import DataLoader
+from data_loader import load_data, load_test_data
+from configs import *
+import numpy as np
+
+torch.cuda.empty_cache()
+
+# 
+
+
+class MLP(nn.Module):
+    def __init__(self, num_classes, num_models):
+        super(MLP, self).__init__()
+        self.layers = nn.Sequential(
+            nn.Linear(num_classes * num_models, 1024),
+            nn.LayerNorm(1024),
+            nn.LeakyReLU(negative_slope=0.01, inplace=True),
+            nn.Dropout(0.8),
+            nn.Linear(1024, 2048),
+            nn.LeakyReLU(negative_slope=0.01, inplace=True),
+            nn.Dropout(0.5),
+            nn.Linear(2048, 2048),
+            nn.LeakyReLU(negative_slope=0.01, inplace=True),
+            nn.Dropout(0.5),
+            nn.Linear(2048, num_classes),
+        )
+
+    def forward(self, x):
+        x = x.view(x.size(0), -1)
+        x = self.layers(x)
+        return x
+
+
+def mlp_meta(num_classes, num_models):
+    model = MLP(num_classes, num_models)
+    return model
+
+
+# Hyperparameters
+input_dim = 3 * 224 * 224  # Modify this based on your input size
+hidden_dim = 256
+output_dim = NUM_CLASSES
+
+# Create the data loaders using your data_loader functions50
+train_loader, val_loader = load_data(COMBINED_DATA_DIR + "1", preprocess, BATCH_SIZE)
+test_loader = load_test_data("data/test/Task 1", preprocess, BATCH_SIZE)
+
+model_paths = [
+    "output/checkpoints/bestsqueezenetSE3.pth",
+    "output/checkpoints/EfficientNetB3WithDropout.pth",
+    "output/checkpoints/MobileNetV2WithDropout2.pth",
+]
+
+
+# Define a function to load pretrained models
+def load_pretrained_model(path, model):
+    model.load_state_dict(torch.load(path))
+    return model.to(DEVICE)
+
+
+def rand_bbox(size, lam):
+    W = size[2]
+    H = size[3]
+    cut_rat = np.sqrt(1.0 - lam)
+    cut_w = np.int_(W * cut_rat)
+    cut_h = np.int_(H * cut_rat)
+
+    # uniform
+    cx = np.random.randint(W)
+    cy = np.random.randint(H)
+
+    bbx1 = np.clip(cx - cut_w // 2, 0, W)
+    bby1 = np.clip(cy - cut_h // 2, 0, H)
+    bbx2 = np.clip(cx + cut_w // 2, 0, W)
+    bby2 = np.clip(cy + cut_h // 2, 0, H)
+
+    return bbx1, bby1, bbx2, bby2
+
+
+def cutmix_data(input, target, alpha=1.0):
+    if alpha > 0:
+        lam = np.random.beta(alpha, alpha)
+    else:
+        lam = 1
+
+    batch_size = input.size()[0]
+    index = torch.randperm(batch_size)
+    rand_index = torch.randperm(input.size()[0])
+
+    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
+    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
+
+    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
+    targets_a = target
+    targets_b = target[rand_index]
+
+    return input, targets_a, targets_b, lam
+
+
+def cutmix_criterion(criterion, outputs, targets_a, targets_b, lam):
+    return lam * criterion(outputs, targets_a) + (1 - lam) * criterion(
+        outputs, targets_b
+    )
+
+
+# Load pretrained models
+model1 = load_pretrained_model(
+    model_paths[0], SqueezeNet1_0WithSE(num_classes=NUM_CLASSES)
+).to(DEVICE)
+model2 = load_pretrained_model(
+    model_paths[1], EfficientNetB3WithDropout(num_classes=NUM_CLASSES)
+).to(DEVICE)
+model3 = load_pretrained_model(
+    model_paths[2], MobileNetV2WithDropout(num_classes=NUM_CLASSES)
+).to(DEVICE)
+
+models = [model1, model2, model3]
+
+# Create the meta learner
+meta_learner_model = mlp_meta(NUM_CLASSES, len(models)).to(DEVICE)
+meta_optimizer = torch.optim.Adam(meta_learner_model.parameters(), lr=0.001)
+meta_loss_fn = torch.nn.CrossEntropyLoss()
+
+# Define the Cosine Annealing Learning Rate Scheduler
+scheduler = CosineAnnealingLR(
+    meta_optimizer, T_max=700
+)  # T_max is the number of epochs for the cosine annealing.
+
+# Define loss function and optimizer for the meta learner
+criterion = nn.CrossEntropyLoss().to(DEVICE)
+
+# Record learning rate
+lr_hist = []
+
+# Training loop
+num_epochs = 160
+for epoch in range(num_epochs):
+    print("[Epoch: {}]".format(epoch + 1))
+    print("Total number of batches: {}".format(len(train_loader)))
+    for batch_idx, data in enumerate(train_loader, 0):
+        print("Batch: {}".format(batch_idx + 1))
+        inputs, labels = data
+        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
+        inputs, targets_a, targets_b, lam = cutmix_data(inputs, labels, alpha=1)
+
+        # Forward pass through the three pretrained models
+        features1 = model1(inputs)
+        features2 = model2(inputs)
+        features3 = model3(inputs)
+
+        # Stack the features from the three models
+        stacked_features = torch.cat((features1, features2, features3), dim=1).to(
+            DEVICE
+        )
+
+        # Forward pass through the meta learner
+        meta_output = meta_learner_model(stacked_features)
+
+        # Compute the loss
+        loss = cutmix_criterion(criterion, meta_output, targets_a, targets_b, lam)
+
+        # Compute the accuracy
+        _, predicted = torch.max(meta_output, 1)
+        total = labels.size(0)
+        correct = (predicted == labels).sum().item()
+
+        # Backpropagation and optimization
+        meta_optimizer.zero_grad()
+        loss.backward()
+        meta_optimizer.step()
+
+        lr_hist.append(meta_optimizer.param_groups[0]["lr"])
+
+        scheduler.step()
+
+    print("Train Loss: {}".format(loss.item()))
+    print("Train Accuracy: {}%".format(100 * correct / total))
+
+    # Validation
+    meta_learner_model.eval()
+    correct = 0
+    total = 0
+    val_loss = 0
+    with torch.no_grad():
+        for data in val_loader:
+            inputs, labels = data
+            inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
+            features1 = model1(inputs)
+            features2 = model2(inputs)
+            features3 = model3(inputs)
+            stacked_features = torch.cat((features1, features2, features3), dim=1).to(
+                DEVICE
+            )
+            outputs = meta_learner_model(stacked_features)
+            loss = criterion(outputs, labels)  # Use the validation loss
+            val_loss += loss.item()
+            _, predicted = torch.max(outputs, 1)
+            total += labels.size(0)
+            correct += (predicted == labels).sum().item()
+
+    print(
+        "Validation Loss: {}".format(val_loss / len(val_loader))
+    )  # Calculate the average loss
+    print("Validation Accuracy: {}%".format(100 * correct / total))
+
+
+print("Finished Training")
+
+# Test the ensemble
+print("Testing the ensemble")
+meta_learner_model.eval()
+correct = 0
+total = 0
+with torch.no_grad():
+    for data in test_loader:
+        inputs, labels = data
+        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
+        features1 = model1(inputs)
+        features2 = model2(inputs)
+        features3 = model3(inputs)
+        stacked_features = torch.cat((features1, features2, features3), dim=1)
+        outputs = meta_learner_model(stacked_features)
+        _, predicted = torch.max(outputs, 1)
+        total += labels.size(0)
+        correct += (predicted == labels).sum().item()
+
+print(
+    "Accuracy of the ensemble network on the test images: {}%".format(
+        100 * correct / total
+    )
+)
+
+
+# Plot the learning rate history
+
+plt.plot(lr_hist)
+plt.xlabel("Iterations")
+plt.ylabel("Learning Rate")
+plt.title("Learning Rate History")
+plt.show()
+
+
+# Save the model
+torch.save(meta_learner_model.state_dict(), "output/checkpoints/ensemble.pth")

eval.py

@@ -11,88 +11,172 @@ from sklearn.metrics import (
     f1_score,
     confusion_matrix,
     ConfusionMatrixDisplay,
-    roc_curve,
-    auc,
 )
 from sklearn.preprocessing import label_binarize
+from torchvision import transforms
 from configs import *
-from data_loader import load_data  # Import the load_data function
+
+# EfficientNet: 0.901978973407545
+# MobileNet: 0.8731189445475158
+# SquuezeNet:  0.8559218559218559
+
 
 # Constants
 DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+NUM_AUGMENTATIONS = 10  # Number of augmentations to perform
+
+model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
+model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
+
+best_weights = [0.901978973407545, 0.8731189445475158, 0.8559218559218559]
 
 # Load the model
-MODEL = MODEL.to(DEVICE)
-MODEL.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
-MODEL.eval()
+model = WeightedVoteEnsemble([model1, model2, model3], best_weights)
+# model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
+model.load_state_dict(torch.load('output/checkpoints/WeightedVoteEnsemble.pth', map_location=DEVICE))
+model.eval()
+
+# define augmentations for TTA
+tta_transforms = transforms.Compose(
+    [
+        transforms.RandomHorizontalFlip(p=0.5),
+        transforms.RandomVerticalFlip(p=0.5),
+    ]
+)
 
-def predict_image(image_path, model, transform):
-    model.eval()
-    correct_predictions = 0
 
-    # Get a list of image files
-    images = list(pathlib.Path(image_path).rglob("*.png"))
+def perform_tta(model, image, tta_transforms):
+    augmented_predictions = []
+    augmented_scores = []
 
-    total_predictions = len(images)
+    for _ in range(NUM_AUGMENTATIONS):
+        augmented_image = tta_transforms(image)
+        output = model(augmented_image)
+        predicted_class = torch.argmax(output, dim=1).item()
+        augmented_predictions.append(predicted_class)
+        augmented_scores.append(output.softmax(dim=1).cpu().numpy())
 
+    # max voting
+    final_predicted_class_max = max(
+        set(augmented_predictions), key=augmented_predictions.count
+    )
+
+    # average probabilities
+    final_predicted_scores_avg = np.mean(np.array(augmented_scores), axis=0)
+
+    # rotate and average probabilities
+    rotation_transforms = [
+        transforms.RandomRotation(degrees=i) for i in range(0, 360, 30)
+    ]
+    rotated_scores = []
+    for rotation_transform in rotation_transforms:
+        augmented_image = rotation_transform(image)
+        output = model(augmented_image)
+        rotated_scores.append(output.softmax(dim=1).cpu().numpy())
+
+    final_predicted_scores_rotation = np.mean(np.array(rotated_scores), axis=0)
+
+    return (
+        final_predicted_class_max,
+        final_predicted_scores_avg,
+        final_predicted_scores_rotation,
+    )
+
+
+def predict_image_with_tta(image_path, model, transform, tta_transforms):
+    model.eval()
+    correct_predictions = 0
     true_classes = []
-    predicted_labels = []
-    predicted_scores = []  # To store predicted class probabilities
+    predicted_labels_max = []
+    predicted_labels_avg = []
+    predicted_labels_rotation = []
 
     with torch.no_grad():
+        images = list(pathlib.Path(image_path).rglob("*.png"))
+        total_predictions = len(images)
+
         for image_file in images:
-            print("---------------------------")
-            # Check the true label of the image by checking the sequence of the folder in Task 1
             true_class = CLASSES.index(image_file.parts[-2])
-            print("Image path:", image_file)
-            print("True class:", true_class)
-            image = Image.open(image_file).convert("RGB")
-            image = transform(image).unsqueeze(0)
-            image = image.to(DEVICE)
-            output = model(image)
-            predicted_class = torch.argmax(output, dim=1).item()
-            # Print the predicted class
-            print("Predicted class:", predicted_class)
-            # Append true and predicted labels to their respective lists
-            true_classes.append(true_class)
-            predicted_labels.append(predicted_class)
-            predicted_scores.append(
-                output.softmax(dim=1).cpu().numpy()
-            )  # Store predicted class probabilities
 
-            # Check if the prediction is correct
-            if predicted_class == true_class:
-                correct_predictions += 1
+            original_image = Image.open(image_file).convert("RGB")
+            original_image = transform(original_image).unsqueeze(0)
+            original_image = original_image.to(DEVICE)
+
+            # Perform TTA with different strategies
+            final_predicted_class_max, _, _ = perform_tta(
+                model, original_image, tta_transforms
+            )
+            _, final_predicted_scores_avg, _ = perform_tta(
+                model, original_image, tta_transforms
+            )
+            _, _, final_predicted_scores_rotation = perform_tta(
+                model, original_image, tta_transforms
+            )
 
-    # Calculate accuracy and f1 score
-    accuracy = accuracy_score(true_classes, predicted_labels)
-    print("Accuracy:", accuracy)
-    f1 = f1_score(true_classes, predicted_labels, average="weighted")
-    print("Weighted F1 Score:", f1)
+            true_classes.append(true_class)
+            predicted_labels_max.append(final_predicted_class_max)
+            predicted_labels_avg.append(np.argmax(final_predicted_scores_avg))
+            predicted_labels_rotation.append(np.argmax(final_predicted_scores_rotation))
 
-    # Convert the lists to tensors
-    predicted_labels_tensor = torch.tensor(predicted_labels)
-    true_classes_tensor = torch.tensor(true_classes)
+            if final_predicted_class_max == true_class:
+                correct_predictions += 1
 
-    # Calculate the confusion matrix
-    conf_matrix = confusion_matrix(true_classes, predicted_labels)
-
-    # Plot the confusion matrix
-    ConfusionMatrixDisplay(confusion_matrix=conf_matrix, display_labels=CLASSES).plot(cmap=plt.cm.Blues)
-    plt.title("Confusion Matrix")
-    plt.show()
+    # accuracy for each strategy
+    accuracy_max = accuracy_score(true_classes, predicted_labels_max)
+    accuracy_avg = accuracy_score(true_classes, predicted_labels_avg)
+    accuracy_rotation = accuracy_score(true_classes, predicted_labels_rotation)
+
+    print("Accuracy (Max Voting):", accuracy_max)
+    print("Accuracy (Average Probabilities):", accuracy_avg)
+    print("Accuracy (Rotation and Average):", accuracy_rotation)
+
+    # final prediction using ensemble (choose the strategy with the highest accuracy)
+    final_predicted_labels = []
+    for i in range(len(true_classes)):
+        max_strategy_accuracy = max(accuracy_max, accuracy_avg, accuracy_rotation)
+        if accuracy_max == max_strategy_accuracy:
+            final_predicted_labels.append(predicted_labels_max[i])
+        elif accuracy_avg == max_strategy_accuracy:
+            final_predicted_labels.append(predicted_labels_avg[i])
+        else:
+            final_predicted_labels.append(predicted_labels_rotation[i])
+
+    # calculate accuracy and f1 score(ensemble)
+    accuracy_ensemble = accuracy_score(true_classes, final_predicted_labels)
+    f1_ensemble = f1_score(true_classes, final_predicted_labels, average="weighted")
+
+    print("Ensemble Accuracy:", accuracy_ensemble)
+    print("Ensemble Weighted F1 Score:", f1_ensemble)
 
     # Classification report
     class_names = [str(cls) for cls in range(NUM_CLASSES)]
     report = classification_report(
-        true_classes, predicted_labels, target_names=class_names
+        true_classes, final_predicted_labels, target_names=class_names
+    )
+    print("Classification Report of", MODEL.__class__.__name__, ":\n", report)
+    
+    # confusion matrix and classification report for the ensemble
+    conf_matrix_ensemble = confusion_matrix(true_classes, final_predicted_labels)
+    ConfusionMatrixDisplay(
+        confusion_matrix=conf_matrix_ensemble, display_labels=range(NUM_CLASSES)
+    ).plot(cmap=plt.cm.Blues)
+    plt.title("Confusion Matrix (Ensemble)")
+    plt.show()
+
+    class_names = [str(cls) for cls in range(NUM_CLASSES)]
+    report_ensemble = classification_report(
+        true_classes, final_predicted_labels, target_names=class_names
     )
-    print("Classification Report:\n", report)
+    print("Classification Report (Ensemble):\n", report_ensemble)
 
     # Calculate precision and recall for each class
     true_classes_binary = label_binarize(true_classes, classes=range(NUM_CLASSES))
     precision, recall, _ = precision_recall_curve(
-        true_classes_binary.ravel(), np.array(predicted_scores).ravel()
+        true_classes_binary.ravel(), np.array(final_predicted_scores_rotation).ravel()
     )
 
     # Plot precision-recall curve
@@ -103,6 +187,4 @@ def predict_image(image_path, model, transform):
     plt.ylabel("Precision")
     plt.show()
 
-
-# Call predict_image function with your image path
-predict_image("data/test/Task 1/", MODEL, preprocess)
+predict_image_with_tta("data/test/Task 1/", model, preprocess, tta_transforms)

eval_orig.py

@@ -0,0 +1,181 @@
+import os
+import torchvision
+import shap
+import torch
+import numpy as np
+import pathlib
+from PIL import Image
+import matplotlib.pyplot as plt
+from matplotlib import rcParams
+from sklearn.metrics import (
+    classification_report,
+    precision_recall_curve,
+    accuracy_score,
+    f1_score,
+    confusion_matrix,
+    ConfusionMatrixDisplay,
+    roc_curve,
+    auc,
+    average_precision_score,
+)
+from sklearn.preprocessing import label_binarize
+from configs import *
+from data_loader import load_data  # Import the load_data function
+
+# MobileNet: 0.8731189445475158
+# EfficientNet: 0.873118944547516
+# SquuezeNet: 0.8865856365856365
+
+
+rcParams['font.family'] = 'Times New Roman'
+
+# Load the model
+model = MODEL.to(DEVICE)
+model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
+model.eval()
+
+# model2 = EfficientNetB3WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+# model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB3WithDropout.pth"))
+# model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+# model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+# model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+# model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
+
+# model1.eval()
+# model2.eval()
+# model3.eval()
+
+# # Load the model
+# model = WeightedVoteEnsemble([model1, model2, model3], [0.38, 0.34, 0.28])
+# # model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
+# model.load_state_dict(
+#     torch.load("output/checkpoints/WeightedVoteEnsemble.pth", map_location=DEVICE)
+# )
+# model.eval()
+
+
+def predict_image(image_path, model, transform):
+    model.eval()
+    correct_predictions = 0
+
+    # Get a list of image files
+    images = list(pathlib.Path(image_path).rglob("*.png"))
+
+    total_predictions = len(images)
+
+    true_classes = []
+    predicted_labels = []
+    predicted_scores = []  # To store predicted class probabilities
+
+    with torch.no_grad():
+        for image_file in images:
+            print("---------------------------")
+            # Check the true label of the image by checking the sequence of the folder in Task 1
+            true_class = CLASSES.index(image_file.parts[-2])
+            print("Image path:", image_file)
+            print("True class:", true_class)
+            image = Image.open(image_file).convert("RGB")
+            image = transform(image).unsqueeze(0)
+            image = image.to(DEVICE)
+            output = model(image)
+            predicted_class = torch.argmax(output, dim=1).item()
+            # Print the predicted class
+            print("Predicted class:", predicted_class)
+            # Append true and predicted labels to their respective lists
+            true_classes.append(true_class)
+            predicted_labels.append(predicted_class)
+            predicted_scores.append(
+                output.softmax(dim=1).cpu().numpy()
+            )  # Store predicted class probabilities
+
+            # Check if the prediction is correct
+            if predicted_class == true_class:
+                correct_predictions += 1
+
+    # Calculate accuracy and f1 score
+    accuracy = accuracy_score(true_classes, predicted_labels)
+    print("Accuracy:", accuracy)
+    f1 = f1_score(true_classes, predicted_labels, average="weighted")
+    print("Weighted F1 Score:", f1)
+
+    # Convert the lists to tensors
+    predicted_labels_tensor = torch.tensor(predicted_labels)
+    true_classes_tensor = torch.tensor(true_classes)
+
+    # Calculate the confusion matrix
+    conf_matrix = confusion_matrix(true_classes, predicted_labels)
+
+    # Plot the confusion matrix
+    ConfusionMatrixDisplay(
+        confusion_matrix=conf_matrix, display_labels=range(NUM_CLASSES)
+    ).plot(cmap=plt.cm.Blues)
+    plt.title("Confusion Matrix")
+    plt.show()
+
+    # Classification report
+    class_names = [str(cls) for cls in range(NUM_CLASSES)]
+    report = classification_report(
+        true_classes, predicted_labels, target_names=class_names
+    )
+    print("Classification Report:\n", report)
+
+    # Calculate precision and recall for each class
+    true_classes_binary = label_binarize(true_classes, classes=range(NUM_CLASSES))
+    precision, recall, _ = precision_recall_curve(
+        true_classes_binary.ravel(), np.array(predicted_scores).ravel()
+    )
+
+    fpr, tpr, _ = roc_curve(
+        true_classes_binary.ravel(), np.array(predicted_scores).ravel()
+    )
+    auc_roc = auc(fpr, tpr)
+    print("AUC-ROC:", auc_roc)
+
+    # Calculate PRC AUC
+    precision, recall, _ = precision_recall_curve(
+        true_classes_binary.ravel(), np.array(predicted_scores).ravel()
+    )
+    auc_prc = average_precision_score(
+        true_classes_binary.ravel(), np.array(predicted_scores).ravel()
+    )
+    print("AUC PRC:", auc_prc)
+
+    # Plot precision-recall curve
+    plt.figure(figsize=(10, 6))
+    plt.plot(recall, precision)
+    plt.title("Precision-Recall Curve")
+    plt.xlabel("Recall")
+    plt.ylabel("Precision")
+    # Show the AUC value on the plot
+    plt.text(
+        0.6,
+        0.2,
+        "AUC-PRC = {:.3f}".format(auc_prc),
+        bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),
+    )
+    plt.savefig("docs/efficientnet/prc.png")
+    plt.show()
+    
+    # Plot ROC curve
+    plt.figure(figsize=(10, 6))
+    plt.plot(fpr, tpr)
+    plt.title("ROC Curve")
+    plt.xlabel("False Positive Rate")
+    plt.ylabel("True Positive Rate")
+    # Show the AUC value on the plot
+    plt.text(
+        0.6,
+        0.2,
+        "AUC-ROC = {:.3f}".format(auc_roc),
+        bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),
+    )
+    plt.savefig("docs/efficientnet/roc.png")
+    plt.show()
+
+
+predict_image("data/test/Task 1/", model, preprocess)
+
+
+# 89 EfficientNetB2WithDropout / 0.873118944547516
+# 89 MobileNetV2WithDropout / 0.8731189445475158
+# 89 SqueezeNet1_0WithSE / .8865856365856365

extract-ensemble.py

@@ -0,0 +1,110 @@
+from pytorch_grad_cam import GradCAMPlusPlus
+from pytorch_grad_cam.utils.image import show_cam_on_image, preprocess_image
+import cv2
+import numpy as np
+import torch
+import torch.nn as nn  # Replace with your model
+from configs import *
+
+# Load your model (change this according to your model definition)
+model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
+model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
+
+model1.eval()
+model2.eval()
+model3.eval()
+
+
+# Find the target layer (modify this based on your model architecture)
+# EfficientNetB2WithDropout - model.features[-1]
+# SqueezeNet1_0WithSE - model.features
+# MobileNetV2WithDropout - model.features[-1]
+
+target_layer_efficientnet = None
+for child in model2.features[-1]:
+    if isinstance(child, nn.Conv2d):
+        target_layer_efficientnet = child
+
+if target_layer_efficientnet is None:
+    raise ValueError(
+        "Invalid EfficientNet layer name: {}".format(target_layer_efficientnet)
+    )
+
+target_layer_squeezenet = None
+for child in model1.features:
+    if isinstance(child, nn.Conv2d):
+        target_layer_squeezenet = child
+
+if target_layer_squeezenet is None:
+    raise ValueError(
+        "Invalid SqueezeNet layer name: {}".format(target_layer_squeezenet)
+    )
+
+target_layer_mobilenet = None
+for child in model3.features[-1]:
+    if isinstance(child, nn.Conv2d):
+        target_layer_mobilenet = child
+
+if target_layer_mobilenet is None:
+    raise ValueError("Invalid MobileNet layer name: {}".format(target_layer_mobilenet))
+
+# Load and preprocess the image
+image_path = r"data\test\Task 1\Cerebral Palsy\89.png"
+rgb_img = cv2.imread(image_path, 1)
+rgb_img = np.float32(rgb_img) / 255
+input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+input_tensor = input_tensor.to(DEVICE)
+input_tensor.requires_grad = True  # Enable gradients for the input tensor
+
+
+# Create a GradCAMPlusPlus object
+efficientnet_cam = GradCAMPlusPlus(model=model2, target_layers=[target_layer_efficientnet], use_cuda=True)
+squeezenet_cam = GradCAMPlusPlus(model=model1, target_layers=[target_layer_squeezenet], use_cuda=True)
+mobilenet_cam = GradCAMPlusPlus(model=model3, target_layers=[target_layer_mobilenet], use_cuda=True)
+
+
+efficientnet_grayscale_cam = efficientnet_cam(input_tensor=input_tensor)[0]
+squeezenet_grayscale_cam = squeezenet_cam(input_tensor=input_tensor)[0]
+mobilenet_grayscale_cam = mobilenet_cam(input_tensor=input_tensor)[0]
+
+# Apply a colormap to the grayscale heatmap
+efficientnet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * efficientnet_grayscale_cam), cv2.COLORMAP_JET)
+squeezenet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * squeezenet_grayscale_cam), cv2.COLORMAP_JET)
+mobilenet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * mobilenet_grayscale_cam), cv2.COLORMAP_JET)
+
+# normalized_efficientnet_heatmap = efficientnet_heatmap_colored / np.max(efficientnet_heatmap_colored)
+# normalized_squeezenet_heatmap = squeezenet_heatmap_colored / np.max(squeezenet_heatmap_colored)
+# normalized_mobilenet_heatmap = mobilenet_heatmap_colored / np.max(mobilenet_heatmap_colored)
+
+# # Ensure heatmap_colored has the same dtype as rgb_img
+# normalized_efficientnet_heatmap = normalized_efficientnet_heatmap.astype(np.float32) / 255
+# normalized_squeezenet_heatmap = normalized_squeezenet_heatmap.astype(np.float32) / 255
+# normalized_mobilenet_heatmap = normalized_mobilenet_heatmap.astype(np.float32) / 255
+
+efficientnet_heatmap_colored = efficientnet_heatmap_colored.astype(np.float32) / 255
+squeezenet_heatmap_colored = squeezenet_heatmap_colored.astype(np.float32) / 255
+mobilenet_heatmap_colored = mobilenet_heatmap_colored.astype(np.float32) / 255
+
+# Adjust the alpha value to control transparency
+alpha = (
+    0.1  # You can change this value to make the original image more or less transparent
+)
+
+
+# [0.38, 0.34, 0.28]
+weighted_heatmap = (
+    efficientnet_heatmap_colored * 0.38
+    + squeezenet_heatmap_colored * 0.34
+    + mobilenet_heatmap_colored * 0.28
+)
+
+
+# Overlay the colored heatmap on the original image
+final_output = cv2.addWeighted(rgb_img, 0.3, weighted_heatmap, 0.7, 0)
+
+# Save the final output
+cv2.imwrite("cam.jpg", (final_output * 255).astype(np.uint8))

extract.py

@@ -0,0 +1,50 @@
+from pytorch_grad_cam import GradCAMPlusPlus
+from pytorch_grad_cam.utils.image import show_cam_on_image, preprocess_image
+import cv2
+import numpy as np
+import torch
+import torch.nn as nn  # Replace with your model
+from configs import *
+
+# Load your model (replace with your model class)
+model = MODEL  # Replace with your model
+model.load_state_dict(torch.load(MODEL_SAVE_PATH))
+model.eval()
+model = model.to(DEVICE)
+
+# Find the target layer (modify this based on your model architecture)
+target_layer = None
+for child in model.features[-1]:
+    if isinstance(child, nn.Conv2d):
+        target_layer = child
+
+if target_layer is None:
+    raise ValueError("Invalid layer name: {}".format(target_layer))
+
+# Load and preprocess the image
+image_path = r'data\test\Task 1\Parkinson Disease\V14PE02.png'
+rgb_img = cv2.imread(image_path, 1)
+rgb_img = np.float32(rgb_img) / 255
+input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+input_tensor = input_tensor.to(DEVICE)
+
+# Create a GradCAMPlusPlus object
+cam = GradCAMPlusPlus(model=model, target_layers=[target_layer], use_cuda=True)
+
+# Generate the GradCAM heatmap
+grayscale_cam = cam(input_tensor=input_tensor)[0]
+
+# Apply a colormap to the grayscale heatmap
+heatmap_colored = cv2.applyColorMap(np.uint8(255 * grayscale_cam), cv2.COLORMAP_JET)
+
+# Ensure heatmap_colored has the same dtype as rgb_img
+heatmap_colored = heatmap_colored.astype(np.float32) / 255
+
+# Adjust the alpha value to control transparency
+alpha = 0.3  # You can change this value to make the original image more or less transparent
+
+# Overlay the colored heatmap on the original image
+final_output = cv2.addWeighted(rgb_img, 0.3, heatmap_colored, 0.7, 0)
+
+# Save the final output
+cv2.imwrite('cam.jpg', (final_output * 255).astype(np.uint8))

genetric_algorithm.py

@@ -0,0 +1,248 @@
+import os
+import optuna
+from optuna.trial import TrialState
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from configs import *
+import data_loader
+from torch.utils.tensorboard import SummaryWriter
+import numpy as np
+import pygad
+import pygad.torchga
+
+torch.cuda.empty_cache()
+model = MODEL.to(DEVICE)
+
+EPOCHS = 10
+N_TRIALS = 20
+TIMEOUT = 1800
+EARLY_STOPPING_PATIENCE = (
+    4  # Number of epochs with no improvement to trigger early stopping
+)
+NUM_GENERATIONS = 10
+SOL_PER_POP = 10  # Number of solutions in the population
+NUM_GENES = 2
+NUM_PARENTS_MATING = 4
+
+# Create a TensorBoard writer
+writer = SummaryWriter(log_dir="output/tensorboard/tuning")
+
+
+# Function to create or modify data loaders with the specified batch size
+def create_data_loaders(batch_size):
+    train_loader, valid_loader = data_loader.load_data(
+        COMBINED_DATA_DIR + "1",
+        preprocess,
+        batch_size=batch_size,
+    )
+    return train_loader, valid_loader
+
+
+# Objective function for optimization
+def objective(trial):
+    global data_inputs, data_outputs
+
+    batch_size = trial.suggest_categorical("batch_size", [16, 32, 64])
+    train_loader, valid_loader = create_data_loaders(batch_size)
+
+    lr = trial.suggest_float("lr", 1e-5, 1e-3, log=True)
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+
+    gamma = trial.suggest_float("gamma", 0.1, 0.9, step=0.1)
+    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
+
+    past_trials = 0  # Number of trials already completed
+
+    # Print best hyperparameters:
+    if past_trials > 0:
+        print("\nBest Hyperparameters:")
+        print(f"{study.best_trial.params}")
+
+    print(f"\n[INFO] Trial: {trial.number}")
+    print(f"Batch Size: {batch_size}")
+    print(f"Learning Rate: {lr}")
+    print(f"Gamma: {gamma}\n")
+
+    early_stopping_counter = 0
+    best_accuracy = 0.0
+
+    for epoch in range(EPOCHS):
+        model.train()
+        for batch_idx, (data, target) in enumerate(train_loader, 0):
+            data, target = data.to(DEVICE), target.to(DEVICE)
+            optimizer.zero_grad()
+            output = model(data)
+            loss = criterion(output, target)
+            loss.backward()
+            optimizer.step()
+
+        scheduler.step()
+
+        model.eval()
+        correct = 0
+        with torch.no_grad():
+            for batch_idx, (data, target) in enumerate(valid_loader, 0):
+                data, target = data.to(DEVICE), target.to(DEVICE)
+                output = model(data)
+                pred = output.argmax(dim=1, keepdim=True)
+                correct += pred.eq(target.view_as(pred)).sum().item()
+
+        accuracy = correct / len(valid_loader.dataset)
+
+        # Log hyperparameters and accuracy to TensorBoard
+        writer.add_scalar("Accuracy", accuracy, trial.number)
+        writer.add_hparams(
+            {"batch_size": batch_size, "lr": lr, "gamma": gamma},
+            {"accuracy": accuracy},
+        )
+
+        print(f"[EPOCH {epoch + 1}] Accuracy: {accuracy:.4f}")
+
+        trial.report(accuracy, epoch)
+
+        if accuracy > best_accuracy:
+            best_accuracy = accuracy
+            early_stopping_counter = 0
+        else:
+            early_stopping_counter += 1
+
+        # Early stopping check
+        if early_stopping_counter >= EARLY_STOPPING_PATIENCE:
+            print(f"\nEarly stopping at epoch {epoch + 1}")
+            break
+
+    if trial.number > 10 and trial.params["lr"] < 1e-3 and best_accuracy < 0.7:
+        return float("inf")
+
+    past_trials += 1
+
+    return best_accuracy
+
+
+# Custom genetic algorithm
+def run_genetic_algorithm(fitness_func):
+    # Initial population
+    population = np.random.rand(SOL_PER_POP, NUM_GENES)  # Random initialization
+
+    # Run for a fixed number of generations
+    for generation in range(NUM_GENERATIONS):
+        # Calculate fitness for each solution in the population
+        fitness = np.array(
+            [fitness_func(solution, idx) for idx, solution in enumerate(population)]
+        )
+
+        # Get the index of the best solution
+        best_idx = np.argmax(fitness)
+        best_solution = population[best_idx]
+        best_fitness = fitness[best_idx]
+
+        # Print the best solution and fitness for this generation
+        print(f"Generation {generation + 1}:")
+        print("Best Solution:")
+        print("Learning Rate = {lr}".format(lr=best_solution[0]))
+        print("Gamma = {gamma}".format(gamma=best_solution[1]))
+        print("Best Fitness = {fitness}".format(fitness=best_fitness))
+
+        # Perform selection and crossover to create the next generation
+        population = selection_and_crossover(population, fitness)
+
+
+# Selection and crossover logic
+def selection_and_crossover(population, fitness):
+    # Perform tournament selection
+    parents = []
+    for _ in range(SOL_PER_POP):
+        tournament_idxs = np.random.choice(range(SOL_PER_POP), NUM_PARENTS_MATING)
+        tournament_fitness = [fitness[idx] for idx in tournament_idxs]
+        selected_parent_idx = tournament_idxs[np.argmax(tournament_fitness)]
+        parents.append(population[selected_parent_idx])
+
+    # Perform single-point crossover
+    offspring = []
+    for i in range(0, SOL_PER_POP, 2):
+        if i + 1 < SOL_PER_POP:
+            crossover_point = np.random.randint(0, NUM_GENES)
+            offspring.extend(
+                [
+                    np.concatenate(
+                        (parents[i][:crossover_point], parents[i + 1][crossover_point:])
+                    )
+                ]
+            )
+            offspring.extend(
+                [
+                    np.concatenate(
+                        (parents[i + 1][:crossover_point], parents[i][crossover_point:])
+                    )
+                ]
+            )
+
+    return np.array(offspring)
+
+
+# Modify callback function to log best accuracy
+def callback_generation(ga_instance):
+    global study
+
+    # Fetch the parameters of the best solution
+    solution, solution_fitness, _ = ga_instance.best_solution()
+    best_learning_rate, best_gamma = solution
+
+    # Report the best accuracy to Optuna study
+    study.set_user_attr("best_accuracy", solution_fitness)
+
+    # Print generation number and best fitness
+    print(
+        "Generation = {generation}".format(generation=ga_instance.generations_completed)
+    )
+    print("Best Fitness = {fitness}".format(fitness=solution_fitness))
+    print("Best Learning Rate = {lr}".format(lr=best_learning_rate))
+    print("Best Gamma = {gamma}".format(gamma=best_gamma))
+
+
+if __name__ == "__main__":
+    global study
+    pruner = optuna.pruners.HyperbandPruner()
+    study = optuna.create_study(
+        direction="maximize",
+        pruner=pruner,
+        study_name="hyperparameter_tuning",
+    )
+
+    # Define data_inputs and data_outputs
+    # You need to populate these with your own data
+
+    # Define the loss function
+    loss_function = nn.CrossEntropyLoss()
+
+    def fitness_func(solution, sol_idx):
+        global data_inputs, data_outputs, model, loss_function
+
+        learning_rate, momentum = solution
+
+        # Update optimizer with the current learning rate and momentum
+        optimizer = torch.optim.SGD(
+            model.parameters(), lr=learning_rate, momentum=momentum
+        )
+
+        # Load the model weights
+        model_weights_dict = pygad.torchga.model_weights_as_dict(
+            model=model, weights_vector=solution
+        )
+        model.load_state_dict(model_weights_dict)
+
+        # Forward pass
+        predictions = model(data_inputs)
+
+        # Calculate cross-entropy loss
+        loss = loss_function(predictions, data_outputs)
+
+        # Higher fitness for lower loss
+        solution_fitness = 1.0 / (loss.detach().numpy() + 1e-8)
+
+        return solution_fitness
+
+    # Run the custom genetic algorithm
+    run_genetic_algorithm(fitness_func)

lazy_predict.py

@@ -0,0 +1,60 @@
+import os
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import matplotlib.pyplot as plt
+from models import *
+from torch.utils.tensorboard import SummaryWriter
+from configs import *
+import data_loader
+import numpy as np
+from lazypredict.Supervised import LazyClassifier
+from sklearn.utils import shuffle
+
+def extract_features_labels(loader):
+    data = []
+    labels = []
+    for inputs, labels_batch in loader:
+        for img in inputs:
+            data.append(img.view(-1).numpy())
+        labels.extend(labels_batch.numpy())
+    return np.array(data), np.array(labels)
+
+def load_and_preprocess_data():
+    train_loader, valid_loader = data_loader.load_data(
+        RAW_DATA_DIR + str(TASK),
+        AUG_DATA_DIR + str(TASK),
+        EXTERNAL_DATA_DIR + str(TASK),
+        preprocess,
+    )
+    return train_loader, valid_loader
+
+def initialize_model_optimizer_scheduler(train_loader, valid_loader):
+    model = MODEL.to(DEVICE)
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
+    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)
+    return model, criterion, optimizer, scheduler
+
+# Load and preprocess data
+train_loader, valid_loader = load_and_preprocess_data()
+
+# Initialize the model, criterion, optimizer, and scheduler
+model, criterion, optimizer, scheduler = initialize_model_optimizer_scheduler(train_loader, valid_loader)
+
+# Extract features and labels
+X_train, y_train = extract_features_labels(train_loader)
+X_valid, y_valid = extract_features_labels(valid_loader)
+
+# LazyClassifier
+clf = LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
+models, predictions = clf.fit(X_train, X_valid, y_train, y_valid)
+
+print("Models:", models)
+print("Predictions:", predictions)
+
+
+
+
+
+

models.py

@@ -39,3 +39,31 @@ from torchvision.models import efficientnet_v2_m
 from torchvision.models import efficientnet_v2_l
 from torchvision.models import efficientnet_b0
 from torchvision.models import efficientnet_b1
+import torch
+import torch.nn as nn
+
+class WeightedVoteEnsemble(nn.Module):
+    def __init__(self, models, weights):
+        super(WeightedVoteEnsemble, self).__init__()
+        self.models = models
+        self.weights = weights
+
+    def forward(self, x):
+        predictions = [model(x) for model in self.models]
+        weighted_predictions = torch.stack(
+            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
+        )
+        avg_predictions = weighted_predictions.sum(dim=0)
+        return avg_predictions
+
+
+def ensemble_predictions(models, image):
+    all_predictions = []
+
+    with torch.no_grad():
+        for model in models:
+            output = model(image)
+            all_predictions.append(output)
+
+    return torch.stack(all_predictions, dim=0).mean(dim=0)
+

plot-gradcam.py

@@ -0,0 +1,30 @@
+# Plot the gradcam pics of 7 classes from C:\Users\User\Documents\PISTEK\HANDETECT\docs\efficientnet\gradcam folder
+# Each picture is named as <class_name>.jpg
+# Usage: python plot-gradcam.py
+
+import os
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib import rcParams
+
+rcParams['font.family'] = 'Times New Roman'
+
+# Load the gradcam pics
+gradcam_dir = r'C:\Users\User\Documents\PISTEK\HANDETECT\docs\efficientnet\gradcam'
+gradcam_pics = []
+for pic in os.listdir(gradcam_dir):
+    gradcam_pics.append(cv2.imread(os.path.join(gradcam_dir, pic), 1))
+    
+# Plot the gradcam pics
+plt.figure(figsize=(20, 20))
+# Very tight layout
+plt.tight_layout(pad=0.1)
+for i, pic in enumerate(gradcam_pics):
+    plt.subplot(3, 3, i + 1)
+    plt.imshow(pic)
+    plt.axis('off')
+    plt.title(os.listdir(gradcam_dir)[i].split('.')[0], fontsize=13)
+plt.savefig('docs/efficientnet/gradcam.jpg')
+plt.show()
+

predict.py

@@ -10,16 +10,29 @@ from configs import *
 
 
 # Load your model (change this according to your model definition)
-MODEL.load_state_dict(
-    torch.load(MODEL_SAVE_PATH, map_location=DEVICE)
-)  # Load the model on the same device
-MODEL.eval()
-MODEL = MODEL.to(DEVICE)
-MODEL.eval()
+model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
+model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
+
+model1.eval()
+model2.eval()
+model3.eval()
+
+# Load the model
+model = MODEL.to(DEVICE)
+# model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
+model.load_state_dict(
+    torch.load("output/checkpoints/EfficientNetB3WithDropout.pth", map_location=DEVICE)
+)
+model.eval()
+
 torch.set_grad_enabled(False)
 
 
-def predict_image(image_path, model=MODEL, transform=preprocess):
+def predict_image(image_path, model=model, transform=preprocess):
     classes = CLASSES
 
     print("---------------------------")

shap_eval.py

@@ -0,0 +1,49 @@
+import numpy as np
+from lime.lime_image import LimeImageExplainer
+from PIL import Image
+import torch
+import torchvision.transforms as transforms
+import matplotlib.pyplot as plt
+from configs import *
+
+
+model = MODEL.to(DEVICE)
+model.load_state_dict(torch.load(MODEL_SAVE_PATH))
+model.eval()
+
+# Load the image
+image = Image.open(
+    r"data\test\Task 1\Healthy\0a7259b2-e650-43aa-93a0-e8b1063476fc.png"
+).convert("RGB")
+image = preprocess(image)
+image = image.unsqueeze(0)  # Add batch dimension
+image = image.to(DEVICE)
+
+
+# Define a function to predict with the model
+def predict(input_image):
+    input_image = torch.tensor(input_image, dtype=torch.float32)
+    if input_image.dim() == 4:
+        input_image = input_image.permute(0, 3, 1, 2)  # Permute the dimensions
+    input_image = input_image.to(DEVICE)  # Move to the appropriate device
+    with torch.no_grad():
+        output = model(input_image)
+    return output
+
+
+# Create the LIME explainer
+explainer = LimeImageExplainer()
+
+# Explain the model's predictions for the image
+explanation = explainer.explain_instance(
+    image[0].permute(1, 2, 0).numpy(), predict, top_labels=5, num_samples=2000
+)
+
+# Get the image and mask for the explanation
+image, mask = explanation.get_image_and_mask(
+    explanation.top_labels[0], positive_only=False, num_features=5, hide_rest=False
+)
+
+# Display the explanation
+plt.imshow(image)
+plt.show()

test.py

@@ -1,39 +1,223 @@
+import sys
 import torch
-import torch.optim as optim
-import torchvision.models as models
-import torchvision.transforms as transforms
+import torch.nn as nn
+from PIL import Image
+import os
 from configs import *
+from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
+import matplotlib.pyplot as plt
+import random
+from itertools import product
 
-# Load a pre-trained model (e.g., VGG16)
-MODEL = MODEL.to(DEVICE)
-MODEL.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
-MODEL.eval()
+random.seed(RANDOM_SEED)
+torch.cuda.manual_seed(RANDOM_SEED)
+torch.manual_seed(RANDOM_SEED)
+print("PyTorch Seed:", torch.initial_seed())
+print("Random Seed:", random.getstate()[1][0])
+print("PyTorch CUDA Seed:", torch.cuda.initial_seed())
 
-# Prepare an initial image (e.g., a random noise image)
-image = torch.randn(1, 3, 224, 224, requires_grad=True).cuda()
+# Define your model paths
+# Load your pre-trained models
+model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
+model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
 
-# Define a loss function to maximize a specific layer or neuron's activation
-loss_function = torch.nn.CrossEntropyLoss()
+# Define the class labels
+class_labels = CLASSES
 
-# Create an optimizer (e.g., Adam) for updating the image
-optimizer = optim.Adam([image], lr=0.01)
+# Define your test data folder path
+test_data_folder = "data/test/Task 1/"
 
-# Optimization loop
-for _ in range(1000):
-    optimizer.zero_grad()
-    output = MODEL(image)
-    loss = -output[0, 'Healthy']  # Maximize the activation of a specific class
-    loss.backward()
-    optimizer.step()
 
-# Visualize the optimized image
-import matplotlib.pyplot as plt
-import torchvision.transforms as transforms
+# Put models in evaluation mode
+def set_models_eval(models):
+    for model in models:
+        model.eval()
+
+
+# Define the ensemble model using a list of models
+class WeightedVoteEnsemble(nn.Module):
+    def __init__(self, models, weights):
+        super(WeightedVoteEnsemble, self).__init__()
+        self.models = models
+        self.weights = weights
+
+    def forward(self, x):
+        predictions = [model(x) for model in self.models]
+        weighted_predictions = torch.stack(
+            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
+        )
+        avg_predictions = weighted_predictions.sum(dim=0)
+        return avg_predictions
+
+
+def ensemble_predictions(models, image):
+    all_predictions = []
+
+    with torch.no_grad():
+        for model in models:
+            output = model(image)
+            all_predictions.append(output)
+
+    return torch.stack(all_predictions, dim=0).mean(dim=0)
+
+
+# Load a single image and make predictions
+def evaluate_image(models, image_path, transform=preprocess):
+    image = Image.open(image_path).convert("RGB")
+    image = transform(image).unsqueeze(0)
+    image = image.to(DEVICE)
+    outputs = ensemble_predictions(models, image)
+
+    return outputs.argmax(dim=1).item()
+
+
+# Evaluate and plot a confusion matrix for an ensemble of models
+def evaluate_and_plot_confusion_matrix(models, test_data_folder):
+    all_predictions = []
+    true_labels = []
+
+    with torch.no_grad():
+        for class_label in class_labels:
+            class_path = os.path.join(test_data_folder, class_label)
+            for image_file in os.listdir(class_path):
+                image_path = os.path.join(class_path, image_file)
+                # print(image_path)
+                predicted_label = evaluate_image(models, image_path, preprocess)
+                all_predictions.append(predicted_label)
+                true_labels.append(class_labels.index(class_label))
+
+    # Print accuracy
+    accuracy = (
+        (torch.tensor(all_predictions) == torch.tensor(true_labels)).float().mean()
+    )
+    print("Accuracy:", accuracy)
+
+    # Create the confusion matrix
+    cm = confusion_matrix(true_labels, all_predictions)
+
+    # Plot the confusion matrix
+    display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels)
+    display.plot(cmap=plt.cm.Blues, values_format="d")
+
+    # Show the plot
+    plt.show()
+
+    return accuracy
+
+# Set the models to evaluation mode
+set_models_eval([model1, model2, model3])
+
+# Define different weight configurations
+# [SqueezeNet, EfficientNetB2WithDropout, MobileNetV2WithDropout]
+weights_configurations = [
+    # Random set of weights using random.random() and all weights sum to 1
+    [
+        random.randrange(1, 10) / 10,
+        random.randrange(1, 10) / 10,
+        random.randrange(1, 10) / 10,
+    ],
+]
+
+
+## NOTE OF PREVIOUS WEIGHTS
+# Best weights: [0.2, 0.3, 0.5] with accuracy: 0.9428571462631226 at iteration: 15 with torch seed: 28434738589300 and random seed: 3188652458777471118 and torch cuda seed: None
+
+
+best_weights = {
+    "weights": 0,
+    "accuracy": 0,
+    "iteration": 0,
+    "torch_seed": 0,
+    "random_seed": 0,
+    "torch_cuda_seed": 0,
+}
+
+i = 0
+
+# weights_hist = []
+
+target_sum = 1.0
+number_of_numbers = 3
+lower_limit = 0.20
+upper_limit = 0.9
+step = 0.1
+
+valid_combinations = []
+
+# Generate all unique combinations of three numbers with values to two decimal places
+range_values = list(range(int(lower_limit * 100), int(upper_limit * 100) + 1))
+for combo in product(range_values, repeat=number_of_numbers):
+    combo_float = [x / 100.0 for x in combo]
+
+    # Check if the sum of the numbers is equal to 1
+    if sum(combo_float) == target_sum:
+        valid_combinations.append(combo_float)
+
+# Calculate the total number of possibilities
+total_possibilities = len(valid_combinations)
+
+print("Total number of possibilities:", total_possibilities)
+
+valid_combinations = [[0.37, 0.34, 0.29]]
+
+for weights in valid_combinations:
+    # while True:
+    print("---------------------------")
+    print("Iteration:", i)
+    # Should iterate until all possible weights are exhausted
+    # Create an ensemble model with weighted voting
+
+    random.seed(RANDOM_SEED)
+    torch.cuda.manual_seed(RANDOM_SEED)
+    torch.manual_seed(RANDOM_SEED)
+    # print("PyTorch Seed:", torch.initial_seed())
+    # weights_hist.append(weights)
+    weighted_vote_ensemble_model = WeightedVoteEnsemble(
+        # [model1, model2, model3], weights
+        [model1, model2, model3],
+        weights,
+    )
+    # print("Weights:", weights)
+    print("Weights:", weights)
+    # Call the evaluate_and_plot_confusion_matrix function with your models and test data folder
+    accuracy = evaluate_and_plot_confusion_matrix(
+        [weighted_vote_ensemble_model], test_data_folder
+    )
+    # Convert tensor to float
+    accuracy = accuracy.item()
+    if accuracy > best_weights["accuracy"]:
+        # best_weights["weights"] = weights
+        best_weights["weights"] = weights
+        best_weights["accuracy"] = accuracy
+        best_weights["iteration"] = i
+        best_weights["torch_seed"] = torch.initial_seed()
+        seed = random.randrange(sys.maxsize)
+        rng = random.Random(seed)
+        best_weights["random_seed"] = seed
+        best_weights["torch_cuda_seed"] = torch.cuda.initial_seed()
+
+    print(
+        "Best weights:",
+        best_weights["weights"],
+        "with accuracy:",
+        best_weights["accuracy"],
+        "at iteration:",
+        best_weights["iteration"],
+        "with torch seed:",
+        best_weights["torch_seed"],
+        "and random seed:",
+        best_weights["random_seed"],
+        "and torch cuda seed:",
+        best_weights["torch_cuda_seed"],
+    )
+    i += 1
 
-# Convert the tensor to an image
-image = transforms.ToPILImage()(image.squeeze())
 
-# Display the generated image
-plt.imshow(image)
-plt.axis('off')
-plt.show()
+torch.save(
+    weighted_vote_ensemble_model.state_dict(),
+    "output/checkpoints/WeightedVoteEnsemble.pth",
+)

testing.py

@@ -0,0 +1,5 @@
+import torch
+
+print("Torch version:",torch.__version__)
+
+print("Is CUDA enabled?",torch.cuda.is_available())

train-svm.py

@@ -0,0 +1,101 @@
+import os
+import numpy as np
+from sklearn import svm
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
+from skimage.io import imread
+from skimage.transform import resize
+from sklearn.model_selection import train_test_split, RandomizedSearchCV
+from scipy.stats import uniform
+from configs import *
+
+# Set the path to your dataset folder, where each subfolder represents a class
+dataset_path = COMBINED_DATA_DIR + str(1)
+
+
+# Function to load, resize, and convert images to grayscale
+def load_resize_and_convert_to_gray(folder, target_size=(100, 100)):
+    images = []
+    for filename in os.listdir(folder):
+        img_path = os.path.join(folder, filename)
+        if os.path.isfile(img_path):
+            img = imread(img_path, as_gray=True)
+            img = resize(img, target_size, anti_aliasing=True)
+            images.append(img)
+    return images
+
+
+# Load, resize, and convert images to grayscale from folders
+X = []  # List to store images
+y = []  # List to store corresponding labels
+
+class_folders = os.listdir(dataset_path)
+class_folders.sort()  # Sort the class folders to ensure consistent class ordering
+
+for class_folder in class_folders:
+    class_path = os.path.join(dataset_path, class_folder)
+    if os.path.isdir(class_path):
+        images = load_resize_and_convert_to_gray(class_path)
+        X.extend(images)
+        y.extend([class_folder] * len(images))  # Assign labels based on folder name
+
+# Convert data to NumPy arrays
+X = np.array(X)
+y = np.array(y)
+
+# Split the dataset into training and testing sets
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.2, random_state=42
+)
+
+# Define the parameter distributions for random search
+param_dist = {
+    "C": uniform(loc=0, scale=10),  # Randomly sample from [0, 10]
+    "kernel": ["linear", "rbf", "poly"],
+    "gamma": uniform(loc=0.001, scale=0.1),  # Randomly sample from [0.001, 0.1]
+}
+
+# Flatten the images to a 1D array
+X_train_flat = X_train.reshape(X_train.shape[0], -1)
+X_test_flat = X_test.reshape(X_test.shape[0], -1)
+
+# Create an SVM classifier
+svm_classifier = svm.SVC()
+
+# Perform Randomized Search with cross-validation
+random_search = RandomizedSearchCV(
+    svm_classifier,
+    param_distributions=param_dist,
+    n_iter=50,
+    cv=5,
+    n_jobs=-1,
+    verbose=2,
+    random_state=42,
+)
+
+# Fit the Randomized Search on the training data
+random_search.fit(X_train_flat, y_train)
+
+# Print the best hyperparameters
+print("Best Hyperparameters:")
+print(random_search.best_params_)
+
+# Get the best SVM model with the tuned hyperparameters
+best_svm_model = random_search.best_estimator_
+
+# Evaluate the best model on the test set
+y_pred = best_svm_model.predict(X_test_flat)
+
+# Calculate accuracy and other metrics
+accuracy = accuracy_score(y_test, y_pred)
+print("Accuracy:", accuracy)
+
+# Confusion Matrix
+conf_matrix = confusion_matrix(y_test, y_pred)
+print("Confusion Matrix:\n", conf_matrix)
+
+# You can also print other classification metrics like precision, recall, and F1-score
+from sklearn.metrics import classification_report
+
+report = classification_report(y_test, y_pred)
+print("Classification Report:\n", report)

train.py

@@ -3,53 +3,75 @@ import torch
 import torch.nn as nn
 import torch.optim as optim
 import matplotlib.pyplot as plt
+from matplotlib import rcParams
 from models import *
 from torch.utils.tensorboard import SummaryWriter
 from configs import *
 import data_loader
 import torch.nn.functional as F
+import csv
 import numpy as np
+from torchcontrib.optim import SWA
 
 
-class LabelSmoothingLoss(nn.Module):
-    def __init__(self, epsilon=0.1, num_classes=2):
-        super(LabelSmoothingLoss, self).__init__()
-        self.epsilon = epsilon
-        self.num_classes = num_classes
+rcParams["font.family"] = "Times New Roman"
 
-    def forward(self, input, target):
-        target_smooth = (1 - self.epsilon) * target + self.epsilon / self.num_classes
-        return nn.CrossEntropyLoss()(input, target_smooth)
+SWA_START = 5  # Starting epoch for SWA
+SWA_FREQ = 5  # Frequency of updating SWA weights
 
 
-def setup_tensorboard():
-    return SummaryWriter(log_dir="output/tensorboard/training")
+def rand_bbox(size, lam):
+    W = size[2]
+    H = size[3]
+    cut_rat = np.sqrt(1.0 - lam)
+    cut_w = np.int_(W * cut_rat)
+    cut_h = np.int_(H * cut_rat)
+
+    # uniform
+    cx = np.random.randint(W)
+    cy = np.random.randint(H)
 
+    bbx1 = np.clip(cx - cut_w // 2, 0, W)
+    bby1 = np.clip(cy - cut_h // 2, 0, H)
+    bbx2 = np.clip(cx + cut_w // 2, 0, W)
+    bby2 = np.clip(cy + cut_h // 2, 0, H)
 
-def mixup_data(x, y, alpha=1.0):
-    """Returns mixed inputs, pairs of targets, and lambda"""
+    return bbx1, bby1, bbx2, bby2
+
+
+def cutmix_data(input, target, alpha=1.0):
     if alpha > 0:
         lam = np.random.beta(alpha, alpha)
     else:
         lam = 1
 
-    batch_size = x.size()[0]
+    batch_size = input.size()[0]
     index = torch.randperm(batch_size)
+    rand_index = torch.randperm(input.size()[0])
+
+    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
+    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
 
-    mixed_x = lam * x + (1 - lam) * x[index, :]
-    y_a, y_b = y, y[index]
-    return mixed_x, y_a, y_b, lam
+    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
+    targets_a = target
+    targets_b = target[rand_index]
 
+    return input, targets_a, targets_b, lam
 
-def mixup_criterion(criterion, pred, y_a, y_b, lam):
-    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)
+
+def cutmix_criterion(criterion, outputs, targets_a, targets_b, lam):
+    return lam * criterion(outputs, targets_a) + (1 - lam) * criterion(
+        outputs, targets_b
+    )
+
+
+def setup_tensorboard():
+    return SummaryWriter(log_dir="output/tensorboard/training")
 
 
 def load_and_preprocess_data():
     return data_loader.load_data(
-        RAW_DATA_DIR + str(TASK),
-        AUG_DATA_DIR + str(TASK),
-        EXTERNAL_DATA_DIR + str(TASK),
+        COMBINED_DATA_DIR + "1",
         preprocess,
     )
 
@@ -77,13 +99,13 @@ def train_one_epoch(model, criterion, optimizer, train_loader, epoch, alpha):
         inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
         optimizer.zero_grad()
 
-        # Apply mixup
-        inputs, targets_a, targets_b, lam = mixup_data(inputs, labels, alpha)
+        # Apply CutMix
+        inputs, targets_a, targets_b, lam = cutmix_data(inputs, labels, alpha=1)
 
         outputs = model(inputs)
 
-        # Calculate mixup loss
-        loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)
+        # Calculate CutMix loss
+        loss = cutmix_criterion(criterion, outputs, targets_a, targets_b, lam)
 
         loss.backward()
         optimizer.step()
@@ -91,8 +113,8 @@ def train_one_epoch(model, criterion, optimizer, train_loader, epoch, alpha):
 
         if (i + 1) % NUM_PRINT == 0:
             print(
-                "[Epoch %d, Batch %d] Loss: %.6f"
-                % (epoch + 1, i + 1, running_loss / NUM_PRINT)
+                f"[Epoch {epoch + 1}, Batch {i + 1}/{len(train_loader)}] "
+                f"Loss: {running_loss / NUM_PRINT:.6f}"
             )
             running_loss = 0.0
 
@@ -132,18 +154,22 @@ def main_training_loop():
     best_val_loss = float("inf")
     best_val_accuracy = 0.0
     no_improvement_count = 0
+    epoch_metrics = []
 
     AVG_TRAIN_LOSS_HIST = []
     AVG_VAL_LOSS_HIST = []
     TRAIN_ACC_HIST = []
     VAL_ACC_HIST = []
 
+    # Initialize SWA optimizer
+    swa_optimizer = SWA(optimizer, swa_start=SWA_START, swa_freq=SWA_FREQ)
+
     for epoch in range(NUM_EPOCHS):
-        print(f"[Epoch: {epoch + 1}]")
+        print(f"\n[Epoch: {epoch + 1}/{NUM_EPOCHS}]")
         print("Learning rate:", scheduler.get_last_lr()[0])
 
         avg_train_loss, train_accuracy = train_one_epoch(
-            model, criterion, optimizer, train_loader, epoch, MIXUP_ALPHA
+            model, criterion, optimizer, train_loader, epoch, CUTMIX_ALPHA
         )
         AVG_TRAIN_LOSS_HIST.append(avg_train_loss)
         TRAIN_ACC_HIST.append(train_accuracy)
@@ -154,9 +180,27 @@ def main_training_loop():
             "Accuracy": train_accuracy,
         }
         plot_and_log_metrics(train_metrics, epoch, writer=writer, prefix="Train")
+        epoch_metrics.append(
+            {
+                "Epoch": epoch + 1,
+                "Train Loss": avg_train_loss,
+                "Train Accuracy": train_accuracy,
+                "Validation Loss": avg_val_loss,
+                "Validation Accuracy": val_accuracy,
+                "Learning Rate": scheduler.get_last_lr()[0],
+            }
+        )
 
         # Learning rate scheduling
-        scheduler.step()
+
+        if epoch < WARMUP_EPOCHS:
+            # Linear warm-up phase
+            lr = LEARNING_RATE * (epoch + 1) / WARMUP_EPOCHS
+            for param_group in optimizer.param_groups:
+                param_group["lr"] = lr
+        else:
+            # Cosine annealing scheduler after warm-up
+            scheduler.step()
 
         avg_val_loss, val_accuracy = validate_model(model, criterion, valid_loader)
         AVG_VAL_LOSS_HIST.append(avg_val_loss)
@@ -167,7 +211,7 @@ def main_training_loop():
             "Loss": avg_val_loss,
             "Accuracy": val_accuracy,
         }
-        plot_and_log_metrics(train_metrics, epoch, writer=writer, prefix="Train")
+        plot_and_log_metrics(val_metrics, epoch, writer=writer, prefix="Validation")
 
         # Print average training and validation metrics
         print(f"Average Training Loss: {avg_train_loss:.6f}")
@@ -190,11 +234,37 @@ def main_training_loop():
                 )
             )
             break
-    MODEL_SAVE_PATH = "output/checkpoints/model.pth"
+
+        # Update SWA weights
+        if epoch >= SWA_START and epoch % SWA_FREQ == 0:
+            swa_optimizer.update_swa()
+
+    # Apply SWA to the final model weights
+    swa_optimizer.swap_swa_sgd()
+    csv_filename = "training_metrics.csv"
+
+    with open(csv_filename, mode="w", newline="") as csv_file:
+        fieldnames = [
+            "Epoch",
+            "Train Loss",
+            "Train Accuracy",
+            "Validation Loss",
+            "Validation Accuracy",
+            "Learning Rate",
+        ]
+
+        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for metric in epoch_metrics:
+            writer.writerow(metric)
+
+    print(f"Metrics saved to {csv_filename}")
+
     # Ensure the parent directory exists
     os.makedirs(os.path.dirname(MODEL_SAVE_PATH), exist_ok=True)
     torch.save(model.state_dict(), MODEL_SAVE_PATH)
-    print("Model saved at", MODEL_SAVE_PATH)
+    print("\nModel saved at", MODEL_SAVE_PATH)
 
     # Plot loss and accuracy curves
     plt.figure(figsize=(12, 4))

tuning.py

@@ -8,59 +8,115 @@ import torch.utils.data
 from configs import *
 import data_loader
 from torch.utils.tensorboard import SummaryWriter
+import time
+import numpy as np
+
+torch.cuda.empty_cache()
+
+print(f"Using device: {DEVICE}")
 
-DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 EPOCHS = 10
-N_TRIALS = 1000
-TIMEOUT = 14400  
+# N_TRIALS = 10
+# TIMEOUT = 5000
+
+EARLY_STOPPING_PATIENCE = (
+    4  # Number of epochs with no improvement to trigger early stopping
+)
+
 
 # Create a TensorBoard writer
 writer = SummaryWriter(log_dir="output/tensorboard/tuning")
 
 
+# Function to create or modify data loaders with the specified batch size
 def create_data_loaders(batch_size):
-    # Create or modify data loaders with the specified batch size
     train_loader, valid_loader = data_loader.load_data(
-        RAW_DATA_DIR + str(TASK),
-        AUG_DATA_DIR + str(TASK),
-        EXTERNAL_DATA_DIR + str(TASK),
+        COMBINED_DATA_DIR + "1",
         preprocess,
         batch_size=batch_size,
     )
     return train_loader, valid_loader
 
 
-def objective(trial, model=MODEL):
-    # Generate the model.
-    model = model.to(DEVICE)
+def rand_bbox(size, lam):
+    W = size[2]
+    H = size[3]
+    cut_rat = np.sqrt(1.0 - lam)
+    cut_w = np.int_(W * cut_rat)
+    cut_h = np.int_(H * cut_rat)
+
+    # uniform
+    cx = np.random.randint(W)
+    cy = np.random.randint(H)
+
+    bbx1 = np.clip(cx - cut_w // 2, 0, W)
+    bby1 = np.clip(cy - cut_h // 2, 0, H)
+    bbx2 = np.clip(cx + cut_w // 2, 0, W)
+    bby2 = np.clip(cy + cut_h // 2, 0, H)
+
+    return bbx1, bby1, bbx2, bby2
+
+
+def cutmix_data(input, target, alpha=1.0):
+    if alpha > 0:
+        lam = np.random.beta(alpha, alpha)
+    else:
+        lam = 1
+
+    batch_size = input.size()[0]
+    index = torch.randperm(batch_size)
+    rand_index = torch.randperm(input.size()[0])
+
+    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
+    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
+
+    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
+    targets_a = target
+    targets_b = target[rand_index]
+
+    return input, targets_a, targets_b, lam
 
-    # Suggest batch size for tuning.
-    batch_size = trial.suggest_categorical("batch_size", [16, 32, 64])
 
-    # Create data loaders with the suggested batch size.
+def cutmix_criterion(criterion, outputs, targets_a, targets_b, lam):
+    return lam * criterion(outputs, targets_a) + (1 - lam) * criterion(
+        outputs, targets_b
+    )
+
+
+# Objective function for optimization
+def objective(trial, model=MODEL):
+    model = model.to(DEVICE)
+    batch_size = trial.suggest_categorical("batch_size", [16, 32])
     train_loader, valid_loader = create_data_loaders(batch_size)
 
-    # Generate the optimizer.
-    lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True)
+    lr = trial.suggest_float("lr", 1e-5, 1e-3, log=True)
     optimizer = optim.Adam(model.parameters(), lr=lr)
     criterion = nn.CrossEntropyLoss()
 
-    # Suggest the gamma parameter for the learning rate scheduler.
-    gamma = trial.suggest_float("gamma", 0.1, 1.0, step=0.1)
+    gamma = trial.suggest_float("gamma", 0.1, 0.9, step=0.1)
+    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
+
+    past_trials = 0  # Number of trials already completed
+
+    # Print best hyperparameters:
+    if past_trials > 0:
+        print("\nBest Hyperparameters:")
+        print(f"{study.best_trial.params}")
+
+    print(f"\n[INFO] Trial: {trial.number}")
+    print(f"Batch Size: {batch_size}")
+    print(f"Learning Rate: {lr}")
+    print(f"Gamma: {gamma}\n")
 
-    # Create a learning rate scheduler with the suggested gamma.
-    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma)
+    early_stopping_counter = 0
+    best_accuracy = 0.0
 
-    # Training of the model.
     for epoch in range(EPOCHS):
-        print(f"[Epoch: {epoch} | Trial: {trial.number}]")
         model.train()
         for batch_idx, (data, target) in enumerate(train_loader, 0):
             data, target = data.to(DEVICE), target.to(DEVICE)
             optimizer.zero_grad()
-            if (
-                model.__class__.__name__ == "GoogLeNet"
-            ):  # the shit GoogLeNet has a different output
+            if model.__class__.__name__ == "GoogLeNet":
                 output = model(data).logits
             else:
                 output = model(data)
@@ -68,21 +124,22 @@ def objective(trial, model=MODEL):
             loss.backward()
             optimizer.step()
 
-        # Update the learning rate using the scheduler.
         scheduler.step()
 
-        # Validation of the model.
         model.eval()
         correct = 0
         with torch.no_grad():
             for batch_idx, (data, target) in enumerate(valid_loader, 0):
                 data, target = data.to(DEVICE), target.to(DEVICE)
+                data, targets_a, targets_b, lam = cutmix_data(data, target, alpha=1)
                 output = model(data)
-                # Get the index of the max log-probability.
                 pred = output.argmax(dim=1, keepdim=True)
                 correct += pred.eq(target.view_as(pred)).sum().item()
 
         accuracy = correct / len(valid_loader.dataset)
+        if accuracy >= 1.0:
+            print(f"Desired accuracy of 1.0 achieved. Stopping early.")
+            return float("inf")
 
         # Log hyperparameters and accuracy to TensorBoard
         writer.add_scalar("Accuracy", accuracy, trial.number)
@@ -91,36 +148,56 @@ def objective(trial, model=MODEL):
             {"accuracy": accuracy},
         )
 
-        # Print hyperparameters and accuracy
-        print("Hyperparameters: ", trial.params)
-        print("Accuracy: ", accuracy)
+        print(f"[EPOCH {epoch + 1}] Accuracy: {accuracy:.4f}")
+
         trial.report(accuracy, epoch)
 
-        # Handle pruning based on the intermediate value.
-        if trial.should_prune():
-            raise optuna.exceptions.TrialPruned()
+        if accuracy > best_accuracy:
+            best_accuracy = accuracy
+            early_stopping_counter = 0
+        else:
+            early_stopping_counter += 1
+
+        # Early stopping check
+        if early_stopping_counter >= EARLY_STOPPING_PATIENCE:
+            print(f"\nEarly stopping at epoch {epoch + 1}")
+            break
 
-    if trial.number > 10 and trial.params["lr"] < 1e-3 and accuracy < 0.7:
-        return float("inf")  # Prune the trial
+    if trial.number > 10 and trial.params["lr"] < 1e-3 and best_accuracy < 0.7:
+        return float("inf")
 
-    return accuracy
+    past_trials += 1
+
+    return best_accuracy
 
 
 if __name__ == "__main__":
-    pruner = optuna.pruners.HyperbandPruner()
+    hyperband_pruner = optuna.pruners.HyperbandPruner()
+
+    # Record the start time
+    start_time = time.time()
+
+    # storage = optuna.storages.InMemoryStorage()
     study = optuna.create_study(
-        direction="maximize",  # Adjust the direction as per your optimization goal
-        pruner=pruner,
+        direction="maximize",
+        pruner=hyperband_pruner,
         study_name="hyperparameter_tuning",
+        storage="sqlite:///" + MODEL.__class__.__name__ + ".sqlite3",
     )
 
-    # Optimize the hyperparameters
-    study.optimize(objective, n_trials=N_TRIALS, timeout=TIMEOUT)
+    study.optimize(objective)
+
+    # Record the end time
+    end_time = time.time()
+
+    # Calculate the duration of hyperparameter tuning
+    tuning_duration = end_time - start_time
+    print(f"Hyperparameter tuning took {tuning_duration:.2f} seconds.")
 
-    # Print the best trial
     best_trial = study.best_trial
-    print("Best trial:")
-    print("  Value: ", best_trial.value)
-    print("  Params: ")
+    print("\nBest Trial:")
+    print(f"  Trial Number: {best_trial.number}")
+    print(f"  Best Accuracy: {best_trial.value:.4f}")
+    print("  Hyperparameters:")
     for key, value in best_trial.params.items():
-        print("    {}: {}".format(key, value))
+        print(f"    {key}: {value}")

weight_averaging.py

@@ -0,0 +1,235 @@
+import sys
+import torch
+import torch.nn as nn
+from PIL import Image
+import os
+from configs import *
+from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
+import matplotlib.pyplot as plt
+import random
+from itertools import product
+
+random.seed(RANDOM_SEED)
+torch.cuda.manual_seed(RANDOM_SEED)
+torch.manual_seed(RANDOM_SEED)
+print("PyTorch Seed:", torch.initial_seed())
+print("Random Seed:", random.getstate()[1][0])
+print("PyTorch CUDA Seed:", torch.cuda.initial_seed())
+
+print("DEVICE:", DEVICE)
+
+# Define your model paths
+# Load your pre-trained models
+model2 = EfficientNetB3WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB3WithDropout.pth"))
+model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
+model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
+model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
+model4 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
+model4.load_state_dict(torch.load("output\checkpoints\EfficientNetB2WithDropout.pth"))
+
+models = [model1, model2, model3, model4]
+
+# Define the class labels
+class_labels = CLASSES
+
+# Define your test data folder path
+test_data_folder = "data/test/Task 1/"
+
+
+# Put models in evaluation mode
+def set_models_eval(models):
+    for model in models:
+        model.eval()
+
+
+# Define the ensemble model using a list of models
+class WeightedVoteEnsemble(nn.Module):
+    def __init__(self, models, weights):
+        super(WeightedVoteEnsemble, self).__init__()
+        self.models = models
+        self.weights = weights
+
+    def forward(self, x):
+        predictions = [model(x) for model in self.models]
+        weighted_predictions = torch.stack(
+            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
+        )
+        avg_predictions = weighted_predictions.sum(dim=0)
+        return avg_predictions
+
+
+def ensemble_predictions(models, image):
+    all_predictions = []
+
+    with torch.no_grad():
+        for model in models:
+            output = model(image)
+            all_predictions.append(output)
+
+    return torch.stack(all_predictions, dim=0).mean(dim=0)
+
+
+# Load a single image and make predictions
+def evaluate_image(models, image_path, transform=preprocess):
+    image = Image.open(image_path).convert("RGB")
+    image = transform(image).unsqueeze(0)
+    image = image.to(DEVICE)
+    outputs = ensemble_predictions(models, image)
+
+    return outputs.argmax(dim=1).item()
+
+
+# Evaluate and plot a confusion matrix for an ensemble of models
+def evaluate_and_plot_confusion_matrix(models, test_data_folder):
+    all_predictions = []
+    true_labels = []
+
+    with torch.no_grad():
+        for class_label in class_labels:
+            class_path = os.path.join(test_data_folder, class_label)
+            for image_file in os.listdir(class_path):
+                image_path = os.path.join(class_path, image_file)
+                # print(image_path)
+                predicted_label = evaluate_image(models, image_path, preprocess)
+                all_predictions.append(predicted_label)
+                true_labels.append(class_labels.index(class_label))
+
+    # Print accuracy
+    accuracy = (
+        (torch.tensor(all_predictions) == torch.tensor(true_labels)).float().mean()
+    )
+    print("Accuracy:", accuracy)
+
+    # Create the confusion matrix
+    # cm = confusion_matrix(true_labels, all_predictions)
+
+    # # Plot the confusion matrix
+    # display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels)
+    # display.plot(cmap=plt.cm.Blues, values_format="d")
+
+    # # Show the plot
+    # plt.show()
+
+    return accuracy
+
+# Set the models to evaluation mode
+set_models_eval(models)
+
+# Define different weight configurations
+# [SqueezeNet, EfficientNetB2WithDropout, MobileNetV2WithDropout]
+weights_configurations = [
+    # Random set of weights using random.random() and all weights sum to 1
+    [
+        random.randrange(1, 10) / 10,
+        random.randrange(1, 10) / 10,
+        random.randrange(1, 10) / 10,
+    ],
+]
+
+
+## NOTE OF PREVIOUS WEIGHTS
+# Best weights: [0.2, 0.3, 0.5] with accuracy: 0.9428571462631226 at iteration: 15 with torch seed: 28434738589300 and random seed: 3188652458777471118 and torch cuda seed: None
+
+
+best_weights = {
+    "weights": 0,
+    "accuracy": 0,
+    "iteration": 0,
+    "torch_seed": 0,
+    "random_seed": 0,
+    "torch_cuda_seed": 0,
+}
+
+i = 0
+
+# weights_hist = []
+
+target_sum = 1.0
+number_of_numbers = 4
+lower_limit = 0.2
+upper_limit = 0.8
+step = 0.01
+
+valid_combinations = []
+
+# Generate all unique combinations of four numbers with values to two decimal places
+for combination in product(
+    *[range(int(lower_limit * 100), int(upper_limit * 100) + 1)] * number_of_numbers
+):
+    # Convert the combination to a list of floats
+    combination = [float(number) / 100 for number in combination]
+    # Check if the sum of the combination is equal to the target sum
+    if sum(combination) == target_sum:
+        # Add the combination to the list of valid combinations
+        valid_combinations.append(combination)
+                
+
+# Calculate the total number of possibilities
+total_possibilities = len(valid_combinations)
+
+print("Total number of possibilities:", total_possibilities)
+
+# valid_combinations = [[0.3, 0.5, 0.2]]
+# 0.38 for SqueezeNet, 0.34 for EfficientNetB2WithDropout, 0.28 for MobileNetV2WithDropout
+best_weighted_vote_ensemble_model = None
+
+for weights in valid_combinations:
+# while True:
+    print("---------------------------")
+    print("Iteration:", i)
+    # Should iterate until all possible weights are exhausted
+    # Create an ensemble model with weighted voting
+
+    random.seed(RANDOM_SEED)
+    torch.cuda.manual_seed(RANDOM_SEED)
+    torch.manual_seed(RANDOM_SEED)
+    # print("PyTorch Seed:", torch.initial_seed())
+    # weights_hist.append(weights)
+    weighted_vote_ensemble_model = WeightedVoteEnsemble(
+        # [model1, model2, model3], weights
+        models,
+        weights,
+    )
+    # print("Weights:", weights)
+    print("Weights:", weights)
+    # Call the evaluate_and_plot_confusion_matrix function with your models and test data folder
+    accuracy = evaluate_and_plot_confusion_matrix(
+        [weighted_vote_ensemble_model], test_data_folder
+    )
+    # Convert tensor to float
+    accuracy = accuracy.item()
+    if accuracy > best_weights["accuracy"]:
+        # best_weights["weights"] = weights
+        best_weights["weights"] = weights
+        best_weights["accuracy"] = accuracy
+        best_weights["iteration"] = i
+        best_weights["torch_seed"] = torch.initial_seed()
+        seed = random.randrange(sys.maxsize)
+        rng = random.Random(seed)
+        best_weights["random_seed"] = seed
+        best_weights["torch_cuda_seed"] = torch.cuda.initial_seed()
+        best_weighted_vote_ensemble_model = weighted_vote_ensemble_model
+
+    print(
+        "Best weights:",
+        best_weights["weights"],
+        "with accuracy:",
+        best_weights["accuracy"],
+        "at iteration:",
+        best_weights["iteration"],
+        "with torch seed:",
+        best_weights["torch_seed"],
+        "and random seed:",
+        best_weights["random_seed"],
+        "and torch cuda seed:",
+        best_weights["torch_cuda_seed"],
+    )
+    i += 1
+
+
+torch.save(
+    best_weighted_vote_ensemble_model.state_dict(),
+    "output/checkpoints/WeightedVoteEnsemble.pth",
+)