Update

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import predict as predict
-import extract as extract
-import lime_eval as lime_eval
+import extract_gradcam as extract_gradcam
+import extract_lime as extract_lime
 
 
 def upload_file(files):
@@ -24,33 +24,17 @@ def process_file(
         result.append(f"{class_label}: {class_prob}%")
     result = result[:4]
     if gradcam_toggle == True:
-        cam = extract.extract_gradcam(upload_filepath, save_path="gradcam.jpg")
+        cam = extract_gradcam.extract_gradcam(upload_filepath, save_path="gradcam.jpg")
         result.append("gradcam.jpg")
     else:
         result.append(None)
     if lime_toggle == True:
-        lime = lime_eval.generate_lime(upload_filepath, save_path="lime.jpg")
+        lime = extract_lime.generate_lime(upload_filepath, save_path="lime.jpg")
         result.append("lime.jpg")
     else:
         result.append(None)
     return result
-    # else:
-    #     sorted_classes = predict.predict_image(upload_filepath)
-    #     for class_label, class_prob in sorted_classes:
-    #         class_prob = class_prob.item().__round__(2)
-    #         result.append(f"{class_label}: {class_prob}%")
-    #     result = result[:4]
-    #     if gradcam_toggle == 1:
-    #         cam = extract.extract_gradcam(upload_filepath, save_path="gradcam.jpg")
-    #         result.append("gradcam.jpg")
-    #     if lime_toggle == 1:
-    #         lime = lime_eval.generate_lime(upload_filepath, save_path="lime.jpg")
-    #         result.append("lime.jpg")
-    #     return result
-
-
-# Prerun to innitialize the model
-# process_file(None, r"data\test\Task 1\Dystonia\0c08d2ea-8e1c-4ac6-92db-a752388b30cf.png")
+
 
 css = """
 .block {
@@ -108,21 +92,10 @@ with block as demo:
     with gr.Column():
         gr.Label("SpiralSense", elem_id="title-label", show_label=False)
         gr.Label(
-            "Cost-Effective, Portable And Stressless Spiral Drawing Analysing Web Application for Early Detection of Multiple Neurological Disorders with 96% Accuracy",
+            "A Stress-free, Portable, and Cost-effective Machine Learning-Powered Web Application for Early Detection of Multiple Neurological Disorders through Spiral Drawing Analysis",
             elem_id="desc-label",
-            show_label=False
+            show_label=False,
         )
-        # gr.Markdown(
-        #     """
-        #     <h1 style="text-align: center;">SpiralSense</h1>
-        #     <h4 style="text-align: center;">Cost-Effective, Portable And Stressless Spiral Drawing Analysing Web Application for Early Detection of Multiple Neurological Disorders with 96% Accuracy</h4>
-        #     """
-        # )
-        # gr.Markdown(
-        #     """
-        #     <h4 style="text-align: center;">------------------------------------------</h4>
-        #     """
-        # )
         with gr.Row():
             image_input = gr.Image(
                 type="filepath",
@@ -147,11 +120,6 @@ with block as demo:
         with gr.Row():
             submit_button = gr.Button(value="Submit")
         gr.Markdown("<br>")
-            # cancel_button = gr.Button(value="Cancel")
-        # theme="gradio/soft",
-        # fn=process_file,
-        # title="HANDETECT",
-        # outputs=[
         with gr.Row():
             prob1_textbox = gr.outputs.Textbox(label="Probability 1")
             prob2_textbox = gr.outputs.Textbox(label="Probability 2")
@@ -184,7 +152,6 @@ with block as demo:
             show_progress="minimal",
             preprocess=upload_file,
             scroll_to_output=True,
-            # cancels=[cancel_button],
         )
 
 

compute_mean_std.py

@@ -0,0 +1,45 @@
+import torchvision
+from torchvision import transforms
+from torch.utils.data import DataLoader
+import torch
+from configs import *
+
+def main():
+    data_path = COMBINED_DATA_DIR + str(TASK)
+
+    transform_img = transforms.Compose(
+        [
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),  # Convert to tensor
+            transforms.Grayscale(num_output_channels=3),  # Convert to 3 channels
+        ]
+    )
+
+    image_data = torchvision.datasets.ImageFolder(root=data_path, transform=transform_img)
+
+    batch_size = BATCH_SIZE
+
+    loader = DataLoader(image_data, batch_size=batch_size, num_workers=1)
+
+    def batch_mean_and_sd(loader):
+        cnt = 0
+        fst_moment = torch.empty(3)
+        snd_moment = torch.empty(3)
+
+        for images, _ in loader:
+            b, c, h, w = images.shape
+            nb_pixels = b * h * w
+            sum_ = torch.sum(images, dim=[0, 2, 3])
+            sum_of_square = torch.sum(images**2, dim=[0, 2, 3])
+            fst_moment = (cnt * fst_moment + sum_) / (cnt + nb_pixels)
+            snd_moment = (cnt * snd_moment + sum_of_square) / (cnt + nb_pixels)
+            cnt += nb_pixels
+
+        mean, std = fst_moment, torch.sqrt(snd_moment - fst_moment**2)
+        return mean, std
+
+    mean, std = batch_mean_and_sd(loader)
+    print("mean and std: \n", mean, std)
+
+if __name__ == '__main__':
+    main()

configs.py

@@ -1,39 +1,7 @@
-import os
 import torch
 from torchvision import transforms
 from torch.utils.data import Dataset
 from models import *
-import torch.nn as nn
-from torchvision.models import (
-    squeezenet1_0,
-    SqueezeNet1_0_Weights,
-    squeezenet1_1,
-    SqueezeNet1_1_Weights,
-    shufflenet_v2_x2_0,
-    ShuffleNet_V2_X2_0_Weights,
-    mobilenet_v3_small,
-    MobileNet_V3_Small_Weights,
-    efficientnet_v2_s,
-    EfficientNet_V2_S_Weights,
-    efficientnet_b0,
-    EfficientNet_B0_Weights,
-    efficientnet_b1,
-    EfficientNet_B1_Weights,
-    efficientnet_b2,
-    EfficientNet_B2_Weights,
-    efficientnet_b3,
-    EfficientNet_B3_Weights,
-    mobilenet_v3_small,
-    MobileNet_V3_Small_Weights,
-    mobilenet_v3_large,
-    MobileNet_V3_Large_Weights,
-    googlenet,
-    GoogLeNet_Weights,
-    MobileNet_V2_Weights,
-    mobilenet_v2,
-)
-
-import torch.nn.functional as F
 
 # Constants
 RANDOM_SEED = 123
@@ -44,8 +12,8 @@ LEARNING_RATE = 0.0001
 STEP_SIZE = 10
 GAMMA = 0.3
 CUTMIX_ALPHA = 0.3
-DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-# DEVICE = torch.device("cpu")
+# DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+DEVICE = torch.device("cpu")
 NUM_PRINT = 100
 TASK = 1
 WARMUP_EPOCHS = 5
@@ -69,235 +37,13 @@ CLASSES = [
 ]
 
 
-class SE_Block(nn.Module):
-    def __init__(self, channel, reduction=16):
-        super(SE_Block, self).__init__()
-        self.avg_pool = nn.AdaptiveAvgPool2d(1)
-        self.fc = nn.Sequential(
-            nn.Linear(channel, channel // reduction, bias=False),
-            nn.ReLU(inplace=True),
-            nn.Linear(channel // reduction, channel, bias=False),
-            nn.Sigmoid(),  # Sigmoid activation to produce attention scores
-        )
-
-    def forward(self, x):
-        b, c, _, _ = x.size()
-        y = self.avg_pool(x).view(b, c)
-        y = self.fc(y).view(b, c, 1, 1)
-        return x * y.expand_as(x)
-
-
-class SqueezeNet1_0WithSE(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.5):
-        super(SqueezeNet1_0WithSE, self).__init__()
-        squeezenet = squeezenet1_0(weights=SqueezeNet1_0_Weights.DEFAULT)
-        self.features = squeezenet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(512, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-        # Adjust channel for SqueezeNet1.0 (original SqueezeNet1.0 has 1000 classes)
-        num_classes_squeezenet1_0 = 7
-
-        # Add Squeeze-and-Excitation block
-        self.se_block = SE_Block(
-            channel=num_classes_squeezenet1_0
-        )  # Adjust channel as needed
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        # x = self.se_block(x)  # Apply the SE block
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-class SqueezeNet1_1WithSE(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.2):
-        super(SqueezeNet1_1WithSE, self).__init__()
-        squeezenet = squeezenet1_1(weights=SqueezeNet1_1_Weights.DEFAULT)
-        self.features = squeezenet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(512, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-        # Add Squeeze-and-Excitation block
-        self.se_block = SE_Block(channel=num_classes)  # Adjust channel as needed
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = self.se_block(x)  # Apply the SE block
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-class EfficientNetB2WithDropout(nn.Module):
-    #  0.00022015769999619205
-    def __init__(self, num_classes, dropout_prob=0.2):
-        super(EfficientNetB2WithDropout, self).__init__()
-        efficientnet = efficientnet_b2(weights=EfficientNet_B2_Weights.DEFAULT)
-        self.features = efficientnet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(1408, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-class EfficientNetB3WithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.2):
-        super(EfficientNetB3WithDropout, self).__init__()
-        efficientnet = efficientnet_b3(weights=EfficientNet_B3_Weights.DEFAULT)
-        self.features = efficientnet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(1536, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-class ResNet18WithNorm(nn.Module):
-    def __init__(self, num_classes=1000):
-        super(ResNet18WithNorm, self).__init__()
-        resnet = resnet18(pretrained=False)
-
-        # Remove the last block (Block 4)
-        self.features = nn.Sequential(
-            *list(resnet.children())[:-1]  # Exclude the last block
-        )
-
-        self.classifier = nn.Sequential(
-            nn.AdaptiveAvgPool2d((1, 1)),
-            nn.Flatten(),
-            nn.Linear(
-                512, num_classes
-            ),  # Adjust input size for the fully connected layer
-            nn.BatchNorm1d(num_classes),  # Add batch normalization
-        )
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = torch.flatten(x, 1)
-        return x
-
-
-class MobileNetV3LargeWithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.2):
-        super(MobileNetV3LargeWithDropout, self).__init__()
-        mobilenet = mobilenet_v3_large(weights=MobileNet_V3_Large_Weights.DEFAULT)
-        self.features = mobilenet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(960, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-class GoogLeNetWithSE(nn.Module):
-    def __init__(self, num_classes):
-        super(GoogLeNetWithSE, self).__init__()
-        googlenet = googlenet(weights=GoogLeNet_Weights.DEFAULT)
-        # self.features = googlenet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(1024, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-
-        # Add Squeeze-and-Excitation block
-        self.se_block = SE_Block(channel=num_classes)  # Adjust channel as needed
-
-    def forward(self, x):
-        # x = self.features(x)
-        x = self.classifier(x)
-        x = self.se_block(x)  # Apply the SE block
-        x = torch.flatten(x, 1)
-        return x
-
-
-class MobileNetV2WithDropout(nn.Module):
-    def __init__(self, num_classes, dropout_prob=0.2):
-        super(MobileNetV2WithDropout, self).__init__()
-        mobilenet = mobilenet_v2(weights=MobileNet_V2_Weights.DEFAULT)
-        self.features = mobilenet.features
-        self.classifier = nn.Sequential(
-            nn.Conv2d(1280, num_classes, kernel_size=1),
-            nn.BatchNorm2d(num_classes),  # add batch normalization
-            nn.ReLU(inplace=True),
-            nn.AdaptiveAvgPool2d((1, 1)),
-        )
-        self.dropout = nn.Dropout(
-            dropout_prob
-        )  # Add dropout layer with the specified probability
-
-    def forward(self, x):
-        x = self.features(x)
-        x = self.classifier(x)
-        x = F.dropout(x, training=self.training)  # Apply dropout during training
-        x = torch.flatten(x, 1)
-        return x
-
-
-MODEL = EfficientNetB3WithDropout(num_classes=NUM_CLASSES)
+MODEL = EfficientNetB3WithNorm(num_classes=NUM_CLASSES)
 MODEL_SAVE_PATH = r"output/checkpoints/" + MODEL.__class__.__name__ + ".pth"
-# MODEL_SAVE_PATH = r"C:\Users\User\Downloads\bestsqueezenetSE.pth"
 preprocess = transforms.Compose(
     [
         transforms.Resize((224, 224)),
         transforms.ToTensor(),  # Convert to tensor
-        transforms.Grayscale(num_output_channels=3),  # Convert to 3 channels
-        # Normalize 3 channels
-        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        transforms.Normalize(0.8289, 0.2006),
     ]
 )
 
@@ -313,14 +59,3 @@ class CustomDataset(Dataset):
     def __getitem__(self, idx):
         img, label = self.data[idx]
         return img, label
-
-
-def ensemble_predictions(models, image):
-    all_predictions = []
-
-    with torch.no_grad():
-        for model in models:
-            output = model(image)
-            all_predictions.append(output)
-
-    return torch.stack(all_predictions, dim=0).mean(dim=0)

convert.py

@@ -1,17 +0,0 @@
-import torch
-import onnx2tf
-from configs import *
-
-torch.onnx.export(
-    model=model,
-    args=torch.randn(1, 3, 64, 64),
-    f="output/checkpoints/model.onnx",
-    verbose=True,
-    input_names=["input"],
-    output_names=["output"],
-)
-
-onnx2tf.convert(
-    input_onnx_file_path="output/checkpoints/model.onnx",
-    output_folder_path="output/checkpoints/converted/",
-)

ensemble.py

@@ -1,249 +0,0 @@
-import matplotlib.pyplot as plt
-from torch.optim.lr_scheduler import CosineAnnealingLR
-import torch
-import torch.nn as nn
-from torchvision.datasets import ImageFolder
-from torch.utils.data import DataLoader
-from data_loader import load_data, load_test_data
-from configs import *
-import numpy as np
-
-torch.cuda.empty_cache()
-
-# 
-
-
-class MLP(nn.Module):
-    def __init__(self, num_classes, num_models):
-        super(MLP, self).__init__()
-        self.layers = nn.Sequential(
-            nn.Linear(num_classes * num_models, 1024),
-            nn.LayerNorm(1024),
-            nn.LeakyReLU(negative_slope=0.01, inplace=True),
-            nn.Dropout(0.8),
-            nn.Linear(1024, 2048),
-            nn.LeakyReLU(negative_slope=0.01, inplace=True),
-            nn.Dropout(0.5),
-            nn.Linear(2048, 2048),
-            nn.LeakyReLU(negative_slope=0.01, inplace=True),
-            nn.Dropout(0.5),
-            nn.Linear(2048, num_classes),
-        )
-
-    def forward(self, x):
-        x = x.view(x.size(0), -1)
-        x = self.layers(x)
-        return x
-
-
-def mlp_meta(num_classes, num_models):
-    model = MLP(num_classes, num_models)
-    return model
-
-
-# Hyperparameters
-input_dim = 3 * 224 * 224  # Modify this based on your input size
-hidden_dim = 256
-output_dim = NUM_CLASSES
-
-# Create the data loaders using your data_loader functions50
-train_loader, val_loader = load_data(COMBINED_DATA_DIR + "1", preprocess, BATCH_SIZE)
-test_loader = load_test_data("data/test/Task 1", preprocess, BATCH_SIZE)
-
-model_paths = [
-    "output/checkpoints/bestsqueezenetSE3.pth",
-    "output/checkpoints/EfficientNetB3WithDropout.pth",
-    "output/checkpoints/MobileNetV2WithDropout2.pth",
-]
-
-
-# Define a function to load pretrained models
-def load_pretrained_model(path, model):
-    model.load_state_dict(torch.load(path))
-    return model.to(DEVICE)
-
-
-def rand_bbox(size, lam):
-    W = size[2]
-    H = size[3]
-    cut_rat = np.sqrt(1.0 - lam)
-    cut_w = np.int_(W * cut_rat)
-    cut_h = np.int_(H * cut_rat)
-
-    # uniform
-    cx = np.random.randint(W)
-    cy = np.random.randint(H)
-
-    bbx1 = np.clip(cx - cut_w // 2, 0, W)
-    bby1 = np.clip(cy - cut_h // 2, 0, H)
-    bbx2 = np.clip(cx + cut_w // 2, 0, W)
-    bby2 = np.clip(cy + cut_h // 2, 0, H)
-
-    return bbx1, bby1, bbx2, bby2
-
-
-def cutmix_data(input, target, alpha=1.0):
-    if alpha > 0:
-        lam = np.random.beta(alpha, alpha)
-    else:
-        lam = 1
-
-    batch_size = input.size()[0]
-    index = torch.randperm(batch_size)
-    rand_index = torch.randperm(input.size()[0])
-
-    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
-    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
-
-    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
-    targets_a = target
-    targets_b = target[rand_index]
-
-    return input, targets_a, targets_b, lam
-
-
-def cutmix_criterion(criterion, outputs, targets_a, targets_b, lam):
-    return lam * criterion(outputs, targets_a) + (1 - lam) * criterion(
-        outputs, targets_b
-    )
-
-
-# Load pretrained models
-model1 = load_pretrained_model(
-    model_paths[0], SqueezeNet1_0WithSE(num_classes=NUM_CLASSES)
-).to(DEVICE)
-model2 = load_pretrained_model(
-    model_paths[1], EfficientNetB3WithDropout(num_classes=NUM_CLASSES)
-).to(DEVICE)
-model3 = load_pretrained_model(
-    model_paths[2], MobileNetV2WithDropout(num_classes=NUM_CLASSES)
-).to(DEVICE)
-
-models = [model1, model2, model3]
-
-# Create the meta learner
-meta_learner_model = mlp_meta(NUM_CLASSES, len(models)).to(DEVICE)
-meta_optimizer = torch.optim.Adam(meta_learner_model.parameters(), lr=0.001)
-meta_loss_fn = torch.nn.CrossEntropyLoss()
-
-# Define the Cosine Annealing Learning Rate Scheduler
-scheduler = CosineAnnealingLR(
-    meta_optimizer, T_max=700
-)  # T_max is the number of epochs for the cosine annealing.
-
-# Define loss function and optimizer for the meta learner
-criterion = nn.CrossEntropyLoss().to(DEVICE)
-
-# Record learning rate
-lr_hist = []
-
-# Training loop
-num_epochs = 160
-for epoch in range(num_epochs):
-    print("[Epoch: {}]".format(epoch + 1))
-    print("Total number of batches: {}".format(len(train_loader)))
-    for batch_idx, data in enumerate(train_loader, 0):
-        print("Batch: {}".format(batch_idx + 1))
-        inputs, labels = data
-        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
-        inputs, targets_a, targets_b, lam = cutmix_data(inputs, labels, alpha=1)
-
-        # Forward pass through the three pretrained models
-        features1 = model1(inputs)
-        features2 = model2(inputs)
-        features3 = model3(inputs)
-
-        # Stack the features from the three models
-        stacked_features = torch.cat((features1, features2, features3), dim=1).to(
-            DEVICE
-        )
-
-        # Forward pass through the meta learner
-        meta_output = meta_learner_model(stacked_features)
-
-        # Compute the loss
-        loss = cutmix_criterion(criterion, meta_output, targets_a, targets_b, lam)
-
-        # Compute the accuracy
-        _, predicted = torch.max(meta_output, 1)
-        total = labels.size(0)
-        correct = (predicted == labels).sum().item()
-
-        # Backpropagation and optimization
-        meta_optimizer.zero_grad()
-        loss.backward()
-        meta_optimizer.step()
-
-        lr_hist.append(meta_optimizer.param_groups[0]["lr"])
-
-        scheduler.step()
-
-    print("Train Loss: {}".format(loss.item()))
-    print("Train Accuracy: {}%".format(100 * correct / total))
-
-    # Validation
-    meta_learner_model.eval()
-    correct = 0
-    total = 0
-    val_loss = 0
-    with torch.no_grad():
-        for data in val_loader:
-            inputs, labels = data
-            inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
-            features1 = model1(inputs)
-            features2 = model2(inputs)
-            features3 = model3(inputs)
-            stacked_features = torch.cat((features1, features2, features3), dim=1).to(
-                DEVICE
-            )
-            outputs = meta_learner_model(stacked_features)
-            loss = criterion(outputs, labels)  # Use the validation loss
-            val_loss += loss.item()
-            _, predicted = torch.max(outputs, 1)
-            total += labels.size(0)
-            correct += (predicted == labels).sum().item()
-
-    print(
-        "Validation Loss: {}".format(val_loss / len(val_loader))
-    )  # Calculate the average loss
-    print("Validation Accuracy: {}%".format(100 * correct / total))
-
-
-print("Finished Training")
-
-# Test the ensemble
-print("Testing the ensemble")
-meta_learner_model.eval()
-correct = 0
-total = 0
-with torch.no_grad():
-    for data in test_loader:
-        inputs, labels = data
-        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
-        features1 = model1(inputs)
-        features2 = model2(inputs)
-        features3 = model3(inputs)
-        stacked_features = torch.cat((features1, features2, features3), dim=1)
-        outputs = meta_learner_model(stacked_features)
-        _, predicted = torch.max(outputs, 1)
-        total += labels.size(0)
-        correct += (predicted == labels).sum().item()
-
-print(
-    "Accuracy of the ensemble network on the test images: {}%".format(
-        100 * correct / total
-    )
-)
-
-
-# Plot the learning rate history
-
-plt.plot(lr_hist)
-plt.xlabel("Iterations")
-plt.ylabel("Learning Rate")
-plt.title("Learning Rate History")
-plt.show()
-
-
-# Save the model
-torch.save(meta_learner_model.state_dict(), "output/checkpoints/ensemble.pth")

eval.py

@@ -1,190 +0,0 @@
-import os
-import torch
-import numpy as np
-import pathlib
-from PIL import Image
-import matplotlib.pyplot as plt
-from sklearn.metrics import (
-    classification_report,
-    precision_recall_curve,
-    accuracy_score,
-    f1_score,
-    confusion_matrix,
-    ConfusionMatrixDisplay,
-)
-from sklearn.preprocessing import label_binarize
-from torchvision import transforms
-from configs import *
-
-# EfficientNet: 0.901978973407545
-# MobileNet: 0.8731189445475158
-# SquuezeNet:  0.8559218559218559
-
-
-# Constants
-DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-NUM_AUGMENTATIONS = 10  # Number of augmentations to perform
-
-model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
-model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
-model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
-model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
-
-best_weights = [0.901978973407545, 0.8731189445475158, 0.8559218559218559]
-
-# Load the model
-model = WeightedVoteEnsemble([model1, model2, model3], best_weights)
-# model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
-model.load_state_dict(torch.load('output/checkpoints/WeightedVoteEnsemble.pth', map_location=DEVICE))
-model.eval()
-
-# define augmentations for TTA
-tta_transforms = transforms.Compose(
-    [
-        transforms.RandomHorizontalFlip(p=0.5),
-        transforms.RandomVerticalFlip(p=0.5),
-    ]
-)
-
-
-def perform_tta(model, image, tta_transforms):
-    augmented_predictions = []
-    augmented_scores = []
-
-    for _ in range(NUM_AUGMENTATIONS):
-        augmented_image = tta_transforms(image)
-        output = model(augmented_image)
-        predicted_class = torch.argmax(output, dim=1).item()
-        augmented_predictions.append(predicted_class)
-        augmented_scores.append(output.softmax(dim=1).cpu().numpy())
-
-    # max voting
-    final_predicted_class_max = max(
-        set(augmented_predictions), key=augmented_predictions.count
-    )
-
-    # average probabilities
-    final_predicted_scores_avg = np.mean(np.array(augmented_scores), axis=0)
-
-    # rotate and average probabilities
-    rotation_transforms = [
-        transforms.RandomRotation(degrees=i) for i in range(0, 360, 30)
-    ]
-    rotated_scores = []
-    for rotation_transform in rotation_transforms:
-        augmented_image = rotation_transform(image)
-        output = model(augmented_image)
-        rotated_scores.append(output.softmax(dim=1).cpu().numpy())
-
-    final_predicted_scores_rotation = np.mean(np.array(rotated_scores), axis=0)
-
-    return (
-        final_predicted_class_max,
-        final_predicted_scores_avg,
-        final_predicted_scores_rotation,
-    )
-
-
-def predict_image_with_tta(image_path, model, transform, tta_transforms):
-    model.eval()
-    correct_predictions = 0
-    true_classes = []
-    predicted_labels_max = []
-    predicted_labels_avg = []
-    predicted_labels_rotation = []
-
-    with torch.no_grad():
-        images = list(pathlib.Path(image_path).rglob("*.png"))
-        total_predictions = len(images)
-
-        for image_file in images:
-            true_class = CLASSES.index(image_file.parts[-2])
-
-            original_image = Image.open(image_file).convert("RGB")
-            original_image = transform(original_image).unsqueeze(0)
-            original_image = original_image.to(DEVICE)
-
-            # Perform TTA with different strategies
-            final_predicted_class_max, _, _ = perform_tta(
-                model, original_image, tta_transforms
-            )
-            _, final_predicted_scores_avg, _ = perform_tta(
-                model, original_image, tta_transforms
-            )
-            _, _, final_predicted_scores_rotation = perform_tta(
-                model, original_image, tta_transforms
-            )
-
-            true_classes.append(true_class)
-            predicted_labels_max.append(final_predicted_class_max)
-            predicted_labels_avg.append(np.argmax(final_predicted_scores_avg))
-            predicted_labels_rotation.append(np.argmax(final_predicted_scores_rotation))
-
-            if final_predicted_class_max == true_class:
-                correct_predictions += 1
-
-    # accuracy for each strategy
-    accuracy_max = accuracy_score(true_classes, predicted_labels_max)
-    accuracy_avg = accuracy_score(true_classes, predicted_labels_avg)
-    accuracy_rotation = accuracy_score(true_classes, predicted_labels_rotation)
-
-    print("Accuracy (Max Voting):", accuracy_max)
-    print("Accuracy (Average Probabilities):", accuracy_avg)
-    print("Accuracy (Rotation and Average):", accuracy_rotation)
-
-    # final prediction using ensemble (choose the strategy with the highest accuracy)
-    final_predicted_labels = []
-    for i in range(len(true_classes)):
-        max_strategy_accuracy = max(accuracy_max, accuracy_avg, accuracy_rotation)
-        if accuracy_max == max_strategy_accuracy:
-            final_predicted_labels.append(predicted_labels_max[i])
-        elif accuracy_avg == max_strategy_accuracy:
-            final_predicted_labels.append(predicted_labels_avg[i])
-        else:
-            final_predicted_labels.append(predicted_labels_rotation[i])
-
-    # calculate accuracy and f1 score(ensemble)
-    accuracy_ensemble = accuracy_score(true_classes, final_predicted_labels)
-    f1_ensemble = f1_score(true_classes, final_predicted_labels, average="weighted")
-
-    print("Ensemble Accuracy:", accuracy_ensemble)
-    print("Ensemble Weighted F1 Score:", f1_ensemble)
-
-    # Classification report
-    class_names = [str(cls) for cls in range(NUM_CLASSES)]
-    report = classification_report(
-        true_classes, final_predicted_labels, target_names=class_names
-    )
-    print("Classification Report of", MODEL.__class__.__name__, ":\n", report)
-    
-    # confusion matrix and classification report for the ensemble
-    conf_matrix_ensemble = confusion_matrix(true_classes, final_predicted_labels)
-    ConfusionMatrixDisplay(
-        confusion_matrix=conf_matrix_ensemble, display_labels=range(NUM_CLASSES)
-    ).plot(cmap=plt.cm.Blues)
-    plt.title("Confusion Matrix (Ensemble)")
-    plt.show()
-
-    class_names = [str(cls) for cls in range(NUM_CLASSES)]
-    report_ensemble = classification_report(
-        true_classes, final_predicted_labels, target_names=class_names
-    )
-    print("Classification Report (Ensemble):\n", report_ensemble)
-
-    # Calculate precision and recall for each class
-    true_classes_binary = label_binarize(true_classes, classes=range(NUM_CLASSES))
-    precision, recall, _ = precision_recall_curve(
-        true_classes_binary.ravel(), np.array(final_predicted_scores_rotation).ravel()
-    )
-
-    # Plot precision-recall curve
-    plt.figure(figsize=(10, 6))
-    plt.plot(recall, precision)
-    plt.title("Precision-Recall Curve")
-    plt.xlabel("Recall")
-    plt.ylabel("Precision")
-    plt.show()
-
-predict_image_with_tta("data/test/Task 1/", model, preprocess, tta_transforms)

eval_orig.py → evaluate.py

@@ -1,6 +1,3 @@
-import os
-import torchvision
-import shap
 import torch
 import numpy as np
 import pathlib
@@ -19,14 +16,10 @@ from sklearn.metrics import (
     auc,
     average_precision_score,
     cohen_kappa_score,
+    
 )
 from sklearn.preprocessing import label_binarize
 from configs import *
-from data_loader import load_data  # Import the load_data function
-
-# MobileNet: 0.8731189445475158
-# EfficientNet: 0.873118944547516
-# SquuezeNet: 0.8865856365856365
 
 
 rcParams["font.family"] = "Times New Roman"
@@ -126,7 +119,7 @@ def predict_image(image_path, model, transform):
     plt.title("Confusion Matrix")
     manager = plt.get_current_fig_manager()
     manager.full_screen_toggle()
-    plt.savefig("docs/efficientnet/confusion_matrix.png")
+    plt.savefig("docs/evaluation/confusion_matrix.png")
     plt.show()
 
     # Classification report
@@ -170,7 +163,7 @@ def predict_image(image_path, model, transform):
         "AUC-PRC = {:.3f}".format(auc_prc),
         bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),
     )
-    plt.savefig("docs/efficientnet/prc.png")
+    plt.savefig("docs/evaluation/prc.png")
     plt.show()
 
     # Plot ROC curve
@@ -186,7 +179,7 @@ def predict_image(image_path, model, transform):
         "AUC-ROC = {:.3f}".format(auc_roc),
         bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),
     )
-    plt.savefig("docs/efficientnet/roc.png")
+    plt.savefig("docs/evaluation/roc.png")
     plt.show()
     
     

extract-ensemble.py

@@ -1,110 +0,0 @@
-from pytorch_grad_cam import GradCAMPlusPlus
-from pytorch_grad_cam.utils.image import show_cam_on_image, preprocess_image
-import cv2
-import numpy as np
-import torch
-import torch.nn as nn  # Replace with your model
-from configs import *
-
-# Load your model (change this according to your model definition)
-model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
-model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
-model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
-model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
-
-model1.eval()
-model2.eval()
-model3.eval()
-
-
-# Find the target layer (modify this based on your model architecture)
-# EfficientNetB2WithDropout - model.features[-1]
-# SqueezeNet1_0WithSE - model.features
-# MobileNetV2WithDropout - model.features[-1]
-
-target_layer_efficientnet = None
-for child in model2.features[-1]:
-    if isinstance(child, nn.Conv2d):
-        target_layer_efficientnet = child
-
-if target_layer_efficientnet is None:
-    raise ValueError(
-        "Invalid EfficientNet layer name: {}".format(target_layer_efficientnet)
-    )
-
-target_layer_squeezenet = None
-for child in model1.features:
-    if isinstance(child, nn.Conv2d):
-        target_layer_squeezenet = child
-
-if target_layer_squeezenet is None:
-    raise ValueError(
-        "Invalid SqueezeNet layer name: {}".format(target_layer_squeezenet)
-    )
-
-target_layer_mobilenet = None
-for child in model3.features[-1]:
-    if isinstance(child, nn.Conv2d):
-        target_layer_mobilenet = child
-
-if target_layer_mobilenet is None:
-    raise ValueError("Invalid MobileNet layer name: {}".format(target_layer_mobilenet))
-
-# Load and preprocess the image
-image_path = r"data\test\Task 1\Cerebral Palsy\89.png"
-rgb_img = cv2.imread(image_path, 1)
-rgb_img = np.float32(rgb_img) / 255
-input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
-input_tensor = input_tensor.to(DEVICE)
-input_tensor.requires_grad = True  # Enable gradients for the input tensor
-
-
-# Create a GradCAMPlusPlus object
-efficientnet_cam = GradCAMPlusPlus(model=model2, target_layers=[target_layer_efficientnet], use_cuda=True)
-squeezenet_cam = GradCAMPlusPlus(model=model1, target_layers=[target_layer_squeezenet], use_cuda=True)
-mobilenet_cam = GradCAMPlusPlus(model=model3, target_layers=[target_layer_mobilenet], use_cuda=True)
-
-
-efficientnet_grayscale_cam = efficientnet_cam(input_tensor=input_tensor)[0]
-squeezenet_grayscale_cam = squeezenet_cam(input_tensor=input_tensor)[0]
-mobilenet_grayscale_cam = mobilenet_cam(input_tensor=input_tensor)[0]
-
-# Apply a colormap to the grayscale heatmap
-efficientnet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * efficientnet_grayscale_cam), cv2.COLORMAP_JET)
-squeezenet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * squeezenet_grayscale_cam), cv2.COLORMAP_JET)
-mobilenet_heatmap_colored = cv2.applyColorMap(np.uint8(255 * mobilenet_grayscale_cam), cv2.COLORMAP_JET)
-
-# normalized_efficientnet_heatmap = efficientnet_heatmap_colored / np.max(efficientnet_heatmap_colored)
-# normalized_squeezenet_heatmap = squeezenet_heatmap_colored / np.max(squeezenet_heatmap_colored)
-# normalized_mobilenet_heatmap = mobilenet_heatmap_colored / np.max(mobilenet_heatmap_colored)
-
-# # Ensure heatmap_colored has the same dtype as rgb_img
-# normalized_efficientnet_heatmap = normalized_efficientnet_heatmap.astype(np.float32) / 255
-# normalized_squeezenet_heatmap = normalized_squeezenet_heatmap.astype(np.float32) / 255
-# normalized_mobilenet_heatmap = normalized_mobilenet_heatmap.astype(np.float32) / 255
-
-efficientnet_heatmap_colored = efficientnet_heatmap_colored.astype(np.float32) / 255
-squeezenet_heatmap_colored = squeezenet_heatmap_colored.astype(np.float32) / 255
-mobilenet_heatmap_colored = mobilenet_heatmap_colored.astype(np.float32) / 255
-
-# Adjust the alpha value to control transparency
-alpha = (
-    0.1  # You can change this value to make the original image more or less transparent
-)
-
-
-# [0.38, 0.34, 0.28]
-weighted_heatmap = (
-    efficientnet_heatmap_colored * 0.38
-    + squeezenet_heatmap_colored * 0.34
-    + mobilenet_heatmap_colored * 0.28
-)
-
-
-# Overlay the colored heatmap on the original image
-final_output = cv2.addWeighted(rgb_img, 0.3, weighted_heatmap, 0.7, 0)
-
-# Save the final output
-cv2.imwrite("cam.jpg", (final_output * 255).astype(np.uint8))

extract.py → extract_gradcam.py

@@ -3,6 +3,7 @@ from pytorch_grad_cam.utils.image import show_cam_on_image, preprocess_image
 import cv2
 import numpy as np
 import torch
+import time
 import torch.nn as nn  # Replace with your model
 from configs import *
 import os, random
@@ -21,30 +22,36 @@ for child in model.features[-1]:
 if target_layer is None:
     raise ValueError("Invalid layer name: {}".format(target_layer))
 
-
+print(target_layer)
 
 def extract_gradcam(image_path=None, save_path=None):
     if image_path is None:
         for disease in CLASSES:
             print("Processing", disease)
-            for image_path in os.listdir(r'data\test\Task 1\{}'.format(disease)):
+            for image_path in os.listdir(r"data\test\Task 1\{}".format(disease)):
                 print("Processing", image_path)
-                image_path = r'data\test\Task 1\{}\{}'.format(disease, image_path)
-                image_name = image_path.split('.')[0].split('\\')[-1]
-                print("Processing", image_name)
+                image_path = r"data\test\Task 1\{}\{}".format(disease, image_path)
+                image_name = image_path.split(".")[0].split("\\")[-1]
                 rgb_img = cv2.imread(image_path, 1)
                 rgb_img = np.float32(rgb_img) / 255
-                input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+                input_tensor = preprocess_image(
+                    rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]
+                )
                 input_tensor = input_tensor.to(DEVICE)
+                input_tensor.requires_grad = True
 
                 # Create a GradCAMPlusPlus object
-                cam = GradCAMPlusPlus(model=model, target_layers=[target_layer], use_cuda=True)
+                cam = GradCAMPlusPlus(
+                    model=model, target_layers=[target_layer], use_cuda=True
+                )
 
                 # Generate the GradCAM heatmap
                 grayscale_cam = cam(input_tensor=input_tensor)[0]
 
                 # Apply a colormap to the grayscale heatmap
-                heatmap_colored = cv2.applyColorMap(np.uint8(255 * grayscale_cam), cv2.COLORMAP_JET)
+                heatmap_colored = cv2.applyColorMap(
+                    np.uint8(255 * grayscale_cam), cv2.COLORMAP_JET
+                )
 
                 # Ensure heatmap_colored has the same dtype as rgb_img
                 heatmap_colored = heatmap_colored.astype(np.float32) / 255
@@ -56,34 +63,46 @@ def extract_gradcam(image_path=None, save_path=None):
                 final_output = cv2.addWeighted(rgb_img, 0.3, heatmap_colored, 0.7, 0)
 
                 # Save the final output
-                os.makedirs(f'docs/efficientnet/gradcam/{disease}', exist_ok=True)
-                cv2.imwrite(f'docs/efficientnet/gradcam/{disease}/{image_name}.jpg', (final_output * 255).astype(np.uint8))
+                os.makedirs(f"docs/evaluation/gradcam/{disease}", exist_ok=True)
+                cv2.imwrite(
+                    f"docs/evaluation/gradcam/{disease}/{image_name}.jpg",
+                    (final_output * 255).astype(np.uint8),
+                )
     else:
-            rgb_img = cv2.imread(image_path, 1)
-            rgb_img = np.float32(rgb_img) / 255
-            input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
-            input_tensor = input_tensor.to(DEVICE)
+        rgb_img = cv2.imread(image_path, 1)
+        rgb_img = np.float32(rgb_img) / 255
+        input_tensor = preprocess_image(
+            rgb_img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]
+        )
+        input_tensor = input_tensor.to(DEVICE)
+        input_tensor.requires_grad = True
+
+        # Create a GradCAMPlusPlus object
+        cam = GradCAMPlusPlus(model=model, target_layers=[target_layer])
+
+        # Generate the GradCAM heatmap
+        grayscale_cam = cam(input_tensor=input_tensor)[0]
 
-            # Create a GradCAMPlusPlus object
-            cam = GradCAMPlusPlus(model=model, target_layers=[target_layer])
+        # Apply a colormap to the grayscale heatmap
+        heatmap_colored = cv2.applyColorMap(
+            np.uint8(255 * grayscale_cam), cv2.COLORMAP_JET
+        )
 
-            # Generate the GradCAM heatmap
-            grayscale_cam = cam(input_tensor=input_tensor)[0]
+        # Ensure heatmap_colored has the same dtype as rgb_img
+        heatmap_colored = heatmap_colored.astype(np.float32) / 255
 
-            # Apply a colormap to the grayscale heatmap
-            heatmap_colored = cv2.applyColorMap(np.uint8(255 * grayscale_cam), cv2.COLORMAP_JET)
+        # Adjust the alpha value to control transparency
+        alpha = 0.3  # You can change this value to make the original image more or less transparent
 
-            # Ensure heatmap_colored has the same dtype as rgb_img
-            heatmap_colored = heatmap_colored.astype(np.float32) / 255
+        # Overlay the colored heatmap on the original image
+        final_output = cv2.addWeighted(rgb_img, 0.3, heatmap_colored, 0.7, 0)
 
-            # Adjust the alpha value to control transparency
-            alpha = 0.3  # You can change this value to make the original image more or less transparent
+        # Save the final output
+        cv2.imwrite(save_path, (final_output * 255).astype(np.uint8))
 
-            # Overlay the colored heatmap on the original image
-            final_output = cv2.addWeighted(rgb_img, 0.3, heatmap_colored, 0.7, 0)
+        return save_path
 
-            # Save the final output
-            cv2.imwrite(save_path, (final_output * 255).astype(np.uint8))
-            
-            return save_path            
-            
+# start = time.time()
+# extract_gradcam()
+# end = time.time()
+# print("Time taken:", end - start)

lime_eval.py → extract_lime.py

@@ -1,12 +1,11 @@
+import os
 import numpy as np
 from lime.lime_image import LimeImageExplainer
 from PIL import Image
 import torch
-import torchvision.transforms as transforms
 import matplotlib.pyplot as plt
-from matplotlib.colors import Normalize
 from configs import *
-from sklearn.preprocessing import minmax_scale
+import time
 
 
 model = MODEL.to(DEVICE)
@@ -34,7 +33,6 @@ def generate_lime(image_path=None, save_path=None):
                 print("Processing", image_path)
                 image_path = r"data\test\Task 1\{}\{}".format(disease, image_path)
                 image_name = image_path.split(".")[0].split("\\")[-1]
-                print("Processing", image_name)
                 image = Image.open(image_path).convert("RGB")
                 image = preprocess(image)
                 image = image.unsqueeze(0)  # Add batch dimension
@@ -67,9 +65,9 @@ def generate_lime(image_path=None, save_path=None):
                 image = (image - np.min(image)) / (np.max(image) - np.min(image))
 
                 # image = Image.fromarray(image)
-                os.makedirs(f"docs/efficientnet/lime/{disease}", exist_ok=True)
-                # image.save(f'docs/efficientnet/lime/{disease}/{image_name}.jpg')
-                plt.imsave(f"docs/efficientnet/lime/{disease}/{image_name}.jpg", image)
+                os.makedirs(f"docs/evaluation/lime/{disease}", exist_ok=True)
+                # image.save(f'docs/evaluation/lime/{disease}/{image_name}.jpg')
+                plt.imsave(f"docs/evaluation/lime/{disease}/{image_name}.jpg", image)
 
     else:
         image = None
@@ -103,6 +101,15 @@ def generate_lime(image_path=None, save_path=None):
         image = (image - np.min(image)) / (np.max(image) - np.min(image))
 
         # image = Image.fromarray(image)
-        # os.makedirs(f"docs/efficientnet/lime/{disease}", exist_ok=True)
-        # image.save(f'docs/efficientnet/lime/{disease}/{image_name}.jpg')
+        # os.makedirs(f"docs/evaluation/lime/{disease}", exist_ok=True)
+        # image.save(f'docs/evaluation/lime/{disease}/{image_name}.jpg')
         plt.imsave(save_path, image)
+
+
+# start = time.time()
+
+# generate_lime()
+
+# end = time.time()
+
+# print("Time taken:", end - start)

genetric_algorithm.py

@@ -1,34 +1,38 @@
 import os
 import optuna
-from optuna.trial import TrialState
 import torch
 import torch.nn as nn
 import torch.optim as optim
+import torch.utils.data
 from configs import *
 import data_loader
 from torch.utils.tensorboard import SummaryWriter
+import time
 import numpy as np
-import pygad
-import pygad.torchga
+import random
 
 torch.cuda.empty_cache()
-model = MODEL.to(DEVICE)
+RANDOM_SEED1=42
+random.seed(RANDOM_SEED1)
+torch.cuda.manual_seed(RANDOM_SEED1)
+torch.manual_seed(RANDOM_SEED1)
+print("PyTorch Seed:", torch.initial_seed())
+print("Random Seed:", random.getstate()[1][0])
+print("PyTorch CUDA Seed:", torch.cuda.initial_seed())
+
 
-EPOCHS = 10
-N_TRIALS = 20
-TIMEOUT = 1800
-EARLY_STOPPING_PATIENCE = (
-    4  # Number of epochs with no improvement to trigger early stopping
-)
-NUM_GENERATIONS = 10
-SOL_PER_POP = 10  # Number of solutions in the population
-NUM_GENES = 2
-NUM_PARENTS_MATING = 4
+# Define the constants for genetic algorithm
+POPULATION_SIZE = 5
+MUTATION_RATE = 0.05
+CROSSOVER_RATE = 0.7
+NUM_GENERATIONS = 5
 
+EPOCHS = 5
+
+EARLY_STOPPING_PATIENCE = 4
 # Create a TensorBoard writer
 writer = SummaryWriter(log_dir="output/tensorboard/tuning")
 
-
 # Function to create or modify data loaders with the specified batch size
 def create_data_loaders(batch_size):
     train_loader, valid_loader = data_loader.load_data(
@@ -38,211 +42,204 @@ def create_data_loaders(batch_size):
     )
     return train_loader, valid_loader
 
+# Create a TensorBoard writer
+writer = SummaryWriter(log_dir="output/tensorboard/tuning")
+model = MODEL.to(DEVICE)
+# model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
 
-# Objective function for optimization
-def objective(trial):
-    global data_inputs, data_outputs
-
-    batch_size = trial.suggest_categorical("batch_size", [16, 32, 64])
-    train_loader, valid_loader = create_data_loaders(batch_size)
-
-    lr = trial.suggest_float("lr", 1e-5, 1e-3, log=True)
+def fitness_function(individual,model):
+    batch_size, lr,= individual
+    
+    # Assuming you have a model, optimizer, and loss function defined
+    model = model.to(DEVICE)
     optimizer = optim.Adam(model.parameters(), lr=lr)
     criterion = nn.CrossEntropyLoss()
+    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)
+    # Define your data loaders using the given batch_size
+    train_loader, valid_loader = create_data_loaders(batch_size)
+
+    # Training loop
+    for epoch in range(EPOCHS):
+        model.train()
+        for batch_idx, (data, target) in enumerate(train_loader, 0):
+            data, target = data.to(DEVICE), target.to(DEVICE)
+            optimizer.zero_grad()
+            if model.__class__.__name__ == "GoogLeNet":
+                output = model(data).logits
+            else:
+                output = model(data)
+            loss = criterion(output, target)
+            loss.backward()
+            optimizer.step()
+        scheduler.step()
+        # Validation loop
+        model.eval()
+        correct = 0
+        with torch.no_grad():
+            for batch_idx, (data, target) in enumerate(valid_loader, 0):
+                data, target = data.to(DEVICE), target.to(DEVICE)
+                data, targets_a, targets_b, lam = cutmix_data(data, target, alpha=1)
+                output = model(data)
+                pred = output.argmax(dim=1, keepdim=True)
+                correct += pred.eq(target.view_as(pred)).sum().item()
+
+        accuracy = correct / len(valid_loader.dataset)
+        print(f"Epoch {epoch + 1}/{EPOCHS}, Accuracy: {accuracy:.4f}")
+    return accuracy,
+
+def rand_bbox(size, lam):
+    W = size[2]
+    H = size[3]
+    cut_rat = np.sqrt(1.0 - lam)
+    cut_w = np.int_(W * cut_rat)
+    cut_h = np.int_(H * cut_rat)
+
+    # uniform
+    cx = np.random.randint(W)
+    cy = np.random.randint(H)
+
+    bbx1 = np.clip(cx - cut_w // 2, 0, W)
+    bby1 = np.clip(cy - cut_h // 2, 0, H)
+    bbx2 = np.clip(cx + cut_w // 2, 0, W)
+    bby2 = np.clip(cy + cut_h // 2, 0, H)
+
+    return bbx1, bby1, bbx2, bby2
+
+def cutmix_data(input, target, alpha=1.0):
+    if alpha > 0:
+        lam = np.random.beta(alpha, alpha)
+    else:
+        lam = 1
 
-    gamma = trial.suggest_float("gamma", 0.1, 0.9, step=0.1)
-    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
+    batch_size = input.size()[0]
+    index = torch.randperm(batch_size)
+    rand_index = torch.randperm(input.size()[0])
 
-    past_trials = 0  # Number of trials already completed
+    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
+    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
 
-    # Print best hyperparameters:
-    if past_trials > 0:
-        print("\nBest Hyperparameters:")
-        print(f"{study.best_trial.params}")
+    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
+    targets_a = target
+    targets_b = target[rand_index]
 
-    print(f"\n[INFO] Trial: {trial.number}")
-    print(f"Batch Size: {batch_size}")
-    print(f"Learning Rate: {lr}")
-    print(f"Gamma: {gamma}\n")
+    return input, targets_a, targets_b, lam
 
-    early_stopping_counter = 0
-    best_accuracy = 0.0
+def cutmix_criterion(criterion, outputs, targets_a, targets_b, lam):
+    return lam * criterion(outputs, targets_a) + (1 - lam) * criterion(outputs, targets_b)
 
+# Function to create or modify data loaders with the specified batch size
+def create_data_loaders(batch_size):
+    print(f"Batch Size (before conversion): {batch_size}")
+    batch_size = int(batch_size)  # Ensure batch_size is an integer
+    print(f"Batch Size (after conversion): {batch_size}")
+    train_loader, valid_loader = data_loader.load_data(
+        COMBINED_DATA_DIR + "1",
+        preprocess,
+        batch_size=batch_size,
+    )
+    return train_loader, valid_loader
+
+# Genetic algorithm initialization functions
+def create_individual():
+    lr = abs(np.random.uniform(0.0006, 0.0009))
+    print(f"Generated lr: {lr}")
+    return creator.Individual([
+        int(np.random.choice([32])),  # Choose a valid batch size
+        lr,  # lr in log scale between 1e-4 and 1e-2
+    ])
+# Genetic algorithm evaluation function
+def evaluate_individual(individual, model=MODEL):
+    batch_size, lr, = individual
+    lr=abs(lr)
+    # Assuming you have a model, optimizer, and loss function defined
+    model = model.to(DEVICE)
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+
+    # Define your data loaders using the given batch_size
+    train_loader, valid_loader = create_data_loaders(batch_size)
+
+    # Training loop
     for epoch in range(EPOCHS):
         model.train()
         for batch_idx, (data, target) in enumerate(train_loader, 0):
             data, target = data.to(DEVICE), target.to(DEVICE)
             optimizer.zero_grad()
-            output = model(data)
-            loss = criterion(output, target)
+            # Apply CutMix
+            data, targets_a, targets_b, lam = cutmix_data(data, target, alpha=1)
+            if model.__class__.__name__ == "GoogLeNet":
+                output = model(data).logits
+            else:
+                output = model(data)
+            loss = cutmix_criterion(criterion, output, targets_a, targets_b, lam)
             loss.backward()
             optimizer.step()
 
-        scheduler.step()
-
+        # Validation loop
         model.eval()
         correct = 0
         with torch.no_grad():
             for batch_idx, (data, target) in enumerate(valid_loader, 0):
                 data, target = data.to(DEVICE), target.to(DEVICE)
+                data, targets_a, targets_b, lam = cutmix_data(data, target, alpha=1)
                 output = model(data)
                 pred = output.argmax(dim=1, keepdim=True)
                 correct += pred.eq(target.view_as(pred)).sum().item()
 
         accuracy = correct / len(valid_loader.dataset)
 
-        # Log hyperparameters and accuracy to TensorBoard
-        writer.add_scalar("Accuracy", accuracy, trial.number)
-        writer.add_hparams(
-            {"batch_size": batch_size, "lr": lr, "gamma": gamma},
-            {"accuracy": accuracy},
-        )
-
-        print(f"[EPOCH {epoch + 1}] Accuracy: {accuracy:.4f}")
-
-        trial.report(accuracy, epoch)
-
-        if accuracy > best_accuracy:
-            best_accuracy = accuracy
-            early_stopping_counter = 0
-        else:
-            early_stopping_counter += 1
-
-        # Early stopping check
-        if early_stopping_counter >= EARLY_STOPPING_PATIENCE:
-            print(f"\nEarly stopping at epoch {epoch + 1}")
-            break
-
-    if trial.number > 10 and trial.params["lr"] < 1e-3 and best_accuracy < 0.7:
-        return float("inf")
-
-    past_trials += 1
-
-    return best_accuracy
-
-
-# Custom genetic algorithm
-def run_genetic_algorithm(fitness_func):
-    # Initial population
-    population = np.random.rand(SOL_PER_POP, NUM_GENES)  # Random initialization
-
-    # Run for a fixed number of generations
-    for generation in range(NUM_GENERATIONS):
-        # Calculate fitness for each solution in the population
-        fitness = np.array(
-            [fitness_func(solution, idx) for idx, solution in enumerate(population)]
-        )
-
-        # Get the index of the best solution
-        best_idx = np.argmax(fitness)
-        best_solution = population[best_idx]
-        best_fitness = fitness[best_idx]
-
-        # Print the best solution and fitness for this generation
-        print(f"Generation {generation + 1}:")
-        print("Best Solution:")
-        print("Learning Rate = {lr}".format(lr=best_solution[0]))
-        print("Gamma = {gamma}".format(gamma=best_solution[1]))
-        print("Best Fitness = {fitness}".format(fitness=best_fitness))
-
-        # Perform selection and crossover to create the next generation
-        population = selection_and_crossover(population, fitness)
-
-
-# Selection and crossover logic
-def selection_and_crossover(population, fitness):
-    # Perform tournament selection
-    parents = []
-    for _ in range(SOL_PER_POP):
-        tournament_idxs = np.random.choice(range(SOL_PER_POP), NUM_PARENTS_MATING)
-        tournament_fitness = [fitness[idx] for idx in tournament_idxs]
-        selected_parent_idx = tournament_idxs[np.argmax(tournament_fitness)]
-        parents.append(population[selected_parent_idx])
-
-    # Perform single-point crossover
-    offspring = []
-    for i in range(0, SOL_PER_POP, 2):
-        if i + 1 < SOL_PER_POP:
-            crossover_point = np.random.randint(0, NUM_GENES)
-            offspring.extend(
-                [
-                    np.concatenate(
-                        (parents[i][:crossover_point], parents[i + 1][crossover_point:])
-                    )
-                ]
-            )
-            offspring.extend(
-                [
-                    np.concatenate(
-                        (parents[i + 1][:crossover_point], parents[i][crossover_point:])
-                    )
-                ]
-            )
-
-    return np.array(offspring)
-
-
-# Modify callback function to log best accuracy
-def callback_generation(ga_instance):
-    global study
-
-    # Fetch the parameters of the best solution
-    solution, solution_fitness, _ = ga_instance.best_solution()
-    best_learning_rate, best_gamma = solution
-
-    # Report the best accuracy to Optuna study
-    study.set_user_attr("best_accuracy", solution_fitness)
-
-    # Print generation number and best fitness
-    print(
-        "Generation = {generation}".format(generation=ga_instance.generations_completed)
-    )
-    print("Best Fitness = {fitness}".format(fitness=solution_fitness))
-    print("Best Learning Rate = {lr}".format(lr=best_learning_rate))
-    print("Best Gamma = {gamma}".format(gamma=best_gamma))
+        # Log accuracy or other metrics as needed
+        writer.add_scalar("Accuracy", accuracy, epoch)
+
+        print(f"Epoch {epoch + 1}/{EPOCHS}, Accuracy: {accuracy:.4f}")
 
+    # Return the accuracy (or any other metric you want to optimize)
+    return (accuracy,)
 
 if __name__ == "__main__":
-    global study
     pruner = optuna.pruners.HyperbandPruner()
+
+    start_time = time.time()
     study = optuna.create_study(
         direction="maximize",
         pruner=pruner,
-        study_name="hyperparameter_tuning",
+        study_name="hyperparameter_optimization",
+        storage="sqlite:///" + MODEL.__class__.__name__ + ".sqlite3",
     )
 
-    # Define data_inputs and data_outputs
-    # You need to populate these with your own data
+    from deap import base, creator, tools, algorithms
 
-    # Define the loss function
-    loss_function = nn.CrossEntropyLoss()
+    creator.create("FitnessMax", base.Fitness, weights=(1.0,))
 
-    def fitness_func(solution, sol_idx):
-        global data_inputs, data_outputs, model, loss_function
+    creator.create("Individual", list, fitness=creator.FitnessMax)
 
-        learning_rate, momentum = solution
+    toolbox = base.Toolbox()
+    toolbox.register("individual", create_individual)
+    toolbox.register("population", tools.initRepeat, list, toolbox.individual)
+    toolbox.register("evaluate", fitness_function, model=model)
+    toolbox.register("mate", tools.cxTwoPoint)
+    toolbox.register("mutate", tools.mutGaussian, mu=0, sigma=0.1, indpb=MUTATION_RATE)
+    toolbox.register("select", tools.selTournament, tournsize=3)
 
-        # Update optimizer with the current learning rate and momentum
-        optimizer = torch.optim.SGD(
-            model.parameters(), lr=learning_rate, momentum=momentum
-        )
+    population = toolbox.population(n=POPULATION_SIZE)
 
-        # Load the model weights
-        model_weights_dict = pygad.torchga.model_weights_as_dict(
-            model=model, weights_vector=solution
-        )
-        model.load_state_dict(model_weights_dict)
+    for ind in population:
+        print(type(ind))
+        fitness_value = evaluate_individual(ind, model)
+        ind.fitness.values = (fitness_value[0],)
 
-        # Forward pass
-        predictions = model(data_inputs)
+    algorithms.eaSimple(population, toolbox, cxpb=CROSSOVER_RATE, mutpb=MUTATION_RATE, ngen=NUM_GENERATIONS, stats=None, halloffame=None, verbose=True)
 
-        # Calculate cross-entropy loss
-        loss = loss_function(predictions, data_outputs)
+    best_individual = tools.selBest(population, 1)[0]
+    best_batch_size, best_lr = best_individual
 
-        # Higher fitness for lower loss
-        solution_fitness = 1.0 / (loss.detach().numpy() + 1e-8)
+    best_accuracy = evaluate_individual(best_individual, model)
 
-        return solution_fitness
+    print("Best Hyperparameters:")
+    print(f"Batch Size: {best_batch_size}")
+    print(f"Learning Rate: {best_lr}")
+    print(f"Best Accuracy: {best_accuracy[0]}")
 
-    # Run the custom genetic algorithm
-    run_genetic_algorithm(fitness_func)
+    end_time = time.time()
+    tuning_duration = end_time - start_time
+    print(f"Hyperparameter optimization took {tuning_duration:.2f} seconds.")

lazy_predict.py

@@ -1,60 +0,0 @@
-import os
-import torch
-import torch.nn as nn
-import torch.optim as optim
-import matplotlib.pyplot as plt
-from models import *
-from torch.utils.tensorboard import SummaryWriter
-from configs import *
-import data_loader
-import numpy as np
-from lazypredict.Supervised import LazyClassifier
-from sklearn.utils import shuffle
-
-def extract_features_labels(loader):
-    data = []
-    labels = []
-    for inputs, labels_batch in loader:
-        for img in inputs:
-            data.append(img.view(-1).numpy())
-        labels.extend(labels_batch.numpy())
-    return np.array(data), np.array(labels)
-
-def load_and_preprocess_data():
-    train_loader, valid_loader = data_loader.load_data(
-        RAW_DATA_DIR + str(TASK),
-        AUG_DATA_DIR + str(TASK),
-        EXTERNAL_DATA_DIR + str(TASK),
-        preprocess,
-    )
-    return train_loader, valid_loader
-
-def initialize_model_optimizer_scheduler(train_loader, valid_loader):
-    model = MODEL.to(DEVICE)
-    criterion = nn.CrossEntropyLoss()
-    optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
-    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)
-    return model, criterion, optimizer, scheduler
-
-# Load and preprocess data
-train_loader, valid_loader = load_and_preprocess_data()
-
-# Initialize the model, criterion, optimizer, and scheduler
-model, criterion, optimizer, scheduler = initialize_model_optimizer_scheduler(train_loader, valid_loader)
-
-# Extract features and labels
-X_train, y_train = extract_features_labels(train_loader)
-X_valid, y_valid = extract_features_labels(valid_loader)
-
-# LazyClassifier
-clf = LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
-models, predictions = clf.fit(X_train, X_valid, y_train, y_valid)
-
-print("Models:", models)
-print("Predictions:", predictions)
-
-
-
-
-
-

lrp-eval.py

@@ -1,16 +0,0 @@
-import torch
-from torchvision.models import vgg16, VGG16_Weights
-from src.lrp import LRPModel
-from configs import *
-from PIL import Image
-
-
-image = Image.open(r'data\test\Task 1\Alzheimer Disease\0d846ee1-c90d-4ed5-8467-3550dd653858.png').convert("RGB")
-image = preprocess(image).unsqueeze(0)
-image = image.to(DEVICE)
-model = MODEL.to(DEVICE)
-print(dict(model.named_modules()))
-model.load_state_dict(torch.load(MODEL_SAVE_PATH, map_location=DEVICE))
-model.eval()
-lrp_model = LRPModel(model)
-r = lrp_model.forward(image)

models.py

@@ -1,69 +1,22 @@
-#######################################################
-# This file stores all the models used in the project.#
-#######################################################
-
-# Import all models from torchvision.models
-from torchvision.models import resnet50
-from torchvision.models import resnet18
-from torchvision.models import squeezenet1_0
-from torchvision.models import vgg16
-from torchvision.models import alexnet
-from torchvision.models import densenet121
-from torchvision.models import googlenet
-from torchvision.models import inception_v3
-from torchvision.models import mobilenet_v2
-from torchvision.models import mobilenet_v3_small
-from torchvision.models import mobilenet_v3_large
-from torchvision.models import shufflenet_v2_x0_5
-from torchvision.models import vgg11
-from torchvision.models import vgg11_bn
-from torchvision.models import vgg13
-from torchvision.models import vgg13_bn
-from torchvision.models import vgg16_bn
-from torchvision.models import vgg19_bn
-from torchvision.models import vgg19
-from torchvision.models import wide_resnet50_2
-from torchvision.models import wide_resnet101_2
-from torchvision.models import mnasnet0_5
-from torchvision.models import mnasnet0_75
-from torchvision.models import mnasnet1_0
-from torchvision.models import mnasnet1_3
-from torchvision.models import resnext50_32x4d
-from torchvision.models import resnext101_32x8d
-from torchvision.models import shufflenet_v2_x1_0
-from torchvision.models import shufflenet_v2_x1_5
-from torchvision.models import shufflenet_v2_x2_0
-from torchvision.models import squeezenet1_1
-from torchvision.models import efficientnet_v2_s
-from torchvision.models import efficientnet_v2_m
-from torchvision.models import efficientnet_v2_l
-from torchvision.models import efficientnet_b0
-from torchvision.models import efficientnet_b1
+from torchvision.models import efficientnet_b3, EfficientNet_B3_Weights
 import torch
 import torch.nn as nn
 
-class WeightedVoteEnsemble(nn.Module):
-    def __init__(self, models, weights):
-        super(WeightedVoteEnsemble, self).__init__()
-        self.models = models
-        self.weights = weights
 
-    def forward(self, x):
-        predictions = [model(x) for model in self.models]
-        weighted_predictions = torch.stack(
-            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
+class EfficientNetB3WithNorm(nn.Module):
+    def __init__(self, num_classes):
+        super(EfficientNetB3WithNorm, self).__init__()
+        efficientnet = efficientnet_b3(weights=EfficientNet_B3_Weights.DEFAULT)
+        self.features = efficientnet.features
+        self.classifier = nn.Sequential(
+            nn.Conv2d(1536, num_classes, kernel_size=1),
+            nn.BatchNorm2d(num_classes),  # add batch normalization
+            nn.ReLU(inplace=True),
+            nn.AdaptiveAvgPool2d((1, 1)),
         )
-        avg_predictions = weighted_predictions.sum(dim=0)
-        return avg_predictions
-
-
-def ensemble_predictions(models, image):
-    all_predictions = []
-
-    with torch.no_grad():
-        for model in models:
-            output = model(image)
-            all_predictions.append(output)
-
-    return torch.stack(all_predictions, dim=0).mean(dim=0)
 
+    def forward(self, x):
+        x = self.features(x)
+        x = self.classifier(x)
+        x = torch.flatten(x, 1)
+        return x

plot-gradcam.py

@@ -1,65 +0,0 @@
-# Plot a table, each column is a test image, separate to 7 tables (one for each disease), each column have 4 rows, one is disease name, one is gradcam, one is lime, one is original image
-
-import os
-import cv2
-import numpy as np
-import torch
-import torchvision.transforms as transforms
-import matplotlib.pyplot as plt
-from matplotlib.colors import Normalize
-from configs import *
-from sklearn.preprocessing import minmax_scale
-
-plt.rcParams["font.family"] = "Times New Roman"
-
-# Plot a table, each column is a test image, separate to 7 plot (one for each disease), each column have 4 rows, one is disease name, one is gradcam, one is lime, one is original image, the images are in 'docs/efficientnet/gradcam' and 'docs/efficientnet/lime' and 'data/test/Task 1'
-
-
-def plot_table():
-    diseases = CLASSES
-    diseases.sort()
-    # diseases = ["Atelectasis", "Cardiomegaly", "Consolidation", "Edema", "Effusion", "Emphysema", "Fibrosis", "Hernia", "Infiltration", "Mass", "Nodule", "Pleural_Thickening", "Pneumonia", "Pneumothorax"]
-    print(diseases)
-    fig, axs = plt.subplots(4, 14, figsize=(20, 10))
-    fig.tight_layout()
-    for i, disease in enumerate(diseases):
-        # Create a new plot
-        print("Processing", disease)
-        axs[0, i].axis("off")
-        axs[0, i].set_title(disease)
-        axs[1, i].axis("off")
-        axs[1, i].set_title("GradCAM")
-        axs[2, i].axis("off")
-        axs[2, i].set_title("LIME")
-        axs[3, i].axis("off")
-        axs[3, i].set_title("Original")
-        # For each image in test folder, there are corresponding ones in gradcam folder and lime folder, plot it accordingly
-        for j, image_path in enumerate(os.listdir(r"data\test\Task 1\{}".format(disease))):
-            print("Processing", image_path)
-            image_path = r"data\test\Task 1\{}\{}".format(disease, image_path)
-            image_name = image_path.split(".")[0].split("\\")[-1]
-            print("Processing", image_name)
-            # Plot the original image
-            image = cv2.imread(image_path, 1)
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-            axs[3, i].imshow(image)
-            # Plot the gradcam image
-            image = cv2.imread(
-                f"docs/efficientnet/gradcam/{disease}/{image_name}.jpg", 1
-            )
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-            axs[1, i].imshow(image)
-            # Plot the lime image
-            image = cv2.imread(
-                f"docs/efficientnet/lime/{disease}/{image_name}.jpg", 1
-            )
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-            axs[2, i].imshow(image)
-            # # Plot the disease name
-            # axs[0, i].text(0.5, 0.5, disease, horizontalalignment="center")
-        plt.savefig("docs/efficientnet/table.png")
-        plt.show()
-    
-if __name__ == "__main__":
-    plot_table()
-    

plot_structure.py

@@ -0,0 +1,20 @@
+# from torchview import draw_graph
+from torchviz import make_dot
+from configs import *
+import os
+# import graphviz
+
+# when running on VSCode run the below command
+# svg format on vscode does not give desired result
+# graphviz.set_jupyter_format("png")
+
+model = EfficientNetB3WithNorm(num_classes=7)
+
+batch_size = 2
+# device='meta' -> no memory is consumed for visualization
+# model_graph = draw_graph(model, input_size=(32, 3, 224, 224), save_graph=True, filename="model_graph.png")
+# model_graph.visual_graph
+
+model_graph = make_dot(
+    model(torch.randn(batch_size, 3, 224, 224)), params=dict(model.named_parameters())
+).render("torchviz", format="png")

plot_training_metrics.py

@@ -0,0 +1,36 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+
+# Load data from the CSV file
+df = pd.read_csv('training_metrics.csv')
+
+# Extract data
+epochs = df['Epoch']
+train_loss = df['Train Loss']
+train_accuracy = df['Train Accuracy']
+validation_loss = df['Validation Loss']
+validation_accuracy = df['Validation Accuracy']
+
+# Create subplots for loss and accuracy
+plt.figure(figsize=(12, 5))
+
+# Loss subplot
+plt.subplot(1, 2, 1)
+plt.plot(epochs, train_loss, label='Train Loss', marker='o')
+plt.plot(epochs, validation_loss, label='Validation Loss', marker='o')
+plt.title('Training and Validation Loss')
+plt.xlabel('Epoch')
+plt.ylabel('Loss')
+plt.legend()
+
+# Accuracy subplot
+plt.subplot(1, 2, 2)
+plt.plot(epochs, train_accuracy, label='Train Accuracy', marker='o')
+plt.plot(epochs, validation_accuracy, label='Validation Accuracy', marker='o')
+plt.title('Training and Validation Accuracy')
+plt.xlabel('Epoch')
+plt.ylabel('Accuracy')
+plt.legend()
+
+plt.tight_layout()
+plt.show()

shap_eval.py

@@ -1,37 +0,0 @@
-# Import necessary libraries
-import shap
-import torch
-import numpy as np
-
-# Load your EfficientNetB3 model
-from torchvision import models
-
-# Load your test data
-from data_loader import load_test_data  # Replace with your actual data loader function
-from configs import *
-
-# Define your EfficientNetB3 model and load its pre-trained weights
-model = MODEL
-
-# Set your model to evaluation mode
-model.eval()
-
-# Load your test data using your data loader
-test_loader = load_test_data(TEST_DATA_DIR + "1", preprocess)  # Replace with your test data loader
-
-# Choose a specific image from the test dataset
-image, _ = next(iter(test_loader))
-
-# Make sure your model and input data are on the same device (CPU or GPU)
-device = DEVICE
-model = model.to(device)
-image = image.to(device)
-
-# Initialize an explainer for your model using SHAP's DeepExplainer
-explainer = shap.DeepExplainer(model, data=test_loader)
-
-# Calculate SHAP values for your chosen image
-shap_values = explainer(image)
-
-# Summarize the feature importance for the specific image
-shap.summary_plot(shap_values, image)

test.py

@@ -1,223 +0,0 @@
-import sys
-import torch
-import torch.nn as nn
-from PIL import Image
-import os
-from configs import *
-from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
-import matplotlib.pyplot as plt
-import random
-from itertools import product
-
-random.seed(RANDOM_SEED)
-torch.cuda.manual_seed(RANDOM_SEED)
-torch.manual_seed(RANDOM_SEED)
-print("PyTorch Seed:", torch.initial_seed())
-print("Random Seed:", random.getstate()[1][0])
-print("PyTorch CUDA Seed:", torch.cuda.initial_seed())
-
-# Define your model paths
-# Load your pre-trained models
-model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
-model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
-model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
-model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
-
-# Define the class labels
-class_labels = CLASSES
-
-# Define your test data folder path
-test_data_folder = "data/test/Task 1/"
-
-
-# Put models in evaluation mode
-def set_models_eval(models):
-    for model in models:
-        model.eval()
-
-
-# Define the ensemble model using a list of models
-class WeightedVoteEnsemble(nn.Module):
-    def __init__(self, models, weights):
-        super(WeightedVoteEnsemble, self).__init__()
-        self.models = models
-        self.weights = weights
-
-    def forward(self, x):
-        predictions = [model(x) for model in self.models]
-        weighted_predictions = torch.stack(
-            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
-        )
-        avg_predictions = weighted_predictions.sum(dim=0)
-        return avg_predictions
-
-
-def ensemble_predictions(models, image):
-    all_predictions = []
-
-    with torch.no_grad():
-        for model in models:
-            output = model(image)
-            all_predictions.append(output)
-
-    return torch.stack(all_predictions, dim=0).mean(dim=0)
-
-
-# Load a single image and make predictions
-def evaluate_image(models, image_path, transform=preprocess):
-    image = Image.open(image_path).convert("RGB")
-    image = transform(image).unsqueeze(0)
-    image = image.to(DEVICE)
-    outputs = ensemble_predictions(models, image)
-
-    return outputs.argmax(dim=1).item()
-
-
-# Evaluate and plot a confusion matrix for an ensemble of models
-def evaluate_and_plot_confusion_matrix(models, test_data_folder):
-    all_predictions = []
-    true_labels = []
-
-    with torch.no_grad():
-        for class_label in class_labels:
-            class_path = os.path.join(test_data_folder, class_label)
-            for image_file in os.listdir(class_path):
-                image_path = os.path.join(class_path, image_file)
-                # print(image_path)
-                predicted_label = evaluate_image(models, image_path, preprocess)
-                all_predictions.append(predicted_label)
-                true_labels.append(class_labels.index(class_label))
-
-    # Print accuracy
-    accuracy = (
-        (torch.tensor(all_predictions) == torch.tensor(true_labels)).float().mean()
-    )
-    print("Accuracy:", accuracy)
-
-    # Create the confusion matrix
-    cm = confusion_matrix(true_labels, all_predictions)
-
-    # Plot the confusion matrix
-    display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels)
-    display.plot(cmap=plt.cm.Blues, values_format="d")
-
-    # Show the plot
-    plt.show()
-
-    return accuracy
-
-# Set the models to evaluation mode
-set_models_eval([model1, model2, model3])
-
-# Define different weight configurations
-# [SqueezeNet, EfficientNetB2WithDropout, MobileNetV2WithDropout]
-weights_configurations = [
-    # Random set of weights using random.random() and all weights sum to 1
-    [
-        random.randrange(1, 10) / 10,
-        random.randrange(1, 10) / 10,
-        random.randrange(1, 10) / 10,
-    ],
-]
-
-
-## NOTE OF PREVIOUS WEIGHTS
-# Best weights: [0.2, 0.3, 0.5] with accuracy: 0.9428571462631226 at iteration: 15 with torch seed: 28434738589300 and random seed: 3188652458777471118 and torch cuda seed: None
-
-
-best_weights = {
-    "weights": 0,
-    "accuracy": 0,
-    "iteration": 0,
-    "torch_seed": 0,
-    "random_seed": 0,
-    "torch_cuda_seed": 0,
-}
-
-i = 0
-
-# weights_hist = []
-
-target_sum = 1.0
-number_of_numbers = 3
-lower_limit = 0.20
-upper_limit = 0.9
-step = 0.1
-
-valid_combinations = []
-
-# Generate all unique combinations of three numbers with values to two decimal places
-range_values = list(range(int(lower_limit * 100), int(upper_limit * 100) + 1))
-for combo in product(range_values, repeat=number_of_numbers):
-    combo_float = [x / 100.0 for x in combo]
-
-    # Check if the sum of the numbers is equal to 1
-    if sum(combo_float) == target_sum:
-        valid_combinations.append(combo_float)
-
-# Calculate the total number of possibilities
-total_possibilities = len(valid_combinations)
-
-print("Total number of possibilities:", total_possibilities)
-
-valid_combinations = [[0.37, 0.34, 0.29]]
-
-for weights in valid_combinations:
-    # while True:
-    print("---------------------------")
-    print("Iteration:", i)
-    # Should iterate until all possible weights are exhausted
-    # Create an ensemble model with weighted voting
-
-    random.seed(RANDOM_SEED)
-    torch.cuda.manual_seed(RANDOM_SEED)
-    torch.manual_seed(RANDOM_SEED)
-    # print("PyTorch Seed:", torch.initial_seed())
-    # weights_hist.append(weights)
-    weighted_vote_ensemble_model = WeightedVoteEnsemble(
-        # [model1, model2, model3], weights
-        [model1, model2, model3],
-        weights,
-    )
-    # print("Weights:", weights)
-    print("Weights:", weights)
-    # Call the evaluate_and_plot_confusion_matrix function with your models and test data folder
-    accuracy = evaluate_and_plot_confusion_matrix(
-        [weighted_vote_ensemble_model], test_data_folder
-    )
-    # Convert tensor to float
-    accuracy = accuracy.item()
-    if accuracy > best_weights["accuracy"]:
-        # best_weights["weights"] = weights
-        best_weights["weights"] = weights
-        best_weights["accuracy"] = accuracy
-        best_weights["iteration"] = i
-        best_weights["torch_seed"] = torch.initial_seed()
-        seed = random.randrange(sys.maxsize)
-        rng = random.Random(seed)
-        best_weights["random_seed"] = seed
-        best_weights["torch_cuda_seed"] = torch.cuda.initial_seed()
-
-    print(
-        "Best weights:",
-        best_weights["weights"],
-        "with accuracy:",
-        best_weights["accuracy"],
-        "at iteration:",
-        best_weights["iteration"],
-        "with torch seed:",
-        best_weights["torch_seed"],
-        "and random seed:",
-        best_weights["random_seed"],
-        "and torch cuda seed:",
-        best_weights["torch_cuda_seed"],
-    )
-    i += 1
-
-
-torch.save(
-    weighted_vote_ensemble_model.state_dict(),
-    "output/checkpoints/WeightedVoteEnsemble.pth",
-)

test-speed.py → test_speed.py

@@ -1,3 +1,4 @@
+import os
 from gradio_client import Client
 import time
 import csv
@@ -6,7 +7,7 @@ from matplotlib import rcParams
 from configs import *
 from PIL import Image
 
-client = Client("https://cycool29-handetect.hf.space/")
+client = Client("https://cycool29-spiralsense.hf.space/")
 
 list_of_times = []
 
@@ -25,33 +26,35 @@ for disease in CLASSES:
         image_path = r"data\test\Task 1\{}\{}".format(disease, image_path)
         start_time = time.time()
         result = client.predict(
-                        image_path,	
-                        api_name="/predict"
+            image_path,
+            False,  
+            False,  
+            fn_index=0,
         )
         time_taken = time.time() - start_time
         list_of_times.append(time_taken)
         print("Time taken:", time_taken)
 
         # Log to csv
-        with open('log.csv', 'a', newline='') as file:
+        with open("log.csv", "a", newline="") as file:
             writer = csv.writer(file)
             writer.writerow([disease])
             writer.writerow([image_path])
             writer.writerow([time_taken])
-            
 
-print("Average time taken:", sum(list_of_times)/len(list_of_times))
+
+print("Average time taken:", sum(list_of_times) / len(list_of_times))
 print("Max time taken:", max(list_of_times))
 print("Min time taken:", min(list_of_times))
 print("Total time taken:", sum(list_of_times))
-print("Median time taken:", sorted(list_of_times)[len(list_of_times)//2])
+print("Median time taken:", sorted(list_of_times)[len(list_of_times) // 2])
 
 # Plot the histogram
 plt.hist(list_of_times, bins=10)
 plt.xlabel("Time taken (s)")
 plt.ylabel("Frequency")
-plt.title("Time taken to process each image")
-plt.savefig("docs/efficientnet/time_taken_for_web.png")
+plt.title("Time Taken to Process Each Image (Web)")
+plt.savefig("docs/evaluation/time_taken_for_web.png")
 
 
 # Now is local
@@ -72,22 +75,22 @@ for disease in CLASSES:
         print("Time taken:", time_taken)
 
         # Log to csv
-        with open('log.csv', 'a', newline='') as file:
+        with open("log.csv", "a", newline="") as file:
             writer = csv.writer(file)
             writer.writerow([disease])
             writer.writerow([image_path])
             writer.writerow([time_taken])
 
 
-print("Average time taken local:", sum(list_of_times)/len(list_of_times))
+print("Average time taken local:", sum(list_of_times) / len(list_of_times))
 print("Max time taken local:", max(list_of_times))
 print("Min time taken local:", min(list_of_times))
 print("Total time taken local:", sum(list_of_times))
-print("Median time taken local:", sorted(list_of_times)[len(list_of_times)//2])
+print("Median time taken local:", sorted(list_of_times)[len(list_of_times) // 2])
 
 # Plot the histogram
 plt.hist(list_of_times, bins=10)
-plt.xlabel("Time taken (s) local")
-plt.ylabel("Frequency local")
-plt.title("Time taken to process each image local")
-plt.savefig("docs/efficientnet/time_taken_for_local.png")
+plt.xlabel("Time taken (s)")
+plt.ylabel("Frequency")
+plt.title("Time taken to Process Each Image (Local)")
+plt.savefig("docs/evaluation/time_taken_for_local.png")

train-svm.py

@@ -1,101 +0,0 @@
-import os
-import numpy as np
-from sklearn import svm
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
-from skimage.io import imread
-from skimage.transform import resize
-from sklearn.model_selection import train_test_split, RandomizedSearchCV
-from scipy.stats import uniform
-from configs import *
-
-# Set the path to your dataset folder, where each subfolder represents a class
-dataset_path = COMBINED_DATA_DIR + str(1)
-
-
-# Function to load, resize, and convert images to grayscale
-def load_resize_and_convert_to_gray(folder, target_size=(100, 100)):
-    images = []
-    for filename in os.listdir(folder):
-        img_path = os.path.join(folder, filename)
-        if os.path.isfile(img_path):
-            img = imread(img_path, as_gray=True)
-            img = resize(img, target_size, anti_aliasing=True)
-            images.append(img)
-    return images
-
-
-# Load, resize, and convert images to grayscale from folders
-X = []  # List to store images
-y = []  # List to store corresponding labels
-
-class_folders = os.listdir(dataset_path)
-class_folders.sort()  # Sort the class folders to ensure consistent class ordering
-
-for class_folder in class_folders:
-    class_path = os.path.join(dataset_path, class_folder)
-    if os.path.isdir(class_path):
-        images = load_resize_and_convert_to_gray(class_path)
-        X.extend(images)
-        y.extend([class_folder] * len(images))  # Assign labels based on folder name
-
-# Convert data to NumPy arrays
-X = np.array(X)
-y = np.array(y)
-
-# Split the dataset into training and testing sets
-X_train, X_test, y_train, y_test = train_test_split(
-    X, y, test_size=0.2, random_state=42
-)
-
-# Define the parameter distributions for random search
-param_dist = {
-    "C": uniform(loc=0, scale=10),  # Randomly sample from [0, 10]
-    "kernel": ["linear", "rbf", "poly"],
-    "gamma": uniform(loc=0.001, scale=0.1),  # Randomly sample from [0.001, 0.1]
-}
-
-# Flatten the images to a 1D array
-X_train_flat = X_train.reshape(X_train.shape[0], -1)
-X_test_flat = X_test.reshape(X_test.shape[0], -1)
-
-# Create an SVM classifier
-svm_classifier = svm.SVC()
-
-# Perform Randomized Search with cross-validation
-random_search = RandomizedSearchCV(
-    svm_classifier,
-    param_distributions=param_dist,
-    n_iter=50,
-    cv=5,
-    n_jobs=-1,
-    verbose=2,
-    random_state=42,
-)
-
-# Fit the Randomized Search on the training data
-random_search.fit(X_train_flat, y_train)
-
-# Print the best hyperparameters
-print("Best Hyperparameters:")
-print(random_search.best_params_)
-
-# Get the best SVM model with the tuned hyperparameters
-best_svm_model = random_search.best_estimator_
-
-# Evaluate the best model on the test set
-y_pred = best_svm_model.predict(X_test_flat)
-
-# Calculate accuracy and other metrics
-accuracy = accuracy_score(y_test, y_pred)
-print("Accuracy:", accuracy)
-
-# Confusion Matrix
-conf_matrix = confusion_matrix(y_test, y_pred)
-print("Confusion Matrix:\n", conf_matrix)
-
-# You can also print other classification metrics like precision, recall, and F1-score
-from sklearn.metrics import classification_report
-
-report = classification_report(y_test, y_pred)
-print("Classification Report:\n", report)

weight_averaging.py

@@ -1,235 +0,0 @@
-import sys
-import torch
-import torch.nn as nn
-from PIL import Image
-import os
-from configs import *
-from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
-import matplotlib.pyplot as plt
-import random
-from itertools import product
-
-random.seed(RANDOM_SEED)
-torch.cuda.manual_seed(RANDOM_SEED)
-torch.manual_seed(RANDOM_SEED)
-print("PyTorch Seed:", torch.initial_seed())
-print("Random Seed:", random.getstate()[1][0])
-print("PyTorch CUDA Seed:", torch.cuda.initial_seed())
-
-print("DEVICE:", DEVICE)
-
-# Define your model paths
-# Load your pre-trained models
-model2 = EfficientNetB3WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB3WithDropout.pth"))
-model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
-model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
-model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))
-model4 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
-model4.load_state_dict(torch.load("output\checkpoints\EfficientNetB2WithDropout.pth"))
-
-models = [model1, model2, model3, model4]
-
-# Define the class labels
-class_labels = CLASSES
-
-# Define your test data folder path
-test_data_folder = "data/test/Task 1/"
-
-
-# Put models in evaluation mode
-def set_models_eval(models):
-    for model in models:
-        model.eval()
-
-
-# Define the ensemble model using a list of models
-class WeightedVoteEnsemble(nn.Module):
-    def __init__(self, models, weights):
-        super(WeightedVoteEnsemble, self).__init__()
-        self.models = models
-        self.weights = weights
-
-    def forward(self, x):
-        predictions = [model(x) for model in self.models]
-        weighted_predictions = torch.stack(
-            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
-        )
-        avg_predictions = weighted_predictions.sum(dim=0)
-        return avg_predictions
-
-
-def ensemble_predictions(models, image):
-    all_predictions = []
-
-    with torch.no_grad():
-        for model in models:
-            output = model(image)
-            all_predictions.append(output)
-
-    return torch.stack(all_predictions, dim=0).mean(dim=0)
-
-
-# Load a single image and make predictions
-def evaluate_image(models, image_path, transform=preprocess):
-    image = Image.open(image_path).convert("RGB")
-    image = transform(image).unsqueeze(0)
-    image = image.to(DEVICE)
-    outputs = ensemble_predictions(models, image)
-
-    return outputs.argmax(dim=1).item()
-
-
-# Evaluate and plot a confusion matrix for an ensemble of models
-def evaluate_and_plot_confusion_matrix(models, test_data_folder):
-    all_predictions = []
-    true_labels = []
-
-    with torch.no_grad():
-        for class_label in class_labels:
-            class_path = os.path.join(test_data_folder, class_label)
-            for image_file in os.listdir(class_path):
-                image_path = os.path.join(class_path, image_file)
-                # print(image_path)
-                predicted_label = evaluate_image(models, image_path, preprocess)
-                all_predictions.append(predicted_label)
-                true_labels.append(class_labels.index(class_label))
-
-    # Print accuracy
-    accuracy = (
-        (torch.tensor(all_predictions) == torch.tensor(true_labels)).float().mean()
-    )
-    print("Accuracy:", accuracy)
-
-    # Create the confusion matrix
-    # cm = confusion_matrix(true_labels, all_predictions)
-
-    # # Plot the confusion matrix
-    # display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels)
-    # display.plot(cmap=plt.cm.Blues, values_format="d")
-
-    # # Show the plot
-    # plt.show()
-
-    return accuracy
-
-# Set the models to evaluation mode
-set_models_eval(models)
-
-# Define different weight configurations
-# [SqueezeNet, EfficientNetB2WithDropout, MobileNetV2WithDropout]
-weights_configurations = [
-    # Random set of weights using random.random() and all weights sum to 1
-    [
-        random.randrange(1, 10) / 10,
-        random.randrange(1, 10) / 10,
-        random.randrange(1, 10) / 10,
-    ],
-]
-
-
-## NOTE OF PREVIOUS WEIGHTS
-# Best weights: [0.2, 0.3, 0.5] with accuracy: 0.9428571462631226 at iteration: 15 with torch seed: 28434738589300 and random seed: 3188652458777471118 and torch cuda seed: None
-
-
-best_weights = {
-    "weights": 0,
-    "accuracy": 0,
-    "iteration": 0,
-    "torch_seed": 0,
-    "random_seed": 0,
-    "torch_cuda_seed": 0,
-}
-
-i = 0
-
-# weights_hist = []
-
-target_sum = 1.0
-number_of_numbers = 4
-lower_limit = 0.2
-upper_limit = 0.8
-step = 0.01
-
-valid_combinations = []
-
-# Generate all unique combinations of four numbers with values to two decimal places
-for combination in product(
-    *[range(int(lower_limit * 100), int(upper_limit * 100) + 1)] * number_of_numbers
-):
-    # Convert the combination to a list of floats
-    combination = [float(number) / 100 for number in combination]
-    # Check if the sum of the combination is equal to the target sum
-    if sum(combination) == target_sum:
-        # Add the combination to the list of valid combinations
-        valid_combinations.append(combination)
-                
-
-# Calculate the total number of possibilities
-total_possibilities = len(valid_combinations)
-
-print("Total number of possibilities:", total_possibilities)
-
-# valid_combinations = [[0.3, 0.5, 0.2]]
-# 0.38 for SqueezeNet, 0.34 for EfficientNetB2WithDropout, 0.28 for MobileNetV2WithDropout
-best_weighted_vote_ensemble_model = None
-
-for weights in valid_combinations:
-# while True:
-    print("---------------------------")
-    print("Iteration:", i)
-    # Should iterate until all possible weights are exhausted
-    # Create an ensemble model with weighted voting
-
-    random.seed(RANDOM_SEED)
-    torch.cuda.manual_seed(RANDOM_SEED)
-    torch.manual_seed(RANDOM_SEED)
-    # print("PyTorch Seed:", torch.initial_seed())
-    # weights_hist.append(weights)
-    weighted_vote_ensemble_model = WeightedVoteEnsemble(
-        # [model1, model2, model3], weights
-        models,
-        weights,
-    )
-    # print("Weights:", weights)
-    print("Weights:", weights)
-    # Call the evaluate_and_plot_confusion_matrix function with your models and test data folder
-    accuracy = evaluate_and_plot_confusion_matrix(
-        [weighted_vote_ensemble_model], test_data_folder
-    )
-    # Convert tensor to float
-    accuracy = accuracy.item()
-    if accuracy > best_weights["accuracy"]:
-        # best_weights["weights"] = weights
-        best_weights["weights"] = weights
-        best_weights["accuracy"] = accuracy
-        best_weights["iteration"] = i
-        best_weights["torch_seed"] = torch.initial_seed()
-        seed = random.randrange(sys.maxsize)
-        rng = random.Random(seed)
-        best_weights["random_seed"] = seed
-        best_weights["torch_cuda_seed"] = torch.cuda.initial_seed()
-        best_weighted_vote_ensemble_model = weighted_vote_ensemble_model
-
-    print(
-        "Best weights:",
-        best_weights["weights"],
-        "with accuracy:",
-        best_weights["accuracy"],
-        "at iteration:",
-        best_weights["iteration"],
-        "with torch seed:",
-        best_weights["torch_seed"],
-        "and random seed:",
-        best_weights["random_seed"],
-        "and torch cuda seed:",
-        best_weights["torch_cuda_seed"],
-    )
-    i += 1
-
-
-torch.save(
-    best_weighted_vote_ensemble_model.state_dict(),
-    "output/checkpoints/WeightedVoteEnsemble.pth",
-)