test.py

import sys
import torch
import torch.nn as nn
from PIL import Image
import os
from configs import *
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
import random
from itertools import product

random.seed(RANDOM_SEED)
torch.cuda.manual_seed(RANDOM_SEED)
torch.manual_seed(RANDOM_SEED)
print("PyTorch Seed:", torch.initial_seed())
print("Random Seed:", random.getstate()[1][0])
print("PyTorch CUDA Seed:", torch.cuda.initial_seed())

# Define your model paths
# Load your pre-trained models
model2 = EfficientNetB2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
model2.load_state_dict(torch.load("output/checkpoints/EfficientNetB2WithDropout.pth"))
model1 = SqueezeNet1_0WithSE(num_classes=NUM_CLASSES).to(DEVICE)
model1.load_state_dict(torch.load("output/checkpoints/SqueezeNet1_0WithSE.pth"))
model3 = MobileNetV2WithDropout(num_classes=NUM_CLASSES).to(DEVICE)
model3.load_state_dict(torch.load("output\checkpoints\MobileNetV2WithDropout.pth"))

# Define the class labels
class_labels = CLASSES

# Define your test data folder path
test_data_folder = "data/test/Task 1/"


# Put models in evaluation mode
def set_models_eval(models):
    for model in models:
        model.eval()


# Define the ensemble model using a list of models
class WeightedVoteEnsemble(nn.Module):
    def __init__(self, models, weights):
        super(WeightedVoteEnsemble, self).__init__()
        self.models = models
        self.weights = weights

    def forward(self, x):
        predictions = [model(x) for model in self.models]
        weighted_predictions = torch.stack(
            [w * pred for w, pred in zip(self.weights, predictions)], dim=0
        )
        avg_predictions = weighted_predictions.sum(dim=0)
        return avg_predictions


def ensemble_predictions(models, image):
    all_predictions = []

    with torch.no_grad():
        for model in models:
            output = model(image)
            all_predictions.append(output)

    return torch.stack(all_predictions, dim=0).mean(dim=0)


# Load a single image and make predictions
def evaluate_image(models, image_path, transform=preprocess):
    image = Image.open(image_path).convert("RGB")
    image = transform(image).unsqueeze(0)
    image = image.to(DEVICE)
    outputs = ensemble_predictions(models, image)

    return outputs.argmax(dim=1).item()


# Evaluate and plot a confusion matrix for an ensemble of models
def evaluate_and_plot_confusion_matrix(models, test_data_folder):
    all_predictions = []
    true_labels = []

    with torch.no_grad():
        for class_label in class_labels:
            class_path = os.path.join(test_data_folder, class_label)
            for image_file in os.listdir(class_path):
                image_path = os.path.join(class_path, image_file)
                # print(image_path)
                predicted_label = evaluate_image(models, image_path, preprocess)
                all_predictions.append(predicted_label)
                true_labels.append(class_labels.index(class_label))

    # Print accuracy
    accuracy = (
        (torch.tensor(all_predictions) == torch.tensor(true_labels)).float().mean()
    )
    print("Accuracy:", accuracy)

    # Create the confusion matrix
    cm = confusion_matrix(true_labels, all_predictions)

    # Plot the confusion matrix
    display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels)
    display.plot(cmap=plt.cm.Blues, values_format="d")

    # Show the plot
    plt.show()

    return accuracy

# Set the models to evaluation mode
set_models_eval([model1, model2, model3])

# Define different weight configurations
# [SqueezeNet, EfficientNetB2WithDropout, MobileNetV2WithDropout]
weights_configurations = [
    # Random set of weights using random.random() and all weights sum to 1
    [
        random.randrange(1, 10) / 10,
        random.randrange(1, 10) / 10,
        random.randrange(1, 10) / 10,
    ],
]


## NOTE OF PREVIOUS WEIGHTS
# Best weights: [0.2, 0.3, 0.5] with accuracy: 0.9428571462631226 at iteration: 15 with torch seed: 28434738589300 and random seed: 3188652458777471118 and torch cuda seed: None


best_weights = {
    "weights": 0,
    "accuracy": 0,
    "iteration": 0,
    "torch_seed": 0,
    "random_seed": 0,
    "torch_cuda_seed": 0,
}

i = 0

# weights_hist = []

target_sum = 1.0
number_of_numbers = 3
lower_limit = 0.20
upper_limit = 0.9
step = 0.1

valid_combinations = []

# Generate all unique combinations of three numbers with values to two decimal places
range_values = list(range(int(lower_limit * 100), int(upper_limit * 100) + 1))
for combo in product(range_values, repeat=number_of_numbers):
    combo_float = [x / 100.0 for x in combo]

    # Check if the sum of the numbers is equal to 1
    if sum(combo_float) == target_sum:
        valid_combinations.append(combo_float)

# Calculate the total number of possibilities
total_possibilities = len(valid_combinations)

print("Total number of possibilities:", total_possibilities)

valid_combinations = [[0.37, 0.34, 0.29]]

for weights in valid_combinations:
    # while True:
    print("---------------------------")
    print("Iteration:", i)
    # Should iterate until all possible weights are exhausted
    # Create an ensemble model with weighted voting

    random.seed(RANDOM_SEED)
    torch.cuda.manual_seed(RANDOM_SEED)
    torch.manual_seed(RANDOM_SEED)
    # print("PyTorch Seed:", torch.initial_seed())
    # weights_hist.append(weights)
    weighted_vote_ensemble_model = WeightedVoteEnsemble(
        # [model1, model2, model3], weights
        [model1, model2, model3],
        weights,
    )
    # print("Weights:", weights)
    print("Weights:", weights)
    # Call the evaluate_and_plot_confusion_matrix function with your models and test data folder
    accuracy = evaluate_and_plot_confusion_matrix(
        [weighted_vote_ensemble_model], test_data_folder
    )
    # Convert tensor to float
    accuracy = accuracy.item()
    if accuracy > best_weights["accuracy"]:
        # best_weights["weights"] = weights
        best_weights["weights"] = weights
        best_weights["accuracy"] = accuracy
        best_weights["iteration"] = i
        best_weights["torch_seed"] = torch.initial_seed()
        seed = random.randrange(sys.maxsize)
        rng = random.Random(seed)
        best_weights["random_seed"] = seed
        best_weights["torch_cuda_seed"] = torch.cuda.initial_seed()

    print(
        "Best weights:",
        best_weights["weights"],
        "with accuracy:",
        best_weights["accuracy"],
        "at iteration:",
        best_weights["iteration"],
        "with torch seed:",
        best_weights["torch_seed"],
        "and random seed:",
        best_weights["random_seed"],
        "and torch cuda seed:",
        best_weights["torch_cuda_seed"],
    )
    i += 1


torch.save(
    weighted_vote_ensemble_model.state_dict(),
    "output/checkpoints/WeightedVoteEnsemble.pth",
)