work in progress

.gitignore

@@ -0,0 +1,4 @@
+/env
+__pycache__/
+
+test.jpg

app.py

@@ -0,0 +1,218 @@
+import streamlit as st
+import cv2
+import sys
+import argparse
+import numpy as np
+import os
+import json
+import torch
+import torch.nn.functional as F
+import detectron2.data.transforms as T
+import torchvision
+from collections import OrderedDict
+from scipy import spatial
+import matplotlib.pyplot as plt
+
+from detectron2.engine import DefaultPredictor
+from detectron2.utils.visualizer import Visualizer
+from detectron2.config import get_cfg
+from detectron2 import model_zoo
+from detectron2.data import Metadata
+from detectron2.structures.boxes import Boxes
+from detectron2.structures import Instances
+
+from plots.plot_pca_point import plot_pca_point
+from plots.plot_histogram_dist import plot_histogram_dist
+from plots.plot_gradcam import plot_gradcam
+
+def extract_features(model, img, box):
+    height, width = img.shape[1:3]
+    inputs = [{"image": img, "height": height, "width": width}]
+    with torch.no_grad():
+        img = model.preprocess_image(inputs) 
+        features = model.backbone(img.tensor)
+        features_ = [features[f] for f in model.roi_heads.box_in_features]
+
+        box_features = model.roi_heads.box_pooler(features_, [box])
+
+        output_features = F.avg_pool2d(box_features, [7, 7])
+        output_features = output_features.view(-1, 256)
+
+        return output_features
+
+def forward_model_full(model, cfg, cv_img):
+    height, width = cv_img.shape[:2]
+    transform_gen = T.ResizeShortestEdge(
+        [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST
+    )
+
+    image = transform_gen.get_transform(cv_img).apply_image(cv_img)
+    image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
+    inputs = [{"image": image, "height": height, "width": width}]
+
+    with torch.no_grad():
+        images = model.preprocess_image(inputs)
+        features = model.backbone(images.tensor) 
+        proposals, _ = model.proposal_generator(images, features, None)
+
+        features_ = [features[f] for f in model.roi_heads.box_in_features]
+        
+        box_features = model.roi_heads.box_pooler(features_, [x.proposal_boxes for x in proposals])
+        box_head = model.roi_heads.box_head(box_features)
+        predictions = model.roi_heads.box_predictor(box_head)
+        
+        output_features = F.avg_pool2d(box_features, [7, 7])
+        output_features = output_features.view(-1, 256)
+
+        probs = model.roi_heads.box_predictor.predict_probs(predictions, proposals)
+        
+        pred_instances, pred_inds = model.roi_heads.box_predictor.inference(predictions, proposals)
+        pred_instances = model.roi_heads.forward_with_given_boxes(features, pred_instances)
+
+        pred_instances = model._postprocess(pred_instances, inputs, images.image_sizes)
+        
+        instances = pred_instances[0]["instances"]
+
+        instances.set("probs", probs[0][pred_inds])
+        instances.set("features", output_features[pred_inds])
+        
+        return instances, cv_img
+
+
+def load_model():
+    cfg = get_cfg()
+
+    cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
+    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3
+    cfg.MODEL.WEIGHTS = MODEL
+    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = TH
+
+
+    metadata = Metadata()
+    metadata.set(
+        evaluator_type="coco",
+        thing_classes=["neoplastic", "aphthous", "traumatic"],
+        thing_dataset_id_to_contiguous_id={"1": 0, "2": 1, "3": 2}
+    )
+
+    predictor = DefaultPredictor(cfg)
+    model = predictor.model
+
+    return dict(
+        predictor=predictor,
+        model=model,
+        metadata=metadata,
+        cfg=cfg
+    )
+
+
+
+
+def compute_similarities(features, database):
+    similarities = dict()
+    dist_fn = getattr(spatial.distance, DISTANCE)
+    for file_name, elems in  database.items():
+        for elem in elems:
+            similarities[file_name] = dict(
+                dist=dist_fn(elem["features"], features),
+                file_name=file_name,
+                box=elem["roi"],
+                type=elem["type"]
+            )
+    similarities = OrderedDict(sorted(similarities.items(), key=lambda e: e[1]["dist"]))
+    return similarities
+
+
+def draw_box(file_name, box, type, model, resize_input=False):
+    height, width, channels = img.shape 
+    
+    pred_v = Visualizer(img[:, :, ::-1], model["metadata"], scale=1)
+    instances = Instances((height, width), pred_boxes=Boxes(torch.tensor(box).unsqueeze(0)), pred_classes=torch.tensor([type]))
+    pred_v = pred_v.draw_instance_predictions(instances)
+
+    pred = pred_v.get_image()[:, :, ::-1]
+    pred = cv2.resize(pred, (800, 800))
+
+    return pred
+
+
+def explain(img, model):
+    database = json.load(open(FEATURES_DATABASE))
+    instances, input = forward_model_full(model["model"], model["cfg"], img)
+    
+    instances.remove("pred_masks")
+    
+    pred_v = Visualizer(cv2.cvtColor(input, cv2.COLOR_BGR2RGB), model["metadata"], scale=1)
+    pred_v = pred_v.draw_instance_predictions(instances.to("cpu"))
+
+    pred = pred_v.get_image()[:, :, ::-1]
+    pred = cv2.resize(pred, (800, 800))
+    pred = cv2.cvtColor(pred, cv2.COLOR_BGR2RGB)
+    
+    tabs = st.tabs(["Detection"] + [f"Lesion #{i}" for i in range(0, len(instances))])
+    lesion_tabs = tabs[1:]
+
+    with tabs[0]:
+        st.header("Detected lesions")
+        state.text("All done...")
+        tooltip.success("Use the tabs for a detailed explanation of each lesion")
+        st.image(pred)
+
+    
+    for i, (tab, box, type, scores, features) in enumerate(zip(lesion_tabs, instances.pred_boxes, instances.pred_classes, instances.probs, instances.features)):
+        healthy_prob = scores[-1].item()
+        scores = scores[:-1]
+        features = features.tolist()
+
+        with tab:
+            st.header(f"Lesion #{i}")
+            lesion_img = draw_box(img, box.cpu(), type, model)
+            lesion_img = cv2.cvtColor(lesion_img, cv2.COLOR_BGR2RGB)
+
+            classes = ["healty", "neoplastic", "aphthous", "traumatic"]
+            y_pos = np.arange(len(classes))
+            probs = [healthy_prob] + scores.cpu().numpy().tolist()
+
+            probs_fig = plt.figure()
+            plt.bar(y_pos, probs, align="center")
+            plt.xticks(y_pos, classes)
+            plt.ylabel("Probability")
+            plt.title("Class")
+
+
+            st.subheader("Classification")
+            col1, col2 = st.columns(2)
+            
+            col1.image(lesion_img)
+            col2.pyplot(probs_fig)
+
+            st.subheader("Feature space")
+            col1, col2 = st.columns(2)
+
+            fig = plot_pca_point(point=features, features_database=FEATURES_DATABASE, pca_model=PCA_MODEL, fig_h=800, fig_w=600, fig_dpi=100)
+            col1.pyplot(fig)
+
+            fig = plot_histogram_dist(point=features, features_database=FEATURES_DATABASE, fig_h=800, fig_w=600, fig_dpi=100)
+            col2.pyplot(fig)
+
+            st.subheader("Gradcam++")
+            fig = plot_gradcam(model=MODEL, file=FILE, instance=i, fig_h=1600, fig_w=1200, fig_dpi=200, th=TH, layer="backbone.bottom_up.res5.2.conv3")
+            st.pyplot(fig)
+
+FILE = "./test.jpg"
+MODEL = "./models/model.pth"
+PCA_MODEL = "./models/pca.pkl"
+FEATURES_DATABASE = "./assets/features/features.json"
+
+DISTANCE = "cosine"
+TH = 0.5
+
+state = st.empty()
+tooltip = st.empty()
+
+state.write("Loading model...")
+model = load_model()
+
+img = cv2.imread(FILE)
+img = cv2.resize(img, (800, 800))
+explain(img, model)

plots/gradcam/detectron2_gradcam.py

@@ -0,0 +1,109 @@
+# Author: Alexander Riedel 
+# License: Unlicensed
+# Link: https://github.com/alexriedel1/detectron2-GradCAM
+
+from plots.gradcam.gradcam import GradCAM, GradCamPlusPlus
+import detectron2.data.transforms as T
+import torch
+from detectron2.checkpoint import DetectionCheckpointer
+from detectron2.config import get_cfg
+from detectron2.data import DatasetCatalog, MetadataCatalog
+from detectron2.data.detection_utils import read_image
+from detectron2.modeling import build_model
+from detectron2.data.datasets import register_coco_instances
+
+class Detectron2GradCAM():
+  """
+      Attributes
+    ----------
+    config_file : str
+        detectron2 model config file path
+    cfg_list : list
+        List of additional model configurations
+    root_dir : str [optional]
+        directory of coco.josn and dataset images for custom dataset registration
+    custom_dataset : str [optional]
+        Name of the custom dataset to register
+    """
+  def __init__(self, config_file, cfg_list, root_dir=None, custom_dataset=None):
+      # load config from file
+      cfg = get_cfg()
+      cfg.merge_from_file(config_file)
+
+      if custom_dataset:
+          register_coco_instances(custom_dataset, {}, root_dir + "coco.json", root_dir)
+          cfg.DATASETS.TRAIN = (custom_dataset,)
+          MetadataCatalog.get(custom_dataset)
+          DatasetCatalog.get(custom_dataset)
+
+      if torch.cuda.is_available():
+          cfg.MODEL.DEVICE = "cuda"
+      else:
+          cfg.MODEL.DEVICE = "cpu"
+
+      cfg.merge_from_list(cfg_list)
+      cfg.freeze()
+
+      self.cfg =  cfg
+
+  def _get_input_dict(self, original_image):
+      height, width = original_image.shape[:2]
+      transform_gen = T.ResizeShortestEdge(
+          [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST
+      )
+      image = transform_gen.get_transform(original_image).apply_image(original_image)
+      image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)).requires_grad_(True)
+      inputs = {"image": image, "height": height, "width": width}
+      return inputs
+
+  def get_cam(self, img, target_instance, layer_name, grad_cam_type="GradCAM"):
+      """
+      Calls the GradCAM++ instance
+
+      Parameters
+      ----------
+      img : str
+          Path to inference image
+      target_instance : int
+          The target instance index
+      layer_name : str
+          Convolutional layer to perform GradCAM on
+      grad_cam_type : str
+          GradCAM or GradCAM++ (for multiple instances of the same object, GradCAM++ can be favorable)
+
+      Returns
+      -------
+      image_dict : dict
+        {"image" : <image>, "cam" : <cam>, "output" : <output>, "label" : <label>}
+        <image> original input image
+        <cam> class activation map resized to original image shape
+        <output> instances object generated by the model
+        <label> label of the 
+      cam_orig : numpy.ndarray
+        unprocessed raw cam
+      """
+      model = build_model(self.cfg)
+      checkpointer = DetectionCheckpointer(model)
+      checkpointer.load(self.cfg.MODEL.WEIGHTS)
+
+      image = read_image(img, format="BGR")
+      input_image_dict = self._get_input_dict(image)
+
+      if grad_cam_type == "GradCAM":
+        grad_cam = GradCAM(model, layer_name)
+
+      elif grad_cam_type == "GradCAM++":
+        grad_cam = GradCamPlusPlus(model, layer_name)
+      
+      else:
+        raise ValueError('Grad CAM type not specified')
+
+      with grad_cam as cam:
+        cam, cam_orig, output = cam(input_image_dict, target_category=target_instance)
+      
+      image_dict = {}
+      image_dict["image"] = image
+      image_dict["cam"] = cam
+      image_dict["output"] = output[0]
+      image_dict["label"] = MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0]).thing_classes[output[0]["instances"].pred_classes[target_instance]]
+      return image_dict, cam_orig

plots/gradcam/gradcam.py

@@ -0,0 +1,168 @@
+# Author: Alexander Riedel 
+# License: Unlicensed
+# Link: https://github.com/alexriedel1/detectron2-GradCAM
+
+import cv2
+import numpy as np
+
+class GradCAM():
+    """
+    Class to implement the GradCam function with it's necessary Pytorch hooks.
+
+    Attributes
+    ----------
+    model : detectron2 GeneralizedRCNN Model
+        A model using the detectron2 API for inferencing
+    layer_name : str
+        name of the convolutional layer to perform GradCAM with
+    """
+
+    def __init__(self, model, target_layer_name):
+        self.model = model
+        self.target_layer_name = target_layer_name
+        self.activations = None
+        self.gradient = None
+        self.model.eval()
+        self.activations_grads = []
+        self._register_hook()
+
+    def _get_activations_hook(self, module, input, output):
+        self.activations = output
+
+    def _get_grads_hook(self, module, input_grad, output_grad):
+        self.gradient = output_grad[0]
+
+    def _register_hook(self):
+        for (name, module) in self.model.named_modules():
+            if name == self.target_layer_name:
+                self.activations_grads.append(module.register_forward_hook(self._get_activations_hook))
+                self.activations_grads.append(module.register_backward_hook(self._get_grads_hook))
+                return True
+        print(f"Layer {self.target_layer_name} not found in Model!")
+
+    def _release_activations_grads(self):
+      for handle in self.activations_grads:
+            handle.remove()
+    
+    def _postprocess_cam(self, raw_cam, img_width, img_height):
+        cam_orig = np.sum(raw_cam, axis=0)  # [H,W]
+        cam_orig = np.maximum(cam_orig, 0)  # ReLU
+        cam_orig -= np.min(cam_orig)
+        cam_orig /= np.max(cam_orig)
+        cam = cv2.resize(cam_orig, (img_width, img_height))
+        return cam, cam_orig
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self._release_activations_grads()
+
+    def __call__(self, inputs, target_category):
+        """
+        Calls the GradCAM++ instance
+
+        Parameters
+        ----------
+        inputs : dict
+            The input in the standard detectron2 model input format
+            https://detectron2.readthedocs.io/en/latest/tutorials/models.html#model-input-format
+
+        target_category : int, optional
+            The target category index. If `None` the highest scoring class will be selected
+
+        Returns
+        -------
+        cam : np.array()
+          Gradient weighted class activation map
+        output : list
+          list of Instance objects representing the detectron2 model output
+        """
+        self.model.zero_grad()
+        output = self.model.forward([inputs])
+
+        if target_category == None:
+          target_category =  np.argmax(output[0]['instances'].scores.cpu().data.numpy(), axis=-1)
+
+        score = output[0]['instances'].scores[target_category]
+        #box0 = output[0]['instances'].pred_boxes[0].tensor[0][target_category]
+        #print(box0)
+        #box0.backward()
+        score.backward()
+
+        gradient = self.gradient[0].cpu().data.numpy()  # [C,H,W]
+        activations = self.activations[0].cpu().data.numpy()  # [C,H,W]
+        weight = np.mean(gradient, axis=(1, 2))  # [C]
+
+        cam = activations * weight[:, np.newaxis, np.newaxis]  # [C,H,W]
+        cam, cam_orig = self._postprocess_cam(cam, inputs["width"], inputs["height"])
+
+        return cam, cam_orig, output
+
+class GradCamPlusPlus(GradCAM):
+    """
+    Subclass to implement the GradCam++ function with it's necessary PyTorch hooks.
+    ...
+
+    Attributes
+    ----------
+    model : detectron2 GeneralizedRCNN Model
+        A model using the detectron2 API for inferencing
+    target_layer_name : str
+        name of the convolutional layer to perform GradCAM++ with
+
+    """
+    def __init__(self, model, target_layer_name):
+        super(GradCamPlusPlus, self).__init__(model, target_layer_name)
+
+    def __call__(self, inputs, target_category):
+        """
+        Calls the GradCAM++ instance
+
+        Parameters
+        ----------
+        inputs : dict
+            The input in the standard detectron2 model input format
+            https://detectron2.readthedocs.io/en/latest/tutorials/models.html#model-input-format
+
+        target_category : int, optional
+            The target category index. If `None` the highest scoring class will be selected
+
+        Returns
+        -------
+        cam : np.array()
+          Gradient weighted class activation map
+        output : list
+          list of Instance objects representing the detectron2 model output
+        """
+        self.model.zero_grad()
+        output = self.model.forward([inputs])
+
+        if target_category == None:
+          target_category =  np.argmax(output[0]['instances'].scores.cpu().data.numpy(), axis=-1)
+
+        score = output[0]['instances'].scores[target_category]
+        score.backward()
+
+        gradient = self.gradient[0].cpu().data.numpy()  # [C,H,W]
+        activations = self.activations[0].cpu().data.numpy()  # [C,H,W]
+
+        #from https://github.com/jacobgil/pytorch-grad-cam/blob/master/pytorch_grad_cam/grad_cam_plusplus.py
+        grads_power_2 = gradient**2
+        grads_power_3 = grads_power_2 * gradient
+        # Equation 19 in https://arxiv.org/abs/1710.11063
+        sum_activations = np.sum(activations, axis=(1, 2))
+        eps = 0.000001
+        aij = grads_power_2 / (2 * grads_power_2 +
+                               sum_activations[:, None, None] * grads_power_3 + eps)
+        # Now bring back the ReLU from eq.7 in the paper,
+        # And zero out aijs where the activations are 0
+        aij = np.where(gradient != 0, aij, 0)
+
+        weights = np.maximum(gradient, 0) * aij
+        weight = np.sum(weights, axis=(1, 2))
+
+        cam = activations * weight[:, np.newaxis, np.newaxis]  # [C,H,W]
+        cam, cam_orig = self._postprocess_cam(cam, inputs["width"], inputs["height"])
+
+        return cam, cam_orig, output

plots/make_plots.py

@@ -0,0 +1,226 @@
+import os
+import argparse
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import plotszoo
+
+
+def get_hyperparameters(data_augmentation, sampler):
+    hp = ["lr", "rpn_loss_weight", "roi_heads_loss_weight", "rois_per_image"]
+    if data_augmentation == "full":
+        hp.extend(["random_brightness", "random_contrast"])
+    if data_augmentation == "full" or data_augmentation == "crop-flip":
+        hp.extend(["random_crop"])
+    if sampler == "RepeatFactorTrainingSampler":
+        hp.extend(["repeat_factor_th"])
+    
+    return ["config/"+i for i in hp]
+
+
+def plot_study():
+    query = {"$or": [{"config.wandb_tag": {"$eq": tag}} for tag in args.tags_study_replicas]}
+    data = plotszoo.data.WandbData(args.username, args.project, query, verbose=args.verbose)
+    data.pull_scalars(force_update=args.update_scalars)
+
+
+    group_keys = ["config/sampler", "config/data_augmentation"]
+
+    fig, axes = plt.subplots(nrows=2, ncols=2)
+
+    yticks_fn = lambda index: "Sampler: %s Data Augmentation: %s" % (index[0], index[1])
+
+    test_detection_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/test/results/detection_accuracy")
+    test_classification_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/test/results/classification_accuracy")
+
+    test_detection_df = test_detection_plot.plot(axes[0][0], title="Test Detection Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+    test_classification_df = test_classification_plot.plot(axes[0][1], title="Test Classification Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+
+    train_detection_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/train/results/detection_accuracy")
+    train_classification_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/train/results/classification_accuracy")
+
+    train_detection_df = train_detection_plot.plot(axes[1][0], title="Train Detection Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+    train_classification_df = train_classification_plot.plot(axes[1][1], title="Train Classification Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+
+    test_detection_df.to_excel(os.path.join(args.output_directory, "study/test_detection.xlsx"))
+    test_classification_df.to_excel(os.path.join(args.output_directory, "study/test_classification.xlsx"))
+    train_detection_df.to_excel(os.path.join(args.output_directory, "study/train_detection.xlsx"))
+    train_classification_df.to_excel(os.path.join(args.output_directory, "study/train_classification.xlsx"))
+    
+
+    for ax in axes.flatten():
+        ax.set_xlim(xmin=0.5)
+
+    fig.set_size_inches(30, 10)
+    fig.tight_layout()
+
+    plotszoo.utils.savefig(fig, os.path.join(args.output_directory, "study.png"))
+
+
+def plot_optimization_history(ax, data, dataset):
+        running_max = dict(accuracy=float("-inf"), detection_accuracy=float("-inf"), classification_accuracy=float("-inf"))
+        plots=dict(best_accuracy=[], best_detection_accuracy=[], best_classification_accuracy=[], accuracy=[], detection_accuracy=[], classification_accuracy=[])
+        plot_index = []
+        for i, row in data.scalars.iterrows():
+            if row["summary/"+dataset+"/results/accuracy"] > running_max["accuracy"]:
+                running_max = dict(
+                    accuracy=row["summary/"+dataset+"/results/accuracy"],
+                    detection_accuracy=row["summary/"+dataset+"/results/detection_accuracy"],
+                    classification_accuracy=row["summary/"+dataset+"/results/classification_accuracy"]
+                )
+            plots["accuracy"].append(row["summary/"+dataset+"/results/accuracy"])
+            plots["detection_accuracy"].append(row["summary/"+dataset+"/results/detection_accuracy"])
+            plots["classification_accuracy"].append(row["summary/"+dataset+"/results/classification_accuracy"])
+           
+            plots["best_accuracy"].append(running_max["accuracy"])
+            plots["best_detection_accuracy"].append(running_max["detection_accuracy"])
+            plots["best_classification_accuracy"].append(running_max["classification_accuracy"])
+
+            plot_index.append(i)
+        
+        ax.plot(plot_index, plots["best_accuracy"], "k", label="Best "+dataset+" Accuracy")
+        ax.plot(plot_index, plots["best_detection_accuracy"], "b--", label="Best "+dataset+" Detection Accuracy")
+        ax.plot(plot_index, plots["best_classification_accuracy"], "g--", label="Best "+dataset+" Classification Accuracy")
+
+        ax.scatter(plot_index, plots["accuracy"], c="k", alpha=0.5)
+        ax.scatter(plot_index, plots["detection_accuracy"], c="b", alpha=0.5)
+        ax.scatter(plot_index, plots["classification_accuracy"], c="g", alpha=0.5)
+
+        ax.legend(loc="lower right")
+        
+
+
+def plot_optimization():
+    for tag, params in args.tags_optimization.items():
+        query = {"config.wandb_tag": {"$eq": tag}}
+
+        parameters = get_hyperparameters(**params)
+        parameters.extend(["summary/train/results/detection_accuracy", "summary/train/results/classification_accuracy"])
+
+        data = plotszoo.data.WandbData(args.username, args.project, query, verbose=args.verbose)
+        data.pull_scalars(force_update=args.update_scalars)
+        assert len(data.scalars) > 0, "No data, check the tag name"
+        data.pull_series(force_update=args.update_series)
+
+        data.astype(["summary/train/results/accuracy", "summary/train/results/detection_accuracy", "summary/train/results/classification_accuracy"], float)
+        data.dropna(["summary/train/results/accuracy"])        
+
+        data.create_scalar_from_series("start_time", lambda s: s["_timestamp"].min())
+
+        fig, axes = plt.subplots(1, len(parameters), sharey=False)
+
+        parallel_plot = plotszoo.scalars.ScalarsParallelCoordinates(data, parameters, "summary/train/results/accuracy")
+
+        parallel_plot.plot(axes)
+
+        fig.set_size_inches(32, 10)
+        plotszoo.utils.savefig(fig, os.path.join(args.output_directory, tag, "optim_parallel.png"))
+
+        fig, ax = plt.subplots(2, 1)
+
+        plot_optimization_history(ax[0], data, "train")
+        plot_optimization_history(ax[1], data, "test")
+        
+        fig.set_size_inches(20, 10)
+        plotszoo.utils.savefig(fig, os.path.join(args.output_directory, tag, "optim_history.png"))
+
+        parameters.remove("summary/train/results/detection_accuracy")
+        parameters.remove("summary/train/results/classification_accuracy")
+
+        args_names = [p.split("/")[1].replace("_","-") for p in parameters]
+        best_run = data.scalars["summary/train/results/accuracy"].idxmax()
+        best_args = "".join(["--%s %s " % (n, data.scalars[k][best_run]) for n, k in zip(args_names, parameters)])
+        best_args += "".join(["--%s %s " % (k.replace("_", "-"), v) for k, v in params.items()])
+        print(best_run)
+        print("Tag: %s" % tag)
+        print(data.scalars.loc[best_run][["summary/train/results/detection_accuracy", "summary/train/results/classification_accuracy"]])
+        print("HP: %s" % best_args)
+        print()
+        
+        best_args_f = open(os.path.join(args.output_directory, tag, "best_args.txt"), "w")
+        best_args_f.write(best_args)
+        best_args_f.close()
+
+def plot_replicas():
+    query = {"$or": [{"config.wandb_tag": {"$eq": tag}} for tag in args.tags_best_replicas]}
+    data = plotszoo.data.WandbData(args.username, args.project, query, verbose=args.verbose)
+    data.pull_scalars(force_update=args.update_scalars)
+
+    group_keys = ["config/sampler"]
+
+    fig, axes = plt.subplots(nrows=2, ncols=1)
+
+    yticks_fn = lambda index: "Sampler: %s" % (index, )
+
+    detection_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/test/results/detection_accuracy")
+    classification_plot = plotszoo.scalars.grouped.GroupedScalarsBarchart(data, group_keys, "summary/test/results/classification_accuracy")
+
+    detection_df = detection_plot.plot(axes[0], title="Test Detection Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+    classification_df = classification_plot.plot(axes[1], title="Test Classification Accuracy", nbins=20, grid=True, yticks_fn=yticks_fn)
+
+    for ax in axes: ax.set_xlim(xmin=0.5)
+
+    fig.set_size_inches(20, 10)
+    fig.tight_layout()
+
+    classification_df.to_excel(os.path.join(args.output_directory, "result/classification.xlsx"))
+    detection_df.to_excel(os.path.join(args.output_directory, "result/detection.xlsx"))
+
+    print(classification_df)
+    print(detection_df)
+
+    plotszoo.utils.savefig(fig, os.path.join(args.output_directory, "results.png"))
+
+
+def plot_tables():
+    query = {"$or": [{"config.wandb_tag": {"$eq": tag}} for tag in args.tags_best_replicas]}
+    data = plotszoo.data.WandbData(args.username, args.project, query, verbose=args.verbose)
+    data.pull_scalars(force_update=args.update_scalars)
+
+    group_keys = ["config/sampler"]
+    classes = ["neoplastic", "aphthous", "traumatic"]
+    metrics = ["precision", "recall", "f1-score"]
+
+    grouped_df = data.scalars.groupby(group_keys).agg(np.mean)
+    for group in grouped_df.index:
+        data_df = grouped_df.loc[group]
+        table = np.zeros((len(classes), len(metrics)))
+        for i, c in enumerate(classes):
+            for j, m in enumerate(metrics):
+                table[i, j] = data_df["summary/test/report/%s/%s" % (c, m)]*100
+        
+        table_df = pd.DataFrame(table, columns=metrics, index=classes)
+        table_df.to_csv(os.path.join(args.output_directory, "%s_table.csv" % (group)))
+        print("Sampler: %s" % (group))
+        print(table_df)
+        print()
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--output-directory", type=str, default="./plots")
+parser.add_argument("--username", type=str, default="mlpi")
+parser.add_argument("--project", type=str, default="oral-ai")
+parser.add_argument("--tags-study-replicas", type=str, default=["study-3"], nargs="+")
+parser.add_argument("--tags-optimization", type=dict, default={
+    "hp-optimization-none-trainingsampler-5": dict(
+        data_augmentation="none",
+        sampler="TrainingSampler"
+    ),
+    "hp-optimization-none-repeatfactortrainingsampler-5": dict(
+        data_augmentation="none",
+        sampler="RepeatFactorTrainingSampler"
+    )
+}, nargs="+")
+parser.add_argument("--tags-best-replicas", type=str, default=["best-replicas-7"], nargs="+")
+parser.add_argument("--update-scalars", action="store_true")
+parser.add_argument("--update-series", action="store_true")
+parser.add_argument("--verbose", action="store_true")
+
+
+args = parser.parse_args()
+
+plot_study()
+plot_optimization()
+plot_replicas()
+#plot_tables()

plots/plot_features.py

@@ -0,0 +1,68 @@
+import argparse
+import numpy as np
+import json
+
+from matplotlib import pyplot as plt
+from matplotlib.colors import ListedColormap
+
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+from scipy import spatial
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--features-database", type=str, required=True)
+parser.add_argument("--decomposition", type=str, default="TSNE", choices=["TSNE", "PCA"])
+parser.add_argument("--output", type=str, default="")
+parser.add_argument("--fig-h", type=int, default=1080)
+parser.add_argument("--fig-w", type=int, default=720)
+parser.add_argument("--fig-dpi", type=int, default=100)
+parser.add_argument("--distance", type=str, default="cosine")
+
+parser.add_argument("--point", type=str, default="")
+
+args = parser.parse_args()
+point = None
+if args.point != "":
+    point = json.loads(args.point)
+
+
+dist_fn = getattr(spatial.distance, args.distance)
+features_database = json.load(open(args.features_database, "r"))
+
+features = []
+classes = []
+for name, feature_list in features_database.items():
+    for feature in feature_list:
+        features.append(feature["features"])
+        classes.append(feature["type"])
+
+if point is not None:
+    features.append(point)
+
+features = np.array(features)
+classes = np.array(classes)
+
+if args.decomposition == "TSNE":
+    decomposition = TSNE(n_components=2, metric=dist_fn)
+elif args.decomposition == "PCA":
+    decomposition = PCA(n_components=2)
+transformed = decomposition.fit_transform(features)
+
+if point is not None:
+    transformed = transformed[:-1,:]
+    transformed_point = transformed[-1,:]
+
+plt.figure(figsize=(args.fig_h/args.fig_dpi, args.fig_w/args.fig_dpi), dpi=args.fig_dpi)
+cmap = ListedColormap(["r","b","g"])
+scatter = plt.scatter(transformed[:, 0], transformed[:, 1], c=classes, cmap=cmap, s=10)
+
+if point is not None:
+    plt.scatter(transformed_point[0], transformed_point[1], marker="x", s=200, c="k")
+
+plt.legend(handles=scatter.legend_elements()[0], labels=["neoplastic", "aphthous", "traumatic"])
+
+if args.output == "":
+    plt.show()
+else:
+    plt.savefig(args.output, dpi=args.fig_dpi)

plots/plot_gradcam.py

@@ -0,0 +1,69 @@
+import argparse
+import torch
+import matplotlib
+import matplotlib.pyplot as plt
+from types import SimpleNamespace
+
+from detectron2.utils.visualizer import Visualizer
+from detectron2.data import Metadata
+from detectron2 import model_zoo
+
+from plots.gradcam.detectron2_gradcam import Detectron2GradCAM
+
+
+def plot_gradcam(**kwargs):
+    kwargs = SimpleNamespace(**kwargs)
+    
+    config_file = model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
+
+    cfg_list = [
+    "MODEL.ROI_HEADS.SCORE_THRESH_TEST", str(kwargs.th),
+    "MODEL.ROI_HEADS.NUM_CLASSES", "3",
+    "MODEL.WEIGHTS", kwargs.model
+    ]
+
+    metadata = Metadata()
+    metadata.set(
+        evaluator_type="coco",
+        thing_classes=["neoplastic", "aphthous", "traumatic"],
+        thing_dataset_id_to_contiguous_id={"1": 0, "2": 1, "3": 2}
+    )
+
+
+    cam_extractor = Detectron2GradCAM(config_file, cfg_list)
+    image_dict, cam_orig = cam_extractor.get_cam(img=kwargs.file, target_instance=kwargs.instance, layer_name=kwargs.layer, grad_cam_type="GradCAM++")
+
+    with torch.no_grad():
+        fig = plt.figure(figsize=(kwargs.fig_h/kwargs.fig_dpi, kwargs.fig_w/kwargs.fig_dpi), dpi=kwargs.fig_dpi)
+        v = Visualizer(image_dict["image"], metadata, scale=1.0)
+        img = image_dict["output"]["instances"][kwargs.instance]
+        img.remove("pred_masks")
+        
+        out = v.draw_instance_predictions(img.to("cpu"))
+
+        plt.gca().set_axis_off()
+        plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, 
+            hspace = 0, wspace = 0)
+        plt.margins(0,0)
+        plt.imshow(out.get_image(), interpolation='none')
+        plt.imshow(image_dict["cam"], cmap='jet', alpha=0.5)
+
+        return fig
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--model", type=str, required=True)
+    parser.add_argument("--layer", type=str, default="backbone.bottom_up.res5.2.conv3")
+    parser.add_argument("--th", type=float, default=0.5)
+    parser.add_argument("--file", type=str, required=True)
+    parser.add_argument("--instance", type=int, required=True)
+    parser.add_argument("--output", type=str, default="")
+    parser.add_argument("--fig-h", type=int, default=1080)
+    parser.add_argument("--fig-w", type=int, default=720)
+    parser.add_argument("--fig-dpi", type=int, default=100)
+
+    args = parser.parse_args()
+
+    plot_gradcam(**vars(args))

plots/plot_histogram_dist.py

@@ -0,0 +1,58 @@
+import argparse
+import numpy as np
+import json
+import pickle
+from scipy import spatial
+
+from matplotlib import pyplot as plt
+from matplotlib.colors import ListedColormap
+
+
+def plot_histogram_dist(features_database, fig_h, fig_w, fig_dpi, point, distance="cosine"):
+    features_database = json.load(open(features_database, "r"))
+    dist_fn = getattr(spatial.distance, distance)
+    class_names = ["neoplastic", "aphthous", "traumatic"]
+    cmap = ListedColormap(["r","b","g"])
+
+    dists = dict()
+    for name, feature_list in features_database.items():
+        for feature in feature_list:
+            if feature["type"] not in dists:
+                dists[feature["type"]] = []
+            
+            dists[feature["type"]].append(dist_fn(point, feature["features"]))
+
+        
+    fig, axes = plt.subplots(len(dists))
+
+    for k, ax in zip(dists.keys(), axes):
+        dist = dists[k]
+        ax.set_title(class_names[k])
+        ax.set_xlim(0, 1)
+        n, bins, patches = ax.hist(dist, "auto", density=True, color=cmap(k))
+    
+    fig.tight_layout(pad=3.0)
+
+    return fig
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--features-database", type=str, required=True)
+    parser.add_argument("--output", type=str, default="")
+    parser.add_argument("--fig-h", type=int, default=1080)
+    parser.add_argument("--fig-w", type=int, default=720)
+    parser.add_argument("--fig-dpi", type=int, default=100)
+    parser.add_argument("--distance", type=str, default="cosine")
+
+
+    parser.add_argument("--point", type=str, required=True)
+
+    args = parser.parse_args()
+
+    point = json.loads(args.point)
+
+    dict_args = vars(args)
+    del dict_args["point"]
+    plot_histogram_dist(**dict_args, point=point)

plots/plot_matrix_distance.py

@@ -0,0 +1,56 @@
+import json
+import argparse
+from scipy import spatial
+import numpy as np
+from tqdm import tqdm
+from matplotlib import pyplot as plt
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--rows", type=str, required=True)
+parser.add_argument("--cols", type=str, required=True)
+parser.add_argument("--distance", type=str, default="cosine")
+parser.add_argument("--output", type=str, default="")
+
+args = parser.parse_args()
+
+rows_features = json.load(open(args.rows, "r"))
+cols_features = json.load(open(args.cols, "r"))
+dist_fn = getattr(spatial.distance, args.distance)
+
+
+rows_features_rois = []
+cols_features_rois = []
+
+for row_feature in rows_features.values():
+    for roi_feature in row_feature:
+        rows_features_rois.append(roi_feature)
+
+for col_feature in cols_features.values():
+    for roi_feature in col_feature:
+        cols_features_rois.append(roi_feature)
+
+
+rows_features_rois = sorted(rows_features_rois, key=lambda e: e["type"])
+cols_features_rois = sorted(cols_features_rois, key=lambda e: e["type"])
+
+
+matrix = np.zeros((len(rows_features_rois), len(cols_features_rois)))
+for i, row in tqdm(enumerate(rows_features_rois), total=len(rows_features_rois)):
+    for j, col in enumerate(cols_features_rois):
+        matrix[i, j] = dist_fn(row["features"], col["features"])
+
+fig, ax = plt.subplots()
+
+ax.set_xlabel(args.rows)
+ax.set_ylabel(args.cols)
+
+pos = ax.imshow(matrix)
+fig.colorbar(pos, ax=ax)
+
+if args.output == "":
+    plt.show()
+else:
+    plt.savefig(args.output)
+
+

plots/plot_pca_point.py

@@ -0,0 +1,57 @@
+import argparse
+import numpy as np
+import json
+import pickle
+
+from matplotlib import pyplot as plt
+from matplotlib.colors import ListedColormap
+
+from sklearn.decomposition import PCA
+
+
+def plot_pca_point(features_database, pca_model, fig_h, fig_w, fig_dpi, point):
+    features_database = json.load(open(features_database, "r"))
+    pca = pickle.load(open(pca_model, "rb"))
+
+    features = []
+    classes = []
+    for name, feature_list in features_database.items():
+        for feature in feature_list:
+            features.append(feature["features"])
+            classes.append(feature["type"])
+
+    features = np.array(features)
+    classes = np.array(classes)
+
+    features = pca.transform(features)
+    point = pca.transform(np.atleast_2d(point))
+
+    fig = plt.figure(figsize=(fig_h/fig_dpi, fig_w/fig_dpi), dpi=fig_dpi)
+    cmap = ListedColormap(["r","b","g"])
+    scatter = plt.scatter(features[:, 0], features[:, 1], c=classes, cmap=cmap, s=10)
+
+    plt.scatter(point[:, 0], point[:, 1], marker="x", s=200, c="k")
+
+    plt.legend(handles=scatter.legend_elements()[0], labels=["neoplastic", "aphthous", "traumatic"])
+
+    return fig
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--features-database", type=str, required=True)
+    parser.add_argument("--pca-model", type=str, required=True)
+    parser.add_argument("--output", type=str, default="")
+    parser.add_argument("--fig-h", type=int, default=1080)
+    parser.add_argument("--fig-w", type=int, default=720)
+    parser.add_argument("--fig-dpi", type=int, default=100)
+
+    parser.add_argument("--point", type=str, required=True)
+
+    args = parser.parse_args()
+
+    point = json.loads(args.point)
+
+    dict_args = vars(args)
+    del dict_args["point"]
+    plot_pca_point(**dict_args, point=point)