Source code for eli5.keras.gradcam

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from typing import Union, Optional, Tuple, List

import numpy as np
import keras
import keras.backend as K
from keras.models import Model
from keras.layers import Layer

[docs]def gradcam(weights, activations): # type: (np.ndarray, np.ndarray) -> np.ndarray """ Generate a localization map (heatmap) using Gradient-weighted Class Activation Mapping (Grad-CAM) ( The values for the parameters can be obtained from :func:`eli5.keras.gradcam.gradcam_backend`. Parameters ---------- weights : numpy.ndarray Activation weights, vector with one weight per map, rank 1. activations : numpy.ndarray Forward activation map values, vector of matrices, rank 3. Returns ------- lmap : numpy.ndarray A Grad-CAM localization map, rank 2, with values normalized in the interval [0, 1]. Notes ----- We currently make two assumptions in this implementation * We are dealing with images as our input to ``model``. * We are doing a classification. ``model``'s output is a class scores or probabilities vector. Credits * Jacob Gildenblat for "". * Author of "" for fixes to Jacob's implementation. * Kotikalapudi, Raghavendra and contributors for "". """ # For reusability, this function should only use numpy operations # Instead of backend library operations # Perform a weighted linear combination # we need to multiply (dim1, dim2, maps,) by (maps,) over the first two axes # and add each result to (dim1, dim2,) results array # there does not seem to be an easy way to do this: # see: spatial_shape = activations.shape[:2] # -> (dim1, dim2) lmap = np.zeros(spatial_shape, dtype=np.float64) # iterate through each activation map for i, w in enumerate(weights): # weight * spatial map # add result to the entire localization map (NOT pixel by pixel) lmap += w * activations[..., i] lmap = np.maximum(lmap, 0) # ReLU # normalize lmap to [0, 1] ndarray # add eps to avoid division by zero in case lmap is 0's # this also means that lmap max will be slightly less than the 'true' max lmap = lmap / (np.max(lmap)+K.epsilon()) return lmap
[docs]def gradcam_backend(model, # type: Model doc, # type: np.ndarray targets, # type: Optional[List[int]] activation_layer # type: Layer ): # type: (...) -> Tuple[np.ndarray, np.ndarray, np.ndarray, int, float] """ Compute the terms and by-products required by the Grad-CAM formula. Parameters ---------- model : keras.models.Model Differentiable network. doc : numpy.ndarray Input to the network. targets : list, optional Index into the network's output, indicating the output node that will be used as the "loss" during differentiation. activation_layer : keras.layers.Layer Keras layer instance to differentiate with respect to. See :func:`eli5.keras.explain_prediction` for description of the ``model``, ``doc``, ``targets`` parameters. Returns ------- (weights, activations, gradients, predicted_idx, predicted_val) : (numpy.ndarray, ..., int, float) Values of variables. """ # score for class in targets predicted_idx = _get_target_prediction(targets, model) predicted_val = K.gather(model.output[0,:], predicted_idx) # access value by index # output of target activation layer, i.e. activation maps of a convolutional layer activation_output = activation_layer.output # score for class w.r.p.t. activation layer grads = _calc_gradient(predicted_val, [activation_output]) # Global Average Pooling of gradients to get the weights # note that axes are in range [-rank(x), rank(x)) (we start from 1, not 0) # TODO: decide whether this should go in gradcam_backend() or gradcam() weights = K.mean(grads, axis=(1, 2)) evaluate = K.function([model.input], [weights, activation_output, grads, predicted_val, predicted_idx] ) # evaluate the graph / do actual computations weights, activations, grads, predicted_val, predicted_idx = evaluate([doc]) # put into suitable form weights = weights[0] predicted_val = predicted_val[0] predicted_idx = predicted_idx[0] activations = activations[0, ...] grads = grads[0, ...] return weights, activations, grads, predicted_idx, predicted_val
def _calc_gradient(ys, xs): # (K.variable, list) -> K.variable """ Return the gradient of scalar ``ys`` with respect to each of list ``xs``, (must be singleton) and apply grad normalization. """ # differentiate ys (scalar) with respect to each variable in xs grads = K.gradients(ys, xs) # grads gives a python list with a tensor (containing the derivatives) for each xs # to use grads with other operations and with K.function # we need to work with the actual tensors and not the python list grads, = grads # grads should be a singleton list (because xs is a singleton) # validate that the gradients were calculated successfully (no None's) # # if grads is None: raise ValueError('Gradient calculation resulted in None values. ' 'Check that the model is differentiable and try again. ' 'ys: {}. xs: {}. grads: {}'.format( ys, xs, grads)) # this seems to make the heatmap less noisy grads = K.l2_normalize(grads) return grads def _get_target_prediction(targets, model): # type: (Optional[list], Model) -> K.variable """ Get a prediction ID based on ``targets``, from the model ``model`` (with a rank 2 tensor for its final layer). Returns a rank 1 K.variable tensor. """ if isinstance(targets, list): # take the first prediction from the list if len(targets) == 1: target = targets[0] _validate_target(target, model.output_shape) predicted_idx = K.constant([target], dtype='int64') else: raise ValueError('More than one prediction target ' 'is currently not supported ' '(found a list that is not length 1): ' '{}'.format(targets)) elif targets is None: predicted_idx = K.argmax(model.output, axis=-1) else: raise TypeError('Invalid argument "targets" (must be list or None): %s' % targets) return predicted_idx def _validate_target(target, output_shape): # type: (int, tuple) -> None """ Check whether ``target``, an integer index into the model's output is valid for the given ``output_shape``. """ if isinstance(target, int): output_nodes = output_shape[1:][0] if not (0 <= target < output_nodes): raise ValueError('Prediction target index is ' 'outside the required range [0, {}). ' 'Got {}'.format(output_nodes, target)) else: raise TypeError('Prediction target must be int. ' 'Got: {}'.format(target))