【快速实践】类激活图（CAM，class activation map）可视化

类激活图可视化：有助于了解一张图像的哪一部分让卷积神经网络做出了最终的分类决策

对输入图像生成类激活热力图
类激活热力图是与特定输出类别相关的二维分数网格：对任何输入图像的每个位置都要进行计算，它表示每个位置对该类别的重要程度

我们将使用的具体实现方式是“Grad-CAM: visual explanations from deep networks via gradient based localization”

给定一张输入图像，对于一个卷积层的输出特征图，用类别相对于通道的梯度对这个特征图中的每个通道进行加权

代码示例如下

加载 VGG16 执行图片分类

from keras.preprocessing import image 
from keras.applications.vgg16 import VGG16 
from keras.applications.vgg16 import preprocess_input, decode_predictions 
import numpy as np 
from keras import backend as K import tensorflow as tf
tf.compat.v1.disable_eager_execution()##---------------------------------- 加载 VGG16 网络
model = VGG16(weights='imagenet')  ##---------------------------------- 预测给定图片 
img_path = 'elephant.png'   
img = image.load_img(img_path, target_size=(224, 224))   
x = image.img_to_array(img)   
x = np.expand_dims(x, axis=0)   
x = preprocess_input(x) 
preds = model.predict(x) ##---------------------------------- 解读预测结果
print('Predicted:', decode_predictions(preds, top=3)[0]) 
# 打印出：对这张图像预测的前三个类别
# Predicted:', [(u'n02504458', u'African_elephant', 0.92546833),   ## 非洲象（African elephant，92.5% 的概率）
# (u'n01871265', u'tusker', 0.070257246), 						   ## 长牙动物（tusker，7%的概率）
# (u'n02504013', u'Indian_elephant', 0.0042589349)]				   ## 印度象（Indian elephant，0.4% 的概率）print(np.argmax(preds[0])) 
# 获得 非洲象对应输出索引：386

应用Grad-CAM算法，计算类激活热力图 heatmap

african_elephant_output = model.output[:, 386]                                  # 预测向量中的 “非洲象” 元素    (None,)       
last_conv_layer_output = model.get_layer('block5_conv3').output                 # VGG16 最后一个卷积层         (None, 14, 14, 512)# 类别相对于通道的梯度: “非洲象”类别相对于 block5_conv3 输出特征图的梯度
grads = K.gradients(african_elephant_output, last_conv_layer_output)[0]         # grads.shape:                (None, 14, 14, 512)
pooled_grads = K.mean(grads, axis=(0, 1, 2))                                    # pooled_grads.shape:         (512,)# 获得当前图 conv_layer_output_value 和 pooled_grads_value 实际值
iterate = K.function([model.input], [pooled_grads, last_conv_layer_output[0]])
pooled_grads_value, conv_layer_output_value = iterate([x]) 
# 计算加权值
for i in range(512):   conv_layer_output_value[:, :, i] *= pooled_grads_value[i]heatmap = np.mean(conv_layer_output_value, axis=-1)

可视化

import matplotlib.pyplot as plt
heatmap = np.maximum(heatmap, 0) 
heatmap /= np.max(heatmap) 
plt.matshow(heatmap)
plt.show()  ##  ---------------------------------------------------- 下方左图# 缩放 heatmap 后，在原图上叠加显示
import cv2 
img = cv2.imread(img_path) 
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) 
superimposed_img = heatmap * 0.4 + img
cv2.imwrite('elephant_cam.jpg', superimposed_img)  ## -------------- 下方右图

完整代码参考

from keras.preprocessing import image 
from keras.applications.vgg16 import VGG16 
from keras.applications.vgg16 import preprocess_input, decode_predictions 
import numpy as np 
from keras import backend as K 
import tensorflow as tf
tf.compat.v1.disable_eager_execution()##---------------------------------- 加载 VGG16 网络
model = VGG16(weights='imagenet')  ##---------------------------------- 预测给定图片 
img_path = 'elephant.png'   
img = image.load_img(img_path, target_size=(224, 224))   
x = image.img_to_array(img)   
x = np.expand_dims(x, axis=0)   
x = preprocess_input(x) 
preds = model.predict(x) ##---------------------------------- 解读预测结果
print('Predicted:', decode_predictions(preds, top=3)[0]) 
# 打印出：对这张图像预测的前三个类别
# Predicted:', [(u'n02504458', u'African_elephant', 0.92546833),      ## 非洲象（African elephant，92.5% 的概率）
# (u'n01871265', u'tusker', 0.070257246), 		          ## 长牙动物（tusker，7%的概率）
# (u'n02504013', u'Indian_elephant', 0.0042589349)]                   ## 印度象（Indian elephant，0.4% 的概率）print(np.argmax(preds[0])) 
# 获得 非洲象对应输出索引：386african_elephant_output = model.output[:, 386]                                  # 预测向量中的 “非洲象” 元素    (None,)       
last_conv_layer_output = model.get_layer('block5_conv3').output                 # VGG16 最后一个卷积层         (None, 14, 14, 512)# 类别相对于通道的梯度: “非洲象”类别相对于 block5_conv3 输出特征图的梯度
grads = K.gradients(african_elephant_output, last_conv_layer_output)[0]         # grads.shape:                (None, 14, 14, 512)
pooled_grads = K.mean(grads, axis=(0, 1, 2))                                    # pooled_grads.shape:         (512,)# 获得当前图 conv_layer_output_value 和 pooled_grads_value 实际值
iterate = K.function([model.input], [pooled_grads, last_conv_layer_output[0]])
pooled_grads_value, conv_layer_output_value = iterate([x]) 
# 计算加权值
for i in range(512):   conv_layer_output_value[:, :, i] *= pooled_grads_value[i]heatmap = np.mean(conv_layer_output_value, axis=-1) import matplotlib.pyplot as plt
heatmap = np.maximum(heatmap, 0) 
heatmap /= np.max(heatmap) 
plt.matshow(heatmap)
# plt.show()# 缩放 heatmap 后，在原图上叠加显示
import cv2 
img = cv2.imread(img_path) 
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) 
superimposed_img = heatmap * 0.4 + img
cv2.imwrite('elephant_cam.jpg', superimposed_img)