李沐--动手学深度学习批量规范化

1.理论

2.从零开始实现批量规范化

import torch
from torch import nn
from d2l import torch as d2l
from torch.utils.hooks import RemovableHandle
#从零开始实现批量规范化
def batch_norm(X,gamma,beta,moving_mean,moving_var,eps,momentum):#通过is_grad_enabled来判断当前模式是训练模式还是预测模式if not torch.is_grad_enabled():#如果是在预测模式下，直接使用传入的移动平均所得的均值和方差X_hat = (X - moving_mean) / torch.sqrt(moving_var + eps)else:assert len(X.shape) in (2,4)if len(X.shape) == 2:#使用全连接层的情况,计算特征维上的均值和方差mean = X.mean(dim = 0)var = ((X - mean) ** 2).mean(dim=0)else:#使用二维卷积层的情况，计算通道维上（axis=1）的均值和方差。#这里需要保持X的形状以便后面可以做广播运算mean = X.mean(dim=(0,2,3),keepdim = True)var = ((X - mean) ** 2).mean(dim = (0,2,3),keepdim = True)#训练模式下，用当前的均值和方差做标准化X_hat = (X - mean) / torch.sqrt(var + eps)#更新移动平均的均值和方差moving_mean = momentum * moving_mean + (1.0 - momentum) * meanmoving_var = momentum * moving_var + (1.0 - momentum) * varY = gamma * X_hat + beta #缩放和移位return Y,moving_mean.data,moving_var.data#创建一个正确的BatchNorm层。这个层将保持适当的参数：拉伸gamma和偏移beta,这两个参数将在训练过程中更新。层将保存均值和方差的移动平均值，以便在模型预测期间随后使用。
class BatchNorm(nn.Module):#num_features: 完全连接层的输出数量或卷积层的输出通道数#num_dims:2表示完全连接层，4表示卷积层def __init__(self,num_features,num_dims):super().__init__()if num_dims == 2:shape = (1,num_features)else:shape = (1,num_features,1,1)#参与求梯度和迭代的拉伸和偏移参数，分别初始化成1和0self.gamma = nn.Parameter(torch.ones(shape))self.beta = nn.Parameter(torch.zeros(shape))#非模型参数的变量初始化为0和1self.moving_mean = torch.zeros(shape)self.moving_var = torch.ones(shape)def forward(self,X):#如果X不在内存上，将moving_mean 和 moving_var复制到X所在的显存上if self.moving_mean.device != X.device:self.moving_mean = self.moving_mean.to(X.device)#把要用的moving_mean挪到要运行的设备上self.moving_var = self.moving_var.to(X.device)#保存更新过的moving_mean 和 moving_varY,self.moving_mean,self.moving_var = batch_norm(X,self.gamma,self.beta,self.moving_mean,self.moving_var,eps=1e-5,momentum=0.9)return Y#更好理解如何应用BatchNorm，下面我们将其应用于LeNet模型
#批量规范化是在卷积层或全连接层之后、相应的激活函数之前应用的。
net = nn.Sequential(nn.Conv2d(1,6,kernel_size=5),BatchNorm(6,num_dims=4),nn.Sigmoid(),nn.AvgPool2d(kernel_size=2,stride=2),nn.Conv2d(6,16,kernel_size=5),BatchNorm(16,num_dims=4),nn.Sigmoid(),nn.AvgPool2d(kernel_size=2,stride=2),nn.Flatten(),nn.Linear(16*4*4,120),BatchNorm(120,num_dims=2),nn.Sigmoid(),nn.Linear(120,84),BatchNorm(84,num_dims=2),nn.Sigmoid(),nn.Linear(84,10))#在Fashion-MNIST数据集上训练网络。这个代码与第一次训练LeNet时几乎完全相同，主要区别在于学习率大得多。
lr,num_epochs,batch_size = 1.0,10,256
train_iter,test_iter = d2l.load_data_fashion_mnist(batch_size)
d2l.train_ch6(net,train_iter,test_iter,num_epochs,lr,d2l.try_gpu())
d2l.plt.show()
print(net[1].gamma.reshape((-1,)),net[1].beta.reshape((-1)))

3.调用框架实现

import torch
from torch import nn
from d2l import torch as d2l
from torch.utils.hooks import RemovableHandle#调用框架实现
#除了使用刚刚定义的BatchNorm，也可以直接使用深度学习框架中定义的BatchNorm
net = nn.Sequential(nn.Conv2d(1,6,kernel_size=5),nn.BatchNorm2d(6),nn.Sigmoid(),nn.AvgPool2d(kernel_size=2,stride=2),nn.Conv2d(6,16,kernel_size=5),nn.BatchNorm2d(16),nn.Sigmoid(),nn.AvgPool2d(kernel_size=2,stride=2),nn.Flatten(),nn.Linear(256,120),nn.BatchNorm1d(120),nn.Sigmoid(),nn.Linear(120,84),nn.BatchNorm1d(84),nn.Sigmoid(),nn.Linear(84,10))#使用相同超参数来训练模型。请注意，通常高级API变体运行速度快得多，因为它的代码已编译为C++或CUDA，而我们的自定义代码由Python实现
lr,num_epochs,batch_size = 1.0,10,256
train_iter,test_iter = d2l.load_data_fashion_mnist(batch_size)
d2l.train_ch6(net,train_iter,test_iter,num_epochs,lr,d2l.try_gpu())
d2l.plt.show()