PyTorch之数据加载和处理

from __future__ import print_function, division
import os
import torch
import pandas as pd              #用于更容易地进行csv解析
from skimage import io, transform    #用于图像的IO和变换
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utilsimport warnings
warnings.filterwarnings("ignore")plt.ion()"""
读取数据集
将csv中的标注点数据读入（N，2）数组中，其中N是特征点的数量。读取数据代码如下：
"""
landmarks_frame = pd.read_csv('data/faces/faces/face_landmarks.csv')n = 65
img_name = landmarks_frame.iloc[n, 0]   #提取第n行，第0列中的数据
landmarks = landmarks_frame.iloc[n, 1:].values
landmarks = landmarks.astype('float').reshape(-1, 2)print('Image name: {}'.format(img_name))
print('Landmarks shape: {}'.format(landmarks.shape))
print('First 4 Landmarks: {}'.format(landmarks[:4]))"""
写一个简单的函数来展示一张图片和它对应的标注点作为例子。
"""def show_landmarks(image, landmarks):"""显示带有地标的图片"""plt.imshow(image)plt.scatter(landmarks[:, 0], landmarks[:, 1], s=10, marker='.', c='r')plt.pause(5)  # pause a bit so that plots are updatedplt.figure()
show_landmarks(io.imread(os.path.join('data/faces/faces/', img_name)),landmarks)
plt.show()
"""建立数据集类
"""
class FaceLandmarksDataset(Dataset):"""面部标记数据集."""def __init__(self, csv_file, root_dir, transform=None):"""csv_file（string）：带注释的csv文件的路径。root_dir（string）：包含所有图像的目录。transform（callable， optional）：一个样本上的可用的可选变换"""self.landmarks_frame = pd.read_csv(csv_file)self.root_dir = root_dirself.transform = transformdef __len__(self):return len(self.landmarks_frame)def __getitem__(self, idx):img_name = os.path.join(self.root_dir,self.landmarks_frame.iloc[idx, 0])image = io.imread(img_name)landmarks = self.landmarks_frame.iloc[idx, 1:]landmarks = np.array([landmarks])landmarks = landmarks.astype('float').reshape(-1, 2)sample = {'image': image, 'landmarks': landmarks}if self.transform:sample = self.transform(sample)return sample"""
数据可视化
"""face_dataset = FaceLandmarksDataset(csv_file='data/faces/faces/face_landmarks.csv',root_dir='data/faces/faces/')
fig = plt.figure()for i in range(len(face_dataset)):sample = face_dataset[i]print(i, sample['image'].shape, sample['landmarks'].shape)ax = plt.subplot(1, 4, i + 1)plt.tight_layout()ax.set_title('Sample #{}'.format(i))ax.axis('off')show_landmarks(**sample)if i == 3:plt.show()break"""
数据变换,数据预处理
让我们创建三个转换: 
* Rescale：缩放图片 
* RandomCrop：对图片进行随机裁剪。这是一种数据增强操作 
* ToTensor：把numpy格式图片转为torch格式图片 (我们需要交换坐标轴).
"""class Rescale(object):"""将样本中的图像重新缩放到给定大小。.Args:output_size（tuple或int）：所需的输出大小。 如果是元组，则输出为与output_size匹配。 如果是int，则匹配较小的图像边缘到output_size保持纵横比相同。"""def __init__(self, output_size):assert isinstance(output_size, (int, tuple))self.output_size = output_sizedef __call__(self, sample):image, landmarks = sample['image'], sample['landmarks']h, w = image.shape[:2]if isinstance(self.output_size, int):if h > w:new_h, new_w = self.output_size * h / w, self.output_sizeelse:new_h, new_w = self.output_size, self.output_size * w / helse:new_h, new_w = self.output_sizenew_h, new_w = int(new_h), int(new_w)img = transform.resize(image, (new_h, new_w))# h and w are swapped for landmarks because for images,# x and y axes are axis 1 and 0 respectivelylandmarks = landmarks * [new_w / w, new_h / h]return {'image': img, 'landmarks': landmarks}class RandomCrop(object):"""随机裁剪样本中的图像.Args:output_size（tuple或int）：所需的输出大小。 如果是int，方形裁剪是。"""def __init__(self, output_size):assert isinstance(output_size, (int, tuple))if isinstance(output_size, int):self.output_size = (output_size, output_size)else:assert len(output_size) == 2self.output_size = output_sizedef __call__(self, sample):image, landmarks = sample['image'], sample['landmarks']h, w = image.shape[:2]new_h, new_w = self.output_sizetop = np.random.randint(0, h - new_h)left = np.random.randint(0, w - new_w)image = image[top: top + new_h,left: left + new_w]landmarks = landmarks - [left, top]return {'image': image, 'landmarks': landmarks}class ToTensor(object):"""将样本中的ndarrays转换为Tensors."""def __call__(self, sample):image, landmarks = sample['image'], sample['landmarks']# 交换颜色轴因为# numpy包的图片是: H * W * C# torch包的图片是: C * H * Wimage = image.transpose((2, 0, 1))return {'image': torch.from_numpy(image),'landmarks': torch.from_numpy(landmarks)}scale = Rescale(256)
crop = RandomCrop(128)
composed = transforms.Compose([Rescale(256),RandomCrop(224)])"""
应用转换
"""# 在样本上应用上述的每个变换。
fig = plt.figure()
sample = face_dataset[65]
for i, tsfrm in enumerate([scale, crop, composed]):transformed_sample = tsfrm(sample)ax = plt.subplot(1, 3, i + 1)plt.tight_layout()ax.set_title(type(tsfrm).__name__)show_landmarks(**transformed_sample)plt.show()"""
torch.utils.data.DataLoader是一个提供
每次这个数据集被采样时: 及时地从文件中读取图片 * 对读取的图片应用转换 * 由于其中一步操作是随机的 (randomcrop) , 数据被增强了
功能的迭代器
"""transformed_dataset = FaceLandmarksDataset(csv_file='data/faces/faces/face_landmarks.csv',root_dir='data/faces/faces/',transform=transforms.Compose([Rescale(256),RandomCrop(224),ToTensor()]))dataloader = DataLoader(transformed_dataset, batch_size=4,shuffle=True, num_workers=4)# 辅助功能：显示批次
def show_landmarks_batch(sample_batched):"""Show image with landmarks for a batch of samples."""images_batch, landmarks_batch = \sample_batched['image'], sample_batched['landmarks']batch_size = len(images_batch)im_size = images_batch.size(2)grid_border_size = 2grid = utils.make_grid(images_batch)plt.imshow(grid.numpy().transpose((1, 2, 0)))for i in range(batch_size):plt.scatter(landmarks_batch[i, :, 0].numpy() + i * im_size + (i + 1) * grid_border_size,landmarks_batch[i, :, 1].numpy() + grid_border_size,s=10, marker='.', c='r')plt.title('Batch from dataloader')for i_batch, sample_batched in enumerate(dataloader):print(i_batch, sample_batched['image'].size(),sample_batched['landmarks'].size())# 观察第4批次并停止。if i_batch == 3:plt.figure()show_landmarks_batch(sample_batched)plt.axis('off')plt.ioff()plt.show()break

Pytorch小试牛刀

"""
使用NumPy，手动实现网络的 前向和反向传播，来拟合随机数据
"""import numpy as np# N是批量大小; D_in是输入维度;
# 49/5000 H是隐藏的维度; D_out是输出维度。
N, D_in, H, D_out = 64, 1000, 100, 10# 创建随机输入和输出数据
x = np.random.randn(N, D_in)
y = np.random.randn(N, D_out)# 随机初始化权重
w1 = np.random.randn(D_in, H)
w2 = np.random.randn(H, D_out)learning_rate = 1e-6
for t in range(500):# 前向传递：计算预测值yh = x.dot(w1)h_relu = np.maximum(h, 0)y_pred = h_relu.dot(w2)# 计算和打印损失lossloss = np.square(y_pred - y).sum()print(t, loss)# 反向传播，计算w1和w2对loss的梯度grad_y_pred = 2.0 * (y_pred - y)grad_w2 = h_relu.T.dot(grad_y_pred)grad_h_relu = grad_y_pred.dot(w2.T)grad_h = grad_h_relu.copy()grad_h[h < 0] = 0grad_w1 = x.T.dot(grad_h)# 更新权重w1 -= learning_rate * grad_w1w2 -= learning_rate * grad_w2"""
使用PyTorch的tensor，手动在网络中实现前向传播和反向传播
"""import torchdtype = torch.float
device = torch.device("cpu")
# device = torch.device（“cuda：0”）＃取消注释以在GPU上运行# N是批量大小; D_in是输入维度;
# H是隐藏的维度; D_out是输出维度。
N, D_in, H, D_out = 64, 1000, 100, 10#创建随机输入和输出数据
x = torch.randn(N, D_in, device=device, dtype=dtype)
y = torch.randn(N, D_out, device=device, dtype=dtype)# 随机初始化权重
w1 = torch.randn(D_in, H, device=device, dtype=dtype)
w2 = torch.randn(H, D_out, device=device, dtype=dtype)learning_rate = 1e-6
for t in range(500):# 前向传递：计算预测yh = x.mm(w1)h_relu = h.clamp(min=0)y_pred = h_relu.mm(w2)  #(tensor).mm == (numpy).dot# 计算和打印损失loss = (y_pred - y).pow(2).sum().item()print(t, loss)# Backprop计算w1和w2相对于损耗的梯度grad_y_pred = 2.0 * (y_pred - y)grad_w2 = h_relu.t().mm(grad_y_pred)grad_h_relu = grad_y_pred.mm(w2.t())grad_h = grad_h_relu.clone()grad_h[h < 0] = 0grad_w1 = x.t().mm(grad_h)# 使用梯度下降更新权重w1 -= learning_rate * grad_w1w2 -= learning_rate * grad_w2"""
使用PyTorch的Tensors和autograd来实现我们的两层的神经网络
"""import torchdtype = torch.float
device = torch.device("cpu")
# device = torch.device（“cuda：0”）＃取消注释以在GPU上运行# N是批量大小; D_in是输入维度;
# H是隐藏的维度; D_out是输出维度。
N, D_in, H, D_out = 64, 1000, 100, 10# 创建随机Tensors以保持输入和输出。
# 设置requires_grad = False表示我们不需要计算渐变
# 在向后传球期间对于这些Tensors。
x = torch.randn(N, D_in, device=device, dtype=dtype)
y = torch.randn(N, D_out, device=device, dtype=dtype)# 为权重创建随机Tensors。
# 设置requires_grad = True表示我们想要计算渐变
# 在向后传球期间尊重这些张贴。
w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)
w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)learning_rate = 1e-6
for t in range(500):# 前向传播：使用tensors上的操作计算预测值y;# 由于w1和w2有requires_grad=True，涉及这些张量的操作将让PyTorch构建计算图，# 从而允许自动计算梯度。由于我们不再手工实现反向传播，所以不需要保留中间值的引用。y_pred = x.mm(w1).clamp(min=0).mm(w2)# 使用Tensors上的操作计算和打印丢失。# loss是一个形状为()的张量# loss.item() 得到这个张量对应的python数值loss = (y_pred - y).pow(2).sum()print(t, loss.item())# 使用autograd计算反向传播。这个调用将计算loss对所有requires_grad=True的tensor的梯度。# 这次调用后，w1.grad和w2.grad将分别是loss对w1和w2的梯度张量。loss.backward()# 使用梯度下降更新权重。对于这一步，我们只想对w1和w2的值进行原地改变；不想为更新阶段构建计算图，# 所以我们使用torch.no_grad()上下文管理器防止PyTorch为更新构建计算图with torch.no_grad():w1 -= learning_rate * w1.gradw2 -= learning_rate * w2.grad# 反向传播后手动将梯度设置为零w1.grad.zero_()w2.grad.zero_()"""
自定义nn模块:
自定义Module的子类构建两层网络：
"""import torchclass TwoLayerNet(torch.nn.Module):def __init__(self, D_in, H, D_out):"""在构造函数中，我们实例化了两个nn.Linear模块，并将它们作为成员变量。"""super(TwoLayerNet, self).__init__()self.linear1 = torch.nn.Linear(D_in, H)self.linear2 = torch.nn.Linear(H, D_out)def forward(self, x):"""在前向传播的函数中，我们接收一个输入的张量，也必须返回一个输出张量。我们可以使用构造函数中定义的模块以及张量上的任意的（可微分的）操作。"""h_relu = self.linear1(x).clamp(min=0)y_pred = self.linear2(h_relu)return y_pred# N是批大小； D_in 是输入维度；
# H 是隐藏层维度； D_out 是输出维度
N, D_in, H, D_out = 64, 1000, 100, 10# 产生输入和输出的随机张量
x = torch.randn(N, D_in)
y = torch.randn(N, D_out)# 通过实例化上面定义的类来构建我们的模型。
model = TwoLayerNet(D_in, H, D_out)# 构造损失函数和优化器。
# SGD构造函数中对model.parameters()的调用，
# 将包含模型的一部分，即两个nn.Linear模块的可学习参数。
loss_fn = torch.nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)
for t in range(500):# 前向传播：通过向模型传递x计算预测值yy_pred = model(x)#计算并输出lossloss = loss_fn(y_pred, y)print(t, loss.item())# 清零梯度，反向传播，更新权重optimizer.zero_grad()loss.backward()optimizer.step()"""
控制流和权重共享
使用普通的Python流控制来实现循环，并且我们可以通过在定义转发时多次重用同一个模块来实现最内层之间的权重共享。
"""import random
import torchclass DynamicNet(torch.nn.Module):def __init__(self, D_in, H, D_out):"""在构造函数中，我们构造了三个nn.Linear实例，它们将在前向传播时被使用。"""super(DynamicNet, self).__init__()self.input_linear = torch.nn.Linear(D_in, H)self.middle_linear = torch.nn.Linear(H, H)self.output_linear = torch.nn.Linear(H, D_out)def forward(self, x):"""对于模型的前向传播，我们随机选择0、1、2、3，并重用了多次计算隐藏层的middle_linear模块。由于每个前向传播构建一个动态计算图，我们可以在定义模型的前向传播时使用常规Python控制流运算符，如循环或条件语句。在这里，我们还看到，在定义计算图形时多次重用同一个模块是完全安全的。这是Lua Torch的一大改进，因为Lua Torch中每个模块只能使用一次。"""h_relu = self.input_linear(x).clamp(min=0)for _ in range(random.randint(0, 3)):   #权重共享的实现h_relu = self.middle_linear(h_relu).clamp(min=0)y_pred = self.output_linear(h_relu)return y_pred# N是批大小；D是输入维度
# H是隐藏层维度；D_out是输出维度
N, D_in, H, D_out = 64, 1000, 100, 10# 产生输入和输出随机张量
x = torch.randn(N, D_in)
y = torch.randn(N, D_out)# 实例化上面定义的类来构造我们的模型
model = DynamicNet(D_in, H, D_out)# 构造我们的损失函数（loss function）和优化器（Optimizer）。
# 用平凡的随机梯度下降训练这个奇怪的模型是困难的，所以我们使用了momentum方法。
criterion = torch.nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)
for t in range(500):# 前向传播：通过向模型传入x计算预测的y。y_pred = model(x)# 计算并打印损失loss = criterion(y_pred, y)print(t, loss.item())# 清零梯度，反向传播，更新权重optimizer.zero_grad()loss.backward()optimizer.step()

PyTorch之迁移学习

from __future__ import print_function, division
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copyplt.ion()   # interactive mode"""
加载数据
"""#训练集数据扩充和归一化
#在验证集上仅需要归一化
data_transforms = {'train': transforms.Compose([transforms.RandomResizedCrop(224), #随机裁剪一个area然后再resizetransforms.RandomHorizontalFlip(), #随机水平翻转transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),'val': transforms.Compose([transforms.Resize(256),transforms.CenterCrop(224), #图像裁剪transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
}data_dir = 'data/hymenoptera_data'
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),data_transforms[x])for x in ['train', 'val']}
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,shuffle=True, num_workers=4)for x in ['train', 'val']}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
class_names = image_datasets['train'].classesdevice = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")"""
可视化部分图像数据
"""def imshow(inp, title=None):"""Imshow for Tensor."""inp = inp.numpy().transpose((1, 2, 0))mean = np.array([0.485, 0.456, 0.406])std = np.array([0.229, 0.224, 0.225])inp = std * inp + meaninp = np.clip(inp, 0, 1)plt.imshow(inp)if title is not None:plt.title(title)plt.pause(1)  # pause a bit so that plots are updated# 获取一批训练数据
inputs, classes = next(iter(dataloaders['train']))# 批量制作网格
out = torchvision.utils.make_grid(inputs)imshow(out, title=[class_names[x] for x in classes])"""
训练模型
"""def train_model(model, criterion, optimizer, scheduler, num_epochs=25):since = time.time()best_model_wts = copy.deepcopy(model.state_dict())best_acc = 0.0for epoch in range(num_epochs):print('Epoch {}/{}'.format(epoch, num_epochs - 1))print('-' * 10)# 每个epoch都有一个训练和验证阶段for phase in ['train', 'val']:if phase == 'train':scheduler.step()model.train()  # Set model to training modeelse:model.eval()   # Set model to evaluate moderunning_loss = 0.0running_corrects = 0# 迭代数据.for inputs, labels in dataloaders[phase]:inputs = inputs.to(device)labels = labels.to(device)# 零参数梯度optimizer.zero_grad()# 前向# track history if only in trainwith torch.set_grad_enabled(phase == 'train'):outputs = model(inputs)_, preds = torch.max(outputs, 1)loss = criterion(outputs, labels)# 后向+仅在训练阶段进行优化if phase == 'train':loss.backward()optimizer.step()# 统计running_loss += loss.item() * inputs.size(0)running_corrects += torch.sum(preds == labels.data)epoch_loss = running_loss / dataset_sizes[phase]epoch_acc = running_corrects.double() / dataset_sizes[phase]print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))# 深度复制moif phase == 'val' and epoch_acc > best_acc:best_acc = epoch_accbest_model_wts = copy.deepcopy(model.state_dict())print()time_elapsed = time.time() - sinceprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))print('Best val Acc: {:4f}'.format(best_acc))# 加载最佳模型权重model.load_state_dict(best_model_wts)return model"""
可视化模型的预测结果
"""
#一个通用的展示少量预测图片的函数
def visualize_model(model, num_images=6):was_training = model.trainingmodel.eval()images_so_far = 0fig = plt.figure()with torch.no_grad():for i, (inputs, labels) in enumerate(dataloaders['val']):inputs = inputs.to(device)labels = labels.to(device)outputs = model(inputs)_, preds = torch.max(outputs, 1)for j in range(inputs.size()[0]):images_so_far += 1ax = plt.subplot(num_images//2, 2, images_so_far)ax.axis('off')ax.set_title('predicted: {}'.format(class_names[preds[j]]))imshow(inputs.cpu().data[j])if images_so_far == num_images:model.train(mode=was_training)returnmodel.train(mode=was_training)"""
场景1：微调ConvNet
加载预训练模型并重置最终完全连接的图层。
"""model_ft = models.resnet18(pretrained=True)
#直接加载pre-train模型中预先训练好的参数
num_ftrs = model_ft.fc.in_features
#提取fc层中固定的参数
model_ft.fc = nn.Linear(num_ftrs, 2)
#resnet网络最后一层分类层fc是对1000种类型进行划分，对于自己的数据集，如果只有2类model_ft = model_ft.to(device)
#模型加载到指定设备上。(device = 'cpu')criterion = nn.CrossEntropyLoss()# 观察所有参数都正在优化
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)# 每7个epochs衰减LR通过设置gamma=0.1
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)#训练和评估模型
model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,num_epochs=25)#模型评估效果可视化
visualize_model(model_ft)"""
场景2：ConvNet作为固定特征提取器
在这里需要冻结除最后一层之外的所有网络。通过设置requires_grad == False backward()来冻结参数，这样在反向传播backward()的时候他们的梯度就不会被计算。
"""model_conv = torchvision.models.resnet18(pretrained=True)
for param in model_conv.parameters():param.requires_grad = False#默认情况下，新构造模块的参数需要_grad=True
num_ftrs = model_conv.fc.in_features
model_conv.fc = nn.Linear(num_ftrs, 2)model_conv = model_conv.to(device)criterion = nn.CrossEntropyLoss()# 观察到只有最后一层的参数被优化为
# 与以前相反
optimizer_conv = optim.SGD(model_conv.fc.parameters(), lr=0.001, momentum=0.9)# LR每7个周期衰减0.1倍
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)#训练和评估
model_conv = train_model(model_conv, criterion, optimizer_conv,exp_lr_scheduler, num_epochs=25)
#模型评估效果可视化
visualize_model(model_conv)plt.ioff()
plt.show()

保存和加载模型

import torch
import torch.nn as nn
import torch.functional as F
import torch.utils.data as Data
import torchvision
import torch.optim as optim"""
什么是状态字典：state_dict?
"""class TheModelClass(nn.Module):def __init__(self):super(TheModelClass, self).__init__()self.conv1 = nn.Conv2d(3, 6, 5)self.pool = nn.MaxPool2d(2, 2)self.conv2 = nn.Conv2d(6, 16, 5)self.fc1 = nn.Linear(16 * 5 * 5, 120)self.fc2 = nn.Linear(120, 84)self.fc3 = nn.Linear(84, 10)def forward(self, x):x = self.pool(F.relu(self.conv1(x)))x = self.pool(F.relu(self.conv2(x)))x = x.view(-1, 16 * 5 * 5)x = F.relu(self.fc1(x))x = F.relu(self.fc2(x))x = self.fc3(x)return x# 初始化模型
model = TheModelClass()# 初始化优化器
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)# 打印模型的状态字典
print("Model's state_dict:")
for param_tensor in model.state_dict():print(param_tensor, "\t", model.state_dict()[param_tensor].size())# 打印优化器的状态字典
print("Optimizer's state_dict:")
for var_name in optimizer.state_dict():print(var_name, "\t", optimizer.state_dict()[var_name])# """
# 保存和加载推理模型
# """
#
# #2.1 保存/加载state_dict（推荐使用）
#
# #保存
# torch.save(model.state_dict(), PATH)    #(path是存储路径)
#
# #加载
# model = TheModelClass(*args, **kwargs)
# model.load_state_dict(torch.load(PATH))
# model.eval()
# #在 PyTorch 中最常见的模型保存使‘.pt’或者是‘.pth’作为模型文件扩展名。
# #请记住，在运行推理之前，务必调用model.eval()去设置 dropout 和 batch normalization 层为评估模式。如果不这么做，可能导致模型推断结果不一致。
#
# #2.2 保存/加载完整模型
#
# #保存
# torch.save(model, PATH)
#
# #加载
#
# # 模型类必须在此之前被定义
# model = torch.load(PATH)
# model.eval()
#
# """
# 保存和加载 Checkpoint 用于推理/继续训练
# """
#
# #保存
# torch.save({
#             'epoch': epoch,
#             'model_state_dict': model.state_dict(),
#             'optimizer_state_dict': optimizer.state_dict(),
#             'loss': loss,
#             ...
#             }, PATH)
#
# #加载
#
# model = TheModelClass(*args, **kwargs)
# optimizer = TheOptimizerClass(*args, **kwargs)
#
# checkpoint = torch.load(PATH)
# model.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# epoch = checkpoint['epoch']
# loss = checkpoint['loss']
#
# model.eval()
# # - or -
# model.train()
# #PyTorch 中常见的保存checkpoint 是使用 .tar 文件扩展名。
#
# """
# 在一个文件中保存多个模型
# """
#
# #保存
# torch.save({
#             'modelA_state_dict': modelA.state_dict(),
#             'modelB_state_dict': modelB.state_dict(),
#             'optimizerA_state_dict': optimizerA.state_dict(),
#             'optimizerB_state_dict': optimizerB.state_dict(),
#             ...
#             }, PATH)
#
# #加载
# modelA = TheModelAClass(*args, **kwargs)
# modelB = TheModelBClass(*args, **kwargs)
# optimizerA = TheOptimizerAClass(*args, **kwargs)
# optimizerB = TheOptimizerBClass(*args, **kwargs)
#
# checkpoint = torch.load(PATH)
# modelA.load_state_dict(checkpoint['modelA_state_dict'])
# modelB.load_state_dict(checkpoint['modelB_state_dict'])
# optimizerA.load_state_dict(checkpoint['optimizerA_state_dict'])
# optimizerB.load_state_dict(checkpoint['optimizerB_state_dict'])
#
# modelA.eval()
# modelB.eval()
# # - or -
# modelA.train()
# modelB.train()
# #PyTorch 中常见的保存 checkpoint 是使用 .tar 文件扩展名。
# #请记住在运行推理之前，务必调用model.eval()去设置 dropout 和 batch normalization 为评估。如果不这样做，有可能得到不一致的推断结果。
# # 如果你想要恢复训练，请调用model.train()以确保这些层处于训练模式。
#
# """
# 使用在不同模型参数下的热启动模式
# """
#
# #保存
# torch.save(modelA.state_dict(), PATH)
#
# #加载
# modelB = TheModelBClass(*args, **kwargs)
# modelB.load_state_dict(torch.load(PATH), strict=False)
#
# """
# 保存到 CPU、加载到 CPU
# """
#
# #保存
# torch.save(model.state_dict(), PATH)
#
# #加载
# device = torch.device('cpu')
# model = TheModelClass(*args, **kwargs)
# model.load_state_dict(torch.load(PATH, map_location=device))
#
# """
# 保存到 GPU、加载到 GPU
# """
#
# #保存
# torch.save(model.state_dict(), PATH)
#
# #加载
# device = torch.device("cuda")
# model = TheModelClass(*args, **kwargs)
# model.load_state_dict(torch.load(PATH, map_location="cuda:0"))  # Choose whatever GPU device number you want
# model.to(device)
# # 确保在你提供给模型的任何输入张量上调用input = input.to(device)
#
# """
# 保存 torch.nn.DataParallel 模型
# """
#
# #保存：
# torch.save(model.module.state_dict(), PATH)