深度学习本科课程 实验3 网络优化

一、在多分类任务实验中实现momentum、rmsprop、adam优化器

1.1 任务内容

  1. 在手动实现多分类的任务中手动实现三种优化算法,并补全Adam中计算部分的内容
  2. 在torch.nn实现多分类的任务中使用torch.nn实现各种优化器,并对比其效果

1.2 任务思路及代码

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import transforms
import time
from torch.nn import CrossEntropyLoss
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 如果有gpu则在gpu上计算 加快计算速度
print(f'当前使用的device为{device}')
# 多分类任务
mnist_train = torchvision.datasets.FashionMNIST(root='./FashionMNIST', train=True, download=True, transform=transforms.ToTensor())
mnist_test = torchvision.datasets.FashionMNIST(root='./FashionMNIST', train=False, download=True, transform=transforms.ToTensor())
batch_size = 256
train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)
test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=0)
# 定义绘图函数
import matplotlib.pyplot as plt
def draw(name, trainl, testl,xlabel='Epoch',ylabel='Loss'):plt.rcParams["font.sans-serif"]=["SimHei"] #设置字体plt.rcParams["axes.unicode_minus"]=False #该语句解决图像中的“-”负号的乱码问题plt.figure(figsize=(8, 3))plt.title(name[-1]) # 命名color = ['g','r','b','c']if trainl is not None:plt.subplot(121)for i in range(len(name)-1):plt.plot(trainl[i], c=color[i],label=name[i])plt.xlabel(xlabel)plt.ylabel(ylabel)plt.legend()if testl is not None:plt.subplot(122)for i in range(len(name)-1):plt.plot(testl[i], c=color[i], label=name[i])plt.xlabel(xlabel)plt.ylabel(ylabel)plt.legend()
# 自定义实现
class Net():def __init__(self):# 设置隐藏层和输出层的节点数num_inputs, num_hiddens, num_outputs = 28 * 28, 256, 10  # 十分类问题self.w_1 = torch.tensor(np.random.normal(0, 0.01, (num_hiddens, num_inputs)), dtype=torch.float32,requires_grad=True)self.b_1 = torch.zeros(num_hiddens, dtype=torch.float32, requires_grad=True)self.w_2 = torch.tensor(np.random.normal(0, 0.01, (num_outputs, num_hiddens)), dtype=torch.float32,requires_grad=True)self.b_2 = torch.zeros(num_outputs, dtype=torch.float32, requires_grad=True)self.params=[self.w_1, self.b_1, self.w_2, self.b_2]self.w = [self.w_1,self.w_2]# 定义模型结构self.input_layer = lambda x: x.view(x.shape[0], -1)self.hidden_layer = lambda x: self.my_relu(torch.matmul(x, self.w_1.t()) + self.b_1)self.output_layer = lambda x: nn.functional.softmax(torch.matmul(x, self.w_2.t()) + self.b_2, dim=1)self.momentum_states = [torch.zeros_like(param) for param in self.params]def my_relu(self, x):return torch.max(input=x, other=torch.tensor(0.0))# 定义前向传播def forward(self, x):x = self.input_layer(x)x = self.hidden_layer(x)x = self.output_layer(x)return xdef my_cross_entropy_loss(y_hat, labels):def log_softmax(y_hat):max_v = torch.max(y_hat, dim=1).values.unsqueeze(dim=1)return y_hat - max_v - torch.log(torch.exp(y_hat-max_v).sum(dim=1).unsqueeze(dim=1))return (-log_softmax(y_hat))[range(len(y_hat)), labels].mean()# nn实现
class MyNet_NN(nn.Module):def __init__(self,dropout=0.0):super(MyNet_NN, self).__init__()# 设置隐藏层和输出层的节点数self.num_inputs, self.num_hiddens, self.num_outputs = 28 * 28, 256, 10  # 十分类问题# 定义模型结构self.input_layer = nn.Flatten()self.hidden_layer = nn.Linear(28*28,256)self.drop = nn.Dropout(dropout)self.output_layer = nn.Linear(256,10)# 使用relu激活函数self.relu = nn.ReLU()# 定义前向传播def forward(self, x):x = self.drop(self.input_layer(x))x = self.drop(self.hidden_layer(x))x = self.relu(x)x = self.output_layer(x)return x
def train_and_test(model=Net(),init_states=None,optimizer=optim.SGD,epochs=10,lr=0.01,L2=False,lambd=0):train_all_loss = []  test_all_loss = []  train_ACC, test_ACC = [], [] begintime = time.time()criterion = CrossEntropyLoss() for epoch in range(epochs):train_l,train_acc_num = 0, 0for data, labels in train_iter:pred = model.forward(data)train_each_loss = criterion(pred, labels)  # 若L2为True则表示需要添加L2范数惩罚项if L2 == True:train_each_loss += lambd * l2_penalty(model.w)train_l += train_each_loss.item()train_each_loss.backward()  # 反向传播if init_states == None: optimizer(model.params, lr, 128)  # 使用小批量随机梯度下降迭代模型参数else:states = init_states(model.params)optimizer(model.params,states,lr=lr)# 梯度清零train_acc_num += (pred.argmax(dim=1)==labels).sum().item()for param in model.params:param.grad.data.zero_()# print(train_each_loss)train_all_loss.append(train_l)  # 添加损失值到列表中train_ACC.append(train_acc_num / len(mnist_train)) # 添加准确率到列表中with torch.no_grad():is_train = False  test_l, test_acc_num = 0, 0for data, labels in test_iter:pred = model.forward(data)test_each_loss = criterion(pred, labels)test_l += test_each_loss.item()test_acc_num += (pred.argmax(dim=1)==labels).sum().item()test_all_loss.append(test_l)test_ACC.append(test_acc_num / len(mnist_test))   # # 添加准确率到列表中print('epoch: %d\t train loss:%.5f\t test loss:%.5f\t train acc: %.2f\t test acc: %.2f'% (epoch + 1, train_l, test_l, train_ACC[-1],test_ACC[-1]))endtime = time.time()print("%d轮 总用时: %.3f秒" % ( epochs, endtime - begintime))return train_all_loss,test_all_loss,train_ACC,test_ACC
def train_and_test_NN(model=MyNet_NN(),epochs=10,lr=0.01,weight_decay=0.0,optimizer=None):MyModel = modelprint(MyModel)if optimizer == None:optimizer = SGD(MyModel.parameters(), lr=lr,weight_decay=weight_decay) criterion = CrossEntropyLoss() # 损失函数criterion = criterion.to(device)train_all_loss = []  test_all_loss = []  train_ACC, test_ACC = [], []begintime = time.time()for epoch in range(epochs):train_l, train_epoch_count, test_epoch_count = 0, 0, 0for data, labels in train_iter:data, labels = data.to(device), labels.to(device)pred = MyModel(data)train_each_loss = criterion(pred, labels.view(-1))  # 计算每次的损失值optimizer.zero_grad()  train_each_loss.backward()  optimizer.step()  train_l += train_each_loss.item()train_epoch_count += (pred.argmax(dim=1)==labels).sum()train_ACC.append(train_epoch_count/len(mnist_train))train_all_loss.append(train_l) with torch.no_grad():test_loss, test_epoch_count= 0, 0for data, labels in test_iter:data, labels = data.to(device), labels.to(device)pred = MyModel(data)test_each_loss = criterion(pred,labels)test_loss += test_each_loss.item()test_epoch_count += (pred.argmax(dim=1)==labels).sum()test_all_loss.append(test_loss)test_ACC.append(test_epoch_count.cpu()/len(mnist_test))print('epoch: %d\t train loss:%.5f\t test loss:%.5f\t train acc:%5f test acc:%.5f:' % (epoch + 1, train_all_loss[-1], test_all_loss[-1],train_ACC[-1],test_ACC[-1]))endtime = time.time()print("torch.nn实现前馈网络-多分类任务 %d轮 总用时: %.3f秒" % (epochs, endtime - begintime))# 返回训练集和测试集上的 损失值 与 准确率return train_all_loss,test_all_loss,train_ACC,test_ACC
# 手动实现momentum
def init_momentum(params):w1,b1,w2,b2 = torch.zeros(params[0].shape),torch.zeros(params[1].shape),torch.zeros(params[2].shape),torch.zeros(params[3].shape)return (w1,b1,w2,b2)def sgd_momentum(params, states, lr=0.01, momentum=0.9):for p, v in zip(params, states):with torch.no_grad():v[:] = momentum * v - p.gradp[:] += lr*vp.grad.data.zero_()net11 = Net()
trainL11, testL11, trainAcc11, testAcc11 = train_and_test(model=net11,epochs=10,init_states=init_momentum, optimizer=sgd_momentum)
# nn实现Momentum
net12 = MyNet_NN()
net12 = net12.to(device)
momentum_optimizer = optim.SGD(net12.parameters(), lr=0.01, momentum=0.9)
trainL12, testL12, trainAcc12, testAcc12 = train_and_test_NN(model=net12,epochs=10,optimizer=momentum_optimizer)    
# 手动实现RMSpropdef init_rmsprop(params):s_w1, s_b1, s_w2, s_b2 = torch.zeros(params[0].shape), torch.zeros(params[1].shape),\torch.zeros(params[2].shape), torch.zeros(params[3].shape)return (s_w1, s_b1, s_w2, s_b2)def rmsprop(params,states,lr=0.01,gamma=0.9):gamma, eps = gamma, 1e-6for p, s in zip(params,states):with torch.no_grad():s[:] = gamma * s + (1 - gamma) * torch.square(p.grad)p[:] -= lr * p.grad / torch.sqrt(s + eps)p.grad.data.zero_()net21= Net()
trainL21, testL21, trainAcc21, testAcc21 = train_and_test(model=net21,epochs=10,init_states=init_rmsprop, optimizer=rmsprop)
# nn实现RMSprop
net22 = MyNet_NN()
net22 = net22.to(device)
optim_RMSprop = torch.optim.RMSprop(net22.parameters(), lr=0.01, alpha=0.9, eps=1e-6)
trainL22, testL22, trainAcc22, testAcc22 = train_and_test_NN(model=net22,epochs=10,optimizer=optim_RMSprop)    
# 手动实现Adam
def init_adam_states(params):v_w1, v_b1, v_w2, v_b2 = torch.zeros(params[0].shape), torch.zeros(params[1].shape),\torch.zeros(params[2].shape), torch.zeros(params[3].shape)s_w1, s_b1, s_w2, s_b2 = torch.zeros(params[0].shape), torch.zeros(params[1].shape),\torch.zeros(params[2].shape), torch.zeros(params[3].shape)return ((v_w1, s_w1), (v_b1, s_b1),(v_w2, s_w2), (v_b2, s_b2))# 根据Adam算法思想手动实现Adam
Adam_t = 0.01
def Adam(params, states, lr=0.01, t=Adam_t):global Adam_tbeta1, beta2, eps = 0.9, 0.999, 1e-6for p, (v, s) in zip(params, states):with torch.no_grad():v[:] = beta1 * v + (1 - beta1) * p.grads[:] = beta2 * s + (1 - beta2) * (p.grad**2)v_bias_corr = v / (1 - beta1 ** Adam_t)s_bias_corr = s / (1 - beta2 ** Adam_t)p.data -= lr * v_bias_corr / (torch.sqrt(s_bias_corr + eps))p.grad.data.zero_()Adam_t += 1net31 = Net()
trainL31, testL31, trainAcc31, testAcc31 = train_and_test(model=net31,epochs=10,init_states=init_adam_states, optimizer=Adam)   
# nn实现adam
net32 = MyNet_NN()
net32 = net32.to(device)optim_Adam = torch.optim.Adam(net32.parameters(), lr=0.01, betas=(0.9,0.999),eps=1e-6)
trainL32, testL32, trainAcc32, testAcc32 = train_and_test_NN(model=net32,epochs=10,optimizer=optim_Adam)    
name11= ['RMSprop','Momentum','Adam','手动实现不同的优化器-Loss变化']
train11 = [trainL11,trainL21,trainL31]
test11= [testL11, testL21, testL31]
draw(name11, train11, test11)
name12= ['RMSprop','Momentum','Adam','torch.nn实现不同的优化器-Loss变化']
train12 = [trainL12,trainL22,trainL32]
test12 = [testL12, testL22, testL32]
draw(name12, train12, test12)

二、在多分类任务实验中分别手动实现和用torch.nn实现𝑳𝟐正则化

2.1 任务内容

探究惩罚项的权重对实验结果的影响(可用loss曲线进行展示)

2.2 任务思路及代码

# 定义L2范数惩罚项
def l2_penalty(w):cost = 0for i in range(len(w)):cost += (w[i]**2).sum()return cost / batch_size / 2
# 手动实现
net221 = Net()
trainL221, testL221, trainAcc221, testAcc221 = train_and_test(model=net221,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01,L2=True,lambd=0)net222 = Net()
trainL222, testL222, trainAcc222, testAcc222 = train_and_test(model=net222,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01,L2=True,lambd=2)# 可视化比较
name221 = ['lambd= 0','lambd=2','手动实现不同的惩罚权重lambd-Loss变化']
trains221 = [trainL221,trainL222]
tests221= [testL221,testL222]
draw(name221, trains221, tests221)
## nn实现
net223 = MyNet_NN()
net223 = net223.to(device)
momentum_optimizer = optim.SGD(net223.parameters(), lr=0.01, momentum=0.9)
trainL223, testL223, trainAcc223, testAcc223 = train_and_test_NN(model=net223,epochs=10,optimizer=momentum_optimizer,lr=0.01,weight_decay=0.0)net224 = MyNet_NN()
net224 = net223.to(device)
momentum_optimizer = optim.SGD(net224.parameters(), lr=0.01, momentum=0.9)
trainL224, testL224, trainAcc224, testAcc224 = train_and_test_NN(model=net224,epochs=10,optimizer=momentum_optimizer,lr=0.01,weight_decay=0.01)# 可视化比较
name222 = ['weight_decay=0','weight_decay = 0.01','torch.nn实现不同的惩罚权重lambd-Loss变化']
trains222 = [trainL223,trainL224]
tests222= [testL223,testL224]
draw(name222, trains222, tests222)

三、在多分类任务实验中分别手动实现和用torch.nn实现dropout

3.1 任务内容

探究不同丢弃率对实验结果的影响(可用loss曲线进行展示)

3.2 任务思路及代码

# 为手动模型添加dropout项
class MyNet():def __init__(self,dropout=0.0):# 设置隐藏层和输出层的节点数# global dropoutself.dropout = dropoutprint('dropout: ',self.dropout)self.is_train = Nonenum_inputs, num_hiddens, num_outputs = 28 * 28, 256, 10  # 十分类问题w_1 = torch.tensor(np.random.normal(0, 0.01, (num_hiddens, num_inputs)), dtype=torch.float32,requires_grad=True)b_1 = torch.zeros(num_hiddens, dtype=torch.float32, requires_grad=True)w_2 = torch.tensor(np.random.normal(0, 0.01, (num_outputs, num_hiddens)), dtype=torch.float32,requires_grad=True)b_2 = torch.zeros(num_outputs, dtype=torch.float32, requires_grad=True)self.params = [w_1, b_1, w_2, b_2]self.w = [w_1,w_2]# 定义模型结构self.input_layer = lambda x: x.view(x.shape[0], -1)self.hidden_layer = lambda x: self.my_relu(torch.matmul(x, w_1.t()) + b_1)self.output_layer = lambda x: torch.matmul(x, w_2.t()) + b_2def my_relu(self, x):return torch.max(input=x, other=torch.tensor(0.0))def train(self):self.is_train = Truedef test(self):self.is_test = Falsedef dropout_layer(self, x):dropout =self.dropoutassert 0 <= dropout <= 1 #dropout值必须在0-1之间# dropout==1,所有元素都被丢弃。if dropout == 1:return torch.zeros_like(x)# 在本情况中,所有元素都被保留。if dropout == 0:return xmask = (torch.rand(x.shape) < 1.0 - dropout).float() #rand()返回一个张量,包含了从区间[0, 1)的均匀分布中抽取的一组随机数return mask * x / (1.0 - dropout)# 定义前向传播def forward(self, x):x = self.input_layer(x)if self.is_train: # 如果是训练过程,则需要开启dropout 否则 需要关闭 dropoutx = self.dropout_layer(x) elif self.is_test:x = self.dropout_layer(x)x = self.my_relu(self.hidden_layer(x))x = self.output_layer(x)return x
def train_and_test3(model=MyNet(),init_states=None,optimizer=optim.SGD,epochs=20,lr=0.01,L2=False,lambd=0):train_all_loss = []  test_all_loss = [] train_ACC, test_ACC = [], [] begintime = time.time()criterion = CrossEntropyLoss() # 损失函数model.train() for epoch in range(epochs):train_l,train_acc_num = 0, 0for data, labels in train_iter:pred = model.forward(data)train_each_loss = criterion(pred, labels)  # 计算每次的损失值if L2 == True:train_each_loss += lambd * l2_penalty(model.w)train_l += train_each_loss.item()train_each_loss.backward()  # 反向传播if init_states == None: optimizer(model.params, lr, 128)  # 使用小批量随机梯度下降迭代模型参数else:states = init_states(model.params)optimizer(model.params,states,lr=lr)# 梯度清零train_acc_num += (pred.argmax(dim=1)==labels).sum().item()for param in model.params:param.grad.data.zero_()train_all_loss.append(train_l)  train_ACC.append(train_acc_num / len(mnist_train)) # 添加准确率到列表中model.test() with torch.no_grad():is_train = False  # 表明当前为测试阶段,不需要dropout参与test_l, test_acc_num = 0, 0for data, labels in test_iter:pred = model.forward(data)test_each_loss = criterion(pred, labels)test_l += test_each_loss.item()test_acc_num += (pred.argmax(dim=1)==labels).sum().item()test_all_loss.append(test_l)test_ACC.append(test_acc_num / len(mnist_test))   # # 添加准确率到列表中print('epoch: %d\t train loss:%.5f\t test loss:%.5f\t train acc: %.2f\t test acc: %.2f'% (epoch + 1, train_l, test_l, train_ACC[-1],test_ACC[-1]))endtime = time.time()print("手动实现dropout, %d轮 总用时: %.3f" % ( epochs, endtime - begintime))return train_all_loss,test_all_loss,train_ACC,test_ACC
# 手动实现dropout
net331 = MyNet(dropout = 0.0)
trainL331, testL331, trainAcc331, testAcc331= train_and_test3(model=net331,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01)net332 = MyNet(dropout = 0.3)
trainL332, testL332, trainAcc332, testAcc332= train_and_test3(model=net332,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01)net333 = MyNet(dropout = 0.5)
trainL333, testL333, trainAcc333, testAcc333= train_and_test3(model=net333,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01)net334 = MyNet(dropout = 0.8)
trainL334, testL334, trainAcc334, testAcc334= train_and_test3(model=net334,epochs=10,init_states=init_momentum, optimizer=sgd_momentum,lr=0.01)
name331 = ['dropout=0','dropout=0.3','dropout=0.5','dropout=0.8','手动实现不同的dropout-Loss变化']
train331 = [trainL331,trainL332,trainL333,trainL334]
test331 = [testL331,testL332,testL333,testL334]
draw(name331, train331, test331)
# nn实现dropout
net341 = MyNet_NN(dropout=0)
net341 = net341.to(device)
momentum_optimizer = optim.SGD(net341.parameters(), lr=0.01, momentum=0.9)
trainL341, testL341, trainAcc341, testAcc341= train_and_test_NN(model=net341,epochs=10,optimizer=momentum_optimizer,lr=0.01)net342 = MyNet_NN(dropout=0.3)
net342 = net342.to(device)
momentum_optimizer = optim.SGD(net342.parameters(), lr=0.01, momentum=0.9)
trainL342, testL342, trainAcc342, testAcc342= train_and_test_NN(model=net342,epochs=10,optimizer=momentum_optimizer,lr=0.01)net343 = MyNet_NN(dropout=0.5)
net343 = net341.to(device)
momentum_optimizer = optim.SGD(net343.parameters(), lr=0.01, momentum=0.9)
trainL343, testL343, trainAcc343, testAcc343= train_and_test_NN(model=net343,epochs=10,optimizer=momentum_optimizer,lr=0.01)net344 = MyNet_NN(dropout=0.8)
net344 = net344.to(device)
momentum_optimizer = optim.SGD(net344.parameters(), lr=0.01, momentum=0.9)
trainL344, testL344, trainAcc344, testAcc344= train_and_test_NN(model=net344,epochs=10,optimizer=momentum_optimizer,lr=0.01)
name332 = ['dropout=0','dropout=0.3','dropout=0.5','dropout=0.8','手动实现不同的dropout-Loss变化']
train332 = [trainL341,trainL342,trainL343,trainL344]
test332 = [testL341,testL342,testL343,testL344]
draw(name332, train332, test332)

四、对多分类任务实验中实现早停机制,并在测试集上测试

4.1 任务内容

选择上述实验中效果最好的组合,手动将训练数据划分为训练集和验证集,实现早停机制, 并在测试集上进行测试。训练集:验证集=8:2,早停轮数为5.

4.2 任务思路及代码

# 构建数据集
import random
index = list(range(len(mnist_train)))
random.shuffle(index)# 按照 训练集和验证集 8:2 的比例分配各自下标
train_index, val_index = index[ : 48000], index[48000 : ]train_dataset, train_labels = mnist_train.data[train_index], mnist_train.targets[train_index]
val_dataset, val_labels = mnist_train.data[val_index], mnist_train.targets[val_index]
print('训练集:', train_dataset.shape, train_labels.shape)
print('验证集:', val_dataset.shape,val_labels.shape)T_dataset = torch.utils.data.TensorDataset(train_dataset,train_labels)
V_dataset = torch.utils.data.TensorDataset(val_dataset,val_labels)
T_dataloader = torch.utils.data.DataLoader(dataset=T_dataset,batch_size=128,shuffle=True)
V_dataloader = torch.utils.data.DataLoader(dataset=V_dataset,batch_size=128,shuffle=True)
print('T_dataset',len(T_dataset),'T_dataloader batch_size: 128')
print('V_dataset',len(V_dataset),'V_dataloader batch_size: 128')
def train_and_test_4(model=MyNet(0.0),epochs=10,lr=0.01,weight_decay=0.0):print(model)# 优化函数, 默认情况下weight_decay为0 通过更改weight_decay的值可以实现L2正则化。optimizer = torch.optim.Adam(model.parameters(), lr=0.01, betas=(0.9,0.999),eps=1e-6)criterion = CrossEntropyLoss() # 损失函数train_all_loss = []  # 记录训练集上得loss变化val_all_loss = []  # 记录测试集上的loss变化train_ACC, val_ACC = [], []begintime = time.time()flag_stop = 0for epoch in range(1000):train_l, train_epoch_count, val_epoch_count = 0, 0, 0for data, labels in T_dataloader:data, labels = data.to(torch.float32).to(device), labels.to(device)pred = model(data)train_each_loss = criterion(pred, labels.view(-1))  # 计算每次的损失值optimizer.zero_grad()  # 梯度清零train_each_loss.backward()  # 反向传播optimizer.step()  # 梯度更新train_l += train_each_loss.item()train_epoch_count += (pred.argmax(dim=1)==labels).sum()train_ACC.append(train_epoch_count/len(train_dataset))train_all_loss.append(train_l)  # 添加损失值到列表中with torch.no_grad():val_loss, val_epoch_count= 0, 0for data, labels in V_dataloader:data, labels = data.to(torch.float32).to(device), labels.to(device)pred = model(data)val_each_loss = criterion(pred,labels)val_loss += val_each_loss.item()val_epoch_count += (pred.argmax(dim=1)==labels).sum()val_all_loss.append(val_loss)val_ACC.append(val_epoch_count / len(val_dataset))# 实现早停机制# 若连续五次验证集的损失值连续增大,则停止运行,否则继续运行,if epoch > 5 and val_all_loss[-1] > val_all_loss[-2]:flag_stop += 1if flag_stop == 5 or epoch > 35:print('停止运行,防止过拟合')breakelse:flag_stop = 0if epoch == 0 or (epoch + 1) % 4 == 0:print('epoch: %d | train loss:%.5f | val loss:%.5f | train acc:%5f val acc:%.5f:' % (epoch + 1, train_all_loss[-1], val_all_loss[-1],train_ACC[-1],val_ACC[-1]))endtime = time.time()print("torch.nn实现前馈网络-多分类任务 %d轮 总用时: %.3fs" % (epochs, endtime - begintime))# 返回训练集和测试集上的 损失值 与 准确率return train_all_loss,val_all_loss,train_ACC,val_ACCnet4 = MyNet_NN(dropout=0.5)
net4 = net4.to(device)
trainL4, testL4, trainAcc4, testAcc4= train_and_test_4(model=net4,epochs = 10000,lr=0.1)
draw(['', '早停机制'], [trainL4], [testL4])

实验总结

实验中我们通过两种方式构建了前馈神经网络,一种是手动搭建,另一种是利用PyTorch中的torch.nn模块进行构建。在这两种网络结构的基础上,分别引入了dropout层,以有效地防止模型的过拟合现象。

  1. 首先,在优化器的选择上,我们尝试了不同的优化函数,并对它们在模型训练中的效果进行了比较。不同的优化器具有不同的优点,通过对比它们的性能,我们可以更好地选择适合具体任务的优化器,进一步提升模型的性能。

  2. 其次,我们引入了惩罚权重的概念,通过增加惩罚项来约束模型的复杂度。实验结果表明,适度增加惩罚权重可以在一定程度上增大模型输出的损失,但同时也达到了防止过拟合的效果。这进一步证实了模型复杂度与过拟合之间存在一定的权衡关系。

  3. 通过实验我们观察到,适当设置dropout的概率可以显著减轻模型的过拟合问题。dropout通过在训练过程中随机丢弃一部分神经元的输出,有效降低了模型对于训练数据的过度依赖,提高了模型的泛化能力,从而在测试集上表现更为鲁棒。

  4. 最后,为了进一步提高模型的训练效果,我们引入了早停机制。该机制通过监测在验证集上的测试误差,在发现测试误差上升的情况下停止训练,以防止网络过拟合。早停机制在一定程度上能够避免模型在训练过程中过分拟合训练数据,从而提高了模型的泛化性能。

通过以上实验,我们综合考虑了dropout、惩罚权重、不同优化器以及早停机制等因素,为构建更稳健、泛化能力强的前馈神经网络提供了有益的经验和指导。这些技术手段的灵活运用可以在实际任务中更好地平衡模型的性能和泛化能力。

本文来自互联网用户投稿,该文观点仅代表作者本人,不代表本站立场。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如若转载,请注明出处:http://www.mzph.cn/news/668424.shtml

如若内容造成侵权/违法违规/事实不符,请联系多彩编程网进行投诉反馈email:809451989@qq.com,一经查实,立即删除!

相关文章

笔记本电脑的WIFI模块,突然不显示了,网络也连接不上

问题复现&#xff1a; 早上&#xff0c;在更新完笔记本电脑的系统之后&#xff0c;连网之后&#xff0c;网络突然直接断开&#xff0c;一查看&#xff0c;WiFi模块居然不见了&#xff0c;开机重启也是如此&#xff0c;这种情况常常出现在更新系统之后&#xff0c;WiFi模块驱动就…

RK3399平台开发系列讲解(内存篇)进程内存详解

🚀返回专栏总目录 文章目录 一、虚拟地址映射的物理内存1.1、物理内存1.2、虚拟内存1.2.1、用户态:低特权运行程序1.2.2、内核态:运行的程序需要访问操作系统内核数据二、PageCache三、指标查询命令沉淀、分享、成长,让自己和他人都能有所收获!😄 📢进程消耗的内存包…

自动化报告pptx-python|如何将pandas的表格写入PPTX(二)

本篇延续:自动化报告的前奏|使用python-pptx操作PPT(一) 因为在pptx-python中使用table,需要单个cell逐一输入,于是在想有没有pandas可以直接读入的方式, 有两个开源项目有类似的功能: PandasToPowerpointmspandas其中mspandas写的比较复杂,PandasToPowerpoint比较易懂…

编程笔记 html5cssjs 072 JavaScript BigInt数据类型

编程笔记 html5&css&js 072 JavaScript BigInt数据类型 一、BigInt 数据类型二、BigInt 的创建和使用三、BigInt 操作与方法三、示例小结 JavaScript BigInt 数据类型是一种内置的数据类型&#xff0c;用于表示大于 Number.MAX_SAFE_INTEGER&#xff08;即2^53 - 1&…

ASR 概述

前言 随着企业加强了与客户的线上沟通&#xff0c;企业越发依赖于虚拟助手、聊天机器人以及其他的语音技术&#xff0c;以实现与客户的高效互动。这几类人工智能&#xff0c;都是依赖于自动语音识别技术&#xff0c;简称为 ASR。ASR 涉及到将语音转换为文本&#xff0c;促使计…

docker proxy 【docker 代理】

第一种 创建代理配置文件 mkdir -p /etc/systemd/system/docker.service.d/ cat <<EOF > /etc/systemd/system/docker.service.d/http-proxy.conf Environment"HTTP_PROXYhttp://192.168.21.101:7890" Environment"HTTPS_PROXYhttp://192.168.21.1…

同城外卖跑腿app开发:重新定义城市生活

随着科技的发展和人们生活节奏的加快&#xff0c;同城外卖跑腿app应运而生&#xff0c;成为现代城市生活中的重要组成部分。本文将探讨同城外卖跑腿app开发的意义、市场需求、功能特点以及未来的发展趋势。 一、同城外卖跑腿app开发的意义 同城外卖跑腿app作为一种便捷的生活…

【java批量导出pdf】优化方案

问题情境&#xff1a; 项目中存在web页面点击一键导出&#xff0c;导出所有数据对应的pdf文件&#xff0c;由于有些pdf文件是实时生成的&#xff0c;之前最简答的写法for循环处理速度太慢&#xff0c;超过了nginx配置的最大响应时间了&#xff0c;且对用户交互体验上很不友好&…

sqli.labs靶场(41-53关)

41、第四十一关 -1 union select 1,2,3-- -1 union select 1,database(),(select group_concat(table_name) from information_schema.tables where table_schemadatabase()) -- -1 union select 1,2,(select group_concat(column_name) from information_schema.columns wher…

通用函数

目录 处理null 多数值判断 Oracle从入门到总裁:https://blog.csdn.net/weixin_67859959/article/details/135209645 Oracle 提供了两个简单的数据处理函数&#xff1a; nvl()、decode()。在版本升级的过程中&#xff0c;这两个函数又衍生出了许多子函数 处理null 下面首先…

0基础学习VR全景平台篇第141篇:如何制作卫星航拍全景

大家好&#xff0c;欢迎观看蛙色官方系列全景摄影课程&#xff01; 很多人都看过或者拍摄过航拍全景&#xff0c;其效果相比于普通的地拍的确有着更加震撼的拍摄效果&#xff0c;但是受限于无人机高度&#xff0c;以及禁飞区等等限制&#xff0c;导致很多大场景无法展示完全&a…

npm eslint 禁用

配置文件 ESLint 最主要的配置方式。ESLint 配置文件支持多种格式&#xff0c;同一目录下&#xff0c;ESLint 按 .eslintrc.js, .eslintrc.cjs, .eslintrc.yaml, .eslintrc.yml, .eslintrc.json, package.json 下的 eslintConfig 字段 的顺序查找配置&#xff0c;相同目录下只…

Linux防火墙与iptables五表五链规则介绍

目录 一、防火墙基本认识 1. 安全技术 2. 防火墙分类 3. 防火墙工具介绍 二、iptables 1. 概述 2. 五表五链 3. 语法 3.1 基本语法 3.2 语法总结 4. 管理选项 5. 通用匹配 6. 控制类型 7. iptables应用 7.1 新增防火墙规则 7.2 查看规则表 7.3 黑白名单 7.4 …

【应用容器-Docker】

Docker 是一个开源的容器化平台&#xff0c;可以帮助开发者将应用程序和所有依赖项打包到一个称为容器的可移植单元中。容器化技术可以提供一种轻量级、快速、可靠和可重复部署应用程序的方法。 Docker 的基本概念包括以下几个方面&#xff1a; 1. 镜像&#xff08;Image&…

C++ 调用lua 脚本

需求&#xff1a; 使用Qt/C 调用 lua 脚本 扩展原有功能。 步骤&#xff1a; 1&#xff0c;工程中引入 头文件&#xff0c;库文件。lua二进制下载地址&#xff08;Lua Binaries&#xff09; 2&#xff0c; 调用脚本内函数。 这里调用lua 脚本中的process函数&#xff0c;并…

canvas图片上设置镂空文字效果

查看专栏目录 canvas实例应用100专栏&#xff0c;提供canvas的基础知识&#xff0c;高级动画&#xff0c;相关应用扩展等信息。canvas作为html的一部分&#xff0c;是图像图标地图可视化的一个重要的基础&#xff0c;学好了canvas&#xff0c;在其他的一些应用上将会起到非常重…

C语言-3

定义指针 /*指针的概念:1.为了方便访问内存中的内容&#xff0c;给每一个内存单元&#xff0c;进行编号&#xff0c;那么我们称这个编号为地址&#xff0c;也就是指针。2.指针也是一种数据类型&#xff0c;指针变量有自己的内存&#xff0c;里面存储的是地址&#xff0c;也就是…

Ansible概述、Ansible环境准备、Ansibleadhoc临时命令语法、命令模块、文件模块、用户模块、综合练习

ansible 批量管理服务器的工具2015年被红帽公司收购使用Python语言编写的基于ssh进行管理&#xff0c;所以不需要在被管端安装任何软件ansible在管理远程主机的时候&#xff0c;主要是通过各种模块进行操作的 环境准备 主机名IP地址角色web1192.168.88.11被控制节点&#xf…

【HarmonyOS应用开发】APP应用的通知(十五)

相关介绍 通知旨在让用户以合适的方式及时获得有用的新消息&#xff0c;帮助用户高效地处理任务。应用可以通过通知接口发送通知消息&#xff0c;用户可以通过通知栏查看通知内容&#xff0c;也可以点击通知来打开应用&#xff0c;通知主要有以下使用场景&#xff1a; 显示接收…

BUGKU-WEB Simple_SSTI_1

02 Simple_SSTI_1 题目描述 没啥好说的~ 解题思路 进入场景后&#xff0c;显示&#xff1a; You need pass in a parameter named flag。ctrlu 查看源码 <!DOCTYPE html> <html lang"en"> <head><meta charset"UTF-8"><titl…