import torch
from torch import nn
from d2l import torch as d2l
4.6.1 Re-examining overfitting
For the entire section of theory, see the book for details.
4.6.2 Robustness to perturbations
For the entire section of theory, see the book for details.
4.6.3 Temporary withdrawal method in practice
For the entire section of theory, see the book for details.
4.6.4 Implementation from scratch
def dropout_layer(X, dropout):
assert 0 <= dropout <= 1
# 在本情况中，所有元素都被丢弃
if dropout == 1:
return torch.zeros_like(X)
# 在本情况中，所有元素都被保留
if dropout == 0:
return X
mask = (torch.rand(X.shape) > dropout).float() # 从均匀分布U[0,1]中抽取与神经网络同维度的样本，保留大于 dropout 的样本
return mask * X / (1.0 - dropout)
X= torch.arange(16, dtype = torch.float32).reshape((2, 8))
print(X)
print(dropout_layer(X, 0.))
print(dropout_layer(X, 0.5))
print(dropout_layer(X, 1.))
tensor([[ 0., 1., 2., 3., 4., 5., 6., 7.],
[ 8., 9., 10., 11., 12., 13., 14., 15.]])
tensor([[ 0., 1., 2., 3., 4., 5., 6., 7.],
[ 8., 9., 10., 11., 12., 13., 14., 15.]])
tensor([[ 0., 0., 0., 6., 0., 10., 12., 0.],
[16., 0., 0., 0., 24., 26., 0., 0.]])
tensor([[0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0.]])
- Define model parameters
num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256
-
Define model
Apply the backoff method to the output of each hidden layer after the activation function. A common trick is to set a lower backoff probability close to the input layer.
dropout1, dropout2 = 0.2, 0.5 # 将第一个和第二个隐藏层的暂退概率分别设置为 0.2 和 0.5
class Net(nn.Module):
def __init__(self, num_inputs, num_outputs, num_hiddens1, num_hiddens2,
is_training = True):
super(Net, self).__init__()
self.num_inputs = num_inputs
self.training = is_training
self.lin1 = nn.Linear(num_inputs, num_hiddens1)
self.lin2 = nn.Linear(num_hiddens1, num_hiddens2)
self.lin3 = nn.Linear(num_hiddens2, num_outputs)
self.relu = nn.ReLU()
def forward(self, X):
H1 = self.relu(self.lin1(X.reshape((-1, self.num_inputs))))
# 只有在训练模型时才使用dropout
if self.training == True:
# 在第一个全连接层之后添加一个dropout层
H1 = dropout_layer(H1, dropout1)
H2 = self.relu(self.lin2(H1))
if self.training == True:
# 在第二个全连接层之后添加一个dropout层
H2 = dropout_layer(H2, dropout2)
out = self.lin3(H2)
return out
net = Net(num_inputs, num_outputs, num_hiddens1, num_hiddens2)
- training and testing
num_epochs, lr, batch_size = 10, 0.5, 256
loss = nn.CrossEntropyLoss(reduction='none')
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
trainer = torch.optim.SGD(net.parameters(), lr=lr)
d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)
4.6.5 Simple implementation
net = nn.Sequential(nn.Flatten(),
nn.Linear(784, 256),
nn.ReLU(),
# 在第一个全连接层之后添加一个dropout层
nn.Dropout(dropout1),
nn.Linear(256, 256),
nn.ReLU(),
# 在第二个全连接层之后添加一个dropout层
nn.Dropout(dropout2),
nn.Linear(256, 10))
def init_weights(m):
if type(m) == nn.Linear:
nn.init.normal_(m.weight, std=0.01)
net.apply(init_weights)
Sequential(
(0): Flatten(start_dim=1, end_dim=-1)
(1): Linear(in_features=784, out_features=256, bias=True)
(2): ReLU()
(3): Dropout(p=0.2, inplace=False)
(4): Linear(in_features=256, out_features=256, bias=True)
(5): ReLU()
(6): Dropout(p=0.5, inplace=False)
(7): Linear(in_features=256, out_features=10, bias=True)
)
trainer = torch.optim.SGD(net.parameters(), lr=lr)
d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)
practise
def init_weights(m):
if type(m) == nn.Linear:
nn.init.normal_(m.weight, std=0.01)
num_epochs, lr, batch_size = 10, 0.5, 256
loss = nn.CrossEntropyLoss(reduction='none')
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
(1) What will happen if the temporary withdrawal probabilities of the first and second layers are changed? Specifically, what problems will occur if these two layers are exchanged? Design an experiment to answer these questions, describe the results quantitatively, and summarize qualitative conclusions.
for dropouts in [(0.1, 0.5), (0.1, 0.6), (0.2, 0.5), (0.2, 0.6), (0.5, 0.2)]:
net0 = nn.Sequential(nn.Flatten(),
nn.Linear(784, 256),
nn.ReLU(),
nn.Dropout(dropouts[0]),
nn.Linear(256, 256),
nn.ReLU(),
nn.Dropout(dropouts[1]),
nn.Linear(256, 10))
net0.apply(init_weights)
trainer0 = torch.optim.SGD(net0.parameters(), lr=lr)
d2l.train_ch3(net0, train_iter, test_iter, loss, num_epochs, trainer0)
Although the difference is not big, it is indeed better to have a lower probability of temporary regression near the input layer.
(2) Increase the number of training rounds and compare the results obtained when using the temporary withdrawal method and not using the temporary withdrawal method.
num_epochs1, lr, batch_size = 15, 0.5, 256
net1 = nn.Sequential(nn.Flatten(),
nn.Linear(784, 256),
nn.ReLU(),
nn.Dropout(0.2),
nn.Linear(256, 256),
nn.ReLU(),
nn.Dropout(0.5),
nn.Linear(256, 10))
net2 = nn.Sequential(nn.Flatten(),
nn.Linear(784, 256),
nn.ReLU(),
nn.Linear(256, 256),
nn.ReLU(),
nn.Linear(256, 10))
net1.apply(init_weights)
net2.apply(init_weights)
trainer1 = torch.optim.SGD(net1.parameters(), lr=lr)
d2l.train_ch3(net1, train_iter, test_iter, loss, num_epochs1, trainer1)
trainer2 = torch.optim.SGD(net2.parameters(), lr=lr)
d2l.train_ch3(net2, train_iter, test_iter, loss, num_epochs1, trainer2)
Using the temporary retreat method can indeed alleviate the overfitting phenomenon.
(3) What is the variance of the activation values in each hidden layer when using or not using the fallback method? Draw a graph showing how the variance of the activations in each hidden layer of the two models changes over time.
I can't draw, I guess.
(4) Why is the temporary withdrawal method not usually used during testing?
The purpose of using the temporary retreat method is to reduce the over-fitting phenomenon during the training process. The purpose of the test is not for training, but to obtain a result. The network has been fixed, so of course there is no need to use the temporary retreat method.
(5) Take the model in this section as an example to compare the effects of using the temporary retreat method and weight attenuation. What happens if the temporary retreat method and weight decay are used at the same time? Are the results cumulative? Are benefits diminished (or worse)? Do they cancel each other out?
No, slightly.
(6) What happens if we apply the retreat method to each weight of the weight matrix instead of the activation value?
net3 = nn.Sequential(nn.Flatten(),
nn.Linear(784, 256),
nn.Dropout(0.2),
nn.ReLU(),
nn.Linear(256, 256),
nn.Dropout(0.5),
nn.ReLU(),
nn.Linear(256, 10))
net3.apply(init_weights)
trainer3 = torch.optim.SGD(net3.parameters(), lr=lr)
d2l.train_ch3(net3, train_iter, test_iter, loss, num_epochs, trainer3)
There is no visual difference
(7) Develop a technique for injecting noise at each layer that is different from the standard roll-off technique. Try to develop a method that performs better than the fallback method on the Fashion-MINIST dataset (for a fixed architecture).
No, slightly.