In the practice of pytorch, sometimes the extracted layer structure is not enough, and the parameters inside need to be initialized, so how to extract the parameters of the network and initialize them?
Sometimes the extracted layer structure is not enough, and the parameters inside need to be initialized, so how to extract the parameters of the network and initialize them?
First of all, there are two particularly important attributes about parameters in nn.Module, namely named_parameters() and parameters(). named_parameters() is an iterator that gives the names and parameters of a network layer, and parameters() gives an iterator of all the parameters of a network.
import os
import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.nn.init as init
import argparse
import torch.autograd.variable as variable
class SimpleCNN(nn.Module):
def __init__(self):
super(SimpleCNN,self).__init__() #b,3,32,32
layer1=nn.Sequential()
layer1.add_module('conv1',nn.Conv2d(in_channels=3,out_channels=32,kernel_size=3,stride=1,padding=1))
#b,32,32,32
layer1.add_module('relu1',nn.ReLU(True))
layer1.add_module('pool1',nn.MaxPool2d(2,2))
#b,32,16,16
self.layer1=layer1
layer2=nn.Sequential()
layer1.add_module('conv2',nn.Conv2d(in_channels=32,out_channels=64,kernel_size=3,stride=1,padding=1))
#b,64,16,16
layer2.add_module('relu2',nn.ReLU(True))
layer2.add_module('pool2',nn.MaxPool2d(2,2))
#b,64,8,8
self.layer2=layer2
layer3=nn.Sequential()
layer3.add_module('conv3', nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3 ,stride=1, padding = 1))
#b,128,8,8
layer3.add_module('relu3', nn.ReLU(True))
layer3.add_module('poo13', nn.MaxPool2d(2, 2))#b,128,4,4
self.layer3=layer3
layer4 =nn.Sequential()
layer4.add_module('fc1',nn.Linear(in_features=2048, out_features=512 ))
layer4.add_module('fc_relu1', nn.ReLU(True))
layer4.add_module('fc2 ', nn.Linear(in_features=512, out_features=64 ))
layer4.add_module('fc_relu2', nn.ReLU(True))
layer4.add_module('fc3', nn.Linear(64, 10))
self.layer4 = layer4
def forward(self,x):
conv1=self.layer1(x)
conv2=self.layer2(conv1)
conv3=self.layer3(conv2)
fc_input=conv3.view(conv3.size(0),-1)
fc_output=self.layer4(fc_input)
return fc_output
model=SimpleCNN()
for param in model.named_parameters():
print(param[0])
The name of each layer parameter can be obtained, and the output is
How to initialize the weight? It's very simple, because the weight is a Variable, so you only need to take out the data attribute and do the required processing on it.
for m in model.modules():
if isinstance(m,nn.Conv2d):
init.normal(m.weight.data) #通过正态分布填充张量
init.xavier_normal(m.weight.data)
#xavier均匀分布的方法来init，来自2010年的论文“Understanding the difficulty of training deep feedforward neural networks”
init.kaiming_normal(m.weight.data)
#来自2015年何凯明的论文“Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification”
m.bias.data.fill_(0)
elif isinstance(m,nn.Linear):
m.weight.data.normal_()
Through the above operations, initialize the weights of the convolutional layer using the methods provided in PyTorch, so that we can use any initialization we want, and even we can define the initialization method and initialize the weights ourselves.
For more initialization methods, refer to [torch.nn.init] https://pytorch-cn.readthedocs.io/zh/latest/package_references/nn_init/