Here is a comparison of three parameter initializations, namely: full initialization to 0, random initialization, and abnormal gradient suppression initialization.
The first is the function init_utils.py required for forward and backward propagation, drawing, and loading data:
# -*- coding: utf-8 -*-
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import sklearn.datasets
def sigmoid(x):
"""
Compute the sigmoid of x
Arguments:
x -- A scalar or numpy array of any size.
Return:
s -- sigmoid(x)
"""
s = 1/(1+np.exp(-x))
return s
def relu(x):
"""
Compute the relu of x
Arguments:
x -- A scalar or numpy array of any size.
Return:
s -- relu(x)
"""
s = np.maximum(0,x)
return s
def compute_loss(a3, Y):
"""
Implement the loss function
Arguments:
a3 -- post-activation, output of forward propagation
Y -- "true" labels vector, same shape as a3
Returns:
loss - value of the loss function
"""
m = Y.shape[1]
logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y)
loss = 1./m * np.nansum(logprobs)
return loss
def forward_propagation(X, parameters):
"""
Implements the forward propagation (and computes the loss) presented in Figure 2.
Arguments:
X -- input dataset, of shape (input size, number of examples)
Y -- true "label" vector (containing 0 if cat, 1 if non-cat)
parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3":
W1 -- weight matrix of shape ()
b1 -- bias vector of shape ()
W2 -- weight matrix of shape ()
b2 -- bias vector of shape ()
W3 -- weight matrix of shape ()
b3 -- bias vector of shape ()
Returns:
loss -- the loss function (vanilla logistic loss)
"""
# retrieve parameters
W1 = parameters["W1"]
b1 = parameters["b1"]
W2 = parameters["W2"]
b2 = parameters["b2"]
W3 = parameters["W3"]
b3 = parameters["b3"]
# LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
z1 = np.dot(W1, X) + b1
a1 = relu(z1)
z2 = np.dot(W2, a1) + b2
a2 = relu(z2)
z3 = np.dot(W3, a2) + b3
a3 = sigmoid(z3)
cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3)
return a3, cache
def backward_propagation(X, Y, cache):
"""
Implement the backward propagation presented in figure 2.
Arguments:
X -- input dataset, of shape (input size, number of examples)
Y -- true "label" vector (containing 0 if cat, 1 if non-cat)
cache -- cache output from forward_propagation()
Returns:
gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables
"""
m = X.shape[1]
(z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache
dz3 = 1./m * (a3 - Y)
dW3 = np.dot(dz3, a2.T)
db3 = np.sum(dz3, axis=1, keepdims = True)
da2 = np.dot(W3.T, dz3)
dz2 = np.multiply(da2, np.int64(a2 > 0))
dW2 = np.dot(dz2, a1.T)
db2 = np.sum(dz2, axis=1, keepdims = True)
da1 = np.dot(W2.T, dz2)
dz1 = np.multiply(da1, np.int64(a1 > 0))
dW1 = np.dot(dz1, X.T)
db1 = np.sum(dz1, axis=1, keepdims = True)
gradients = {
"dz3": dz3, "dW3": dW3, "db3": db3,
"da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2,
"da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1}
return gradients
def update_parameters(parameters, grads, learning_rate):
"""
Update parameters using gradient descent
Arguments:
parameters -- python dictionary containing your parameters
grads -- python dictionary containing your gradients, output of n_model_backward
Returns:
parameters -- python dictionary containing your updated parameters
parameters['W' + str(i)] = ...
parameters['b' + str(i)] = ...
"""
L = len(parameters) // 2 # number of layers in the neural networks
# Update rule for each parameter
for k in range(L):
parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)]
parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)]
return parameters
def predict(X, y, parameters):
"""
This function is used to predict the results of a n-layer neural network.
Arguments:
X -- data set of examples you would like to label
parameters -- parameters of the trained model
Returns:
p -- predictions for the given dataset X
"""
m = X.shape[1]
p = np.zeros((1,m), dtype = np.int)
# Forward propagation
a3, caches = forward_propagation(X, parameters)
# convert probas to 0/1 predictions
for i in range(0, a3.shape[1]):
if a3[0,i] > 0.5:
p[0,i] = 1
else:
p[0,i] = 0
# print results
print("Accuracy: " + str(np.mean((p[0,:] == y[0,:]))))
return p
def load_dataset(is_plot=True):
np.random.seed(1)
train_X, train_Y = sklearn.datasets.make_circles(n_samples=300, noise=.05)
np.random.seed(2)
test_X, test_Y = sklearn.datasets.make_circles(n_samples=100, noise=.05)
# Visualize the data
if is_plot:
plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral)
plt.show()
train_X = train_X.T
train_Y = train_Y.reshape((1, train_Y.shape[0]))
test_X = test_X.T
test_Y = test_Y.reshape((1, test_Y.shape[0]))
return train_X, train_Y, test_X, test_Y
def plot_decision_boundary(model, X, y):
# Set min and max values and give it some padding
x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1
y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1
h = 0.01
# Generate a grid of points with distance h between them
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
# Predict the function value for the whole grid
Z = model(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
# Plot the contour and training examples
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
plt.ylabel('x2')
plt.xlabel('x1')
plt.scatter(X[0, :], X[1, :], c=y, cmap=plt.cm.Spectral)
plt.show()
def predict_dec(parameters, X):
"""
Used for plotting decision boundary.
Arguments:
parameters -- python dictionary containing your parameters
X -- input data of size (m, K)
Returns
predictions -- vector of predictions of our model (red: 0 / blue: 1)
"""
# Predict using forward propagation and a classification threshold of 0.5
a3, cache = forward_propagation(X, parameters)
predictions = (a3>0.5)
return predictions
Then start our debugging code.
You can first see what the dataset looks like:
train_X, train_Y, test_X, test_Y = init_utils.load_dataset(is_plot=True)
All initialized to 0
That is, both w and b of each layer are initialized to 0.
code show as below:
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import sklearn.datasets
import init_utils
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# 加载数据集
train_X, train_Y, test_X, test_Y = init_utils.load_dataset(is_plot=False)
# 初始化为0
def initialize_parameters_zeros(layers_dims):
"""
将模型的参数全部设置为0
参数：
layers_dims - 列表，模型的层数和对应每一层的节点的数量
返回
parameters - 包含了所有W和b的字典
W1 - 权重矩阵，维度为（layers_dims[1], layers_dims[0]）
b1 - 偏置向量，维度为（layers_dims[1],1）
···
WL - 权重矩阵，维度为（layers_dims[L], layers_dims[L -1]）
bL - 偏置向量，维度为（layers_dims[L],1）
"""
parameters = {
}
L = len(layers_dims) # 网络层数
for l in range(1, L):
parameters["W" + str(l)] = np.zeros((layers_dims[l], layers_dims[l - 1]))
parameters["b" + str(l)] = np.zeros((layers_dims[l], 1))
# 使用断言确保我的数据格式是正确的
assert (parameters["W" + str(l)].shape == (layers_dims[l], layers_dims[l - 1]))
assert (parameters["b" + str(l)].shape == (layers_dims[l], 1))
return parameters
def model(X, Y, learning_rate=0.01, num_iterations=15000, print_cost=True, initialization="he", is_polt=True):
"""
实现一个三层的神经网络：LINEAR ->RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
参数：
X - 输入的数据，维度为(2, 要训练/测试的数量)
Y - 标签，【0 | 1】，维度为(1，对应的是输入的数据的标签)
learning_rate - 学习速率
num_iterations - 迭代的次数
print_cost - 是否打印成本值，每迭代1000次打印一次
initialization - 字符串类型，初始化的类型【"zeros" | "random" | "he"】
is_polt - 是否绘制梯度下降的曲线图
返回
parameters - 学习后的参数
"""
grads = {
}
costs = []
m = X.shape[1]
layers_dims = [X.shape[0], 10, 5, 1]
# 选择初始化参数的类型
if initialization == "zeros":
parameters = initialize_parameters_zeros(layers_dims)
elif initialization == "random":
parameters = initialize_parameters_random(layers_dims)
elif initialization == "he":
parameters = initialize_parameters_he(layers_dims)
else:
print("错误的初始化参数！程序退出")
exit
# 开始学习
for i in range(0, num_iterations):
# 前向传播
a3, cache = init_utils.forward_propagation(X, parameters)
# 计算成本
cost = init_utils.compute_loss(a3, Y)
# 反向传播
grads = init_utils.backward_propagation(X, Y, cache)
# 更新参数
parameters = init_utils.update_parameters(parameters, grads, learning_rate)
# 记录成本
if i % 1000 == 0:
costs.append(cost)
# 打印成本
if print_cost:
print("第" + str(i) + "次迭代，成本值为：" + str(cost))
# 学习完毕，绘制成本曲线
if is_polt:
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('iterations (per hundreds)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# 返回学习完毕后的参数
return parameters
parameters = model(train_X, train_Y, initialization = "zeros",is_polt=True)
print ("训练集:")
predictions_train = init_utils.predict(train_X, train_Y, parameters)
print ("测试集:")
predictions_test = init_utils.predict(test_X, test_Y, parameters)
Run the code, the result is as follows:
第0次迭代，成本值为：0.6931471805599453
第1000次迭代，成本值为：0.6931471805599453
第2000次迭代，成本值为：0.6931471805599453
第3000次迭代，成本值为：0.6931471805599453
第4000次迭代，成本值为：0.6931471805599453
第5000次迭代，成本值为：0.6931471805599453
第6000次迭代，成本值为：0.6931471805599453
第7000次迭代，成本值为：0.6931471805599453
第8000次迭代，成本值为：0.6931471805599453
第9000次迭代，成本值为：0.6931471805599453
第10000次迭代，成本值为：0.6931471805599455
第11000次迭代，成本值为：0.6931471805599453
第12000次迭代，成本值为：0.6931471805599453
第13000次迭代，成本值为：0.6931471805599453
第14000次迭代，成本值为：0.6931471805599453
训练集:
Accuracy: 0.5
测试集:
Accuracy: 0.5
The loss has not changed at all, which is equivalent to no learning. Because the full initialization is 0, each neuron is doing the same linear calculation, no matter how many rounds of gradient descent is running, it is calculating exactly the same function, so the loss will not decrease, so no matter how many layers of networks or how many neurons It makes no sense, and is essentially no different from the logistic regression of a unit.
The solution is random initialization.
random initialization
w is randomly initialized, and b is indifferent as a constant added later, and is initialized to 0.
Join function:
def initialize_parameters_random(layers_dims):
"""
参数：
layers_dims - 列表，模型的层数和对应每一层的节点的数量
返回
parameters - 包含了所有W和b的字典
W1 - 权重矩阵，维度为（layers_dims[1], layers_dims[0]）
b1 - 偏置向量，维度为（layers_dims[1],1）
···
WL - 权重矩阵，维度为（layers_dims[L], layers_dims[L -1]）
b1 - 偏置向量，维度为（layers_dims[L],1）
"""
np.random.seed(3) # 指定随机种子
parameters = {
}
L = len(layers_dims) # 层数
for l in range(1, L):
parameters['W' + str(l)] = np.random.randn(layers_dims[l], layers_dims[l - 1]) * 10 # 使用10倍缩放
parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))
# 使用断言确保我的数据格式是正确的
assert(parameters["W" + str(l)].shape == (layers_dims[l],layers_dims[l-1]))
assert(parameters["b" + str(l)].shape == (layers_dims[l],1))
return parameters
Change the value of initialization to random to use random initialization:
parameters = model(train_X, train_Y, initialization = "random",is_polt=True)
Run the code, the result is as follows:
第0次迭代，成本值为：inf
第1000次迭代，成本值为：0.6242434241539614
第2000次迭代，成本值为：0.5978811277755388
第3000次迭代，成本值为：0.5636242569764779
第4000次迭代，成本值为：0.5500958254523324
第5000次迭代，成本值为：0.544339206192789
第6000次迭代，成本值为：0.5373584514307651
第7000次迭代，成本值为：0.469574666760224
第8000次迭代，成本值为：0.39766324943219844
第9000次迭代，成本值为：0.3934423376823982
第10000次迭代，成本值为：0.3920158992175907
第11000次迭代，成本值为：0.38913979237487845
第12000次迭代，成本值为：0.3861261344766218
第13000次迭代，成本值为：0.3849694511273874
第14000次迭代，成本值为：0.3827489017191917
训练集:
Accuracy: 0.83
测试集:
Accuracy: 0.86
It can be seen that the loss has dropped significantly, indicating that there is learning and the prediction effect is better. But in the end, the loss did not drop to a very low level, and the decline was slow or even stagnant, and it still needs to be optimized.
Inhibition Gradient Exception Initialization
w is randomly initialized and then multiplied by Var(2/n[l-1]), Var is the variance calculation, and n[l-1] is the number of neurons in the previous layer.
Var(2/n[l-1]) is the most suitable parameter for the ReLU activation function, and the effect is the best.
Join function:
def initialize_parameters_he(layers_dims):
"""
参数：
layers_dims - 列表，模型的层数和对应每一层的节点的数量
返回
parameters - 包含了所有W和b的字典
W1 - 权重矩阵，维度为（layers_dims[1], layers_dims[0]）
b1 - 偏置向量，维度为（layers_dims[1],1）
···
WL - 权重矩阵，维度为（layers_dims[L], layers_dims[L -1]）
b1 - 偏置向量，维度为（layers_dims[L],1）
"""
np.random.seed(3) # 指定随机种子
parameters = {
}
L = len(layers_dims) # 层数
for l in range(1, L):
parameters['W' + str(l)] = np.random.randn(layers_dims[l], layers_dims[l - 1]) * np.sqrt(2 / layers_dims[l - 1])
parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))
#使用断言确保我的数据格式是正确的
assert(parameters["W" + str(l)].shape == (layers_dims[l],layers_dims[l-1]))
assert(parameters["b" + str(l)].shape == (layers_dims[l],1))
return parameters
Modify the initialization value to he, and use the gradient exception initialization:
parameters = model(train_X, train_Y, initialization = "he",is_polt=True)
Run the code, the result is as follows:
第0次迭代，成本值为：0.8830537463419761
第1000次迭代，成本值为：0.6879825919728063
第2000次迭代，成本值为：0.6751286264523371
第3000次迭代，成本值为：0.6526117768893807
第4000次迭代，成本值为：0.6082958970572938
第5000次迭代，成本值为：0.5304944491717495
第6000次迭代，成本值为：0.4138645817071794
第7000次迭代，成本值为：0.3117803464844441
第8000次迭代，成本值为：0.23696215330322562
第9000次迭代，成本值为：0.18597287209206834
第10000次迭代，成本值为：0.15015556280371806
第11000次迭代，成本值为：0.12325079292273546
第12000次迭代，成本值为：0.09917746546525934
第13000次迭代，成本值为：0.08457055954024278
第14000次迭代，成本值为：0.07357895962677369
训练集:
Accuracy: 0.9933333333333333
测试集:
Accuracy: 0.96
It can be seen that this is a normal loss curve, with a smooth decline, and the final loss is low enough. The prediction effect is also better.
Summarize
In a multi-layer neural network, the initialization of parameters must be random initialization, otherwise the network will become meaningless if they are all initialized to 0.
However, random initialization also needs to be controlled to prevent the performance of the network from degrading.