People are lazy, and the screenshots explain the background directly. The following is the entire network structure.
Regarding the settings of the parameters of each layer of the original network, it is recommended to move to the following authors to learn more detailed https://blog.csdn.net/zyqdragon/article/details/72353420
This article mainly implements a complete AlexNet convolutional neural network and performs a speed test on its forward propagation and back propagation. The original data is not loaded for training. It is worth noting that this code is structurally the same as the original network, but there are many details that are inconsistent in the middle, such as the generation size step size of the convolution kernel, etc. But it makes you understand the whole network very clearly. If you don't want to see the decomposition, you can copy all the code to the end of the article and run it to see the effect. No need to modify anything, just run it directly.
-------------------------------------------------- --Enter code decomposition--------------------------------------
# Import the corresponding system library and tensorflow from datetime import datetime import math import time import tensorflow as tf
# A total of 100 groups of batches are tested, 32 in each group batch_size = 32 num_batches = 100
# define a name and output size to display each convolutional or pooling layer
def print_activations(t):
print(t.op.name, '', t.get_shape().as_list())
def inference(images):
'''
:param images: input image size 32*224*244*3
:return: The last layer of pool5 and parameters
'''
parameters = []
with tf.name_scope('conv1') as scope:
kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 64], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(images, kernel, [1, 4, 4, 1], padding = 'SAME')
# s = 4,"NHWC", the data is stored in the order of: [batch, height, width, channels]
biases = tf.Variable(tf.constant(0.0, shape = [64], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(bias, name = scope)
print_activations(conv1) # output the name and shape of conv1
parameters += [kernel, biases]
# After the first layer of convolution, use local response normalization to improve generalization ability
# lrn1 = tf.nn.lrn(conv1, 4, bias = 1.0, alpha = 0.001 / 9, beta = 0.75, name = 'lrn1')
pool1 = tf.nn.max_pool(conv1, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool1')
print_activations(pool1)
with tf.name_scope('conv2') as scope:
kernel = tf.Variable(tf.truncated_normal([5, 5, 64, 192], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [192], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv2)
# lrn2 = tf.nn.lrn(conv2, 4, bias = 1.0, alpha = 0.001 / 9, beta = 0.75, name = 'lrn2')
pool2 = tf.nn.max_pool(conv2, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool2')
print_activations(pool2)
with tf.name_scope('conv3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 192, 384], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [384], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv3 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv3)
with tf.name_scope('conv4') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [256], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv4 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv4)
with tf.name_scope('conv5') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [256], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv5 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv5)
pool5 = tf.nn.max_pool(conv5, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool5')
print_activations(pool5)
return pool5, parameters
At the end of the convolution process, the LRN used here and other literatures have been abandoned because the effect is not obvious, but the speed is reduced by 1/3 after use here, and the impact on the accuracy is not large. Can be used or not.
Next is the time test function.
def time_tensorflow_run(session, target, info_string):
'''
:param session: session created
:param target: the operator to be evaluated
:param info_string: name of the test
:return:
'''
num_steps_burn_in = 10 # Let the program warm up for 10 rounds, because the first few rounds of operations have a lot of video memory loading time, and count the time after 10 rounds
total_duration = 0.0 # count the total time
total_duration_squared = 0.0 # and time mean squared
for i in range(num_batches + num_steps_burn_in):
start_time = time.time()
_ = session.run(target)
duration = time.time() - start_time
if i >= num_steps_burn_in:
if not i % 10:
print('%s: step %d, duration = %.3f' %(datetime.now(), i - num_steps_burn_in, duration))
total_duration += duration
total_duration_squared += duration * duration
mn = total_duration / num_batches # average time
vr = total_duration_squared / num_batches - mn * mn
sd = math. sqrt(vr) # time standard deviation
print('%s: %s across %d steps, %.3f +/- %.3f sec / batch' %(datetime.now(), info_string, num_batches, mn, sd))
main function
# main function
def run_benchmark():
with tf.Graph().as_default(): # define the default graph
# There is no imagenet here, instead random numbers are used for training
image_size = 224
images = tf.Variable(tf.random_normal([batch_size, image_size, image_size, 3], dtype = tf.float32, stddev = 1e-1))
pool5, parameters = inference(images)
init = tf.global_variables_initializer()
sex = tf.Session ()
sess.run(init)
# Forward propagation time calculation
time_tensorflow_run(sess, pool5, "Forward")
# Back propagation time calculation
objective = tf.nn.l2_loss(pool5)
grad = tf.gradients(objective, parameters)
time_tensorflow_run(sess, grad, "Forward-backward")
If you want to run, just one sentence
run_benchmark()
It should be noted that this code LRN was commented out by me. If you want to try the results of LRN, just modify it at the first convolution and the second convolution.
I don't use GPU, and the time on CPU is much slower than that of Mr. Huang Wenjian's GPU. He used LRN forward 0.26 seconds, and mine corresponds to 0.468 seconds.
The results should consist of three parts;
conv1 [32, 56, 56, 64] conv1/pool1 [32, 27, 27, 64] conv2 [32, 27, 27, 192] conv2/pool2 [32, 13, 13, 192] conv3 [32, 13, 13, 384] conv4 [32, 13, 13, 256] conv5 [32, 13, 13, 256] conv5/pool5 [32, 6, 6, 256] 2018-04-24 15:27:52.440312: step 0, duration = 0.467 2018-04-24 15:27:57.211732: step 10, duration = 0.516 2018-04-24 15:28:02.045221: step 20, duration = 0.467 2018-04-24 15:28:06.703669: step 30, duration = 0.464 2018-04-24 15:28:11.335167: step 40, duration = 0.461 2018-04-24 15:28:15.966167: step 50, duration = 0.465 2018-04-24 15:28:20.588067: step 60, duration = 0.464 2018-04-24 15:28:25.376623: step 70, duration = 0.462 2018-04-24 15:28:29.995012: step 80, duration = 0.459 2018-04-24 15:28:34.616423: step 90, duration = 0.461 2018-04-24 15:28:38.799483: Forward across 100 steps, 0.468 +/- 0.018 sec / batch 2018-04-24 15:29:09.741985: step 0, duration = 2.689 2018-04-24 15:29:37.293994: step 10, duration = 2.749 2018-04-24 15:30:04.358533: step 20, duration = 2.696 2018-04-24 15:30:31.860958: step 30, duration = 2.783 2018-04-24 15:30:58.855577: step 40, duration = 2.688 2018-04-24 15:31:25.696813: step 50, duration = 2.689 2018-04-24 15:31:52.615997: step 60, duration = 2.688 2018-04-24 15:32:20.105809: step 70, duration = 2.699 2018-04-24 15:32:47.139107: step 80, duration = 2.704 2018-04-24 15:33:14.350406: step 90, duration = 2.981 2018-04-24 15:33:39.250186: Forward-backward across 100 steps, 2.722 +/- 0.066 sec / batch
And finally all the code:
# This part of the code is mainly to build a complete AlexNet convolutional neural network and perform a speed test on its forward propagation and back propagation. The original data is not loaded for training.
# Import the corresponding system library and tensorflow
from datetime import datetime
import math
import time
import tensorflow as tf
# A total of 100 groups of batches are tested, 32 in each group
batch_size = 32
num_batches = 100
# define a name and output size to display each convolutional or pooling layer
def print_activations(t):
print(t.op.name, '', t.get_shape().as_list())
def inference(images):
'''
:param images: input image size 32*224*244*3
:return: The last layer of pool5 and parameters
'''
parameters = []
with tf.name_scope('conv1') as scope:
kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 64], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(images, kernel, [1, 4, 4, 1], padding = 'SAME')
# s = 4,"NHWC", the data is stored in the order of: [batch, height, width, channels]
biases = tf.Variable(tf.constant(0.0, shape = [64], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(bias, name = scope)
print_activations(conv1) # output the name and shape of conv1
parameters += [kernel, biases]
# After the first layer of convolution, use local response normalization to improve generalization ability
# lrn1 = tf.nn.lrn(conv1, 4, bias = 1.0, alpha = 0.001 / 9, beta = 0.75, name = 'lrn1')
pool1 = tf.nn.max_pool(conv1, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool1')
print_activations(pool1)
with tf.name_scope('conv2') as scope:
kernel = tf.Variable(tf.truncated_normal([5, 5, 64, 192], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [192], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv2)
# lrn2 = tf.nn.lrn(conv2, 4, bias = 1.0, alpha = 0.001 / 9, beta = 0.75, name = 'lrn2')
pool2 = tf.nn.max_pool(conv2, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool2')
print_activations(pool2)
with tf.name_scope('conv3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 192, 384], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [384], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv3 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv3)
with tf.name_scope('conv4') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [256], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv4 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv4)
with tf.name_scope('conv5') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype = tf.float32, stddev = 1e-1), name = 'weights')
conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding = 'SAME')
biases = tf.Variable(tf.constant(0.0, shape = [256], dtype = tf.float32), trainable = True, name = 'biases')
bias = tf.nn.bias_add(conv, biases)
conv5 = tf.nn.relu(bias, name = scope)
parameters += [kernel, biases]
print_activations(conv5)
pool5 = tf.nn.max_pool(conv5, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding = 'VALID', name = 'pool5')
print_activations(pool5)
return pool5, parameters
# The convolution process is over, the LRN used here, and other literatures have been abandoned because the effect is not obvious, but the speed is reduced by 1/3 after use here, and the impact on the accuracy is not large. Can be used or not,
def time_tensorflow_run(session, target, info_string):
'''
:param session: session created
:param target: the operator to be evaluated
:param info_string: name of the test
:return:
'''
num_steps_burn_in = 10 # Let the program warm up for 10 rounds, because the first few rounds of operations have a lot of video memory loading time, and count the time after 10 rounds
total_duration = 0.0 # count the total time
total_duration_squared = 0.0 # and time mean squared
for i in range(num_batches + num_steps_burn_in):
start_time = time.time()
_ = session.run(target)
duration = time.time() - start_time
if i >= num_steps_burn_in:
if not i % 10:
print('%s: step %d, duration = %.3f' %(datetime.now(), i - num_steps_burn_in, duration))
total_duration += duration
total_duration_squared += duration * duration
mn = total_duration / num_batches # average time
vr = total_duration_squared / num_batches - mn * mn
sd = math. sqrt(vr) # time standard deviation
print('%s: %s across %d steps, %.3f +/- %.3f sec / batch' %(datetime.now(), info_string, num_batches, mn, sd))
# main function
def run_benchmark():
with tf.Graph().as_default(): # define the default graph
# There is no imagenet here, instead random numbers are used for training
image_size = 224
images = tf.Variable(tf.random_normal([batch_size, image_size, image_size, 3], dtype = tf.float32, stddev = 1e-1))
pool5, parameters = inference(images)
init = tf.global_variables_initializer()
sex = tf.Session ()
sess.run(init)
# Forward propagation time calculation
time_tensorflow_run(sess, pool5, "Forward")
# Back propagation time calculation
objective = tf.nn.l2_loss(pool5)
grad = tf.gradients(objective, parameters)
time_tensorflow_run(sess, grad, "Forward-backward")
run_benchmark()