Prepare :
① To download the Tensorflow Models library, I directly open the website and download the download under my own directory. In order to facilitate the import of downloaded modules such as cifar 10.py, etc., I directly open the cifar10 directory of the downloaded file, and then create my own Tensorflow_cifar in it -10_cnn.py file, as follows: only the last file is created by yourself, and other files are downloaded from tutorials.
② Download the cifar10 dataset
Be careful to download the last binary version
Start copying...
# Import various modules and required files
import digit10
import cifar10_input
import tensorflow as tf
import numpy as np
import time
# Define batch_size, training epoch max_steps, and default path to download CIFAR-10 data
max_steps = 3000
batch_size = 128
# The directory where the dataset is stored
data_dir = 'D:/jupyter_notebook/tensorflow_practice/CIFAR-10_ex/cifar-10-binary/cifar-10-batches-bin'
# Define the function to initialize the weight. Here, add an L2 loss to the weight, which is equivalent to L2 regularization
def variable_with_weight_loss(shape, stddev, w1):
# Create variables with truncated normal distribution
var = tf.Variable(tf.truncated_normal(shape, stddev=stddev))
if w1 is not None:
# tf.multiply() The elements are multiplied by each other, pay attention to the difference with tf.matmul
# tf.nn.l2_loss(var) Calculate the L2 loss of weight, and then use tf.multiply to implement w1 to control the size of L2 loss
weight_loss = tf.multiply(tf.nn.l2_loss(var), w1, name='weight_loss')
# add_to_collection(name, value), equivalent to dictionary usage, key: name, value: value (list data structure), adding something to name is equivalent to adding elements to the list of value
tf.add_to_collection('losses', weight_loss)
return var
# Download the dataset using the cifar10 class, unzip and expand to its default location
# I downloaded it from the website myself...
# cifar10.maybe_download_and_extract()
# Use the distorted_inputs function in the cifar10_input class to generate the data needed for training, including features and their corresponding labels
# Returns the packaged tensor. Each execution will generate a batch_size number of samples. It should be noted that we have performed Data Augmentation on the data.
# Includes: horizontal flipping, random cropping, setting random brightness and contrast, and normalizing the data (0 mean, 1 variance)
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=batch_size)
# Use the cifar10_input.inputs function to generate test data
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size)
# Create placeholder for data, including features and labels
# Note that before our x = tf.placeholder(tf.float32,[None,784]) all are None, but this cannot be set here
# Because it is used to define the network structure after batch_size, the first value of the data size, the number of samples, needs to be preset
# Color image dataset with 3 color channels
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
# Now create the first convolutional layer
# 5*5 convolution kernel, 3 color channels, 64 convolution kernels, and set the standard of the weight initialization function to 0.05.
# Note: w1 = 0.0 means do not regularize the weights of this layer
weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=5e-2,
w1=0.0)
# Use tf.nn.conv2d for convolution operation
kernel1 = tf.nn.conv2d(image_holder, weight1, strides=[1, 1, 1, 1], padding='SAME')
# Initialize bias1 to 0
bias1 = tf.Variable(tf.constant(0.0, shape=[64]))
# Use tf.nn.bias_add to add the bias and use the relu activation function to process it
conv1 = tf.nn.relu(tf.nn.bias_add(kernel1, bias1))
# Pool processing size 3*3, step size 2*2, note that the size and step size are inconsistent, which can increase the richness of the data
pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME')
# tf.nn.lrn: LRN processing, imitating the 'lateral inhibition' mechanism of the biological nervous system, the response to large values is larger, and the response to small values is inhibited, enhancing the generalization ability of the model
# LRN: Local Response Normalization Local response value normalization
# tf.nn.lrn(input,depth_radius=None,bias=None,alpha=None,beta=None,name=None)
norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
# create the second convolutional layer
# The number of convolution kernels in the previous layer is 64, so this time the input channel becomes 54
weight2 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2,
w1=0.0)
kernel2 = tf.nn.conv2d(norm1, weight2, strides=[1, 1, 1, 1], padding='SAME')
bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
conv2 = tf.nn.relu(tf.nn.bias_add(kernel2, bias2))
norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME')
# Define the first fully connected layer
# Use the reshape function to turn each sample into a 1D vector
# tf.reshape(tensor, shape, name = None), where reshape is in the form of a list, -1 means to calculate that dimension by yourself
# [batch_size, -1] means that there are as many rows as batch_size, the columns are automatically calculated, and each row represents a sample
reshape = tf.reshape(pool2, [batch_size, -1])
# We use the get_shape() function to get the flattened length
# For more usage reference of reshape and get_shape(): https://blog.csdn.net/Li_haiyu/article/details/80063842
# Get the reshaped column
dim = reshape.get_shape()[1].value
# 384 hidden nodes, set a non-zero weight loss value of 0.04, all parameters of this layer are constrained by L2 regularization
weight3 = variable_with_weight_loss(shape=[dim, 384], stddev=0.04, w1=0.004)
bias3 = tf.Variable(tf.constant(0.1, shape=[384]))
# The dimension of reshape is [batch_size, dim], the dimension of weight3 is [dim, 384], and the dimensions match
# local3 dimension [batch_size, 384]
local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3)
# Define the second fully connected layer
# The number of hidden nodes in this fully connected layer is reduced by half to 192, and other hyperparameters remain unchanged
weight4 = variable_with_weight_loss(shape=[384, 192], stddev=0.04, w1=0.004)
bias4 = tf.Variable(tf.constant(0.1, shape=[192]))
# local4 dimension [batch_size, 193]
local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4)
# The last layer, do not add regularization
# Note: The softmax is placed in the calculation loss part, because we do not need to perform softmax processing on the output of the inference to obtain the final classification result (you can directly compare the various numerical values of the inference output)
# Calculating softmax is mainly to calculate loss, so it is more appropriate to integrate it later
weight5 = variable_with_weight_loss(shape=[192, 10], stddev=1 / 192.0, w1=0.0)
bias5 = tf.Variable(tf.constant(0.0, shape=[10]))
logits = tf.add(tf.matmul(local4, weight5), bias5)
# Define the loss function cross entropy, which combines the calculation of the softmax function and the cross entropy loss, ie tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels, name='cross_entropy_per_example')
def loss(logits, labels):
labels = tf.cast(labels, tf.int64)
#
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels,
name='cross_entropy_per_example')
# Here use tf.reduce_mean to calculate the mean of cross entropy
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
# Add the loss of cross entropy to the collection of overall losses
tf.add_to_collection('losses', cross_entropy_mean)
# Sum all the losses in the collection of losses to get the final loss
return tf.add_n(tf.get_collection('losses'), name='total_loss')
# get the final loss
loss = loss(logits, label_holder)
# Optimizer select Adam Optimizer
train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)
# Use the tf.nn.in_top_k function to find the accuracy of top k in the output result. By default, top 1 is used, which is the accuracy of the class with the highest output score.
top_k_op = tf.nn.in_top_k(logits, label_holder, 1)
# Use tf.InteractiveSession to create a default session, then initialize all model parameters
sex = tf.InteractiveSession ()
tf.global_variables_initializer().run()
# This step is to start the image data enhancement thread, a total of 16 threads are used here for acceleration
# Note: If the thread is not started, subsequent inference and training operations cannot be started
tf.train.start_queue_runners()
# Officially start training
for step in range(max_steps):
# timestamp
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
# _ also represents a variable that receives the value of train_op
_, loss_value = sess.run([train_op, loss], feed_dict={image_holder: image_batch, label_holder: label_batch})
# Here is the time difference calculated
duration = time.time() - start_time
# output loss value every 10 iterations, etc.
if step % 10 == 0:
example_per_sec = batch_size / duration
sec_per_batch = float(duration)
# %.0f means keep zero digits after the decimal point, .2f means keep 2 digits after the decimal point
format_str = 'step %d, loss=%.2f ,%.1f examples/sec, %.3f sec/batch'
print(format_str % (step, loss_value, example_per_sec, sec_per_batch))
# Evaluate the accuracy of the model on the test set
# The test set has a total of 10,000 samples
num_examples = 10000
import math
# Calculate the number of iterations
# ceil means round up math.ceil(3.2) = 4
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size
step = 0
while step < num_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch, label_holder: label_holder})
true_count += np.sum(predictions)
step += 1
# Finally print out the accuracy evaluation results
precision = true_count / total_sample_count
print('precision @ 1 = %.3f'%precision)
Attachment: Convolutional Neural Network Structure Table
