深度学习 tensorflow 猫狗大战 自己的数据集

本次学习主要参考优酷：Tensorflow tutorial Cats vs. dogs 系列视频  

视频连接：http://i.youku.com/deeplearning101 

数据集链接：http://pan.baidu.com/s/1dFd8kmt 密码：psor

运行环境：win10,64位，TensorFlow CPU版本，电脑显卡不行，整个过程训练了十个小时（10000个steps）

训练目录：D:\cat_VS_dog\cats_vs_dogs\data\train

生成模型目录：D:\cat_VS_dog\cats_vs_dogs\logs\train （每隔2000步记录一次）

主要用到三个python文件：

1. input_data.py

2. model.py

3. training.py

1.input_data.py

[python] view plain copy

print ?

1. <code class="language-python">#By @Kevin Xu  
2. #[email protected]  
3. #Youtube: https://www.youtube.com/channel/UCVCSn4qQXTDAtGWpWAe4Plw  
4. #  
5. #The aim of this project is to use TensorFlow to process our own data.  
6. #    - input_data.py:  read in data and generate batches  
7. #    - model: build the model architecture  
8. #    - training: train  
9.   
10.   
11. # I used Ubuntu with Python 3.5, TensorFlow 1.0*, other OS should also be good.  
12. # With current settings, 10000 traing steps needed 50 minutes on my laptop.  
13.   
14.   
15.   
16.   
17. # data: cats vs. dogs from Kaggle  
18. # Download link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data  
19. # data size: ~540M  
20.   
21. # How to run?  
22. # 1. run the training.py once  
23. # 2. call the run_training() in the console to train the model.  
24.   
25. # Note:   
26. # it is suggested to restart your kenel to train the model multiple times   
27. #(in order to clear all the variables in the memory)  
28. # Otherwise errors may occur: conv1/weights/biases already exist......  
29.   
30. #%%  
31. import tensorflow as tf  
32. import numpy as np  
33. import os  
34. #%%  
35.   
36. # you need to change this to your data directory  
37. #train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'  
38. train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/'   #My dir--20170727-csq  
39.   
40. def get_files(file_dir):  
41.     ''''' 
42.     Args: 
43.         file_dir: file directory 
44.     Returns: 
45.         list of images and labels 
46.     '''  
47.     cats = []  
48.     label_cats = []  
49.     dogs = []  
50.     label_dogs = []  
51.     for file in os.listdir(file_dir):  
52.         name = file.split(sep='.')  
53.         if name[0]=='cat':  
54.             cats.append(file_dir + file)  
55.             label_cats.append(0)  
56.         else:  
57.             dogs.append(file_dir + file)  
58.             label_dogs.append(1)  
59.     print('There are %d cats\nThere are %d dogs' %(len(cats), len(dogs)))  
60.       
61.     image_list = np.hstack((cats, dogs))  
62.     label_list = np.hstack((label_cats, label_dogs))  
63.       
64.     temp = np.array([image_list, label_list])  
65.     temp = temp.transpose()  
66.     np.random.shuffle(temp)  
67.       
68.     image_list = list(temp[:, 0])  
69.     label_list = list(temp[:, 1])  
70.     label_list = [int(i) for i in label_list]  
71.          
72.     return image_list, label_list  
73.   
74.   
75. #%%  
76. def get_batch(image, label, image_W, image_H, batch_size, capacity):  
77.     ''''' 
78.     Args: 
79.         image: list type 
80.         label: list type 
81.         image_W: image width 
82.         image_H: image height 
83.         batch_size: batch size 
84.         capacity: the maximum elements in queue 
85.     Returns: 
86.         image_batch: 4D tensor [batch_size, width, height, 3], dtype=tf.float32 
87.         label_batch: 1D tensor [batch_size], dtype=tf.int32 
88.     '''  
89.       
90.     image = tf.cast(image, tf.string)  
91.     label = tf.cast(label, tf.int32)  
92.   
93.   
94.     # make an input queue  
95.     input_queue = tf.train.slice_input_producer([image, label])  
96.       
97.     label = input_queue[1]  
98.     image_contents = tf.read_file(input_queue[0])  
99.     image = tf.image.decode_jpeg(image_contents, channels=3)  
100.       
101.     ######################################  
102.     # data argumentation should go to here  
103.     ######################################  
104.       
105.     image = tf.image.resize_image_with_crop_or_pad(image, image_W, image_H)  
106.       
107.     # if you want to test the generated batches of images, you might want to comment the following line.  
108.     image = tf.image.per_image_standardization(image)  
109.       
110.     image_batch, label_batch = tf.train.batch([image, label],  
111.                                                 batch_size= batch_size,  
112.                                                 num_threads= 64,   
113.                                                 capacity = capacity)  
114.       
115.     #you can also use shuffle_batch   
116. #    image_batch, label_batch = tf.train.shuffle_batch([image,label],  
117. #                                                      batch_size=BATCH_SIZE,  
118. #                                                      num_threads=64,  
119. #                                                      capacity=CAPACITY,  
120. #                                                      min_after_dequeue=CAPACITY-1)  
121.       
122.     label_batch = tf.reshape(label_batch, [batch_size])  
123.     image_batch = tf.cast(image_batch, tf.float32)  
124.       
125.     return image_batch, label_batch  
126.   
127. #%% TEST  
128. # To test the generated batches of images  
129. # When training the model, DO comment the following codes  
130.   
131.   
132.   
133.   
134.   
135. import matplotlib.pyplot as plt  
136.   
137. BATCH_SIZE = 2  
138. CAPACITY = 256  
139. IMG_W = 208  
140. IMG_H = 208  
141.   
142.   
143. #train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'  
144. train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/'   #My dir--20170727-csq  
145. image_list, label_list = get_files(train_dir)  
146. image_batch, label_batch = get_batch(image_list, label_list, IMG_W, IMG_H, BATCH_SIZE, CAPACITY)  
147.   
148. with tf.Session() as sess:  
149.     i = 0  
150.     coord = tf.train.Coordinator()  
151.     threads = tf.train.start_queue_runners(coord=coord)  
152.       
153.     try:  
154.         while not coord.should_stop() and i<1:  
155.               
156.             img, label = sess.run([image_batch, label_batch])  
157.               
158.             # just test one batch  
159.             for j in np.arange(BATCH_SIZE):  
160.                 print('label: %d' %label[j])  #j-index of quene of Batch_size  
161.                 plt.imshow(img[j,:,:,:])  
162.                 plt.show()  
163.             i+=1  
164.               
165.     except tf.errors.OutOfRangeError:  
166.         print('done!')  
167.     finally:  
168.         coord.request_stop()  
169.     coord.join(threads)  
170.   
171. #%%</code>  

#By @Kevin Xu
#[email protected]
#Youtube: https://www.youtube.com/channel/UCVCSn4qQXTDAtGWpWAe4Plw
#
#The aim of this project is to use TensorFlow to process our own data.
# - input_data.py: read in data and generate batches
# - model: build the model architecture
# - training: train


# I used Ubuntu with Python 3.5, TensorFlow 1.0*, other OS should also be good.
# With current settings, 10000 traing steps needed 50 minutes on my laptop.




# data: cats vs. dogs from Kaggle
# Download link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data
# data size: ~540M

# How to run?
# 1. run the training.py once
# 2. call the run_training() in the console to train the model.

# Note:
# it is suggested to restart your kenel to train the model multiple times
#(in order to clear all the variables in the memory)
# Otherwise errors may occur: conv1/weights/biases already exist......

#%%
import tensorflow as tf
import numpy as np
import os
#%%

# you need to change this to your data directory
#train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'
train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/' #My dir--20170727-csq

def get_files(file_dir):
'''
Args:
file_dir: file directory
Returns:
list of images and labels
'''
cats = []
label_cats = []
dogs = []
label_dogs = []
for file in os.listdir(file_dir):
name = file.split(sep= '.')
if name[ 0]== 'cat':
cats.append(file_dir + file)
label_cats.append( 0)
else:
dogs.append(file_dir + file)
label_dogs.append( 1)
print( 'There are %d cats\nThere are %d dogs' %(len(cats), len(dogs)))

image_list = np.hstack((cats, dogs))
label_list = np.hstack((label_cats, label_dogs))

temp = np.array([image_list, label_list])
temp = temp.transpose()
np.random.shuffle(temp)

image_list = list(temp[:, 0])
label_list = list(temp[:, 1])
label_list = [int(i) for i in label_list]

return image_list, label_list


#%%
def get_batch(image, label, image_W, image_H, batch_size, capacity):
'''
Args:
image: list type
label: list type
image_W: image width
image_H: image height
batch_size: batch size
capacity: the maximum elements in queue
Returns:
image_batch: 4D tensor [batch_size, width, height, 3], dtype=tf.float32
label_batch: 1D tensor [batch_size], dtype=tf.int32
'''

image = tf.cast(image, tf.string)
label = tf.cast(label, tf.int32)


# make an input queue
input_queue = tf.train.slice_input_producer([image, label])

label = input_queue[ 1]
image_contents = tf.read_file(input_queue[ 0])
image = tf.image.decode_jpeg(image_contents, channels= 3)

######################################
# data argumentation should go to here
######################################

image = tf.image.resize_image_with_crop_or_pad(image, image_W, image_H)

# if you want to test the generated batches of images, you might want to comment the following line.
image = tf.image.per_image_standardization(image)

image_batch, label_batch = tf.train.batch([image, label],
batch_size= batch_size,
num_threads= 64,
capacity = capacity)

#you can also use shuffle_batch
# image_batch, label_batch = tf.train.shuffle_batch([image,label],
# batch_size=BATCH_SIZE,
# num_threads=64,
# capacity=CAPACITY,
# min_after_dequeue=CAPACITY-1)

label_batch = tf.reshape(label_batch, [batch_size])
image_batch = tf.cast(image_batch, tf.float32)

return image_batch, label_batch

#%% TEST
# To test the generated batches of images
# When training the model, DO comment the following codes





import matplotlib.pyplot as plt

BATCH_SIZE = 2
CAPACITY = 256
IMG_W = 208
IMG_H = 208


#train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'
train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/' #My dir--20170727-csq
image_list, label_list = get_files(train_dir)
image_batch, label_batch = get_batch(image_list, label_list, IMG_W, IMG_H, BATCH_SIZE, CAPACITY)

with tf.Session() as sess:
i = 0
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)

try:
while not coord.should_stop() and i< 1:

img, label = sess.run([image_batch, label_batch])

# just test one batch
for j in np.arange(BATCH_SIZE):
print( 'label: %d' %label[j]) #j-index of quene of Batch_size
plt.imshow(img[j,:,:,:])
plt.show()
i+= 1

except tf.errors.OutOfRangeError:
print( 'done!')
finally:
coord.request_stop()
coord.join(threads)

#%%

2.model.py

#By @Kevin Xu
#[email protected]
#Youtube: https://www.youtube.com/channel/UCVCSn4qQXTDAtGWpWAe4Plw
#


#The aim of this project is to use TensorFlow to process our own data.
# - input_data.py: read in data and generate batches
# - model: build the model architecture
# - training: train


# data: cats vs. dogs from Kaggle
# Download link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data
# data size: ~540M


# How to run?
# 1. run the training.py once
# 2. call the run_training() in the console to train the model.




#%%


import tensorflow as tf


#%%
def inference(images, batch_size, n_classes):
'''Build the model
Args:
images: image batch, 4D tensor, tf.float32, [batch_size, width, height, channels]
Returns:
output tensor with the computed logits, float, [batch_size, n_classes]
'''
#conv1, shape = [kernel size, kernel size, channels, kernel numbers]

with tf.variable_scope( 'conv1') as scope:
weights = tf.get_variable( 'weights',
shape = [ 3, 3, 3, 16],
dtype = tf.float32,
initializer=tf.truncated_normal_initializer(stddev= 0.1,dtype=tf.float32))
biases = tf.get_variable( 'biases',
shape=[ 16],
dtype=tf.float32,
initializer=tf.constant_initializer( 0.1))
conv = tf.nn.conv2d(images, weights, strides=[ 1, 1, 1, 1], padding= 'SAME')
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name= scope.name)

#pool1 and norm1
with tf.variable_scope( 'pooling1_lrn') as scope:
pool1 = tf.nn.max_pool(conv1, ksize=[ 1, 3, 3, 1],strides=[ 1, 2, 2, 1],
padding= 'SAME', name= 'pooling1')
norm1 = tf.nn.lrn(pool1, depth_radius= 4, bias= 1.0, alpha= 0.001/ 9.0,
beta= 0.75,name= 'norm1')

#conv2
with tf.variable_scope( 'conv2') as scope:
weights = tf.get_variable( 'weights',
shape=[ 3, 3, 16, 16],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev= 0.1,dtype=tf.float32))
biases = tf.get_variable( 'biases',
shape=[ 16],
dtype=tf.float32,
initializer=tf.constant_initializer( 0.1))
conv = tf.nn.conv2d(norm1, weights, strides=[ 1, 1, 1, 1],padding= 'SAME')
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name= 'conv2')


#pool2 and norm2
with tf.variable_scope( 'pooling2_lrn') as scope:
norm2 = tf.nn.lrn(conv2, depth_radius= 4, bias= 1.0, alpha= 0.001/ 9.0,
beta= 0.75,name= 'norm2')
pool2 = tf.nn.max_pool(norm2, ksize=[ 1, 3, 3, 1], strides=[ 1, 1, 1, 1],
padding= 'SAME',name= 'pooling2')


#local3
with tf.variable_scope( 'local3') as scope:
reshape = tf.reshape(pool2, shape=[batch_size, -1])
dim = reshape.get_shape()[ 1].value
weights = tf.get_variable( 'weights',
shape=[dim, 128],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev= 0.005,dtype=tf.float32))
biases = tf.get_variable( 'biases',
shape=[ 128],
dtype=tf.float32,
initializer=tf.constant_initializer( 0.1))
local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)

#local4
with tf.variable_scope( 'local4') as scope:
weights = tf.get_variable( 'weights',
shape=[ 128, 128],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev= 0.005,dtype=tf.float32))
biases = tf.get_variable( 'biases',
shape=[ 128],
dtype=tf.float32,
initializer=tf.constant_initializer( 0.1))
local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name= 'local4')


# softmax
with tf.variable_scope( 'softmax_linear') as scope:
weights = tf.get_variable( 'softmax_linear',
shape=[ 128, n_classes],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev= 0.005,dtype=tf.float32))
biases = tf.get_variable( 'biases',
shape=[n_classes],
dtype=tf.float32,
initializer=tf.constant_initializer( 0.1))
softmax_linear = tf.add(tf.matmul(local4, weights), biases, name= 'softmax_linear')

return softmax_linear


#%%
def losses(logits, labels):
'''Compute loss from logits and labels
Args:
logits: logits tensor, float, [batch_size, n_classes]
labels: label tensor, tf.int32, [batch_size]

Returns:
loss tensor of float type
'''
with tf.variable_scope( 'loss') as scope:
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits\
(logits=logits, labels=labels, name= 'xentropy_per_example')
loss = tf.reduce_mean(cross_entropy, name= 'loss')
tf.summary.scalar(scope.name+ '/loss', loss)
return loss


#%%
def trainning(loss, learning_rate):
'''Training ops, the Op returned by this function is what must be passed to
'sess.run()' call to cause the model to train.

Args:
loss: loss tensor, from losses()

Returns:
train_op: The op for trainning
'''
with tf.name_scope( 'optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate= learning_rate)
global_step = tf.Variable( 0, name= 'global_step', trainable= False)
train_op = optimizer.minimize(loss, global_step= global_step)
return train_op


#%%
def evaluation(logits, labels):
"""Evaluate the quality of the logits at predicting the label.
Args:
logits: Logits tensor, float - [batch_size, NUM_CLASSES].
labels: Labels tensor, int32 - [batch_size], with values in the
range [0, NUM_CLASSES).
Returns:
A scalar int32 tensor with the number of examples (out of batch_size)
that were predicted correctly.
"""
with tf.variable_scope( 'accuracy') as scope:
correct = tf.nn.in_top_k(logits, labels, 1)
correct = tf.cast(correct, tf.float16)
accuracy = tf.reduce_mean(correct)
tf.summary.scalar(scope.name+ '/accuracy', accuracy)
return accuracy


#%%

3.training.py

#By @Kevin Xu
#[email protected]
#Youtube: https://www.youtube.com/channel/UCVCSn4qQXTDAtGWpWAe4Plw
#
#The aim of this project is to use TensorFlow to process our own data.
# - input_data.py: read in data and generate batches
# - model: build the model architecture
# - training: train


# I used Ubuntu with Python 3.5, TensorFlow 1.0*, other OS should also be good.
# With current settings, 10000 traing steps needed 50 minutes on my laptop.


# data: cats vs. dogs from Kaggle
# Download link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data
# data size: ~540M


# How to run?
# 1. run the training.py once
# 2. call the run_training() in the console to train the model.


# Note:
# it is suggested to restart your kenel to train the model multiple times
#(in order to clear all the variables in the memory)
# Otherwise errors may occur: conv1/weights/biases already exist......




#%%


import os
import numpy as np
import tensorflow as tf
import input_data
import model


#%%


N_CLASSES = 2
IMG_W = 208 # resize the image, if the input image is too large, training will be very slow.
IMG_H = 208
BATCH_SIZE = 16
CAPACITY = 2000
MAX_STEP = 10000 # with current parameters, it is suggested to use MAX_STEP>10k
learning_rate = 0.0001 # with current parameters, it is suggested to use learning rate<0.0001




#%%
def run_training():

# you need to change the directories to yours.
#train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'
train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/' #My dir--20170727-csq
#logs_train_dir = '/home/kevin/tensorflow/cats_vs_dogs/logs/train/'
logs_train_dir = 'D:/cat_VS_dog/cats_vs_dogs/logs/train/'
train, train_label = input_data.get_files(train_dir)

train_batch, train_label_batch = input_data.get_batch(train,
train_label,
IMG_W,
IMG_H,
BATCH_SIZE,
CAPACITY)
train_logits = model.inference(train_batch, BATCH_SIZE, N_CLASSES)
train_loss = model.losses(train_logits, train_label_batch)
train_op = model.trainning(train_loss, learning_rate)
train__acc = model.evaluation(train_logits, train_label_batch)

summary_op = tf.summary.merge_all()
sess = tf.Session()
train_writer = tf.summary.FileWriter(logs_train_dir, sess.graph)
saver = tf.train.Saver()

sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)

try:
for step in np.arange(MAX_STEP):
if coord.should_stop():
break
_, tra_loss, tra_acc = sess.run([train_op, train_loss, train__acc])

if step % 50 == 0:
print( 'Step %d, train loss = %.2f, train accuracy = %.2f%%' %(step, tra_loss, tra_acc* 100.0))
summary_str = sess.run(summary_op)
train_writer.add_summary(summary_str, step)

if step % 2000 == 0 or (step + 1) == MAX_STEP:
checkpoint_path = os.path.join(logs_train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)

except tf.errors.OutOfRangeError:
print( 'Done training -- epoch limit reached')
finally:
coord.request_stop()

coord.join(threads)
sess.close()



#%% Evaluate one image
# when training, comment the following codes.




from PIL import Image
import matplotlib.pyplot as plt


def get_one_image(train):
'''Randomly pick one image from training data
Return: ndarray
'''
n = len(train)
ind = np.random.randint( 0, n)
img_dir = train[ind]


image = Image.open(img_dir)
plt.imshow(image)
image = image.resize([ 208, 208])
image = np.array(image)
return image


def evaluate_one_image():
'''Test one image against the saved models and parameters
'''

# you need to change the directories to yours.
#train_dir = '/home/kevin/tensorflow/cats_vs_dogs/data/train/'
train_dir = 'D:/cat_VS_dog/cats_vs_dogs/data/train/'
train, train_label = input_data.get_files(train_dir)
image_array = get_one_image(train)

with tf.Graph().as_default():
BATCH_SIZE = 1
N_CLASSES = 2

image = tf.cast(image_array, tf.float32)
image = tf.image.per_image_standardization(image)
image = tf.reshape(image, [ 1, 208, 208, 3])
logit = model.inference(image, BATCH_SIZE, N_CLASSES)

logit = tf.nn.softmax(logit)

x = tf.placeholder(tf.float32, shape=[ 208, 208, 3])

# you need to change the directories to yours.
#logs_train_dir = '/home/kevin/tensorflow/cats_vs_dogs/logs/train/'
logs_train_dir = 'D:/cat_VS_dog/cats_vs_dogs/logs/train'

saver = tf.train.Saver()

with tf.Session() as sess:

print( "Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(logs_train_dir)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split( '/')[ -1].split( '-')[ -1]
saver.restore(sess, ckpt.model_checkpoint_path)
print( 'Loading success, global_step is %s' % global_step)
else:
print( 'No checkpoint file found')

prediction = sess.run(logit, feed_dict={x: image_array})
max_index = np.argmax(prediction)
if max_index== 0:
print( 'This is a cat with possibility %.6f' %prediction[:, 0])
else:
print( 'This is a dog with possibility %.6f' %prediction[:, 1])

#%%

代码的详细注释可参考文章：http://blog.csdn.net/hjxu2016/article/details/75305123

代码运行过程中显示几张图片时，彩色图片显示不正常：参考文章 http://blog.csdn.net/hjxu2016/article/details/75305123?locationNum=10&fps=1

的解释如下：
之前将图片转为float32了，因此这里imshow()出来的图片色彩会有点奇怪，因为本来imshow()是显示uint8类型的数据（灰度值在uint8类型下是0~255，转为float32后会超出这个范围，所以色彩有点奇怪），不过这不影响后面模型的训练。

该视频教程最后结果对猫狗分类误差还是存在的，测试几次，存在错将猫分成狗。