环境：ubuntu16.04+tensorflow+cpu
文件路径：/home/qf/tensorflow/tf/tf8
数据集为二类卫星云图像/tf8/data/train
1、处理图像prepare.py
#========================================================================================
import os
import tensorflow as tf
from PIL import Image
#
#orig_picture = '/home/qf/git/myfile/dataset/train/'
orig_picture = '/home/qf/tensorflow/tf/tf7/data/train/'
gen_picture = '/home/qf/tensorflow/tf/tf8/data/train/'
#
#classes = {'bedroom','CALsuburb','industrial','kitchen','livingroom','MITcoast','MITforest','MIThighway',
# 'MITinsidecity','MITmountain','MITopencountry','MITstreet','MITtallbuilding','PARoffice','store'}
classes = {'elliptical','spiral'}
#
#num_samples = 2250
num_samples = 518
#
def create_record():
writer = tf.python_io.TFRecordWriter("/home/qf/tensorflow/tf/tf8/data/train.tfrecords")
for index, name in enumerate(classes):
class_path = orig_picture +"/"+ name+"/"
for img_name in os.listdir(class_path):
img_path = class_path + img_name
img = Image.open(img_path)
img = img.resize((256, 256)) #
img_raw = img.tobytes() #
#print (index,img_raw)
example = tf.train.Example(
features=tf.train.Features(feature={
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[index])),
'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
}))
writer.write(example.SerializeToString())
writer.close()
print('successed!!!')
#=======================================================================================
def read_and_decode(filename):
#
filename_queue = tf.train.string_input_producer([filename])
# create a reader from file queue
reader = tf.TFRecordReader()
#
_, serialized_example = reader.read(filename_queue)
# get feature from serialized example
#
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string)
})
label = features['label']
img = features['img_raw']
img = tf.decode_raw(img, tf.uint8)
img = tf.reshape(img, [256, 256, 3])
#img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(label, tf.int32)
return img, label
#=======================================================================================
if __name__ == '__main__':
create_record()
batch = read_and_decode('/home/qf/tensorflow/tf/tf8/data/train.tfrecords')
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session() as sess: #
sess.run(init_op)
coord=tf.train.Coordinator()
threads= tf.train.start_queue_runners(coord=coord)
for i in range(num_samples):
example, lab = sess.run(batch)#
img=Image.fromarray(example, 'RGB')#
img.save(gen_picture+'/'+str(i)+'samples'+str(lab)+'.jpg')#
print(example, lab)
coord.request_stop()
coord.join(threads)
sess.close()
#======================================================================================== 2、输入数据input_data.py
import os
import math
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
#============================================================================
#-----------------produce image dir and label List------------------------------------
train_dir = '/home/qf/tensorflow/tf/tf8/data/train/'
elliptical = []
label_elliptical = []
spiral = []
label_spiral = []
#step1:Get the image path name and save it to The corresponding list is affixed with a label and
# stored in the label list
def get_files(file_dir, ratio):
for file in os.listdir(file_dir+'/elliptical'):
elliptical.append(file_dir +'/elliptical'+'/'+ file)
label_elliptical.append(0)
for file in os.listdir(file_dir+'/spiral'):
spiral.append(file_dir +'/spiral'+'/'+file)
label_spiral.append(1)
#step2:Disturb the generated image path and label List to combine elliptical and spiral to form a list
image_list = np.hstack((elliptical, spiral))
label_list = np.hstack((label_elliptical, label_spiral))
#Use shuffle to disrupt the order
temp = np.array([image_list, label_list])
temp = temp.transpose()
np.random.shuffle(temp)
#Remove the list from the disturbed temp
#image_list = list(temp[:, 0])
#label_list = list(temp[:, 1])
#label_list = [int(i) for i in label_list]
#return image_list, label_list
#Convert all img and lab to list
all_image_list = list(temp[:, 0])
all_label_list = list(temp[:, 1])
#Divide the resulting List into two parts, one for training tra and one for testing val
n_sample = len(all_label_list)
n_val = int(math.ceil(n_sample*ratio)) #Number of test samples
n_train = n_sample - n_val #Number of training samples
tra_images = all_image_list[0:n_train]
tra_labels = all_label_list[0:n_train]
tra_labels = [int(float(i)) for i in tra_labels]
val_images = all_image_list[n_train:-1]
val_labels = all_label_list[n_train:-1]
val_labels = [int(float(i)) for i in val_labels]
return tra_images, tra_labels, val_images, val_labels
#---------------------------------------------------------------------------
#--------------------produce Batch----------------------------------------------
#step1:The above generated List is passed to get_batch()
# converts the type to produce an input queue
# Use tf.train.slice_input_producer() and then read the image from the queue using tf.read_file()
# batch_size:How many pictures to put in each batch
# capacity:How much is a queue
def get_batch(image, label, image_W, image_H, batch_size, capacity):
#Conversion type
image = tf.cast(image, tf.string)
label = tf.cast(label, tf.int32)
# make an input queue
input_queue = tf.train.slice_input_producer([image, label])
label = input_queue[1]
image_contents = tf.read_file(input_queue[0]) #read img from a queue
#step2:Decode the image. Different types of images cannot be mixed together, either using jpeg or png only.
image = tf.image.decode_jpeg(image_contents, channels=3)
#step3:Data preprocessing, rotate, scale, crop, and normalize the image, making the calculated model more robust.
image = tf.image.resize_image_with_crop_or_pad(image, image_W, image_H)
image = tf.image.per_image_standardization(image)
#step4:produce batch
#image_batch: 4D tensor [batch_size, width, height, 3],dtype=tf.float32
#label_batch: 1D tensor [batch_size], dtype=tf.int32
image_batch, label_batch = tf.train.batch([image, label],
batch_size= batch_size,
num_threads= 32,
capacity = capacity)
#Rearrange the label, the number of rows is [batch_size]
label_batch = tf.reshape(label_batch, [batch_size])
image_batch = tf.cast(image_batch, tf.float32)
return image_batch, label_batch
#========================================================================3、构建模型:model.py
import tensorflow as tf
#=========================================================================
#
#Input parameters:images,image batch,4D tensor,tf.float32,[batch_size, width, height, channels]
#Output parameters:logits,float,[batch_size,,n_classes]
def inference(images, batch_size, n_classes):
#Convolution 1
#64 3x3 convolution kernels (3 channels),padding='SAME',indicating that the plot of the convolution after the #padding is the same as the original image size,and the activation function relu()
with tf.variable_scope('conv1') as scope:
weights = tf.Variable(tf.truncated_normal(shape=[3,3,3,64], stddev = 1.0, dtype = tf.float32),
name = 'weights', dtype = tf.float32)
biases = tf.Variable(tf.constant(value = 0.1, dtype = tf.float32, shape = [64]),
name = 'biases', dtype = tf.float32)
conv = tf.nn.conv2d(images, weights, strides=[1,1,1,1], padding='SAME')
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name= scope.name)
#Pooling layer 1
#3x3 maximum pooling, with strides of 2 steps, pooled lrn() operations, partial response normalization
with tf.variable_scope('pooling1_lrn') as scope:
pool1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1],strides=[1,2,2,1],padding='SAME', name='pooling1')
norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='norm1')
#Convolution 2
#16 3x3 convolution kernels (3 channels), padding='SAME', indicating that the plot of the convolution after the #padding is the same as the original image size, and the activation function relu()
with tf.variable_scope('conv2') as scope:
weights = tf.Variable(tf.truncated_normal(shape=[3,3,64,16], stddev = 0.1, dtype = tf.float32),
name = 'weights', dtype = tf.float32)
biases = tf.Variable(tf.constant(value = 0.1, dtype = tf.float32, shape = [16]),
name = 'biases', dtype = tf.float32)
conv = tf.nn.conv2d(norm1, weights, strides = [1,1,1,1],padding='SAME')
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name='conv2')
#Pooling layer 2
#3x3 maximum pooling with strides of 2 and lrn() operations after pooling
#pool2 and norm2
with tf.variable_scope('pooling2_lrn') as scope:
norm2 = tf.nn.lrn(conv2, depth_radius=4, bias=1.0, alpha=0.001/9.0,beta=0.75,name='norm2')
pool2 = tf.nn.max_pool(norm2, ksize=[1,3,3,1], strides=[1,1,1,1],padding='SAME',name='pooling2')
#Fully connected layer 3
#128 neurons reshape the output of the previous pool layer into a row and activate the function relu()
with tf.variable_scope('local3') as scope:
reshape = tf.reshape(pool2, shape=[batch_size, -1])
dim = reshape.get_shape()[1].value
weights = tf.Variable(tf.truncated_normal(shape=[dim,128], stddev = 0.005, dtype = tf.float32),
name = 'weights', dtype = tf.float32)
biases = tf.Variable(tf.constant(value = 0.1, dtype = tf.float32, shape = [128]),
name = 'biases', dtype=tf.float32)
local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
#Fully connected layer 4
#128 neurons, activation function relu()
with tf.variable_scope('local4') as scope:
weights = tf.Variable(tf.truncated_normal(shape=[128,128], stddev = 0.005, dtype = tf.float32),
name = 'weights',dtype = tf.float32)
biases = tf.Variable(tf.constant(value = 0.1, dtype = tf.float32, shape = [128]),
name = 'biases', dtype = tf.float32)
local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name='local4')
#dropout layer
# with tf.variable_scope('dropout') as scope:
# drop_out = tf.nn.dropout(local4, 0.8)
#Softmax layer
#Output the previous FC layer and do a linear regression to calculate the score for each category.
#Here, it is a category 2, so this layer outputs two scores.
with tf.variable_scope('softmax_linear') as scope:
weights = tf.Variable(tf.truncated_normal(shape=[128, n_classes], stddev = 0.005, dtype = tf.float32),
name = 'softmax_linear', dtype = tf.float32)
biases = tf.Variable(tf.constant(value = 0.1, dtype = tf.float32, shape = [n_classes]),
name = 'biases', dtype = tf.float32)
softmax_linear = tf.add(tf.matmul(local4, weights), biases, name='softmax_linear')
return softmax_linear
#-----------------------------------------------------------------------------
#loss
#Input parameters: logits, the network calculates the output value.
# Labels, true value, here is 0 or 1
#Output parameters:loss
def losses(logits, labels):
with tf.variable_scope('loss') as scope:
cross_entropy =tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels, name='xentropy_per_example')
loss = tf.reduce_mean(cross_entropy, name='loss')
tf.summary.scalar(scope.name+'/loss', loss)
return loss
#--------------------------------------------------------------------------
#trainning
#Input parameters:loss ,learning_rate
#Output parameters:train_op
def trainning(loss, learning_rate):
with tf.name_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate= learning_rate)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = optimizer.minimize(loss, global_step= global_step)
return train_op
#-----------------------------------------------------------------------
#acc
#Input parameters:logits labels
#Output parameters:accuracy
def evaluation(logits, labels):
with tf.variable_scope('accuracy') as scope:
correct = tf.nn.in_top_k(logits, labels, 1)
correct = tf.cast(correct, tf.float16)
accuracy = tf.reduce_mean(correct)
tf.summary.scalar(scope.name+'/accuracy', accuracy)
return accuracy
#=======================================================================4、训练模型train.py
“#======================================================================
#
import os
import numpy as np
import tensorflow as tf
import input_data
import model
#
N_CLASSES = 2 #elliptical,spiral
IMG_W = 256 # resize
IMG_H = 256
BATCH_SIZE =20
CAPACITY = 500
MAX_STEP = 400 #
learning_rate = 0.0001 #
#get batch
train_dir ='/home/qf/tensorflow/tf/tf8/data/train' #
logs_train_dir ='/home/qf/tensorflow/tf/tf8/data/logs' #logs
#logs_test_dir = '/home/qf/tensorflow/tf/tf8/data' #logs
#train, train_label = input_data.get_files(train_dir)
train, train_label, val, val_label = input_data.get_files(train_dir, 0.3)
#
train_batch,train_label_batch = input_data.get_batch(train, train_label, IMG_W, IMG_H, BATCH_SIZE, CAPACITY)
#
val_batch, val_label_batch = input_data.get_batch(val, val_label, IMG_W, IMG_H, BATCH_SIZE, CAPACITY)
#
train_logits = model.inference(train_batch, BATCH_SIZE, N_CLASSES)
train_loss = model.losses(train_logits, train_label_batch)
train_op = model.trainning(train_loss, learning_rate)
train_acc = model.evaluation(train_logits, train_label_batch)
#
test_logits = model.inference(val_batch, BATCH_SIZE, N_CLASSES)
test_loss = model.losses(test_logits, val_label_batch)
test_acc = model.evaluation(test_logits, val_label_batch)
#log summary
summary_op = tf.summary.merge_all()
#
sess = tf.Session()
#
train_writer = tf.summary.FileWriter(logs_train_dir, sess.graph)
#val_writer = tf.summary.FileWriter(logs_test_dir, sess.graph)
#
saver = tf.train.Saver()
#
sess.run(tf.global_variables_initializer())
#
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
#
try:
#
for step in np.arange(MAX_STEP):
if coord.should_stop():
break
#
_, tra_loss, tra_acc = sess.run([train_op, train_loss, train_acc])
#
if step % 10 == 0:
print('Step %d, train loss = %.2f, train accuracy = %.2f%%' %(step, tra_loss, tra_acc*100.0))
summary_str = sess.run(summary_op)
train_writer.add_summary(summary_str, step)
#
if (step + 1) == MAX_STEP:
checkpoint_path = os.path.join(logs_train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
print('checkpoint_path',checkpoint_path)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
#========================================================================5、模型测试：test.py
#!/usr/bin/python
#-*-coding:utf-8 -*-
#=============================================================================
from PIL import Image
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import model
from input_data import get_files
import os
#=======================================================================
#
def get_one_image(train):
#Input parameters:train The path to the training image
#Return parameter: image,randomly select a picture from the training picture
n = len(train)
ind = np.random.randint(0, n)
img_dir = train[ind] #Randomly select the test picture
img = Image.open(img_dir)
plt.imshow(img)
imag = img.resize([256, 256]) #Since the picture has been resized during the preprocessing phase, this
#command can be omitted
image = np.array(imag)
return image
#--------------------------------------------------------------------
#
def evaluate_one_image(path,image_array):
with tf.Graph().as_default():
BATCH_SIZE = 1
N_CLASSES = 2
image = tf.cast(image_array, tf.float32)
image = tf.image.per_image_standardization(image)
image = tf.reshape(image, [1, 256,256, 3])
logit = model.inference(image, BATCH_SIZE, N_CLASSES)
logit = tf.nn.softmax(logit)
x = tf.placeholder(tf.float32, shape=[256, 256, 3])
# you need to change the directories to yours.
logs_train_dir ='/home/qf/tensorflow/tf/tf8/data/logs/'
saver = tf.train.Saver()
with tf.Session() as sess:
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(logs_train_dir)
print('ckpt',ckpt)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
print('ckpt.model_checkpoint_path',ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
else:
print('No checkpoint file found')
prediction = sess.run(logit, feed_dict={x: image_array})
max_index = np.argmax(prediction)
print "max_index",max_index
print "prediction",prediction
pred=prediction[0]
print "pred",pred
order=pred.argsort()[1]
print "order",order
label=labels[order]
print "label",label
print('This is a %s with possibility %.6f' %(label,prediction[:, max_index]))
f=file("/home/qf/tensorflow/tf/tf8/data/pre_label.txt","a+")
f.writelines(path+' '+label+'\n')
#if max_index==0:
# print('This is a elliptical with possibility %.6f' %prediction[:, 0])
#else :
# print('This is a spiral with possibility %.6f' %prediction[:, 1])
#------------------------------------------------------------------------
#############################
if __name__ == '__main__':
'''
##随机获取测试集中的一张图像进行测试
train_dir ='/home/qf/tensorflow/tf/tf8/data/test'
train, train_label, val, val_label = get_files(train_dir, 0.3)
img = get_one_image(val) #Verify the training set or test set by changing the parameter train or val
evaluate_one_image(img)
'''
#===========================================================================
#对测试集中的所有图像进行测试
dir = '/home/qf/tensorflow/tf/tf8/data/test2'
labels = np.loadtxt("/home/qf/tensorflow/tf/tf8/data/label.txt", str, delimiter='\t')
filelist=[]
filenames = os.listdir(dir)
for fn in filenames:
fullfilename = os.path.join(dir,fn)
filelist.append(fullfilename)
for i in range(0, len(filelist)):
img_dir= filelist[i]
print "img_dir",img_dir
img = Image.open(img_dir)
#print "img",img
imag = img.resize([256, 256])
image = np.array(imag)
evaluate_one_image(img_dir,image)
6、计算准确率：compute_p.py--该程序使用python3调用
import pandas as pd
import os
pre_label=pd.read_table('/home/qf/tensorflow/tf/tf8/data/pre_label.txt',header=None,sep=' ' )
true_label=pd.read_table('/home/qf/tensorflow/tf/tf8/data/true_test_label.txt',header=None,sep=',')
[im_num,cloum]=pre_label.shape
#print "pre_label",pre_label
print(im_num)
true_num=0
for i in range(im_num):
im_dir=pre_label.iloc[i,0] #image path
(filepath,im_name) = os.path.split(im_dir) #image name
print ("im_name",im_name)
#print(list(true_label.iloc[:,0]))
im_index=list(true_label.iloc[:,0]).index(im_name) #The line number where the image name is located
#print "im_index",im_index
if pre_label.iloc[i,2]==true_label.iloc[im_index,1]:
true_num+=1
P=true_num/im_num
print(P)