keras上Vgg16的finetune实现代码

本文针对初学者，结合自己动手经验对keras文档中 面向小数据集构建图像分类模型 这一节中设计的代码及实现效果做一个总结，希望对急于实现分类效果的新人有一定帮助！

全文分为三个部分，分别对应keras文档中的三部分完整代码的可运行版（本人机器配置：1050Ti显卡+tensoflow(1.12.0)(有无gpu都可以)+Keras (2.2.4)+Keras-Applications (1.0.6)+Keras-Preprocessing (1.0.5)）

我主要是在作者源码基础上进行了部分修改，使其能正常运行在我的设备上，并且以自己仅有的能力写了predict部分的代码。

全文中的vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5 文件可以在我的资源中下载

第一部分 自己搭建的一个三层卷积网络 对应keras原文中代码：keras原文引用代码(好像要翻墙才能打开)

我调整后的可运行版如下：

# 训练部分代码
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K

from matplotlib import pyplot as plt
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input
import numpy as np

# dimensions of our images.
img_width, img_height = 150, 150

train_data_dir = './train/'
validation_data_dir = './valid/'
nb_train_samples = 3000 #我这里是3类，每类有1000张样本
nb_validation_samples = 600 #每类有200张valid
epochs = 50
batch_size = 16

if K.image_data_format() == 'channels_first':
    input_shape = (3, img_width, img_height)
else:
    input_shape = (img_width, img_height, 3)

model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))

model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

# 如果是2分类问题则使用下面这几句替换上面的
# model.add(Dense(1)) 
# model.add(Activation('sigmoid'))
# model.compile(loss='binary_crossentropy',
#              optimizer='rmsprop',
#              metrics=['accuracy'])


# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)

# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical') # 如果是2分类问题则class_mode='binary'


validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical') # 如果是2分类问题则class_mode='binary'

model.fit_generator(
    train_generator,
    steps_per_epoch=nb_train_samples / batch_size,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=nb_validation_samples / batch_size)

# model.save_weights('first_try.h5') 
# 我保存的是整个网络模型加权重
model.save('first_try_model_car_dog_peo.h5')

# 测试部分代码
from keras.models import load_model
from keras.preprocessing.image import img_to_array, load_img
import numpy as np

img_width=150
img_height=150
test_model = load_model('first_try_model_car_dog_peo.h5')
img = load_img('./test/p1.jpg',False,target_size=(img_width,img_height))
x = img_to_array(img)
x = np.expand_dims(x, axis=0)
preds = test_model.predict_classes(x)
probs = test_model.predict_proba(x)
print(preds, probs) 

在有GPU的情况下，训练一个epoch,大概需要20~30秒

测试输出结果：

[2] [[7.725073e-12 0.000000e+00 1.000000e+00]]

速度大概1秒，但是验证了一下发现，该三层的网络进行2分类效果还不错，但是进行多分类时效果欠佳。

第二部分 使用预训练网络的bottleneck特征 对应keras文档中所引用的代码为：使用VGG16的bottleneck(好像要翻墙才能打开)

我调整后可运行的代码（分为2分类和多分类两个版本） 如下：

# 2分类训练部分代码
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras import applications

# dimensions of our images.
img_width, img_height = 150, 150

top_model_weights_path = 'bottleneck_fc_car_dog_weights.h5'
train_data_dir = './train'
validation_data_dir = './valid'
nb_train_samples = 2000
nb_validation_samples = 400
epochs = 50
batch_size = 16


def save_bottlebeck_features():
    datagen = ImageDataGenerator(rescale=1. / 255)

    # build the VGG16 network
    model = applications.VGG16(include_top=False, weights='vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')

    generator = datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_width, img_height),
        batch_size=batch_size,
        class_mode=None,
        shuffle=False)

    bottleneck_features_train = model.predict_generator(
        generator, nb_train_samples // batch_size)
    np.save('bottleneck_features_train.npy',bottleneck_features_train)

    generator = datagen.flow_from_directory(
        validation_data_dir,
        target_size=(img_width, img_height),
        batch_size=batch_size,
        class_mode=None,
        shuffle=False)

    bottleneck_features_validation = model.predict_generator(
        generator, nb_validation_samples // batch_size)
    np.save('bottleneck_features_validation.npy',bottleneck_features_validation)


def train_top_model():
    train_data = np.load('bottleneck_features_train.npy')
    train_labels = np.array(
        [0] * (nb_train_samples // 2) + [1] * (nb_train_samples // 2))

    validation_data = np.load('bottleneck_features_validation.npy')
    validation_labels = np.array(
        [0] * (nb_validation_samples // 2) + [1] * (nb_validation_samples // 2))

    model = Sequential()
    model.add(Flatten(input_shape=train_data.shape[1:]))
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(1, activation='sigmoid'))

    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy', metrics=['accuracy'])

    # print('train_data:',train_data.shape)
    # print('train_labels:', train_labels.shape) #查看train_labels数量
    # train_data，train_labels样本和标签数量必须相等。
    # 而且样本数量又必须是batch_size的整数倍

    model.fit(train_data, train_labels,
              epochs=epochs,
              batch_size=batch_size,
              validation_data=(validation_data, validation_labels))
    model.save_weights(top_model_weights_path)
    # model.save(top_model_weights_path)

save_bottlebeck_features()
train_top_model()

# 多分类部分的代码
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras import applications

# dimensions of our images.
img_width, img_height = 150, 150

top_model_weights_path = 'bottleneck_fc_car_dog_peo_model.h5'
train_data_dir = './train'
validation_data_dir = './valid'
nb_train_samples = 2992 #其中car有1000个，dog有1000个，people有992个
nb_validation_samples = 592 #其中car有200个，dog有200个，people有192个 
                            #样本数量必须是batch_size的整数倍

epochs = 50
batch_size = 16


def save_bottlebeck_features():
    datagen = ImageDataGenerator(rescale=1. / 255)

    # build the VGG16 network
    model = applications.VGG16(include_top=False, weights='vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')

    generator = datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_width, img_height),
        batch_size=batch_size,
        class_mode='categorical',
        shuffle=False)
    bottleneck_features_train = model.predict_generator(
        generator, nb_train_samples // batch_size)
    np.save('bottleneck_features_train.npy',bottleneck_features_train)

   
    generator = datagen.flow_from_directory(
        validation_data_dir,
        target_size=(img_width, img_height),
        batch_size=batch_size,
        class_mode='categorical',
        shuffle=False)
    bottleneck_features_validation = model.predict_generator(
        generator, nb_validation_samples // batch_size)
    np.save('bottleneck_features_validation.npy',bottleneck_features_validation)


def train_top_model():
    train_data = np.load('bottleneck_features_train.npy')
    
    train_labels = np.array(
        [0] * 1000 + [1] * 1000 + [2] * 992)

    validation_data = np.load('bottleneck_features_validation.npy')

    validation_labels = np.array(
        [0] * 200 + [1] * 200 + [2] * 192)

    train_labels = applications.utils.to_categorical(train_labels, 3)#多分类所添加
    validation_labels = applications.utils.to_categorical(validation_labels, 3)
    #多分类所添加上面两句

    model = Sequential()
    model.add(Flatten(input_shape=train_data.shape[1:]))
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(0.5))
    # model.add(Dense(1, activation='sigmoid'))
    model.add(Dense(3, activation='softmax'))

    # model.compile(optimizer='rmsprop',
    #               loss='binary_crossentropy', metrics=['accuracy'])
    model.compile(optimizer='rmsprop',
                    loss='categorical_crossentropy', metrics=['accuracy'])
    
    # print('train_data:',train_data.shape)
    # print('train_labels:', train_labels.shape)
    # train_data,train_labels样本和标签数量必须相等。
    # 而样本数量又必须是batch_size的整数倍

    model.fit(train_data, train_labels,
              epochs=epochs,
              batch_size=batch_size,
              validation_data=(validation_data, validation_labels))
    # model.save_weights(top_model_weights_path)
    model.save(top_model_weights_path)


save_bottlebeck_features()
train_top_model()

在有GPU上训练速度极快，一个epcoh大概只用1秒

2分类和多分类的测试部分代码相同：

#该部分代码可用但未优化，在有GPU上耗时大约1秒

import numpy as np
import scipy.misc
from keras.applications.vgg16 import VGG16
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input
from keras import backend as K
from keras.models import load_model

K.set_image_dim_ordering('th')


def vgg16(img_path):
    print('Loading VGG16')
    model = VGG16(weights='vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', include_top=False)
    img = image.load_img(img_path, target_size=(150, 150))
    x = image.img_to_array(img)
    print(x.shape)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)

    features = model.predict(x)
    print(features)
    return features


# def load_model(model_def_fname, model_weight_fname):
#     # model = model_from_json(open(model_def_fname).read())
#     # model.load_weights(model_weight_fname)
#     model=load_model('bottleneck_fc_model.h5')
#     return model


def head_model(bottom_features):
    print('loading the top model')
    model = load_model('bottleneck_fc_car_dog_peo_model.h5')
    bottom_features=bottom_features.transpose(0,3,2,1)
    print('shape:',bottom_features.shape)
    prediction = model.predict_classes(bottom_features, batch_size=32, verbose=1)
    probability = model.predict(bottom_features, batch_size=32, verbose=1)
    return prediction, probability


if __name__ == '__main__':
    bottom_features = vgg16('./test/p6.jpg')

    prediction, probability = head_model(bottom_features)

    print(prediction, probability)

    print('the program is done')

其中多分类测试结果如下：

....

1/1 [==============================] - 0s 262ms/step

1/1 [==============================] - 0s 1ms/step
[2] [[0.0000000e+00 1.9230116e-07 9.9999976e-01]]
the program is done

其中[2]为第三类，概率为9.9999976e-01

速度大约1秒

第三部分 在预训练的网络上fine-tune 对应keras文档中引用代码为：在预训练的网络上fine-tune(好像要翻墙才能打开)

代码如下：

# 2分类训练部分的代码

from keras import applications
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras.models import Model

# path to the model weights files.

top_model_weights_path = 'bottleneck_fc_car_dog_weights.h5'
# bottleneck_fc_car_dog_weights.h5 可以用第二部分中的2分类网络产生，不过记住是保存weights权
# 重，而不是网络模型

# dimensions of our images.
img_width, img_height = 150, 150

train_data_dir = './train'
validation_data_dir = './vaild'
nb_train_samples = 2000
nb_validation_samples = 400
epochs = 50
batch_size = 16

# build the VGG16 network

base_model = applications.VGG16(weights='vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', include_top=False,input_shape=(150, 150, 3))
print('Model loaded.')

# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(1, activation='sigmoid'))

# note that it is necessary to start with a fully-trained
# classifier, including the top classifier,
# in order to successfully do fine-tuning
top_model.load_weights(top_model_weights_path)

# add the model on top of the convolutional base
# model.add(top_model)
model = Model(input=base_model.input, output=top_model(base_model.output))

# set the first 25 layers (up to the last conv block)
# to non-trainable (weights will not be updated)
for layer in model.layers[:25]:
    layer.trainable = False

# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='binary_crossentropy',
              optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
              metrics=['accuracy'])

# prepare data augmentation configuration
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='binary')

# fine-tune the model
model.fit_generator(
    train_generator,
    samples_per_epoch=nb_train_samples,
    epochs=epochs,
    validation_data=validation_generator,
    nb_val_samples=nb_validation_samples)

model.save('finetuneVgg16_car_dog_model.h5')

对vgg16 finetune的多分类训练部分代码：

# 多分类代码
from keras import applications
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras.models import Model

# path to the model weights files.
top_model_weights_path = 'bottleneck_fc_car_dog_peo_weights.h5'
# 由第二部分的网络用相同样本训练产生的权重

# dimensions of our images.
img_width, img_height = 150, 150

train_data_dir = './train'
validation_data_dir = './valid'
nb_train_samples = 3000
nb_validation_samples = 600
epochs = 50
batch_size = 16

# build the VGG16 network

base_model = applications.VGG16(weights='vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', include_top=False,input_shape=(150, 150, 3))
print('Model loaded.')

# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(3, activation='softmax'))

# note that it is necessary to start with a fully-trained
# classifier, including the top classifier,
# in order to successfully do fine-tuning
top_model.load_weights(top_model_weights_path)

# add the model on top of the convolutional base
# model.add(top_model)
model = Model(input=base_model.input, output=top_model(base_model.output))

# set the first 25 layers (up to the last conv block)
# to non-trainable (weights will not be updated)
for layer in model.layers[:25]:
    layer.trainable = False

# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='categorical_crossentropy',
              optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
              metrics=['accuracy'])

# prepare data augmentation configuration
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_height, img_width),
    batch_size=batch_size,
    class_mode='categorical')

# fine-tune the model
model.fit_generator(
    train_generator,
    samples_per_epoch=nb_train_samples,
    epochs=epochs,
    validation_data=validation_generator,
    nb_val_samples=nb_validation_samples)

model.save('finetuneVgg16_car_dog_peo_model.h5')

# 测试部分代码
import numpy as np
from keras import backend as K
from keras.models import load_model
K.set_image_dim_ordering('th')
from keras.preprocessing.image import img_to_array, load_img

def pred(img_path):
    img_width = 150
    img_height = 150
    test_model = load_model(img_path)
    img = load_img('./test/d0.jpg', False, target_size=(img_width, img_height))
    x = img_to_array(img)
    x = np.expand_dims(x, axis=0)
    x=x.transpose(0,3,2,1)
    print("x.shape:",x.shape)
    preds = test_model.predict(x)
    # probs = test_model.predict_proba(x)
    print(preds)


if __name__ == '__main__':
   
    pred('finetuneVgg16_car_dog_model.h5')

再给出一份利用 第二部分网络 测试整个文件夹中图片的代码：

# 利用第二部分网络 测试整个文件夹中图片分类
import numpy as np
from keras.applications.vgg16 import VGG16
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input
from keras import backend as K
from keras.models import load_model
import os.path
import time

K.set_image_dim_ordering('th')

if __name__ == '__main__':

    print('Loading VGG16')
    base_model = VGG16(weights='./weights/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', include_top=False)
    print('loading the top model')
    model = load_model('./weights/bottleneck_fc_car_dog_peo_model.h5')

    fileParh = 'D:\\resNet_keras\\dataset\\test2'
    pathDir = os.listdir(fileParh)

    # 进入循环
    for allDir in pathDir:
        t_start = time.time()
        imgpath = fileParh + '\\' + allDir
        print('allDir:', imgpath)
        img = image.load_img(imgpath, target_size=(150, 150))

        x = image.img_to_array(img)
        # print(x.shape)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)
        features = base_model.predict(x)

        bottom_features = features.transpose(0, 3, 2, 1)
        # print('shape:',bottom_features.shape)
        prediction = model.predict_classes(bottom_features, batch_size=32, verbose=1)
        probability = model.predict(bottom_features, batch_size=32, verbose=1)

        t_end = time.time()
        startTime=lambda:int(round(t_start * 1000))
        endTime=lambda:int(round(t_end * 1000))
        print(prediction, probability)
        print(endTime()-startTime())

        #结束循环