caffe自定义模型训练的方法

1.准备数据集

共分为5类:

animal

下载并进行解压

2.数据集进行分类


# coding:utf-8

import numpy as np
import os
import shutil

'''
    数据集的分类
    测试集: 300张
    训练集: 500张
'''

root_caffe = 'D:/Development/caffe/caffe-master/caffe-master/'

type = ['train','test']
data_type = ['animal', 'flower', 'guitar', 'house', 'plane']

#创建文件夹
model_dir = 'models/my_models_recognition/data/'

#数据集存放的路径
database_path = 'H:/database/'

#创建一个模型的总文件夹
path = root_caffe + model_dir
if not os.path.exists(path):
    os.makedirs(path)

for i in range(2):
    if not os.path.exists(path + type[i]):
        os.makedirs(path + type[i])

    for j in range(5):
        if not os.path.exists(path + type[i] + '/' + data_type[j]):
            os.makedirs(path + type[i] + '/' + data_type[j])

        # 读取文件
        for root,dir,file in os.walk(database_path + data_type[j]):
            print(len(file))
            paths = []
            if(i == 0): # 训练集
                list = file[0:500]

                for k in range(500):
                    print(database_path + data_type[j] + '/'+ list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/'+ list[k]  , #图片的原始路径
                                    path + type[i] + '/' + data_type[j] + '/' + list[k]  ) #图片的caffe模型路径

            elif(i==1): # 测试集
                list = file[0:300]

                for k in range(300):
                    print(database_path + data_type[j] + '/' + list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/' + list[k],
                                    path + type[i] + '/' + data_type[j] + '/' + list[k])

3.标签文件的制作


# coding: utf-8

import os  


# In[2]:

#定义Caffe根目录
caffe_root = 'D:/Development/caffe/caffe-master/caffe-master/'

#制作训练标签数据
i = 0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/train.txt','wb') as train_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'+str(dir)): #遍历每一个文件夹中的文件 
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    train_txt.writelines(label)                   #写入标签文件中
                i+=1 #编号加1


#制作测试标签数据
i=0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/test.txt','wb') as test_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'+str(dir)): #遍历每一个文件夹中的文件
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    test_txt.writelines(label)                   #写入标签文件中
                i+=1#编号加1
                
print "成功生成文件列表"

4.数据转化，把图片转化为LMDB格式

convert_imageset_train.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%

SET GLOG_logtostderr=1
D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\train\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\train.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\train\

pause

convert_imageset_test.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%


D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\test\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\test.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\test\

pause

5.编写网络

用的是caffenet的网络，对此网络进行修改

train_val.prototxt:

name: "CaffeNet"
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
#  transform_param { #求均值方法1
#    mirror: true
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

#mean pixel / channel-wise mean instead of mean image
  transform_param { #求均值方法2
    crop_size: 227  #随机截取227*227
    mean_value: 104 #大量图片求的均值
    mean_value: 117
    mean_value: 123
    mirror: true
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/train"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
#  transform_param {
#    mirror: false
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

# mean pixel / channel-wise mean instead of mean image
  transform_param {
    crop_size: 227
    mean_value: 104
    mean_value: 117
    mean_value: 123
    mirror: false
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/test"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 96
    kernel_size: 11
    stride: 4
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm1"
  type: "LRN"
  bottom: "pool1"
  top: "norm1"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "norm1"
  top: "conv2"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 2
    kernel_size: 5
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm2"
  type: "LRN"
  bottom: "pool2"
  top: "norm2"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv3"
  type: "Convolution"
  bottom: "norm2"
  top: "conv3"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}
layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}
layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "pool5"
  top: "fc6"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc7"
  top: "fc7"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 5
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "accuracy"
  type: "Accuracy"
  bottom: "fc8"
  bottom: "label"
  top: "accuracy"
  include {
    phase: TEST
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "fc8"
  bottom: "label"
  top: "loss"
}

6.修改超参数文件

solver.prototxt:

net: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/train_val.prototxt"
test_iter: 30 #1000 /batch
test_interval: 200  #间隔数

base_lr: 0.01
lr_policy: "step"  #每隔stepsize步，对学习率进行改变
gamma: 0.1
stepsize: 1000

display: 100    #每100次显示一次
max_iter: 5000
momentum: 0.9
weight_decay: 0.0005
snapshot: 5000
snapshot_prefix: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/model"
solver_mode: CPU

7.编写train的bat文件

 %train训练数据%
 %超参数文件%

D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\caffe.exe train ^
-solver=D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/solver.prototxt

pause

caffe自定义模型训练的方法

1.准备数据集

2.数据集进行分类

3.标签文件的制作

4.数据转化，把图片转化为LMDB格式

5.编写网络

6.修改超参数文件

猜你喜欢