pytorch advanced learning (5): a detailed introduction to the nanny level of neural network transfer learning applications, how to replace the trained model with the model you need

Code resources and data set resources use the code in Advanced Learning (4), you can eat together~

Pytorch advanced learning (four): use different classification models for data training (alexnet, resnet, vgg, etc.) - Programmer Sought

Dataset: Flower dataset, a total of five categories.

  

Table of contents

 1. Model training accuracy before pre-training

1. CreateDataset.py generates its own dataset

2. TrainModal.py training neural network

3. Running results 

4. Improve method: use transfer learning 

2. Transfer Learning

1. Concept

2. Goals

3. Specific steps

3.1 View the pre-trained network structure

3.2 Modify the number of fc layer output categories

3.3 Download the pre-training parameters of resnet18

3.4 Delete the fc layer parameters in the pre-training parameters

3.5 View the framework of the model you built

3.6 Update your own model parameters

3.7 Freeze parameters other than the fc layer

3.8 Define the loss function and optimizer, and update the gradient of fc layer parameters 

3.9 Load the model and train

3. Complete code

---

 1. Model training accuracy before pre-training

1. CreateDataset.py generates its own dataset

'''
生成训练集和测试集，保存在txt文件中
'''
##相当于模型的输入。后面做数据加载器dataload的时候从里面读他的数据
import os
import random#打乱数据用的

# 百分之60用来当训练集
train_ratio = 0.6

# 用来当测试集
test_ratio = 1-train_ratio

rootdata = r"data"  #数据的根目录

train_list, test_list = [],[]#读取里面每一类的类别
data_list = []

#生产train.txt和test.txt
class_flag = -1
for a,b,c in os.walk(rootdata):
    print(a)
    for i in range(len(c)):
        data_list.append(os.path.join(a,c[i]))

    for i in range(0,int(len(c)*train_ratio)):
        train_data = os.path.join(a, c[i])+'\t'+str(class_flag)+'\n'
        train_list.append(train_data)

    for i in range(int(len(c) * train_ratio),len(c)):
        test_data = os.path.join(a, c[i]) + '\t' + str(class_flag)+'\n'
        test_list.append(test_data)

    class_flag += 1

print(train_list)
random.shuffle(train_list)#打乱次序
random.shuffle(test_list)

with open('train.txt','w',encoding='UTF-8') as f:
    for train_img in train_list:
        f.write(str(train_img))

with open('test.txt','w',encoding='UTF-8') as f:
    for test_img in test_list:
        f.write(test_img)

2. TrainModal.py training neural network

The resnet18 neural network is used, no training parameters are used, epoch=5, and the accuracy rate will be very low at this time.

'''
    加载pytorch自带的模型，从头训练自己的数据
'''
import time
import torch
from torch import nn
from torch.utils.data import DataLoader
from utils import LoadData

# 设置显卡型号为1
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '1'


from torchvision.models import alexnet  #最简单的模型
from torchvision.models import vgg11, vgg13, vgg16, vgg19   # VGG系列
from torchvision.models import resnet18, resnet34,resnet50, resnet101, resnet152    # ResNet系列
from torchvision.models import inception_v3     # Inception 系列

# 定义训练函数，需要
def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    # 从数据加载器中读取batch（一次读取多少张，即批次数），X(图片数据)，y（图片真实标签）。
    for batch, (X, y) in enumerate(dataloader):
        # 将数据存到显卡
        X, y = X.cuda(), y.cuda()

        # 得到预测的结果pred
        pred = model(X)

        # 计算预测的误差
        # print(pred,y)
        loss = loss_fn(pred, y)

        # 反向传播，更新模型参数
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # 每训练10次，输出一次当前信息
        if batch % 10 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")


def test(dataloader, model):
    size = len(dataloader.dataset)
    # 将模型转为验证模式
    model.eval()
    # 初始化test_loss 和 correct， 用来统计每次的误差
    test_loss, correct = 0, 0
    # 测试时模型参数不用更新，所以no_gard()
    # 非训练， 推理期用到
    with torch.no_grad():
        # 加载数据加载器，得到里面的X（图片数据）和y(真实标签）
        for X, y in dataloader:
            # 将数据转到GPU
            X, y = X.cuda(), y.cuda()
            # 将图片传入到模型当中就，得到预测的值pred
            pred = model(X)
            # 计算预测值pred和真实值y的差距
            test_loss += loss_fn(pred, y).item()
            # 统计预测正确的个数
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= size
    correct /= size
    print(f"correct = {correct}, Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")




if __name__=='__main__':
    batch_size = 8

    # # 给训练集和测试集分别创建一个数据集加载器
    train_data = LoadData("train.txt", True)
    valid_data = LoadData("test.txt", False)


    train_dataloader = DataLoader(dataset=train_data, num_workers=4, pin_memory=True, batch_size=batch_size, shuffle=True)
    test_dataloader = DataLoader(dataset=valid_data, num_workers=4, pin_memory=True, batch_size=batch_size)

    # 如果显卡可用，则用显卡进行训练
    device = "cuda" if torch.cuda.is_available() else "cpu"
    print(f"Using {device} device")


    '''
        随着模型的加深，需要训练的模型参数量增加，相同的训练次数下模型训练准确率起来得更慢
    '''
   
    model = resnet18(weights=False, num_classes=5).to(device)    # 43.6%
 


    print(model)
    # 定义损失函数，计算相差多少，交叉熵，
    loss_fn = nn.CrossEntropyLoss()

    # 定义优化器，用来训练时候优化模型参数，随机梯度下降法
    optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)  # 初始学习率


    # 一共训练1次
    epochs = 5
    for t in range(epochs):
        print(f"Epoch {t+1}\n-------------------------------")
        time_start = time.time()
        train(train_dataloader, model, loss_fn, optimizer)
        time_end = time.time()
        print(f"train time: {(time_end-time_start)}")
        test(test_dataloader, model)
    print("Done!")

    # 保存训练好的模型
    torch.save(model.state_dict(), "model_resnet18.pth")
    print("Saved PyTorch Model Success!")

3. Running results 

Given the accuracy rate of 5 epochs, when the pre-trained model is not used for training, the accuracy rate after 5 epoch training is 57%.

Epoch 1
-------------------------------
train time: 21.131392002105713
correct = 0.48411322934719814, Test Error: 
 Accuracy: 48.4%, Avg loss: 0.163418 

Epoch 2
-------------------------------
train time: 17.17208242416382
correct = 0.4927787406123628, Test Error: 
 Accuracy: 49.3%, Avg loss: 0.150533 

Epoch 3
-------------------------------
train time: 17.276950359344482
correct = 0.5343731946851531, Test Error: 
 Accuracy: 53.4%, Avg loss: 0.138617 

Epoch 4
-------------------------------
train time: 16.926883220672607
correct = 0.5557481224725592, Test Error: 
 Accuracy: 55.6%, Avg loss: 0.135120 

Epoch 5
-------------------------------
train time: 17.293556213378906
correct = 0.5678798382437897, Test Error: 
Accuracy: 56.8%, Avg loss: 0.136653 

Done!
Saved PyTorch Model Success!

4. Improve method: use transfer learning 

        If you want to achieve a higher accuracy rate as soon as possible, you need to use the pre-trained model parameters that have been trained on the Internet, but the pre-trained model is trained using a dataset with 1000 categories, so the last fully connected layer The output category is 1000, and the number of categories in our data set is 5 , so we need to modify the number of categories in the fully connected layer and change the model to the framework we need. This is transfer learning.

2. Transfer Learning

1. Concept

Transfer Learning (Transfer)

Migration learning - freezing some parameters and modifying the fully connected layer

Pytorch loads part of the pre-trained model and freezes some layers_pre-trained model freezes batch normalization_csdn_1HAO's blog-CSDN Blog

        Transfer Learning is a machine learning method that takes the model developed for task A as the initial point and reuses it in the process of developing the model for task B.

        What we mainly use here is model-based transfer (Parameter based TL) : using the parameter sharing model of the source domain and the target domain. It refers to the method of finding the shared parameter information between the source domain and the target domain to realize the migration. The assumptions required by this migration method are: The data in the source domain and the data in the target domain can share some model parameters.

2. Goals

We use the training parameters of the pre-trained model, but we need to modify the category 1000 output by the original fully connected (fc) layer to 5. Our transfer learning in this example mainly involves two parts of modification :

• Modification of the neural network framework: change the output of the fc layer of the resnet18 neural network from 1000 -> 5
• Modification of pre-training parameter weights: delete 1000 weights of fc layer and replace them with newly trained 5 parameters

3. Specific steps

3.1 View the pre-trained network structure

  We named the resnet18 network before the unmodified framework pretrain_model, and printed it, we can see that the number of output categories of the fc layer is 1000.

pretrain_model = resnet18(weights=False) # 加载ResNet
print(pretrain_model)

3.2 Modify the number of fc layer output categories

•  The input access to the fc layer of the pretrain_model is accessed in num_ftrs;
• Replace the output layer of the fc layer with 5, use the linear method to linearly connect the input layer and the output layer, and assign it to the fc layer of the pretrain_model;
• Print the network structure of pretrain_model, you can see out_features and replace it with 5.

num_ftrs = pretrain_model.fc.in_features  # 获取全连接层的输入
    pretrain_model.fc = nn.Linear(num_ftrs, 5)  # 全连接层改为不同的输出,5
    print(pretrain_model)

3.3 Download the pre-training parameters of resnet18

• We enter the definition page of resnet, and we can find the download URL of the resnet pre-training model parameters.

URL is: url="https://download.pytorch.org/models/resnet18-f37072fd.pth"

• After downloading the pth file, rename it to resnet18_pretrain, and put it in the project directory.

•  Load the pre-trained parameter file, store it in the dictionary named pretrained_dict, and print to view its content.

    # 预先训练好的参数， 'https://download.pytorch.org/models/resnet18-5c106cde.pth'
    pretrained_dict = torch.load('./resnet18_pretrain.pth')
    # pretrained_dict = torch.load('./resnet34_pretrain.pth')
    print(pretrained_dict)

• You can see that the weights and bias items of the pre-trained model are printed out. I have marked the parameters of the fc layer here, and there are 1000 weights and biases each.

• But we don't need the weights and bias in the fc layer, because we only need to output 5, so we need to discard the parameters of the fc layer. Delete the elements of the pre-trained model with the same layer name as the current model but with a different layer structure. These are the parameters we need to replace .

3.4 Delete the fc layer parameters in the pre-training parameters

Use pop to pop up unnecessary parameters, print and view. 

# # 弹出fc层的参数
    pretrained_dict.pop('fc.weight')
    pretrained_dict.pop('fc.bias')
print(pretrained_dict)

It can be seen that there are no parameters of the fc layer in the parameters, which are the parameters  we will apply to the neural network pretrain_model we built ourselves later .

3.5 View the framework of the model you built

Our own pretrain_model has completed the modification of the fc output category, but since the above pre-training parameters have not been loaded, the parameters of our own neural network are in the initial state at this time, with many 0s and 1s, we can print and see.

# 自己的模型参数变量，在开始时里面参数处于初始状态，所以很多0和1
    model_dict = pretrain_model.state_dict()
    print(model_dict)

 The work that has been completed at this time is:

1. Model_dict is the initialization parameter of pretrain_model. We changed the number of categories of his last fully connected layer to 5 in the front, so the parameters of the fc layer are also 5;
2. pretrained_dict is the pre-training parameter of the resnet we downloaded, and we have deleted the weight of the last fc layer.

3.6 Update your own model parameters

Now, we need to extract the parameters of the same shape as the current model in the pretrained model and store them in the pretrained_dict dictionary. Among them, k represents the parameter name, and v represents the parameter value.

# # 去除一些不需要的参数，加载预训练参数
    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}

Model parameters are updated and loaded. 

# 模型参数列表进行参数更新，加载参数
    model_dict.update(pretrained_dict)

    # 改进过的预训练模型结构，加载刚刚的模型参数列表
    pretrain_model.load_state_dict(model_dict)
    print(pretrain_model)

 After completing this operation, except for the parameters of the last fc layer that have not been updated, the parameters of other previous layers are replaced with the parameters in the pre-trained model.

3.7 Freeze parameters other than the fc layer

We need to train the parameters of the last fc layer separately.

It is necessary to freeze the parameters in other layers that have been loaded before but are not used now.

Freezing method: If the parameter name is not fc.weights and fc.bias, set requires_grad to false. requires_grad means whether to update the parameters. If it is true, update it. Freeze all parameters except the fc layer.

# 将满足条件的参数的 requires_grad 属性设置为False
    for name, value in pretrain_model.named_parameters():
        if (name != 'fc.weight') and (name != 'fc.bias'):
            value.requires_grad = False

3.8 Define the loss function and optimizer, and update the gradient of fc layer parameters 

 The filter function selects the parameters whose attribute requires_grad = True in the model, and then uses the stochastic gradient descent algorithm SDG to update the parameters of the fc layer with gradients, and controls the optimizer to only update the layers that need to be updated .

    # filter 函数将模型中属性 requires_grad = True 的参数选出来
    params_conv = filter(lambda p: p.requires_grad, pretrain_model.parameters())  # 要更新的参数在parms_conv当中
  
# 定义损失函数，计算相差多少，交叉熵，  
loss_fn = nn.CrossEntropyLoss()
# 对参数进行更新
    optimizer = torch.optim.SGD(params_conv, lr=1e-3)  # 初始学习率
    

3.9 Load the model and train

epoch=5, the training results are as follows.

 model = pretrain_model.to(device)

 # 一共训练5次
    epochs = 5
    for t in range(epochs):
        print(f"Epoch {t + 1}\n-------------------------------")
        train(train_dataloader, model, loss_fn, optimizer)
        test(test_dataloader, model)
    print("Done!")

    # 保存训练好的模型
    torch.save(model.state_dict(), "model_resnet34.pth")
    print("Saved PyTorch Model Success!")

It can be seen that the accuracy rate when epoch=5 reaches 82.4%, which is relatively high and meets our requirements.

3. Complete code

'''
    加载pytorch自带的模型，从头训练自己的数据
'''
import time
import torch
from torch import nn
from torch.utils.data import DataLoader
from utils import LoadData

# 设置显卡型号为1
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '1'


from torchvision.models import alexnet  #最简单的模型
from torchvision.models import vgg11, vgg13, vgg16, vgg19   # VGG系列
from torchvision.models import resnet18, resnet34,resnet50, resnet101, resnet152    # ResNet系列
from torchvision.models import inception_v3     # Inception 系列

# 定义训练函数，需要
def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    # 从数据加载器中读取batch（一次读取多少张，即批次数），X(图片数据)，y（图片真实标签）。
    for batch, (X, y) in enumerate(dataloader):
        # 将数据存到显卡
        X, y = X.cuda(), y.cuda()

        # 得到预测的结果pred
        pred = model(X)

        # 计算预测的误差
        # print(pred,y)
        loss = loss_fn(pred, y)

        # 反向传播，更新模型参数
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # 每训练10次，输出一次当前信息
        if batch % 10 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")


def test(dataloader, model):
    size = len(dataloader.dataset)
    # 将模型转为验证模式
    model.eval()
    # 初始化test_loss 和 correct， 用来统计每次的误差
    test_loss, correct = 0, 0
    # 测试时模型参数不用更新，所以no_gard()
    # 非训练， 推理期用到
    with torch.no_grad():
        # 加载数据加载器，得到里面的X（图片数据）和y(真实标签）
        for X, y in dataloader:
            # 将数据转到GPU
            X, y = X.cuda(), y.cuda()
            # 将图片传入到模型当中就，得到预测的值pred
            pred = model(X)
            # 计算预测值pred和真实值y的差距
            test_loss += loss_fn(pred, y).item()
            # 统计预测正确的个数
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= size
    correct /= size
    print(f"correct = {correct}, Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")


if __name__=='__main__':
    batch_size = 8

    # # 给训练集和测试集分别创建一个数据集加载器
    train_data = LoadData("train.txt", True)
    valid_data = LoadData("test.txt", False)


    train_dataloader = DataLoader(dataset=train_data, num_workers=4, pin_memory=True, batch_size=batch_size, shuffle=True)
    test_dataloader = DataLoader(dataset=valid_data, num_workers=4, pin_memory=True, batch_size=batch_size)

    # 如果显卡可用，则用显卡进行训练
    device = "cuda" if torch.cuda.is_available() else "cpu"
    print(f"Using {device} device")


    '''        ResNet系列    '''
    pretrain_model = resnet18(weights=False)    # 43.6%
    num_ftrs = pretrain_model.fc.in_features  # 获取全连接层的输入
    pretrain_model.fc = nn.Linear(num_ftrs, 5)  # 全连接层改为不同的输出,5
    # print(pretrain_model)

    # 加载预训练模型
    pretrained_dict = torch.load('./resnet18_pretrain.pth')
    # 对模型进行输出
    # print(pretrained_dict)

    # # 删除预训练模型跟当前模型层名称相同，层结构却不同的元素;，弹出fc层的参数
    pretrained_dict.pop('fc.weight')
    pretrained_dict.pop('fc.bias')
    # print(pretrained_dict)

    # 自己的模型参数变量，在开始时里面参数处于初始状态，所以很多0和1
    model_dict = pretrain_model.state_dict()
    # print(model_dict)

    # 将预训练模型中与当前模型形状相同的参数提取出来，存储在pretrained_dict字典中。其中，k表示参数名，v表示参数值
    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}

    # 模型参数列表进行参数更新，加载参数
    model_dict.update(pretrained_dict)

    # 改进过的预训练模型结构，加载刚刚的模型参数列表
    pretrain_model.load_state_dict(model_dict)
    # print(pretrain_model)

    '''
           冻结部分层
       '''
    # 将满足条件的参数的 requires_grad 属性设置为False
    for name, value in pretrain_model.named_parameters():
        if (name != 'fc.weight') and (name != 'fc.bias'):
            value.requires_grad = False


    # filter 函数将模型中属性 requires_grad = True 的参数选出来
    params_conv = filter(lambda p: p.requires_grad, pretrain_model.parameters())  # 要更新的参数在parms_conv当中

    # 定义损失函数，计算相差多少，交叉熵，
    loss_fn = nn.CrossEntropyLoss()
    # 对参数进行更新
    optimizer = torch.optim.SGD(params_conv, lr=1e-3)  # 初始学习率

    model = pretrain_model.to(device)

 # 一共训练5次
    epochs = 5
    for t in range(epochs):
        print(f"Epoch {t + 1}\n-------------------------------")
        train(train_dataloader, model, loss_fn, optimizer)
        test(test_dataloader, model)
    print("Done!")

    # 保存训练好的模型
    torch.save(model.state_dict(), "model_resnet34.pth")
    print("Saved PyTorch Model Success!")