Get from read_split_data: training dataset, validation dataset, training label, validation label. All specific detailed paths of
Dataset location: https://download.csdn.net/download/guoguozgw/87437634
import os
#一种轻量级的数据交换格式，
import json
#文件读/写操作
import pickle
import random
import matplotlib.pyplot as plt
def read_split_data(root:str,val_rate:float = 0.2):
random.seed(0)#保证随机结果可重复出现
assert os.path.exists(root),'dataset root:{} does not exist.'.format(root)
#遍历文件夹，一个文件夹对应一个类别
flower_class = [cla for cla in os.listdir(root) if os.path.isdir(os.path.join(root,cla))]
#排序，保证顺序一致
flower_class.sort()
#生成类别名称以及对应的数字索引,将数据转换为字典的类型。将标签分好类之后，其类别是key，对应的唯一值是value
class_indices = dict((k,v) for v,k in enumerate(flower_class))
#将数据编写成json文件
json_str = json.dumps(class_indices,indent=4)
with open('json_str','w') as json_file:
json_file.write(json_str)
train_images_path = [] #存储训练集的所有图片路径
train_images_label = [] #存储训练集所有图片的标签
val_images_path = [] #存储验证机所有图片的路径
val_images_label = [] #存储验证机所有图片的标签
every_class_num = [] #存储每个类别的样本总数
supported = [".jpg", ".JPG", ".png", ".PNG"] # 支持的文件后缀类型
#遍历每一个文件夹下的文件
for cla in flower_class:
cla_path = os.path.join(root,cla)
#遍历获取supported支持的所有文件路径,得到所有图片的路径地址。针对的是某一个类别。
images = [os.path.join(root,cla,i) for i in os.listdir(cla_path) if os.path.splitext(i)[-1] in supported]
#获取该类别对应的索引,此时对应就是数字了。对应的只是一个数字
image_class = class_indices[cla]
#记录该类别的样本数量
every_class_num.append(len(images))
#按比例随机采样验证样本，按照0.2的比例来作为测试集。
val_path = random.sample(images,k=int(len(images)*val_rate))
for img_path in images:
#如果该路径在采样的验证集样本中则存入验证集。否则的话存入到训练集当中。其中label和image是相互对应的。
if img_path in val_path:
val_images_path.append(img_path)
val_images_label.append(image_class)
else:
train_images_path.append(img_path)
train_images_label.append(image_class)
print('该数据集一共有{}多张图片。'.format(sum(every_class_num)))
print('一共有{}张图片是训练集'.format(len(train_images_path)))
print('一共有{}张图片是验证集'.format(len(val_images_path)))
#输出每一个类别对应的图片个数
for i in every_class_num:
print(i)
plot_image = False
if plot_image:
#绘制每一种类别个数柱状图
plt.bar(range(len(flower_class)),every_class_num,align='center')
#将横坐标0，1，2，3，4替换成相应类别的名称
plt.xticks(range(len(flower_class)),flower_class)
#在柱状图上添加数值标签
for i,v in enumerate(every_class_num):
plt.text(x=i,y=v+5,s=str(v),ha='center')
#设置x坐标
plt.xlabel('image class')
plt.ylabel('number of images')
#
plt.title('flower class distribution')
plt.show()
return train_images_path,train_images_label,val_images_path,val_images_label
if __name__ == '__main__':
root = '../11Flowers_Predict/flower_photos'
read_split_data(root)
The final data information is the same, and the path in the code needs to be replaced (replaced with your own path).
From write Dataset class
from PIL import Image
import torch
from torch.utils.data import Dataset
class MyDataSet(Dataset):
'''
自定义数据集
'''
def __init__(self,images_path:list,images_classes:list,transform = None):
super(MyDataSet, self).__init__()
self.images_path = images_path
self.images_classes = images_classes
self.transform = transform
def __len__(self):
return len(self.images_path)
def __getitem__(self, item):
img = Image.open(self.images_path[item])
#RGB为彩色图片，L为灰度图片
if img.mode != 'RGB':
#直接在这里终止程序的运行
raise ValueError('image ：{} is not RGB mode.'.format(self.images_path[item]))
label = self.images_classes[item]
if self.transform is not None:
img = self.transform(img)
return img , label
The preprocessing part of the dataset
import os
import torch
from torchvision import transforms
from utils import read_split_data
from my_dataset import MyDataSet
#数据集所在的位置
root = '../11Flowers_Predict/flower_photos'
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('using {} device.'.format(device))
#接下来这一行是对数据的读取
train_images_path,train_images_label,val_images_path,val_images_label = read_split_data(root)
#设置transform,compose立main必须是列表
data_transform = {
"train": transforms.Compose([transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
"val": transforms.Compose([transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])}
train_data_set = MyDataSet(images_path=train_images_path,
images_classes=train_images_label,
transform=data_transform['train'])
val_data_set = MyDataSet(images_path=val_images_path,
images_classes=val_images_label,
transform=data_transform['val'])
batch_size = 32
#number of workers
#nw = min([os.cpu_count() , batch_size if batch_size>1 else 0,8])
#print('Using {} dataloader workers'.format(nw))
train_loader = torch.utils.data.DataLoader(train_data_set,
batch_size=batch_size,
shuffle=True,
num_workers = 0
)
val_loader = torch.utils.data.DataLoader(val_data_set,
batch_size=batch_size,
shuffle=True,
num_workers = 0)
for step,data in enumerate(train_loader):
images,labels = data
#print(images.shape)
#print(labels)
#print(labels.shape)
return train_loader,val_loader
if __name__ == '__main__':
main()
Start training on the dataset
import torch
from torch import nn
import torchvision
from torchvision import transforms,models
from tqdm import tqdm
from main import *
import time
HP = {
'epochs':25,
'batch_size':32,
'learning_rate':1e-3,
'momentum':0.9,
'test_size':0.05,
'seed':1
}
#创建一个残差网络34层结果，使用预训练参数
model = models.resnet34(pretrained=True)
model.fc = torch.nn.Sequential(
torch.nn.Dropout(0.1),
torch.nn.Linear(model.fc.in_features,5)
)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if device == 'cuda':
torch.backends.cudnn.benchmark = True
print(f'using {
device} device')
#将模型添加到gpu当中
model = model.to(device)
#分类问题使用交叉熵函数损失
criterion = torch.nn.CrossEntropyLoss()
#优化器使用SGD随机梯度下降法
optimizer = torch.optim.SGD(model.parameters(),lr=HP['learning_rate'],momentum=HP['momentum'])
train_loader,val_loader = main()
def train(model,criterion,optimizer,train_loader,val_loader):
#设置总的训练损失和验证损失，以及训练准确度和验证准确度。
total_train_loss = 0
total_val_loss = 0
total_train_accracy = 0
total_val_accracy = 0
model.train()#设置为训练模式
loop = tqdm(enumerate(train_loader),total=len(train_loader))
loop.set_description(f'training')
for step,data in loop:
images,labels = data
#将数据添加到GPU当中
images = images.to(device)
labels = labels.to(device)
output = model(images)
#单个损失
loss = criterion(output,labels)
#计算准确率
accracy = (output.argmax(1)==labels).sum()
#将所有的损失进行相加
total_train_loss += loss.item()
#将所有正确的全部相加起来
total_train_accracy += accracy
#开始进行层数更新
optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
loop_val = tqdm(enumerate(val_loader),total=len(val_loader))
loop_val.set_description(f'valuing')
for step,data in loop_val:
images,labels = data
images = images.to(device)
labels = labels.to(device)
output = model(images)
loss = criterion(output,labels)
accracy_val = (output.argmax(1)==labels).sum()
total_val_loss += loss.item()
total_val_accracy += accracy_val
train_acc = total_train_accracy/(2939)
val_acc = total_val_accracy/(731)
train_loss = total_train_loss/(2939)
val_loss = total_val_loss/(731)
print(f'训练集损失率: {
train_loss:.4f} 训练集准确率: {
train_acc:.4f}')
print(f'验证集损失率: {
val_loss:.4f} 验证集准确率: {
val_acc:.4f}')
if __name__ == '__main__':
time_start = time.time()
for i in range(HP['epochs']):
print(f"Epoch {
i+1}/{
HP['epochs']}")
train(model, criterion, optimizer, train_loader, val_loader)
time_end = time.time()
print(time_end-time_start)
json_str
{
"daisy": 0,
"dandelion": 1,
"roses": 2,
"sunflowers": 3,
"tulips": 4
}
After the training is over, the results of the training can be obtained.
Summary section:
One: For all directories, and the directory contains data sets that have been classified into categories, and the data is not divided into training sets and test sets 1: Function parameters are set
to: path, the probability of division
2: Set a certain random result
3: Judgment Whether the path exists, use assert
4: judge all the folders under the current path according to the passed root, if it is a folder, write it into the list
5: at the same time, this list is also all categories, the list will be updated Sorting
6: Use enumerate to make it a dictionary, where the key corresponds to the category, and the value corresponds to the value
7: (Optional) Use json to write it into the file
8: Create the training set image path, training set label Path, verification set image path, verification set label path, and the number of each category are all in the form of a list.
9: Start traversing the file, and then store it in the above collection.
10: Use join according to the category and root connect them. Cycle according to the category, and then splicing
11: When connecting this category cycle, use random numbers to divide it into the verification data set and the training data set
Two: If the data has been divided into training set and test set, if there are csv files, pandas can be used for data processing
(the shuffle function is a class in sklearn utils),
(for reading csv files, mainly The pandas library is used)
1: For the read csv file, you can first use the head to view the first few data
2: Use the shuffle method in sklearn to scramble the order
3: Use the factorize in pandas to digitize the labels Display (decompose complex calculations into basic operations), the return value
is Yuanzu 4: use unique to return a list, encapsulate the labels into a list
5: then encapsulate them into a dictionary corresponding to each other: key is the category, and value is the number
6: Use the train_test_split method in sklearn to divide the data set, the incoming parameter is (DataFrame, ratio)
7: Use value_count to count the labels
Rewriting of DataSet:
1: Mainly implement the three methods, init, getitem, len
2: Init mainly accepts parameters, paths, categories, and transforms. Here, the image must be processed to each corresponding one. The body of the picture
3: the return is a picture in image format, and a label number
part of test code
#
import os
def main(root:int,images_class: list,transform = None):
print('root:',root)
print('int:', int)
print('images_class:', images_class)
print('list:', list)
def read_split_data(root:str,val_rate:float = 0.2):
print('root:', root)
print('str:', str)
print('val_rate:', val_rate)
print('float:', float)
root = '../11Flowers_Predict/flower_photos'
#遍历文件夹
'''
os.listdir是展示当前所在层的所有文件
os.isdir判断当前这个文件是否属于文件夹
os.path.join()将两个字符串进行连接中间用/
os.path.splittext()返回的是一个元祖
'''
flowers_classes = [cla for cla in os.listdir(root) if os.path.isdir(os.path.join(root,cla))]
print(flowers_classes)
flowers_classes_copy = flowers_classes.copy()
flowers_classes.sort()
print(os.path.isdir('../11Flowers_Predict/flower_photos'))
print(os.path.join(root,'roses'))
print(flowers_classes)
class_ind = dict((k, v) for v, k in enumerate(flowers_classes))
for v,k in enumerate(flowers_classes):
print('此时标号{},对应的类别是{}.'.format(v,k))
for v,k in class_ind.items():
print(v,k)
import json
json_str = json.dumps(class_ind,indent=2)
print(json_str)
with open('json_str','w') as json_file:
json_file.write(json_str)
AA = os.path.splitext('123.jpg')
print(type(os.path.splitext('123.jpg')))
supported = [".jpg", ".JPG", ".png", ".PNG"] # 支持的文件后缀类型
print(AA[-1] in supported)
list = [1,2,3,4]
#main(root,list)
for cla in flowers_classes:
image_class = class_ind[cla]
print(image_class)
import matplotlib.pyplot as plt
every_class_num = [633,898,641,699,799]
plt.bar(flowers_classes,every_class_num,align='center')
# 这个东西就是用来替换的
#plt.xticks(range(len(flowers_classes)),[10,11,12,13,14])
for i,v in enumerate(every_class_num):
plt.text(x=i,y=v,s=str(v))
plt.show()