版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_36022260/article/details/83302023
数据加载和处理教程
了解如何从非平凡的数据集加载和预处理/扩充数据。
导入包：
from __future__ import print_function ,division
import os
import torch
import pandas as pd
from skimage import io, transform
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms ,utils
import warnings
快速读取CSV文件，并在(N,2)数组中获取注释
CSV: 逗号分隔值（Comma-Separated Values，CSV，有时也称为字符分隔值，因为分隔字符也可以不是逗号），其文件以纯文本形式存储表格数据（数字和文本）
warnings.filterwarnings("ignore")
plt.ion()
landmarks_frame = pd.read_csv('faces/face_landmarks.csv')
n = 65
img_name = landmarks_frame.iloc[n,0]
# 访问n+1行第二数起所有数
landmarks = landmarks_frame.iloc[n ,1:].as_matrix()
landmarks = landmarks.astype('float').reshape(-1,2)
print('Image name:{}'.format(img_name) )
print('Landmarks shape{}'.format(landmarks.shape))
print('First 4 Landmarks:{}'.format(landmarks[:4]))OUT:
Image name:person-7.jpg
Landmarks shape(68, 2)
First 4 Landmarks:[[32. 65.]
[33. 76.]
[34. 86.]
[34. 97.]]
一个简单的辅助函数来显示图像及其标记，并用它来显示样本。
def show_landmarks(image,landmarks):
# 展示图像以及标记
plt.imshow(image)
plt.scatter(landmarks[:,0],landmarks[:,1],c='r',markers='.')
plt.pause(0.001)
plt.figure()
plt.show_landmarks(io.imread(os.path.join('faces/')),landmarks)
plt.show()
OUT：
数据集类
torch.utils.data.Dataset是表示数据集的抽象类。您的自定义数据集应继承Dataset并覆盖以下方法：
__len__这样就可以len(dataset)返回数据集的大小。__getitem__支持索引，以便dataset[i]可以用来获取样本
让我们为面部地标数据集创建一个数据集类。我们将阅读csv，__init__但保留图像的读取__getitem__。这是内存效率高的，因为所有图像不会立即存储在内存中，而是根据需要读取。
class FaceLandmarksDatasets(Dataset):
"""Face landmarks datasets"""
def __init__(self,csv_file,root_dir,transform=None):
self.landmarks_frame = pd.read_csv(csv_file)
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.landmarks_frame)
def __getitem__(self,idx):
image_name = os.path.join(self.root_dir,self.landmarks_frame.iloc[idx,0])
image = io.imread(image_name)
landmarks = self.landmarks_frame.iloc[idx,1:].as_matrix()
landmarks = landmarks.astype('float').reshape(-1,2)
sample = {'image':image,'landmarks':landmarks}
if self.transform:
sample = self.transform(sample)
return sample
实例化：
show_landmarks为辅助函数同上。
face_dataset = FaceLandmarksDatasets(csv_file='faces/face_landmarks.csv',root_dir = 'faces/')
fig = plt.figure()
for i in range(len(face_dataset)):
sample = face_dataset[i]
print(i,sample['image'].shape,sample['landmarks'].shape)
ax = plt.subplot(1,4,i+1)
plt.tight_layout()
ax.set_title('Samples {}'.format(i))
ax.axis('off')
show_landmarks(**sample)
if i==3:
plt.show()
break
OUT:
0 (324, 215, 3) (68, 2)
1 (500, 333, 3) (68, 2)
2 (250, 258, 3) (68, 2)
3 (434, 290, 3) (68, 2)
变换
从上面我们可以看到的一个问题是样本的大小不同。大多数神经网络都期望固定大小的图像。因此，我们需要编写一些预处理代码。让我们创建三个变换：
Rescale：缩放图像RandomCrop：随机裁剪图像。这是数据增加。ToTensor：将numpy图像转换为火炬图像（我们需要交换轴）。
请注意以下这些变换如何应用于 image and landmarks.
class Rescale(object):
"""Rescale the image in a sample to a given size"""
def __init__(self,output_size):
assert isinstance(output_size,(int ,tuple))
self.output_size = output_size
def __call__(self,sample):
image ,landmarks = sample['image'] , sample['landmarks']
h , w = image.shape[:2]
if isinstance(self.output_size,int):
if h > w:
new_h , new_w = self.output_size*h / w ,self.output_size
else:
new_h , new_w = self.output_size , self.output_size*h / w
else:
new_h , new_w = self.output_size
new_h , new_w = int(new_h) , int(new_w)
img = transform.resize(image,(new_h,new_w))
landmarks = landmarks*[new_w/w , new_h/h]
return {'image':img , 'landmarks':landmarks}
class RandomCrop(object):
"""Crop randomly the image in a sample"""
def __init__(self,output_size):
assert isinstance(output_size,(int ,tuple))
if isinstance(output_size,int):
self.output_size = (output_size,output_size)
else:
assert len(output_size) == 2
self.output_size = output_size
def __call__(self,sample):
image ,landmarks = sample['image'] , sample['landmarks']
h , w = image.shape[:2]
new_h , new_w = self.output_size
top = np.random.randint(0,h - new_h)
left = np.random.randint(0,w - new_w)
image = image[top:top + new_h ,left:left + new_w]
landmarks = landmarks - [left , top]
return {'image':image , 'landmarks':landmarks}
class ToTensor(object):
"""Convert ndarrays in samples to Tensor"""
def __call__(self,sample):
image , landmarks = sample['image'] , samplem['landmarks']
image = image.transpose(2,0,1)
return {'image':torch.from_numpy(image),'landmarks':torch.from_numpy(landmarks)}
撰写变换
现在，我们在样本上应用变换。
假设我们想要将图像的短边重新缩放到256，然后从中随机裁剪一个224的正方形。也就是说，我们想要撰写 Rescale和RandomCrop转换。 torchvision.transforms.Compose是一个简单的可调用类，它允许我们这样做。
scale = Rescale(256)
crop = RandomCrop(128)
composed = transforms.Compose([Rescale(126),RandomCrop(224)])
# Apply each of above transform on sample
fig = plt.figure()
sample = face_dataset[65]
for i , tsfrm in enumerate([scale,crop,composed]):
transformed_sample = tsfrm(sample)
ax = plt.subplot(1,3,i+1)
plt.tight_layout()
ax.set_title(type(tsfrm).__name__)
show_landmarks(**transformed_sample)
plt.show()
