Un numpy para construir una red neuronal

Este es el día 13 de mi participación en el Desafío de actualización de agosto. Para obtener detalles del evento, consulte: Desafío de actualización de agosto

NumPy para construir mi primera red neuronal

prefacio

Al usar numpy puro para realizar el reconocimiento de gestos, el primero es la composición general de la red y luego se muestra la parte del código. Esta es mi primera red neuronal.

Código completo: GitHub

La red generalmente refleja: capa de entrada, capa oculta, capa de salida. Ya se sabe que la capa de salida tiene diez resultados, que es la probabilidad de 10 números.

Acerca del conjunto de entrenamiento, conjunto de validación, conjunto de prueba

Descenso de gradiente a mano cálculo para la solución de tres parámetros

El primer parámetro es un proceso detallado calculado a mano.

sección de código

La función de activación y la derivada de la función de activación

def tanh(x):
    return np.tanh(x)

def bypass(x):
    return x

def softmax(x):
    exp=np.exp(x-x.max())   # 为了防止指数爆炸(指数函数)
    # 所以现在就要将其中的减去最大值，变成负数，但是最后是相处，
    #结果都是一样的。
    return exp/exp.sum()

def d_softmax(data):
    sm = softmax(data)
    return np.diag(sm) - np.outer(sm, sm)

def d_tanh(data):  # 返回值返回向量
    return 1 / (np.cosh(data)) ** 2

def d_bypass(x):
    return 1

differential = {softmax: d_softmax, tanh: d_tanh,bypass:d_bypass}
d_type = {bypass:'times',softmax:'dot',tanh:'times'}
复制代码

Parámetros de inicialización

dimensions=[28*28,100,10]  # 28*28个神经元,100个神经元是隐藏层，输出的10个数字
# 28*28个神经元先是连接到100个神经元中再连接到10个输出层中
activation =[bypass,tanh,softmax]  # bypass  两个激活函数
distribution=[   # 初始化过程
    {},# 空着
    {'b':[0,0],'w':[-math.sqrt(6/(dimensions[0]+dimensions[1])),math.sqrt(6/(dimensions[0]+dimensions[1]))]},
    {'b':[0,0],'w':[-math.sqrt(6/(dimensions[1]+dimensions[2])),math.sqrt(6/(dimensions[1]+dimensions[2]))]},
]
复制代码

función de parámetro de inicialización

def init_parameters_b(layer):  # 初始化b
    dist= distribution[layer]['b']
    return np.random.rand(dimensions[layer])*(dist[1]-dist[0])+dist[0]    

def init_parameters_w(layer):
    dist=distribution[layer]['w']
    return np.random.rand(dimensions[layer-1],dimensions[layer])*(dist[1]-dist[0])+dist[0]

def init_parameters():
    parameter=[]   # 迭代每一次的结果
    for i in range (len(distribution)):
        layer_parameter={}   # 存放每一次迭代的数据，并且一直存储起来
        for j in distribution[i].keys():
            if j=='b':
                layer_parameter['b']=init_parameters_b(i)
                continue
            if j=='w':
                layer_parameter['w']=init_parameters_w(i)
                continue
        parameter.append(layer_parameter)
    return parameter
复制代码

Parámetros de inicialización iniciales (no entrenados)

parameters=init_parameters()
复制代码

función de predicción

def predict(img,parameters):
    # 参数:图片，参数
    l_in=img
    l_out=activation[0](l_in)  # 第一层的初始化
    for layer in range(1,len(dimensions)):  
        l_in = np.dot(l_out,parameters[layer]['w'])+parameters[layer]['b']  # 不断地进行迭代
        l_out = activation[layer](l_in)
    return l_out
复制代码

La primera predicción (esta predicción es inestable porque se usa el modelo inicializado)

predict(train_img[0],init_parameters())

//结果：
array([0.07210171, 0.07957606, 0.13152407, 0.05420442, 0.08498909,
       0.12788144, 0.14911174, 0.14570486, 0.08225591, 0.07265069])
复制代码

conjunto de entrenamiento, conjunto de validación, conjunto de prueba

dataset_path=Path('D:/Desktop/MNIST')

train_img_path=dataset_path/'train-images-idx3-ubyte/train-images.idx3-ubyte'
train_lab_path=dataset_path/'train-labels-idx1-ubyte/train-labels.idx1-ubyte'
test_img_path=dataset_path/'t10k-images-idx3-ubyte/t10k-images.idx3-ubyte'
test_lab_path=dataset_path/'t10k-labels-idx1-ubyte/t10k-labels.idx1-ubyte'
复制代码

conjuntos separados

train_num = 50000  # 训练
valid_num = 10000  # 验证
test_num = 10000  # 测试

with open(train_img_path, 'rb') as f:
    struct.unpack('>4i', f.read(16))
    temp_img = np.fromfile(f, dtype=np.uint8).reshape(-1, 28 * 28) / 255
    train_img = temp_img[:train_num]  # 将训练集中的数据分了1w出去给验证
    valid_img = temp_img[train_num:]

with open(test_img_path, 'rb') as f:
    struct.unpack('>4i', f.read(16))
    test_img = np.fromfile(f, dtype=np.uint8).reshape(-1, 28 * 28) / 255

with open(train_lab_path, 'rb') as f:
    struct.unpack('>2i', f.read(8))
    temp_lab = np.fromfile(f, dtype=np.uint8)
    train_lab = temp_lab[:train_num]
    valid_lab = temp_lab[train_num:]

with open(test_lab_path, 'rb') as f:
    struct.unpack('>2i', f.read(8))
    test_lab = np.fromfile(f, dtype=np.uint8)
复制代码

Mostrar etiqueta de imagen

def show_train(index):
    plt.imshow(train_img[index].reshape(28, 28), cmap='gray')
    pylab.show()
    print('label:{}'.format(train_lab[index]))


def show_valid(index):
    plt.imshow(valid_img[index].reshape(28, 28), cmap='gray')
    pylab.show()
    print('label:{}'.format(valid_lab[index]))


def show_test(index):
    plt.imshow(test_img[index].reshape(28, 28), cmap='gray')
    pylab.show()
    print('test:{}'.format(test_lab[index]))
复制代码

Usar números aleatorios para predecir resultados

predict(np.random.rand(784),parameters)

//结果：
array([0.0942381 , 0.11644771, 0.05850607, 0.23711087, 0.02732923,
       0.0176975 , 0.19317991, 0.14196864, 0.08510021, 0.02842176])
复制代码

Verifique que su derivada esté escrita correctamente y use la definición para verificar, de lo contrario, si la derivada está incorrecta, será en vano.

h = 0.0001
func = softmax
input_len = 4
for i in range(input_len):  # 两种求导数的方法
    # 一种是定义方法，与一种是直接求导法
    test_input = np.random.rand(input_len)
    derivative = differential[func](test_input)
    value1 = func(test_input)
    test_input[i] += h
    value2 = func(test_input)
    # print((value2 - value1) / h)
    # print(derivative[i] - (value2 - value1) / h)  # 差值

onehot = np.identity(dimensions[-1])  # 10个数字

复制代码

Descenso de gradiente, aquí es pasar a la inversa (en términos simples, derivación en cadena), para hacer que el valor de loss_function sea lo más pequeño posible, es hacer que el valor de y1-y0 sea lo más cercano posible, por lo que es para acerque y1 al valor verdadero, y luego el valor inicial se actualiza una vez, y la actualización se repite continuamente.

def sqr_loss(img, lab, parameters):
    y_pred = predict(img, parameters)
    y = onehot[lab]
    diff = y - y_pred
    return np.dot(diff, diff)

def grad_parameters(img, lab, parameters):
    # 参数:图片，参数
    l_in_list=[img] # 第一个参数是0所以img+0是等于img
    l_out_list=[activation[0](l_in_list[0])]  # 第一个out也是初始化之后的out
    for layer in range(1,len(dimensions)):  
        l_in = np.dot(l_in_list[layer-1], parameters[layer]['w']) + parameters[layer]['b']
        l_out = activation[layer](l_in)
        l_in_list.append(l_in)
        l_out_list.append(l_out)
        
    d_layer =-2*(onehot[lab] - l_out_list[-1])
    grad_result=[None]*len(dimensions)
    for layer in range(len(dimensions)-1,0,-1):  # 反向传播
        if d_type[activation[layer]]=='times':
            d_layer = differential[activation[layer]](l_in_list[layer])*d_layer
        if d_type[activation[layer]]=='dot':
            d_layer = np.dot(differential[activation[layer]](l_in_list[layer]), d_layer)
        # 参数一:
        grad_result[layer]={}
        grad_result[layer]['b'] = d_layer
        grad_result[layer]['w'] = np.outer(l_out_list[layer-1], d_layer)  # 上一层的结果
        d_layer = np.dot(parameters[layer]['w'],d_layer)
    return grad_result 
复制代码

Resultados de parámetros después de la retropropagación

grad_parameters(train_img[0],train_lab[0],init_parameters())
复制代码

Verifique que la derivada parcial de backpropagation sea correcta

# 验证参数b
h = 0.00001  					# 验证反向传递求的对不对，求导过程。
layer=2
parameters = init_parameters()
pname='b'
for i in range(len(parameters[layer][pname])):  	# 两种求导数的方法
    # 一种是定义方法，与一种是直接求导法
    img_i = np.random.randint(train_num)  			# 随机找数字图片
    test_parameters = init_parameters()  			# 随机找
    derivative = grad_parameters(train_img[img_i], train_lab[img_i], test_parameters)[layer][pname]
    value1 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)
    test_parameters[layer][pname][i] += h
    value2 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)
    print(derivative[i]-(value2-value1)/h)

# 验证参数w
h = 0.00001  # 验证方向传递求的对不对，求导过程。
layer=1
parameters = init_parameters()
pname='w'
grad_list=[]
for i in range(len(parameters[layer][pname])):  # 两种求导数的方法
    for j in range(len(parameters[layer][pname][0])): 
        img_i = np.random.randint(train_num)  # 随机找数字图片
        test_parameters = init_parameters()  # 随机找
        derivative = grad_parameters(train_img[img_i], train_lab[img_i], test_parameters)[layer][pname]
        value1 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)
        test_parameters[layer][pname][i][j] += h
        value2 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)
        grad_list.append(derivative[i][j]-(value2-value1)/h)
np.abs(grad_list).max()
复制代码

función de pérdida

def valid_loss(parameters):  # 验证集函数
    loss_accu = 0
    for img_i in range(valid_num):
        loss_accu += sqr_loss(valid_img[img_i], valid_lab[img_i], parameters)
    return loss_accu / (valid_num / 10000)  # 每1w张图片的loss_accu
    # 进行归一化，1w个图片为整体

    
def valid_accuracy(parameters):  # 准确率
    correct = [predict(valid_img[img_i], parameters).argmax() == valid_lab[img_i] for img_i in range(valid_num)]
    return correct.count(True) / len(correct)


def train_loss(parameters):   # 验证集函数
    loss_accu = 0
    for img_i in range(train_num):
        loss_accu += sqr_loss(train_img[img_i], train_lab[img_i], parameters)
    return loss_accu / (train_num / 10000)


def train_accuracy(parameters):  # 准确率
    correct = [predict(train_img[img_i], parameters).argmax() == train_lab[img_i] for img_i in range(train_num)]
    return correct.count(True) / len(correct)

def test_accuracy(parameters):  # 准确率
    correct = [predict(test_img[img_i], parameters).argmax() == test_lab[img_i] for img_i in range(test_num)]
    return correct.count(True) / len(correct)

复制代码

def grad_add(grad1,grad2):
    for layer in range(1,len(grad1)):
        for pname in grad1[layer].keys():
            grad1[layer][pname]+=grad2[layer][pname]
    return grad1
    
def grad_divide(grad,denominator):
    for layer in range(1,len(grad)):
        for pname in grad[layer].keys():
            grad[layer][pname]/=denominator
    return grad
    
def combine_parameters(parameters, grad, learn_rate):  # 新的参数的形成
    parameter_tmp = copy.deepcopy(parameters)
    for layer in range(len(parameter_tmp)):
        for pname in parameter_tmp[layer].keys():
            parameter_tmp[layer][pname] -= learn_rate * grad[layer][pname]
    return parameter_tmp
复制代码

un total de volumen de entrenamiento

batch_size = 100  	# 一百张图片为整体
                    # 100张图片当一小块
                    # 训练一小块
def train_batch(current_batch, parameters):  	# 将这一百张图片的梯度计算起来，拿到一个平均值！
    grad_accu = grad_parameters(train_img[current_batch * batch_size], train_lab[current_batch * batch_size],
                                parameters)
    for img_i in range(1, batch_size):
        grad_temp = grad_parameters(train_img[current_batch * batch_size + img_i],
                                    train_lab[current_batch * batch_size + img_i], parameters)
        grad_add(grad_accu,grad_temp)
            # 获得累加的梯度
    grad_divide(grad_accu,batch_size)    				# 拿到方向
    return grad_accu

parameters = init_parameters()
复制代码

proceso de entrenamiento

from tqdm import tqdm_notebook
current_epoch = 0
train_loss_list = []  # 存储loss的数值
valid_loss_list = []  # 存储验证的loss的数值
train_accu_list = []  # 训练的正确性
valid_accu_list = []  # 验证集的正确性
learn_rate = 10**-0.3  # 学习率到最后要变小
epoch_num = 5  # 训练的次数  训练完一次叫一个epoch
for epoch in tqdm_notebook(range(epoch_num)):
    for i in range(train_num // batch_size):
#         if i % 100 == 99:
#             print('running batch{}/{}'.format(i + 1, train_num // batch_size))
        grad_tmp = train_batch(i, parameters)
        parameters = combine_parameters(parameters, grad_tmp, learn_rate)
    current_epoch += 1
    train_loss_list.append(train_loss(parameters))
    train_accu_list.append(train_accuracy(parameters))
    valid_loss_list.append(valid_loss(parameters))
    valid_accu_list.append(valid_accuracy(parameters))

    
valid_accuracy(parameters)
复制代码

pérdida para conjuntos de validación y entrenamiento

lower = -0
plt.plot(valid_loss_list[lower:], color='black', label='validation loss')
plt.plot(train_loss_list[lower:], color='red', label='train loss')
plt.show()
复制代码

Precisión en conjuntos de validación y entrenamiento

plt.plot(valid_accu_list[lower:], color='black', label='validation accuracy')
plt.plot(train_accu_list[lower:], color='red', label='train accuracy')
plt.show()
复制代码

Al final

Xiaosheng Fanyi, esperando su atención.