BP neural network (Description in Python language)

Preface

The emergence of the BP algorithm has made a major breakthrough in the research of forward multilayer neural networks, and has effectively promoted the rise of artificial neural networks research climax.

Algorithm principle

Code

#   本例是一个两层前向BP神经网络
#   程序使用说明
#   采用有“惯性效应”的调整算法
#   主要函数的功能
#   BPNeuralNetwork定义一个BP神经网络的类
#   setup函数定义各层神经元的个数；setup(self, ni, nh, no)第一个参数是输入层
#   个数，第二个参数是隐含层个数，第三个参数是输出层个数；
#   train(input, output, limit=4000, learn=0.05, correct=0.1)函数时训练
#   函数，correct时惯性系数(默认值是0.1)，它的值越大系数调整的惯性越大。
#   learn是步长；limit是最大迭代次数；
#   input是输入层数据；这里是列表的格式；
#   output是输出层数据；这里是列表的格式；
#   showerrorcurve(self)是误差迭代曲线函数；
#   printcanshu(self)是打印各层权系数矩阵的函数；
#   由于采用的是S形函数，因而输出层各神经元的理想输出值不能达到1 或0，而只能接近于1或0，
#   所以，训练时理想输出值设为0.85或0.15。
#   权重系数w_ij初值的选择。初值可以随机设置，但不能相等。如果初始
#   权值相等，那么隐含层权重系数的调整量始终相同。
#   前向多层神经网络，其误差相对于各权重系数的变化具有非常复杂的“曲面”，
#   具有很多极小点，其中只有一个是“全局最小点”，其它都是“局部最小点”。
#   显然，只有全局最小点才是网络的最佳解。如果初值设置不当就有可能使权重
#   系数在学习结束时收敛到某个局部极小值上。
import math
import random
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False

random.seed(0)


def rand(a, b):
    return (b - a) * random.random() + a


def make_matrix(m, n, fill=0.0):
    mat = []
    for i in range(m):
        mat.append([fill] * n)
    return mat


def sigmoid(x):
    return 1.0 / (1.0 + math.exp(-x))


def sigmoid_derivative(x):
    return x * (1 - x)


class BPNeuralNetwork:
    def __init__(self):
        self.input_n = 0
        self.hidden_n = 0
        self.output_n = 0
        self.cases = []
        self.labels = []
        self.input_cells = []
        self.hidden_cells = []
        self.output_cells = []
        self.input_weights = []
        self.output_weights = []
        self.input_correction = []
        self.output_correction = []
        self.error_r = []

    def setup(self, ni, nh, no):
        self.input_n = ni + 1
        self.hidden_n = nh
        self.output_n = no
        # 初始化神经元
        self.input_cells = [1.0] * self.input_n
        self.hidden_cells = [1.0] * self.hidden_n
        self.output_cells = [1.0] * self.output_n
        # 初始化权系数
        self.input_weights = make_matrix(self.input_n, self.hidden_n)
        self.output_weights = make_matrix(self.hidden_n, self.output_n)
        # 给两层权系数赋随机初始值
        for i in range(self.input_n):
            for h in range(self.hidden_n):
                self.input_weights[i][h] = rand(-0.2, 0.2)
        for h in range(self.hidden_n):
            for o in range(self.output_n):
                self.output_weights[h][o] = rand(-2.0, 2.0)
        # init correction matrix
        self.input_correction = make_matrix(self.input_n, self.hidden_n)
        self.output_correction = make_matrix(self.hidden_n, self.output_n)

    def predict(self, inputs):
        # 激活输入层
        for i in range(self.input_n - 1):
            self.input_cells[i] = inputs[i]
        # 激活隐藏层
        for j in range(self.hidden_n):
            total = 0.0
            for i in range(self.input_n):
                total += self.input_cells[i] * self.input_weights[i][j]
            self.hidden_cells[j] = sigmoid(total)
        # 激活输出层
        for k in range(self.output_n):
            total = 0.0
            for j in range(self.hidden_n):
                total += self.hidden_cells[j] * self.output_weights[j][k]
            self.output_cells[k] = sigmoid(total)
        return self.output_cells[:]

    def back_propagate(self, case, label, learn, correct):

        # 前向网络
        self.predict(case)
        # 计算输出层误差
        output_deltas = [0.0] * self.output_n
        for o in range(self.output_n):
            error = label[o] - self.output_cells[o]
            output_deltas[o] = sigmoid_derivative(self.output_cells[o]) * error
        # 计算隐含层误差
        hidden_deltas = [0.0] * self.hidden_n
        for h in range(self.hidden_n):
            error = 0.0
            for o in range(self.output_n):
                error += output_deltas[o] * self.output_weights[h][o]
            hidden_deltas[h] = sigmoid_derivative(self.hidden_cells[h]) * error
        # 更新输出层的权系数
        for h in range(self.hidden_n):
            for o in range(self.output_n):
                change = output_deltas[o] * self.hidden_cells[h]
                self.output_weights[h][o] += 2*learn * change + correct * self.output_correction[h][o]
                self.output_correction[h][o] = change
        # 更新输入层的权系数
        for i in range(self.input_n):
            for h in range(self.hidden_n):
                change = hidden_deltas[h] * self.input_cells[i]
                self.input_weights[i][h] += learn * change + correct * self.input_correction[i][h]
                self.input_correction[i][h] = change
        # 更新全局误差
        error = 0.0
        for o in range(len(label)):
            error += 0.5 * (label[o] - self.output_cells[o]) ** 2
        return error

    def train(self, cases, labels, limit=10000, learn=0.05, correct=0.1):
        self.cases = cases
        self.labels = labels
        for j in range(limit):
            error = 0.0
            for i in range(len(cases)):
                label = labels[i]
                case = cases[i]
                error += self.back_propagate(case, label, learn, correct)
            self.error_r.append(error)


    def test(self, cases, labels):
        self.train(cases, labels, 10000, 0.05, 0.1)


    def printcanshu(self):
        print('输出层连接权值参数为：')
        print(self.output_weights)
        print('隐含层连接权值参数为：')
        print(self.hidden_cells)
        print('输入层连接权值参数为：')
        print(self.input_weights)


    def errorcal(self):
        n = len(self.cases)
        pridet1 = np.zeros((n, 2))
        c = []
        for i in range(n):
            c.append(self.predict(self.cases[i]))
        for i in range(n):
            for j in range(2):
                if c[i][j] > 0.85:
                    c[i][j] = 1
                elif c[i][j] < 0.15:
                    c[i][j] = 0
        corra = 0
        for i in range(n):
            if c[i][1] == self.labels[i][1] and c[i][0] == self.labels[i][0]:
                corra = corra + 1
        print('网络测试正确率为{}%'.format(corra/n*100))


    def showerrorcurve(self):
        plt.plot(range(1, len(self.error_r)+1), self.error_r)
        plt.xlabel('迭代次数')
        plt.ylabel('误差')
        plt.title('BP神经网络迭代误差曲线')
        plt.savefig('BP神经网络迭代误差曲线.jpg')
        plt.show()




if __name__ == '__main__':
    input = np.array([[7, 6, 7, 7, 8],
                      [2, 1, 5, 5, 7],
                      [0, 4, 9, 8, 0],
                      [8, 7, 5, 7, 6],
                      [0, 8, 9, 9, 0],
                      [0, 3, 7, 7, 5],
                      [8, 7, 9, 4, 8],
                      [7, 1, 1, 8, 6],
                      [1, 0, 0, 4, 2],
                      [2, 5, 9, 1, 8]])
    output = np.array([[1, 1],
                       [1, 0],
                       [0, 1],
                       [1, 1],
                       [0, 0],
                       [1, 0],
                       [0, 0],
                       [0, 0],
                       [1, 0],
                       [0, 1]])
    input = list(input)
    output = list(output)
    nn = BPNeuralNetwork()
    nn.setup(5, 11, 2)
    nn.train(input, output, limit=4000, learn=0.05, correct=0.1)
    nn.errorcal()
    print('输入[2, 3, 8, 6, 7]')
    print('预测结果为：')
    print(nn.predict([2, 3, 8, 6, 7]))
    nn.printcanshu()
    nn.showerrorcurve()

Calculation results

in conclusion

1. Since the sigmoid function is used, the ideal output value of each neuron in the output layer cannot reach 1 or 0, but can only be close to 1 or 0. Therefore, the ideal output value during training is usually set to 0.9 or 0.1. This article is set to 0.85 and 0.15;
2. The network prediction ability of this experiment is strong, and it achieves very good results when iterative calculation is 1000 times;
3. On the basis of this example, one or two more hidden layers can be added