【机器学习】SVM（基于SMO算法）—— phthon3 实现方案

class SVM:
    def __init__(self, C=1, toler=0.001, maxIter=500, kernel_option=('', 1)):
        self.C = C  # 正则化参数
        self.toler = toler  # 容错率
        self.maxIter = maxIter  # 最大迭代次数
        self.kernel_opt = kernel_option  # 选择核函数

    def cal_kernel_value(self, X, X_i, kernel_option):
        '''
        计算样本之间的核函数的值
        :param X: 特征集 m*n
        :param X_i: 特征集的第i个样本， 一个行向量 1*n
        :param kernel_option:  选择的核函数与参数值
        :return: 样本之间的核函数值， m*1
        '''
        m = X.shape[0]
        kernel_value = np.mat(np.zeros((m, 1)))
        if kernel_option[0] == 'rbf':  # 径向基函数（高斯核函数）
            sigma = kernel_option[1]
            for i in range(m):
                diff = X[i, :] - X_i
                kernel_value[i] = np.exp(np.dot(diff, diff.T) / (-2 * sigma**2))
        else:
            kernel_value = np.dot(X, X_i.T)
        return kernel_value

    def cal_kernel(self, X, kernel_option):
        '''
        计算核函数的矩阵
        :param X: 训练集m*n
        :param kernel_option: 选择的核函数
        :return: 核函数的矩阵m*m
        '''
        m = X.shape[0]
        kernel_matrix = np.mat(np.zeros((m, m)))
        for i in range(m):
            kernel_matrix[:, i] = self.cal_kernel_value(X, X[i, :], kernel_option)
        return kernel_matrix

    def SVM_training(self, X, y):
        self.X = np.mat(X)  # 转换为矩阵 m*n
        self.y = np.mat(y)  # m*1
        self.m = len(X)
        self.alpha = np.mat(np.zeros((self.m, 1)))  # 拉格朗日乘子
        self.b = 0  # 偏差b
        self.Ecache = np.mat(np.zeros((self.m, 2))) # 存放误差E
        self.kernel_mat = self.cal_kernel(self.X, self.kernel_opt)  # 载入核函数矩阵

        entireSet = True
        alpha_changed = 0
        iter = 0
        # 当迭代轮次超过最大值或者 遍历全集后alpha值无变化， 则跳出外循环，训练结束
        while iter < self.maxIter and (alpha_changed > 0 or entireSet):
            print('Iter:', iter)
            alpha_changed = 0

            if entireSet:  # 第一次 遍历全集
                for i in range(self.m):
                    alpha_changed += self.innerL(i)
                iter += 1

            else:  # 第二次 遍历非边界值， 直到其中所有的alpha都不能再优化（即alpha_changed==0）， 再遍历全集
                bound_samples = []
                for i in range(self.m):  # 找出所有非边界值的索引
                    if self.alpha[i, 0] > 0 and self.alpha[i, 0] < self.C:
                        bound_samples.append(i)
                # bound_samples = [i for i, a in enumerate(self.alpha[:, 0]) if (a > 0 and a < self.C)]
                # print(bound_samples)
                for i in bound_samples:
                    alpha_changed += self.innerL(i)
                iter += 1

            if entireSet:
                entireSet = False
            elif alpha_changed == 0:
                entireSet = True
        return


    # 获取不等于i的j值
    def Jrand(self, i, m):
        j = i
        while j == i:
            j = np.random.randint(0, m)
        return j

    # 裁剪alpha，使满足KKT条件
    def clipAlpha(self, alphaj, L, H):
        if alphaj < L:
            alphaj = L
        if alphaj > H:
            alphaj = H
        return alphaj

    # 计算误差E
    def cal_E(self, k):
        fxk = float(np.dot(np.multiply(self.alpha, self.y).T, self.kernel_mat[:, k]) + self.b)
        Ek = fxk - float(self.y[k])
        return Ek

    # 更新alpha 和 b 后，更新Ecache
    def update_E(self, k):
        Ek = self.cal_E(k)
        self.Ecache[k] = [1, Ek]

    # 寻找第二个乘子。 满足abs(Ei-Ej)最大，即 步伐最大。
    def select_second_alpha(self, i, Ei):
        j, Ej, maxstep = 0, 0, 0

        self.Ecache[i] = [1, Ei]
        validE = np.nonzero(self.Ecache[:, 0].A)[0]  # 找出所有已经更新进Ecache的E值
        # validE = [i for i, a in enumerate(self.Epache[:, 0]) if a > 0]
        if len(validE) > 1:
            for k in validE:
                if k == i:
                    continue
                Ek = self.cal_E(k)
                deltaE = abs(Ei - Ek)
                if deltaE > maxstep:
                    j = k
                    Ej = Ek
                    maxstep = deltaE
        else:  # 第一次遍历，找一个不等于i的随机数j
            j = self.Jrand(i, self.m)
            Ej = self.cal_E(j)

        return j, Ej

    # 内循环，找出违反KTT条件的alpha，并更新alpha,b,Ecache。 如果没有更新值，返回0，否则返回1
    def innerL(self, i):
        Ei = self.cal_E(i)
        r = self.y[i] * Ei # 拆开来，等价于 y(wx+b)-1
        if (r < -self.toler and self.alpha[i] < self.C) or (r > self.toler and self.alpha[i] > 0):  # 如果没有容错率，则分别是 r<0 和 r>0. 容错率的意思是，在（-toler,toler）之间的点，就当做是满足KKT条件了，而放过不做优化
            j, Ej = self.select_second_alpha(i, Ei)
            alphaIold = self.alpha[i].copy()
            alphaJold = self.alpha[j].copy()

            if self.y[i] == self.y[j]:  # 计算出上下边界
                L = max(0, alphaIold + alphaJold - self.C)
                H = min(self.C, alphaIold + alphaJold)
            else:
                L = max(0, alphaJold - alphaIold)
                H = min(self.C, self.C + alphaJold - alphaIold)

            if L == H:
                # print('L == H')
                return 0

            eta = self.kernel_mat[i, i] + self.kernel_mat[j, j] - 2 * self.kernel_mat[i, j]
            if eta <= 0:  # eta是alphaj的二阶导数。 根据二阶导数性质，只有当二阶导数>0是，原函数才能取到最小值。
                # print('eta <= 0')
                return 0

            alphaJnew = alphaJold + self.y[j] * (Ei - Ej) / eta
            alphaJnew = self.clipAlpha(alphaJnew, L, H)
            if abs(alphaJnew - alphaJold) < 0.00001:  # 改变太小的也认为是没有改变
                # print('alpha NOT moving enough')
                return 0

            self.alpha[j] = alphaJnew
            alphaInew = alphaIold + self.y[i] * self.y[j] * (alphaJold - alphaJnew)
            self.alpha[i] = alphaInew
            bi = float(-Ei + self.y[i] * self.kernel_mat[i, i] * (alphaIold - alphaInew) + self.y[j] * self.kernel_mat[i, j] * (alphaJold - alphaJnew) + self.b)
            bj = float(-Ej + self.y[i] * self.kernel_mat[i, j] * (alphaIold - alphaInew) + self.y[j] * self.kernel_mat[j, j] * (alphaJold - alphaJnew) + self.b)
            if alphaInew < self.C and alphaInew > 0:  # 如果alphaJnew 和 alphaInew 都是非边界点， 那他们代表的是支持向量。bi=bj
                self.b = bi
            elif alphaJnew < self.C and alphaJnew > 0:
                self.b = bj
            else:
                self.b = (bi + bj) / 2
            self.update_E(i)  # 更新Ecache
            self.update_E(j)
            return 1
        return 0

    # 根据求出来的alpha，计算权重
    def cal_w(self):
        w = np.dot(np.multiply(self.alpha, self.y).T, self.X)
        return w

    # 输出预测结果
    def predict(self, X):
        w = self.cal_w()
        pred = np.dot(X, w.T) + self.b
        return [1 if p >= 0 else -1 for p in pred]

    # 计算准确率
    def accuracy(self, X, y):
        predictions = self.predict(X)
        correct = [1 if a == b else 0 for a, b in zip(predictions, y)]
        accuracy = correct.count(1) / len(correct)
        return accuracy