机器学习实战Chp6： SVM-支持向量机--SMO高效优化算法

• 机器学习实战Chp6： SVM-支持向量机–SMO高效优化算法
• 从demo为完整的Platt SMO算法加速优化
• 参考李航《统计学习方法》和周志华的《机器学习》

# -*- coding: utf-8 -*-
"""
Created on Mon Jul 23 19:30:48 2018

@author: muli
"""
# 参考李航《统计学习方法》
# 监督学习一般使用两种类型的目标变量：标称型和数值型
# 标称型：标称型目标变量的结果只在有限目标集中取值，如真与假(标称型目标变量主要用于分类)
# 数值型：数值型目标变量则可以从无限的数值集合中取值，如0.100，42.001等 (数值型目标变量主要用于回归分析)

# 如果所有样本点都可以正确被分类，则我们假设所有点到超平面的距离均大于等于1
#（可以通过将w和b缩放的形式达到该目标），
# 并且称距离唯一的点为支持向量，两个异类支持向量到划分超平面的距离之和为间隔。

from numpy import *
from time import sleep


# 读取数据集
def loadDataSet(fileName):
    dataMat = []; labelMat = []
    fr = open(fileName)
    for line in fr.readlines():
        lineArr = line.strip().split('\t')
        dataMat.append([float(lineArr[0]), float(lineArr[1])])
        labelMat.append(float(lineArr[2]))
    return dataMat,labelMat


# 用于在区间内选择一个整数，i为alpha的下标，m为alpha的个数
def selectJrand(i,m):
    j=i 
    # 只要函数值不等于输入值i就会随机
    # 因为要满足 ∑alpha(i)*label(i)=0,同时改变两个alpha
    while (j==i):
        j = int(random.uniform(0,m))
    return j


# 用来调整大于H或小于L的alpha值
def clipAlpha(aj,H,L):
    if aj > H: 
        aj = H
    if L > aj:
        aj = L
    return aj


########################################################
# 简化版SMO算法
########################################################
# 数据集，类别标签，常数C，容错率toler，退出前的最大循环次数maxIter
def smoSimple(dataMatIn, classLabels, C, toler, maxIter):
    # 数组 转化为 矩阵
    dataMatrix = mat(dataMatIn); 
    labelMat = mat(classLabels).transpose()
    b = 0;
    # 得到矩阵的维数
    m,n = shape(dataMatrix)
    # m个 a 
    alphas = mat(zeros((m,1)))
    # 没有任何alpha改变下的遍历数据集的次数
    iter = 0
    while (iter < maxIter):
        # 用来记录alpha是否被优化
        alphaPairsChanged = 0
        for i in range(m):
            fXi = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[i,:].T)) + b
            # 误差Ei
            Ei = fXi - float(labelMat[i])#if checks if an example violates KKT conditions
            # 如果误差很大，就可以基于该组数据所对应的alpha进行优化
            # 在if语句，测试正间隔和负间隔，同时检查alpha值,保证其不能等于0或C
            if ((labelMat[i]*Ei < -toler) and (alphas[i] < C)) or ((labelMat[i]*Ei > toler) and (alphas[i] > 0)):
                j = selectJrand(i,m)
                fXj = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[j,:].T)) + b
                Ej = fXj - float(labelMat[j])
                # 把两个alpha赋值，这样的好处是不改变原有alphas的值
                alphaIold = alphas[i].copy(); 
                alphaJold = alphas[j].copy();
                # 如果标签向量不相等，保证alpha再0~C之间
                if (labelMat[i] != labelMat[j]):
                    L = max(0, alphas[j] - alphas[i])
                    H = min(C, C + alphas[j] - alphas[i])
                else:
                    L = max(0, alphas[j] + alphas[i] - C)
                    H = min(C, alphas[j] + alphas[i])
                if L==H: 
                    print "L==H"; 
                    continue
                # 是alpha[j]的最优修改量
                # 计算η值，注意η值与书上的定义相反
                # 下面的计算，与书上的定义，有些相反
                eta = 2.0 * dataMatrix[i,:]*dataMatrix[j,:].T - dataMatrix[i,:]*dataMatrix[i,:].T - dataMatrix[j,:]*dataMatrix[j,:].T
                if eta >= 0: 
                    print "eta>=0"; 
                    continue
                alphas[j] -= labelMat[j]*(Ei - Ej)/eta
                # 可参考李航P127，式7.108
                alphas[j] = clipAlpha(alphas[j],H,L)
                if (abs(alphas[j] - alphaJold) < 0.00001): 
                    print "j not moving enough"; continue
                alphas[i] += labelMat[j]*labelMat[i]*(alphaJold - alphas[j])#update i by the same amount as j
                                                                        #the update is in the oppostie direction
                b1 = b - Ei- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[i,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[i,:]*dataMatrix[j,:].T
                b2 = b - Ej- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[j,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[j,:]*dataMatrix[j,:].T
                # 更新 b 值 
                if (0 < alphas[i]) and (C > alphas[i]): 
                    b = b1
                elif (0 < alphas[j]) and (C > alphas[j]): 
                    b = b2
                else: 
                    b = (b1 + b2)/2.0
                alphaPairsChanged += 1
                print "iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)
        if (alphaPairsChanged == 0): 
            iter += 1
        else: 
            iter = 0
        print "iteration number: %d" % iter
    return b,alphas


###################################################################
## 完整版的 Platt SMO 算法
###################################################################
# 数据结构的对象
class optStruct:
    def __init__(self,dataMatIn, classLabels, C, toler):
#    def __init__(self,dataMatIn, classLabels, C, toler, kTup):  # Initialize the structure with the parameters 
        self.X = dataMatIn
        self.labelMat = classLabels
        self.C = C
        self.tol = toler
        self.m = shape(dataMatIn)[0]
        self.alphas = mat(zeros((self.m,1)))
        self.b = 0
        self.eCache = mat(zeros((self.m,2))) #first column is valid flag
#        self.K = mat(zeros((self.m,self.m)))
#        for i in range(self.m):
#            self.K[:,i] = kernelTrans(self.X, self.X[i,:], kTup)


# 模块化 计算fXk的值 
def calcEk(oS, k):
    fXk = float(multiply(oS.alphas,oS.labelMat).T*(oS.X*oS.X[k,:].T)) + oS.b
    Ek = fXk - float(oS.labelMat[k])
    return Ek

def selectJ(i, oS, Ei):         #this is the second choice -heurstic, and calcs Ej
    maxK = -1; maxDeltaE = 0; Ej = 0
    oS.eCache[i] = [1,Ei]  #set valid #choose the alpha that gives the maximum delta E
    validEcacheList = nonzero(oS.eCache[:,0].A)[0]
    if (len(validEcacheList)) > 1:
        for k in validEcacheList:   #loop through valid Ecache values and find the one that maximizes delta E
            if k == i: continue #don't calc for i, waste of time
            Ek = calcEk(oS, k)
            deltaE = abs(Ei - Ek)
            if (deltaE > maxDeltaE):
                maxK = k; maxDeltaE = deltaE; Ej = Ek
        return maxK, Ej
    else:   #in this case (first time around) we don't have any valid eCache values
        j = selectJrand(i, oS.m)
        Ej = calcEk(oS, j)
    return j, Ej


def updateEk(oS, k):#after any alpha has changed update the new value in the cache
    Ek = calcEk(oS, k)
    oS.eCache[k] = [1,Ek]


def innerL(i, oS):
    Ei = calcEk(oS, i)
    if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):
        j,Ej = selectJ(i, oS, Ei) #this has been changed from selectJrand
        alphaIold = oS.alphas[i].copy(); alphaJold = oS.alphas[j].copy();
        if (oS.labelMat[i] != oS.labelMat[j]):
            L = max(0, oS.alphas[j] - oS.alphas[i])
            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])
        else:
            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)
            H = min(oS.C, oS.alphas[j] + oS.alphas[i])
        if L==H: 
            print "L==H"; return 0
#        eta = 2.0 * oS.K[i,j] - oS.K[i,i] - oS.K[j,j] #changed for kernel
        eta = 2.0 *oS.X[i,:]*oS.X[j,:].T-oS.X[i,:]*oS.X[i,:].T-oS.X[j,:]*oS.X[j,:].T
        if eta >= 0: 
            print "eta>=0"; return 0
        oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/eta
        oS.alphas[j] = clipAlpha(oS.alphas[j],H,L)
        updateEk(oS, j) #added this for the Ecache
        if (abs(oS.alphas[j] - alphaJold) < 0.00001): print "j not moving enough"; return 0
        oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])#update i by the same amount as j
        updateEk(oS, i) #added this for the Ecache                    #the update is in the oppostie direction
#        b1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,i] - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[i,j]
#        b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,j]- oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[j,j]
        b1=oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[i,:].T-oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[i,:]*oS.X[j,:].T
        b2=oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[j,:].T-oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[j,:]*oS.X[j,:].T
        if (0 < oS.alphas[i]) and (oS.C > oS.alphas[i]): 
            oS.b = b1
        elif (0 < oS.alphas[j]) and (oS.C > oS.alphas[j]): 
            oS.b = b2
        else: 
            oS.b = (b1 + b2)/2.0
        return 1
    else: 
        return 0


# 相当于是 启动函数    
def smoP(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)):    #full Platt SMO
    oS = optStruct(mat(dataMatIn),mat(classLabels).transpose(),C,toler)
    iter = 0
    entireSet = True; alphaPairsChanged = 0
    while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):
        alphaPairsChanged = 0
        if entireSet:   #go over all
            for i in range(oS.m):        
                alphaPairsChanged += innerL(i,oS)
                print "fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)
            iter += 1
        else:#go over non-bound (railed) alphas
            nonBoundIs = nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]
            for i in nonBoundIs:
                alphaPairsChanged += innerL(i,oS)
                print "non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)
            iter += 1
        # 1: 在所有数据集上单遍扫描
        # 2：在非边界 alpha 中 实现单遍扫描
        # 两种交替执行
        if entireSet: 
            entireSet = False #toggle entire set loop
        elif (alphaPairsChanged == 0): 
            entireSet = True  
        print "iteration number: %d" % iter
    return oS.b,oS.alphas


# 计算w值，参考周志华《机器学习》P124 式6.12
def calcWs(alphas,dataArr,classLabels):
    X = mat(dataArr); labelMat = mat(classLabels).transpose()
    m,n = shape(X)
    w = zeros((n,1))
    for i in range(m):
        w += multiply(alphas[i]*labelMat[i],X[i,:].T)
    return w


# 测试模块
if __name__ == "__main__" :
    dataArr,labelArr = loadDataSet('testSet.txt')
#    print(labelArr)
##     数据集，类别标签，常数C，容错率toler，退出前的最大循环次数maxIter
#    b,alphas=smoSimple(dataArr,labelArr,0.6,0.001,40)
#    print(b)
#    print(alphas)
#    print(alphas[alphas>0])
#    print(shape(alphas[alphas>0]))
#    for i in range(100):
#        if alphas[i]>0.0:
#            print(dataArr[i],labelArr[i])

    b,alphas=smoP(dataArr,labelArr,0.6,0.001,40)
    print(b)
    print(alphas)
    print(alphas[alphas>0])
    print(shape(alphas[alphas>0]))
    for i in range(100):
        if alphas[i]>0.0:
            print(dataArr[i],labelArr[i])
    ws=calcWs(alphas,dataArr,labelArr)
    print(ws)
    print("--------------------------------")
    dataMat=mat(dataArr)
    pre_result=dataMat[2]*mat(ws)+b
    print("预测为："+str(pre_result))
    print("实际结果为："+str(labelArr[2]))