Machine Learning -SVM- handwriting recognition problem

Machine Learning -SVM- handwriting recognition problem

Here we addressed before with KNN still had solved the problem of handwriting recognition (https://www.cnblogs.com/jiading/p/11622019.html), but compared to the KNN, SVM good place that once our model training is completed, we can get a decision hyperplane determined, of course, this hyper-plane w with all the support vectors to describe, which means we only need to release the model includes all support, including vectors on it , all the remaining vectors can be discarded, and each time compared to the needs of all vectors KNN, which greatly reduces the size of the model.

Note that this move is a dichotomous example, if the multi-classification, then the need for other configurations, there are three construction ideas, see here: https: //www.cnblogs.com/cheesezh/p/5265959.html

'''
Created on Nov 4, 2010
Chapter 5 source file for Machine Learing in Action
@author: Peter
'''
from numpy import *
from time import sleep
def selectJrand(i,m):#在某个区间范围内随机选择一个整数
    #i为第一个alhpa的下表，m是所有alpha的数目
    j=i #we want to select any J not equal to i
    while (j==i):
        j = int(random.uniform(0,m))
    return j

def clipAlpha(aj,H,L):#在数值太大的时候对其进行调整
    #aj是H是下限，是L的上限
    if aj > H: 
        aj = H
    if L > aj:
        aj = L
    return aj
def kernelTrans(X, A, kTup): #calc the kernel or transform data to a higher dimensional space
    #X是数据，A是
    m,n = shape(X)
    K = mat(zeros((m,1)))
    #这里为了简单，我们只内置了两种核函数，但是原理是一样的，需要的话再写其他类型就是了
    #线性核函数：k(x,x_i)=x*x_i,它不需要再传入参数，这个其实就是我们之前用的那种，dataMatrix*dataMatrix[j,:].T
    if kTup[0]=='lin': K = X * A.T   #linear kernel，线性核函数
    elif kTup[0]=='rbf':#高斯核函数，传入的参数作为detla
        for j in range(m):
            deltaRow = X[j,:] - A
            K[j] = deltaRow*deltaRow.T#2范数
        K = exp(K/(-1*kTup[1]**2)) #divide in NumPy is element-wise not matrix like Matlab
        #numpy中，/表示对矩阵元素进行计算而不是计算逆（MATLAB）
    else: raise NameError('Houston We Have a Problem -- \
    That Kernel is not recognized')#2333老师玩梗
    return K
#定义了一个类来进行SMO算法
class optStruct:
    def __init__(self,dataMatIn, classLabels, C, toler, kTup):  # Initialize the structure with the parameters 
        #kTup储存核函数信息，它是一个元组，元组第一个元素是一个描述核函数类型的字符串，其他两个元素是核函数可能需要的参数
        self.X = dataMatIn
        self.labelMat = classLabels
        self.C = C
        self.tol = toler
        self.m = shape(dataMatIn)[0]#m是样本个数，也是a的个数
        self.alphas = mat(zeros((self.m,1)))#初始化a序列，都设置为0
        self.b = 0
        #第一列给出的是eCache是否有效的标志位，而第二位是实际的E值
        self.eCache = mat(zeros((self.m,2))) #first column is valid flag
        #如果第一位是1，说明现在的这个Ek是有效的
        self.K = mat(zeros((self.m,self.m)))#使用核函数计算后的数据，就是内积矩阵，方便直接调用内积结果
        for i in range(self.m):
            self.K[:,i] = kernelTrans(self.X, self.X[i,:], kTup)
        
def calcEk(oS, k):#计算第k个样本的Ek
    fXk = float(multiply(oS.alphas,oS.labelMat).T*oS.K[:,k] + oS.b)
    Ek = fXk - float(oS.labelMat[k])
    return Ek
        
def selectJ(i, oS, Ei):         #this is the second choice -heurstic, and calcs Ej
    #选定了a_1之后选择a_2
    #选择a_2
    maxK = -1; maxDeltaE = 0; Ej = 0#选择|E1-E2|最大的E2并返回E2和j
    #先将E1存进去，以便于之后的统一化进行
    oS.eCache[i] = [1,Ei]  #set valid #choose the alpha that gives the maximum delta E
    '''
    numpy函数返回非零元素的目录。
    返回值为元组， 两个值分别为两个维度， 包含了相应维度上非零元素的目录值。   
    可以通过a[nonzero(a)]来获得所有非零值。
    .A的意思是：getArray(),也就是将矩阵转换为数组
    '''
        #获取哪些样本的Ek是有效的，ValidEcacheList里面存的是所有有效的样本行Index
    validEcacheList = nonzero(oS.eCache[:,0].A)[0]
    #对每一个有效的Ecache都比较一遍
    if (len(validEcacheList)) > 1:
        for k in validEcacheList:   #loop through valid Ecache values and find the one that maximizes delta E
            if k == i: continue #don't calc for i, waste of time
                #如果选到了和a1一样的，就继续，因为a1和a2必须选不一样的样本
            Ek = calcEk(oS, k)
            deltaE = abs(Ei - Ek)
            if (deltaE > maxDeltaE):
                maxK = k; maxDeltaE = deltaE; Ej = Ek
        return maxK, Ej
    else:   #in this case (first time around) we don't have any valid eCache values
        j = selectJrand(i, oS.m)
        Ej = calcEk(oS, j)
    return j, Ej
def updateEk(oS, k):#after any alpha has changed update the new value in the cache
    Ek = calcEk(oS, k)
    oS.eCache[k] = [1,Ek]
    #计算Ek并保存在类oS中
#内循环寻找合适的a_2        
def innerL(i, oS):
    Ei = calcEk(oS, i)#为什么这里要重新算呢？因为a_1刚刚更新了，和存储的不一样
    if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):
        #如果a1选的合适的话，不合适就直接结束了
        #剩下的逻辑都一样，只不过不是使用x_ix_j，而是使用核函数
        j,Ej = selectJ(i, oS, Ei) #this has been changed from selectJrand
        alphaIold = oS.alphas[i].copy(); alphaJold = oS.alphas[j].copy();
        if (oS.labelMat[i] != oS.labelMat[j]):
            L = max(0, oS.alphas[j] - oS.alphas[i])
            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])
        else:
            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)
            H = min(oS.C, oS.alphas[j] + oS.alphas[i])
        if L==H: print ("L==H"); return 0
        eta = 2.0 * oS.K[i,j] - oS.K[i,i] - oS.K[j,j] #changed for kernel
        if eta >= 0: print ("eta>=0"); return 0
        oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/eta
        oS.alphas[j] = clipAlpha(oS.alphas[j],H,L)
        updateEk(oS, j) #added this for the Ecache
        if (abs(oS.alphas[j] - alphaJold) < 0.00001): print ("j not moving enough"); return 0
        oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])#update i by the same amount as j
        updateEk(oS, i) #added this for the Ecache                    #the update is in the oppostie direction
        b1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,i] - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[i,j]
        b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,j]- oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[j,j]
        if (0 < oS.alphas[i]) and (oS.C > oS.alphas[i]): oS.b = b1
        elif (0 < oS.alphas[j]) and (oS.C > oS.alphas[j]): oS.b = b2
        else: oS.b = (b1 + b2)/2.0
        return 1
    else: return 0

def smoP(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)):#默认核函数是线性，参数为0（那就是它本身了）
    #这个kTup先不管，之后用
    oS = optStruct(mat(dataMatIn),mat(classLabels).transpose(),C,toler, kTup)#初始化这一结构
    iter = 0
    #entireSet是控制开关，一次循环对所有样本点都遍历，第二次就只遍历非边界点
    entireSet = True; alphaPairsChanged = 0
    while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):
        alphaPairsChanged = 0
        if entireSet:   #go over all
            for i in range(oS.m):        
                alph aPairsChanged += innerL(i,oS)
                print ("fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        else:#go over non-bound (railed) alphas
            #把大于0且小于C的a_i挑出来，这些是非边界点，只从这些点上遍历
            nonBoundIs = nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]
            for i in nonBoundIs:
                alphaPairsChanged += innerL(i,oS)
                print ("non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        #如果第一次是对所有点进行的，那么第二次就只对非边界点进行
        if entireSet: entireSet = False #toggle entire set loop
        #如果对非边界点进行后没有，就在整个样本上进行
        '''
        首先在非边界集上选择能够使函数值足够下降的样本作为第二个变量，
        如果非边界集上没有，则在整个样本集上选择第二个变量，
        如果整个样本集依然不存在，则重新选择第一个变量。
        '''
        elif (alphaPairsChanged == 0): entireSet = True  
        print ("iteration number: %d" % iter)
    return oS.b,oS.alphas
#利用SVM进行分类，返回的是函数间隔，大于0属于1类，小于0属于2类。
def calcWs(alphas,dataArr,classLabels):
    X = mat(dataArr); labelMat = mat(classLabels).transpose()
    m,n = shape(X)
    w = zeros((n,1))
    for i in range(m):
        w += multiply(alphas[i]*labelMat[i],X[i,:].T)
    return w
#将图像转换为向量
def img2vector(filename):
    #一共有1024个特征
    returnVect = zeros((1,1024))
    fr = open(filename)
    for i in range(32):
        lineStr = fr.readline()
        for j in range(32):
            returnVect[0,32*i+j] = int(lineStr[j])
    return returnVect

def loadImages(dirName):
    from os import listdir
    hwLabels = []
    #利用listdir读文件名，这里的label写在了文件名里面
    trainingFileList = listdir(dirName)           #load the training set
    m = len(trainingFileList)
    trainingMat = zeros((m,1024))
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]     #take off .txt
        classNumStr = int(fileStr.split('_')[0])
        if classNumStr == 9: hwLabels.append(-1)
        else: hwLabels.append(1)
        trainingMat[i,:] = img2vector('%s/%s' % (dirName, fileNameStr))
    return trainingMat, hwLabels    
#手写识别问题
def testDigits(kTup=('rbf', 10)):
    dataArr,labelArr = loadImages('trainingDigits')
    b,alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, kTup)
    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()
    svInd=nonzero(alphas.A>0)[0]
    sVs=datMat[svInd] 
    labelSV = labelMat[svInd];
    print ("there are %d Support Vectors" % shape(sVs)[0])
    m,n = shape(datMat)
    errorCount = 0
    for i in range(m):
        kernelEval = kernelTrans(sVs,datMat[i,:],kTup)
        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b
        if sign(predict)!=sign(labelArr[i]): errorCount += 1
    print ("the training error rate is: %f" % (float(errorCount)/m))
    dataArr,labelArr = loadImages('testDigits')
    errorCount = 0
    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()
    m,n = shape(datMat)
    for i in range(m):
        kernelEval = kernelTrans(sVs,datMat[i,:],kTup)
        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b
        if sign(predict)!=sign(labelArr[i]): errorCount += 1    
    print ("the test error rate is: %f" % (float(errorCount)/m) )