The principle of SVM

SVM code is realization of the principle

1) Calculation Kernel

def calcKernelValue(matrix_x, sample_x, kernelOption)：
    	kernelType = kernelOption[0]
    	numSamples = matrix_x.shape[0]
    	kernelValue = mat(zeros((numSamples, 1)))
	
    	if kernelType == 'linear':
        	kernelValue = matrix_x * sample_x.T
    	elif kernelType == 'rbf':
        	sigma = kernelOption[1]
        	if sigma == 0:
            	sigma = 1.0
        	for i in range(numSamples):
            	diff = matrix_x[i, :] - sample_x
            	kernelValue[i] = exp(diff * diff.T / (-2.0 * sigma ** 2))
    	else:
        	raise NameError('Not support kernel type! You can use linear or rbf!')
    	return kernelValue

2) Select the maximum number of iterations aj

Step One: Select aj

def selectAlpha_j(svm, alpha_i, error_i):
    	svm.errorCache[alpha_i] = [1, error_i]  # mark as valid(has been optimized)
    	candidateAlphaList = nonzero(svm.errorCache[:, 0].A)[0]  # mat.A return array
    	maxStep = 0;
    	alpha_j = 0;
    	error_j = 0

    	# find the alpha with max iterative step
    	if len(candidateAlphaList) > 1:
        	for alpha_k in candidateAlphaList:
            	if alpha_k == alpha_i:
                	continue
            	error_k = mysvm.calcError(svm, alpha_k)
            	if abs(error_k - error_i) > maxStep:
                	maxStep = abs(error_k - error_i)
                	alpha_j = alpha_k
                	error_j = error_k
    	# if came in this loop first time, we select alpha j randomly
    	else:
        	alpha_j = alpha_i
        	while alpha_j == alpha_i:
            	alpha_j = int(random.uniform(0, svm.numSamples))
        	error_j = mysvm.calcError(svm, alpha_j)

    	return alpha_j, error_j

Step two: calculate upper and lower bounds

alpha_j, error_j = mysvm.selectAlpha_j(svm, alpha_i, error_i)
	alpha_i_old = svm.alphas[alpha_i].copy()
	alpha_j_old = svm.alphas[alpha_j].copy()

Third step: calculation of ai and aj sample correlation

eta = 2.0 * svm.kernelMat[alpha_i, alpha_j] - svm.kernelMat[alpha_i, alpha_i] 
      - svm.kernelMat[alpha_j, alpha_j]
	if eta >= 0:
    	return 0

Step four: Update aj

svm.alphas[alpha_j] -= svm.train_y[alpha_j] * (error_i - error_j) / eta

Step Five: Make sure aj within the bounds range

if svm.alphas[alpha_j] > H:
    	svm.alphas[alpha_j] = H
	if svm.alphas[alpha_j] < L:
    	svm.alphas[alpha_j] = L

Sixth Step: Update ai aj updated according to

svm.alphas[alpha_i] += svm.train_y[alpha_i] * svm.train_y[alpha_j] \
                       * (alpha_j_old - svm.alphas[alpha_j])

Step Seven: Update Threshold b

b1 = svm.b - error_i - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \
     * svm.kernelMat[alpha_i, alpha_i] \
     - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \
     * svm.kernelMat[alpha_i, alpha_j]
	b2 = svm.b - error_j - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \
     * svm.kernelMat[alpha_i, alpha_j] \
     - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \
     * svm.kernelMat[alpha_j, alpha_j]
	if (0 < svm.alphas[alpha_i]) and (svm.alphas[alpha_i] < svm.C):
    	svm.b = b1
	elif (0 < svm.alphas[alpha_j]) and (svm.alphas[alpha_j] < svm.C):
    	svm.b = b2
	else:
    	svm.b = (b1 + b2) / 2.0