深度学习序列——稀疏自编码器模型(SAP)

1. 模型简介

(1) 模型要能够学习恒等变换

(2) 隐藏层的神经元的活动性满足稀疏性(模型大脑)

(3) 隐藏层的权重矩阵同样满足稀疏性(一种正则化方法，实际上，在稀疏编码取得好的效果有启发作用)

2. 模型实现

# Sparse Auto-Encoder 
#
# Author: HSW
# Date: 2018-05-07
#

import tensorflow as tf
import numpy      as np

def axvier_init(fan_in, fan_out, constant = 1):
    ''' Initial weights '''
    low   = -constant + np.sqrt(6.0 / (fan_in + fan_out))
    high =  constant + np.sqrt(6.0 / (fan_in + fan_out))

    return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)


class SparseAutoEncoder(object):

    def __init__(self, AutoEncoder_Shape = (128, 256, 128), transfer_function = tf.nn.softplus, optimizer = tf.train.AdamOptimizer(), p = 0.05, sparse_beta = 0.1, weights_lambda = 0.00001):
        ''' Constructor Funcion '''
        self.LayerCnt          = len(AutoEncoder_Shape)
        self.AutoEncoder_Shape = AutoEncoder_Shape
        self.transfer          = transfer_function
        self.p                 = p
        self.sparse_beta       = sparse_beta 
        self.weights_lambda    = weights_lambda
        self.weights           = self.init_weights()
        self.layers            = self.init_layers()
        self.total_loss        = self.init_loss()
        self.optimizer         = optimizer.minimize(self.total_loss)
        init                   = tf.global_variables_initializer()
        self.sess              = tf.Session()
        self.sess.run(init) 
        


    def init_weights(self):
        ''' AutoEncoder Weights ''' 
        all_weights = dict()
        for iter in range(self.LayerCnt - 1):
            weightName              = "weight" + str(iter + 1)
            biasName                = "bias"   + str(iter + 1)
            if iter == self.LayerCnt - 2: 
                all_weights[weightName] = tf.Variable(axvier_init(self.AutoEncoder_Shape[iter], self.AutoEncoder_Shape[iter + 1]))
                all_weights[biasName]   = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter + 1]]))
            else:
                all_weights[weightName] = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter], self.AutoEncoder_Shape[iter + 1]]))
                all_weights[biasName]   = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter + 1]]))

        return all_weights

    def init_layers(self):
        ''' Sparse AutoEncoder: Input Layer + Hidden Layer + Output Layer ''' 
        all_layers = dict()
        all_layers['inputs'] = tf.placeholder(tf.float32, [None, self.AutoEncoder_Shape[0]])
        
        for iter in range(self.LayerCnt - 1):
            if iter == 0:
                # inputs Layer => 1th Hidden Layer
                preLayerName          = 'inputs'
                layerName             = 'hidden' + str(iter + 1)
                weightName            = 'weight' + str(iter + 1)
                biasName              = 'bias'   + str(iter + 1) 
                all_layers[layerName] = self.transfer(tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName]))
            elif iter == self.LayerCnt - 2:
                # Last Hidden Layer => outputs Layer
                preLayerName          = 'hidden' + str(iter) 
                layerName             = 'outputs'
                weightName            = 'weight' + str(iter + 1)
                biasName              = 'bias'   + str(iter + 1)
                all_layers[layerName] = tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName])
            else:
                # pre-Hidden Layer => cur-Hidden Layer
                preLayerName          = 'hidden' + str(iter)
                layerName             = 'hidden' + str(iter + 1)
                weightName            = 'weight' + str(iter + 1)
                biasName              = 'bias'   + str(iter + 1)
                all_layers[layerName] = self.transfer(tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName]))

        return all_layers 
                
                
    def init_loss(self):
        ''' Sparse AutoEncoder Loss '''
        # data loss 
        data_loss         = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.layers['outputs'], self.layers['inputs']), 2))
        # sparse loss 
        active_loss       = 0
        weights_loss      = 0
        
        for iter in range(self.LayerCnt - 1):
            weightName    = 'weight' + str(iter + 1) 
            weights_loss += tf.norm(self.weights[weightName], ord='fro', axis=[-2, -1])
            
        for iter in range(self.LayerCnt - 2):
            hiddenName    = 'hidden' + str(iter + 1)
            sparse_loss   = tf.reduce_mean(self.layers[hiddenName], 0)
            active_loss  += sparse_loss * tf.log(sparse_loss / self.p) + (1 - sparse_loss) * tf.log((1 - self.p) / (1 - sparse_loss))

        return data_loss + self.sparse_beta * active_loss + self.weights_lambda * weights_loss 

    def partial_fit(self, inputs):
        ''' For train '''
        cost, opt = self.sess.run((self.total_loss, self.optimizer), feed_dict={self.layers['inputs']: inputs})

        return cost

    def calc_total_cost(self, inputs):
        ''' For evalution '''
        
        return self.sess.run(self.total_loss, feed_dict={self.inputs: inputs})

      

    def transform(self, inputs, layerNum):
        ''' inputs => Encode Feature '''
        hiddenName = 'hidden' + str(layerNum)
        
        return self.sess.run(self.layers[hiddenName], feed_dict={self.layers['inputs']: inputs})

    def generate(self, hidden = None, layerNum = 1):
        ''' Encode Feature => outputs '''
        hiddenName = 'hidden' + str(layerNum)

        return self.sess.run(self.layers['outputs'], feed_dict={self.layers[hiddenName]: hidden})

    def reconstruct(self, inputs):
        ''' inputs => outputs '''

        return self.sess.run(self.layers['outputs'], feed_dict={self.layers['inputs']: inputs})

    def getWeigths(self, layerNum):
        ''' weight of layerNum-th layer '''
        weightName = 'weight' + str(layerNum)
        
        return self.sess.run(self.weights[weightName])

    def getBiases(self, layerNum):
        ''' bias of layerNum-th layer '''
        biasName = 'bias' + str(layerNum)

        return self.sess.run(self.weights[biasName]) 
        
    
if __name__ == "__main__":

    sparseAutoEncoder = SparseAutoEncoder();

    print(sparseAutoEncoder.layers)
    print(sparseAutoEncoder.weights['weight1'])
    print(sparseAutoEncoder.weights['bias1'])
    print(sparseAutoEncoder.weights['weight2'])
    print(sparseAutoEncoder.weights['bias2'])

3. 模型测试