LSTM是RNN的一个延伸算法，一般可以用在文字处理等序列化内容上，下面有一个小的案例，是讲了二进制和十进制之间转换的一个场景，希望对初学者有些帮助：
#!/usr/bin/env python
# _*_ UTF-8 _*_
import tensorflow as tf
import numpy as np
from _operator import add
# 用于映射数字到对应的二进制
int2binary = {}
# 二进制的最大维度为8
binary_dim = 8
# 在二进制的情况下，可以表示的最大的int数是多大
largest_number = pow(2, binary_dim)
# np.unpackbits可以将整数转化成二进制
# np.array([range(largest_number)], dtype=np.uint8).T：定义一个数组np.array,数组的长度为range(largest_number)，即（0,256）的范围的数组；
binary = np.unpackbits(np.array([range(largest_number)], dtype=np.uint8).T, axis = 1)
for i in range(largest_number):
int2binary[i] = binary[i]
# 上面的int2binary记录了所有int值对应的二进制表达。
# 返回numbers中数值的二进制表达
def binary_generation(numbers, reverse = False):
# 得到二进制的x值
binary_x = np.array([int2binary[num] for num in numbers], dtype = np.uint8)
if reverse:
# 将矩阵的列绕垂直轴，左右反转
binary_x = np.fliplr(binary_x)
return binary_x
def batch_generation(batch_size, largest_number):
# np.random.randint(a,b, size = (n,m))：随机生成一个【a，b）范围，尺寸为（n,m）的随机数
# largest_number//2:表示最大值除以2
n1 = np.random.randint(0, largest_number//2, batch_size)
n2 = np.random.randint(0, largest_number//2, batch_size)
# 相加的是两个int型的矩阵
add = n1 + n2
binary_n1 = binary_generation(n1, True)
binary_n2 = binary_generation(n2, True)
batch_y = binary_generation(add, True)
# 将binary_n1, binary_n2两个值摞在一起，
batch_x = np.dstack((binary_n1, binary_n2))
return batch_x, batch_y, n1, n2, add
# batch_x：n1、n2对应的二进制；
# batch_y：add对应的二进制；
# n1：十进制数据
# n2：十进制数据
# add：求和
# 将二进制转化成int型。
def binary2int(binary_array):
out = 0
# 枚举list中的值，通过循环返回序号和value；
for index, x in enumerate(reversed(binary_array)):
out+=x*pow(2, index)
return out
batch_size = 64#
lstm_size = 20 # cell中神经网络的隐藏层有20个节点，一层有20个节点，这一值得数量也决定了ht的向量长度。
lstm_layers = 2# 共有两个cell
x = tf.placeholder(tf.float32, [None, binary_dim, 2], name='input_x')
y_ = tf.placeholder(tf.float32, [None, binary_dim], name='input_y')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# BasicLSTMCell(num_units, forget_bias, state_is_true, activation, reuse, name)：
# num_units表示cell中隐藏层的个数，即cell中神经网络的层数；
# forget_bias:表示遗忘门的偏置；
lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
# 对Dropout进行防过拟合操作。
drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob = keep_prob)
def lstm_cell():
return tf.contrib.rnn.BasicLSTMCell(lstm_size)
# 构建多个cell，一共构建2层，形成一个复合的cell数据：
cell = tf.contrib.rnn.MultiRNNCell([lstm_cell() for _ in range(lstm_layers)])
# 上面模型构建完成
# embedding是指将文本编码成向量；
# 构建全零的初始状态
initial_state = cell.zero_state(batch_size, tf.float32)
# 对rnn进行训练返回outputs, final_state，即有几个节点，每个节点一个LSTM，将数据和cell放入其中，dynamic_rnn会循环的
# 计算每个节点h值final_state，以及输出y值outputs，多次循环，直到出现最终结果；
outputs, final_state = tf.nn.dynamic_rnn(cell, x, initial_state=initial_state)
weights = tf.Variable(tf.truncated_normal([lstm_size, 1], stddev=0.01))
bias = tf.zeros([1])
outputs = tf.reshape(outputs, [-1, lstm_size])
# 将输出值进行一次激活函数处理，然后变形得到预测值
logits = tf.sigmoid(tf.matmul(outputs, weights))
predictions = tf.reshape(logits, [-1, binary_dim])
# 使用均方误差
cost = tf.losses.mean_squared_error(y_, predictions)
optimizer = tf.train.AdamOptimizer().minimize(cost)
steps = 2000
with tf.Session() as sess:
tf.global_variables_initializer().run()
iteration = 1
for i in range(steps):
input_x, input_y, _,_,_ = batch_generation(batch_size, largest_number)
_, loss = sess.run([optimizer, cost], feed_dict={x:input_x, y_:input_y, keep_prob:0.5})
if iteration%100 == 0:
print('Iter:{}, Loss:{}'.format(iteration, loss))
iteration += 1
val_x, val_y, n1, n2, add = batch_generation(batch_size, largest_number)
result = sess.run(predictions, feed_dict={x:val_x, y_:val_y, keep_prob:1.0})
result = np.fliplr(np.round(result))
result = result.astype(np.int32)
for b_x, b_p, a, b, add in zip(np.fliplr(val_x), result, n1, n2, add):
print('{}:{}'.format(b_x[:,0], a))
print('{}:{}'.format(b_x[:,1], b))
print('{}:{}\n'.format(b_p, binary2int(b_p)))
# 1）rnn的参数是共享的；
# 2）rnn模式的常用结构为：
# RNN及其变种：主要是N2N；N21；12N；N2M等结构；
# seq2seq结构：是由两个RNN组成，来实现多对多的输出；
# attention机制：主要是解决C点的信息超载问题;
另外对于RNN/LSTM算法网上还有一个案例，可以参考，供大家练习：
#!/usr/bin/env python
# _*_ UTF-8 _*_
import time
import reader
import numpy as np
import tensorflow as tf
flags = tf.flags
logging = tf.logging
flags.DEFINE_string("model", "small","A type of model. Possible options are: small, medium, large.")
flags.DEFINE_string("data_path", '/home/multiangle/download/simple-examples/data/', "data_path")
flags.DEFINE_bool("use_fp16", False,"Train using 16-bit floats instead of 32bit floats")
FLAGS = flags.FLAGS
def data_type():
return tf.float16 if FLAGS.use_fp16 else tf.float32
class PTBModel(object):
"""The PTB model."""
def __init__(self, is_training, config):
"""
:param is_training: 是否要进行训练.如果is_training=False,则不会进行参数的修正。
"""
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
size = config.hidden_size
vocab_size = config.vocab_size
self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps]) # 输入
self._targets = tf.placeholder(tf.int32, [batch_size, num_steps]) # 预期输出，两者都是index序列，长度为num_step
# Slightly better results can be obtained with forget gate biases
# initialized to 1 but the hyperparameters of the model would need to be
# different than reported in the paper.
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(size, forget_bias=0.0, state_is_tuple=True)
if is_training and config.keep_prob < 1: # 在外面包裹一层dropout
lstm_cell = tf.nn.rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=config.keep_prob)
cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell] * config.num_layers, state_is_tuple=True) # 多层lstm cell 堆叠起来
self._initial_state = cell.zero_state(batch_size, data_type()) # 参数初始化,rnn_cell.RNNCell.zero_state
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size], dtype=data_type())
# vocab size * hidden size, 将单词转成embedding描述
# 将输入seq用embedding表示, shape=[batch, steps, hidden_size]
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
# Simplified version of tensorflow.models.rnn.rnn.py's rnn().
# This builds an unrolled LSTM for tutorial purposes only.
# In general, use the rnn() or state_saving_rnn() from rnn.py.
# The alternative version of the code below is:
# inputs = [tf.squeeze(input_, [1])
# for input_ in tf.split(1, num_steps, inputs)]
# outputs, state = tf.nn.rnn(cell, inputs, initial_state=self._initial_state)
outputs = []
state = self._initial_state # state 表示 各个batch中的状态
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
# cell_out: [batch, hidden_size]
(cell_output, state) = cell(inputs[:, time_step, :], state)
outputs.append(cell_output) # output: shape[num_steps][batch,hidden_size]
# 把之前的list展开，成[batch, hidden_size*num_steps],然后 reshape, 成[batch*numsteps, hidden_size]
output = tf.reshape(tf.concat(1, outputs), [-1, size])
# softmax_w , shape=[hidden_size, vocab_size], 用于将distributed表示的单词转化为one-hot表示
softmax_w = tf.get_variable("softmax_w", [size, vocab_size], dtype=data_type())
softmax_b = tf.get_variable("softmax_b", [vocab_size], dtype=data_type())
# [batch*numsteps, vocab_size] 从隐藏语义转化成完全表示
logits = tf.matmul(output, softmax_w) + softmax_b
# loss , shape=[batch*num_steps]
# 带权重的交叉熵计算
loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
[logits], # output [batch*numsteps, vocab_size]
[tf.reshape(self._targets, [-1])], # target, [batch_size, num_steps] 然后展开成一维【列表】
[tf.ones([batch_size * num_steps], dtype=data_type())]) # weight
self._cost = cost = tf.reduce_sum(loss) / batch_size # 计算得到平均每批batch的误差
self._final_state = state
if not is_training: # 如果没有训练，则不需要更新state的值。
return
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
# clip_by_global_norm: 梯度衰减，具体算法为t_list[i] * clip_norm / max(global_norm, clip_norm)
# 这里gradients求导，ys和xs都是张量
# 返回一个长为len(xs)的张量，其中的每个元素都是\grad{\frac{dy}{dx}}
# clip_by_global_norm 用于控制梯度膨胀,前两个参数t_list, global_norm, 则
# t_list[i] * clip_norm / max(global_norm, clip_norm)
# 其中 global_norm = sqrt(sum([l2norm(t)**2 for t in t_list]))
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), config.max_grad_norm)
# 梯度下降优化，指定学习速率
optimizer = tf.train.GradientDescentOptimizer(self._lr)
# optimizer = tf.train.AdamOptimizer()
# optimizer = tf.train.GradientDescentOptimizer(0.5)
self._train_op = optimizer.apply_gradients(zip(grads, tvars)) # 将梯度应用于变量
self._new_lr = tf.placeholder(tf.float32, shape=[], name="new_learning_rate")
# 用于外部向graph输入新的 lr值
self._lr_update = tf.assign(self._lr, self._new_lr) # 使用new_lr来更新lr的值
def assign_lr(self, session, lr_value):
# 使用 session 来调用 lr_update 操作
session.run(self._lr_update, feed_dict={self._new_lr: lr_value})
@property
def input_data(self):
return self._input_data
@property
def targets(self):
return self._targets
@property
def initial_state(self):
return self._initial_state
@property
def cost(self):
return self._cost
@property
def final_state(self):
return self._final_state
@property
def lr(self):
return self._lr
@property
def train_op(self):
return self._train_op
class SmallConfig(object):
"""Small config."""
init_scale = 0.1 #
learning_rate = 1.0 # 学习速率
max_grad_norm = 5 # 用于控制梯度膨胀，
num_layers = 2 # lstm层数
num_steps = 20 # 单个数据中，序列的长度。
hidden_size = 200 # 隐藏层规模
max_epoch = 4 # epoch<max_epoch时，lr_decay值=1,epoch>max_epoch时,lr_decay逐渐减小
max_max_epoch = 13 # 指的是整个文本循环13遍。
keep_prob = 1.0
lr_decay = 0.5 # 学习速率衰减
batch_size = 20 # 每批数据的规模，每批有20个。
vocab_size = 10000 # 词典规模，总共10K个词
class MediumConfig(object):
"""Medium config."""
init_scale = 0.05
learning_rate = 1.0
max_grad_norm = 5
num_layers = 2
num_steps = 35
hidden_size = 650
max_epoch = 6
max_max_epoch = 39
keep_prob = 0.5
lr_decay = 0.8
batch_size = 20
vocab_size = 10000
class LargeConfig(object):
"""Large config."""
init_scale = 0.04
learning_rate = 1.0
max_grad_norm = 10
num_layers = 2
num_steps = 35
hidden_size = 1500
max_epoch = 14
max_max_epoch = 55
keep_prob = 0.35
lr_decay = 1 / 1.15
batch_size = 20
vocab_size = 10000
class TestConfig(object):
"""Tiny config, for testing."""
init_scale = 0.1
learning_rate = 1.0
max_grad_norm = 1
num_layers = 1
num_steps = 2
hidden_size = 2
max_epoch = 1
max_max_epoch = 1
keep_prob = 1.0
lr_decay = 0.5
batch_size = 20
vocab_size = 10000
def run_epoch(session, model, data, eval_op, verbose=False):
"""Runs the model on the given data."""
# epoch_size 表示批次总数。也就是说，需要向session喂这么多次数据
epoch_size = ((len(data) // model.batch_size) - 1) // model.num_steps # // 表示整数除法
start_time = time.time()
costs = 0.0
iters = 0
state = session.run(model.initial_state)
for step, (x, y) in enumerate(reader.ptb_producer(data, model.batch_size, model.num_steps)):
fetches = [model.cost, model.final_state, eval_op] # 要进行的操作，注意训练时和其他时候eval_op的区别
feed_dict = {} # 设定input和target的值
feed_dict[model.input_data] = x
feed_dict[model.targets] = y
for i, (c, h) in enumerate(model.initial_state):
feed_dict[c] = state[i].c # 这部分有什么用？看不懂
feed_dict[h] = state[i].h
cost, state, _ = session.run(fetches, feed_dict) # 运行session,获得cost和state
costs += cost # 将 cost 累积
iters += model.num_steps
if verbose and step % (epoch_size // 10) == 10:
# 也就是每个epoch要输出10个perplexity值
print("%.3f perplexity: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size,
np.exp(costs / iters),
iters * model.batch_size / (time.time() - start_time)))
return np.exp(costs / iters)
def get_config():
if FLAGS.model == "small":
return SmallConfig()
elif FLAGS.model == "medium":
return MediumConfig()
elif FLAGS.model == "large":
return LargeConfig()
elif FLAGS.model == "test":
return TestConfig()
else:
raise ValueError("Invalid model: %s", FLAGS.model)
if __name__=='__main__':
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
print(FLAGS.data_path)
raw_data = reader.ptb_raw_data(FLAGS.data_path) # 获取原始数据
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale, # 定义如何对参数变量初始化
config.init_scale)
with tf.variable_scope("model", reuse=None,initializer=initializer):
m = PTBModel(is_training=True, config=config) # 训练模型， is_trainable=True
with tf.variable_scope("model", reuse=True,initializer=initializer):
mvalid = PTBModel(is_training=False, config=config) # 交叉检验和测试模型，is_trainable=False
mtest = PTBModel(is_training=False, config=eval_config)
summary_writer = tf.summary.FileWriter('/tmp/lstm_logs',session.graph)
tf.initialize_all_variables().run() # 对参数变量初始化
for i in range(config.max_max_epoch): # 所有文本要重复多次进入模型训练
# learning rate 衰减
# 在 遍数小于max epoch时， lr_decay = 1 ; > max_epoch时， lr_decay = 0.5^(i-max_epoch)
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay) # 设置learning rate
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, train_data, m.train_op,verbose=True) # 训练困惑度
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op()) # 检验困惑度
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest, test_data, tf.no_op()) # 测试困惑度
print("Test Perplexity: %.3f" % test_perplexity)
if __name__ == "__main__":
tf.app.run()