Tensorflow2.0 循环神经网络 LSTM与GRU

LSTM

CELL LSTM循环神经网络单元

有两个记忆参数

self.state0 = [tf.zeros([batch_size, units]), tf.zeros([batch_size, units])]  # 层1 记忆参数

import numpy as np
import tensorflow as tf
from tensorflow import keras

print(tf.__version__, np.__version__)

# 设置随机种子
tf.random.set_seed(22)
tf.random.set_seed(22)

# GPU自增长
gpus = tf.config.experimental.list_physical_devices('gpu')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# 设置参数
total_words = 10000  # 所有单词类型
max_review_len = 80  # 句子最大长度
batch_size = 64

# 数据加载 新闻分类
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

# 数据集 整理
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batch_size, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batch_size, drop_remainder=True)

print("x_train shape:", x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

# 最常出现的单词 独热编码长度
embedding_len = 100


# 定义RNN类
class MyRNN(tf.keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # [b, 64]  c 和 h的参数
        self.state0 = [tf.zeros([batch_size, units]), tf.zeros([batch_size, units])]  # 层1 记忆参数
        self.state1 = [tf.zeros([batch_size, units]), tf.zeros([batch_size, units])]  # 层2 记忆参数

        # embedding层  [b, 80] ==> [b, 80, 100]
        self.embedding = tf.keras.layers.Embedding(total_words,
                                                   embedding_len,
                                                   input_length=max_review_len)
        # 简单循环神经网络单元 [b, 80, 100] ==> h_dim:64
        # self.rnn_cell0 = tf.keras.layers.SimpleRNNCell(units=units, dropout=0.5)
        # self.rnn_cell1 = tf.keras.layers.SimpleRNNCell(units=units, dropout=0.5)

        self.rnn_cell0 = tf.keras.layers.LSTMCell(units=units, dropout=0.5)
        self.rnn_cell1 = tf.keras.layers.LSTMCell(units=units, dropout=0.5)

        # 全连接层 预测结果 [b, 64] ==> [b, 1]  好评与坏评
        self.outlayer = tf.keras.layers.Dense(1)

    # 堆叠网络
    def call(self, inputs, training=None, **kwargs):
        # inputs:[b, 80]
        # training：默认为  训练模式
        # 输入层
        x = inputs
        # [b, 80] ==> [b, 80, 100] predict
        # embedding层
        x = self.embedding(x)
        # rnn [b, 80, 100] ==> [b, 64]
        # 记忆参数
        state0 = self.state0
        state1 = self.state1

        # batch个句子  循环取第i个单词(80个单词) 处理
        for word in tf.unstack(x, axis=1):  # word[b, 100]  [b, axis=1, 100]
            # h1 = x* wxh + h0 * whh
            out0, state0 = self.rnn_cell0(word, state0, training)
            out1, state1 = self.rnn_cell1(out0, state1, training)

        # 输出层
        # out:[b, 64] ==> [b, 1]
        x = self.outlayer(out1)

        # 概率输出
        prob = tf.sigmoid(x)

        return prob


def main():
    units = 64  # 循环神经网络 宽度
    epochs = 4  # 总循环次数
    import time
    t0 = time.time()
    # 创建对象
    model = MyRNN(units)
    # 编译模型
    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'],
                  experimental_run_tf_function=False)

    # 训练模型
    model.fit(db_train, epochs=epochs, validation_data=db_test)

    # 评估模型
    model.evaluate(db_test)
    t1 = time.time()
    print('total time cost:', t1 - t0)


if __name__ == '__main__':
    main()

keras

import numpy as np
import tensorflow as tf
from tensorflow import keras

print(tf.__version__, np.__version__)

# 随机种子
tf.random.set_seed(22)
tf.random.set_seed(22)

# GPU自增长
gpus = tf.config.experimental.list_physical_devices('gpu')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# 设置参数
total_words = 10000  # 所有单词类型
max_review_len = 80  # 句子最大长度
batch_size = 128

# 数据加载 新闻分类
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

# 数据集 整理
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batch_size, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batch_size, drop_remainder=True)

print("x_train shape:", x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

# 最常出现的单词 独热编码长度
embedding_len = 100


class MyRNN(tf.keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # embedding 层
        # [b, 80] ==> [b, 80, 100]
        self.embedding = tf.keras.layers.Embedding(total_words,
                                                   embedding_len,
                                                   input_length=max_review_len)
        # 循环神经网络层 [b, 80, 100] ==> h_dim:64
        self.rnn = keras.Sequential([
            tf.keras.layers.LSTM(units, dropout=0.5, return_sequences=True, unroll=True),
            tf.keras.layers.LSTM(units, dropout=0.5, unroll=True)
        ])

        # 全连接层 [b, 64] ==> [b, 1]
        self.outlayer = tf.keras.layers.Dense(1)

    # 前向传播
    def call(self, inputs, training=None, **kwargs):
        # inputs:[b, 80]
        # training：默认为  训练模式
        x = inputs
        # embedding 层
        # [b, 80] ==> [b, 80, 100]
        x = self.embedding(x)

        # 循环神经网络层
        x = self.rnn(x)

        # 全连接 输出层
        # out:[b, 64] ==> [b, 1]
        x = self.outlayer(x)

        prob = tf.sigmoid(x)

        return prob


def main():
    units = 64  # 循环神经网络宽度
    epochs = 5  # 总循环次数
    # 4 ： 83.29%
    # 5： 82.57%
    import time
    t0 = time.time()

    # 创建网络
    model = MyRNN(units)
    # 编译网络
    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'],
                  experimental_run_tf_function=False)
    # 训练网络
    model.fit(db_train, epochs=epochs, validation_data=db_test)

    # 评估网络
    model.evaluate(db_test)

    t1 = time.time()
    print('total time cost:', t1 - t0)


if __name__ == '__main__':
    main()

GRU

CELL GRU循环神经网络单元

import numpy as np
import tensorflow as tf
from tensorflow import keras

print(tf.__version__, np.__version__)

# 设置随机种子
tf.random.set_seed(22)
tf.random.set_seed(22)

# GPU自增长
gpus = tf.config.experimental.list_physical_devices('gpu')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# 设置参数
total_words = 10000  # 所有单词类型
max_review_len = 80  # 句子最大长度
batch_size = 64

# 数据加载 新闻分类
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

# 数据集 整理
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batch_size, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batch_size, drop_remainder=True)

print("x_train shape:", x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

# 最常出现的单词 独热编码长度
embedding_len = 100


# 定义RNN类
class MyRNN(tf.keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # [b, 64]  c 和 h的参数
        self.state0 = [tf.zeros([batch_size, units])]  # 层1 记忆参数
        self.state1 = [tf.zeros([batch_size, units])]  # 层2 记忆参数

        # embedding层  [b, 80] ==> [b, 80, 100]
        self.embedding = tf.keras.layers.Embedding(total_words,
                                                   embedding_len,
                                                   input_length=max_review_len)
        # 简单循环神经网络单元 [b, 80, 100] ==> h_dim:64
        # self.rnn_cell0 = tf.keras.layers.SimpleRNNCell(units=units, dropout=0.5)
        # self.rnn_cell1 = tf.keras.layers.SimpleRNNCell(units=units, dropout=0.5)

        self.rnn_cell0 = tf.keras.layers.GRUCell(units=units, dropout=0.5)
        self.rnn_cell1 = tf.keras.layers.GRUCell(units=units, dropout=0.5)

        # 全连接层 预测结果 [b, 64] ==> [b, 1]  好评与坏评
        self.outlayer = tf.keras.layers.Dense(1)

    # 堆叠网络
    def call(self, inputs, training=None, **kwargs):
        # inputs:[b, 80]
        # training：默认为  训练模式
        # 输入层
        x = inputs
        # [b, 80] ==> [b, 80, 100] predict
        # embedding层
        x = self.embedding(x)
        # rnn [b, 80, 100] ==> [b, 64]
        # 记忆参数
        state0 = self.state0
        state1 = self.state1

        # batch个句子  循环取第i个单词(80个单词) 处理
        for word in tf.unstack(x, axis=1):  # word[b, 100]  [b, axis=1, 100]
            # h1 = x* wxh + h0 * whh
            out0, state0 = self.rnn_cell0(word, state0, training)
            out1, state1 = self.rnn_cell1(out0, state1, training)

        # 输出层
        # out:[b, 64] ==> [b, 1]
        x = self.outlayer(out1)

        # 概率输出
        prob = tf.sigmoid(x)

        return prob


def main():
    units = 64  # 循环神经网络 宽度
    epochs = 4  # 总循环次数
    # 4: 83.04%
    import time
    t0 = time.time()
    # 创建对象
    model = MyRNN(units)
    # 编译模型
    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'],
                  experimental_run_tf_function=False)

    # 训练模型
    model.fit(db_train, epochs=epochs, validation_data=db_test)

    # 评估模型
    model.evaluate(db_test)
    t1 = time.time()
    print('total time cost:', t1 - t0)


if __name__ == '__main__':
    main()

keras

import numpy as np
import tensorflow as tf
from tensorflow import keras

print(tf.__version__, np.__version__)

# 随机种子
tf.random.set_seed(22)
tf.random.set_seed(22)

# GPU自增长
gpus = tf.config.experimental.list_physical_devices('gpu')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# 设置参数
total_words = 10000  # 所有单词类型
max_review_len = 80  # 句子最大长度
batch_size = 128

# 数据加载 新闻分类
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

# 数据集 整理
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batch_size, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batch_size, drop_remainder=True)

print("x_train shape:", x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

# 最常出现的单词 独热编码长度
embedding_len = 100


class MyRNN(tf.keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # embedding 层
        # [b, 80] ==> [b, 80, 100]
        self.embedding = tf.keras.layers.Embedding(total_words,
                                                   embedding_len,
                                                   input_length=max_review_len)
        # 循环神经网络层 [b, 80, 100] ==> h_dim:64
        self.rnn = keras.Sequential([
            tf.keras.layers.GRU(units, dropout=0.5, return_sequences=True, unroll=True),
            tf.keras.layers.GRU(units, dropout=0.5, unroll=True)
        ])

        # 全连接层 [b, 64] ==> [b, 1]
        self.outlayer = tf.keras.layers.Dense(1)

    # 前向传播
    def call(self, inputs, training=None, **kwargs):
        # inputs:[b, 80]
        # training：默认为  训练模式
        x = inputs
        # embedding 层
        # [b, 80] ==> [b, 80, 100]
        x = self.embedding(x)

        # 循环神经网络层
        x = self.rnn(x)

        # 全连接 输出层
        # out:[b, 64] ==> [b, 1]
        x = self.outlayer(x)

        prob = tf.sigmoid(x)

        return prob


def main():
    units = 64  # 循环神经网络宽度
    epochs = 4  # 总循环次数
    # LSTM: 4： 83.29%
    # LSTM: 5： 82.57%
    # GRU 4: 83.53%
    import time
    t0 = time.time()

    # 创建网络
    model = MyRNN(units)
    # 编译网络
    model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'],
                  experimental_run_tf_function=False)
    # 训练网络
    model.fit(db_train, epochs=epochs, validation_data=db_test)

    # 评估网络
    model.evaluate(db_test)

    t1 = time.time()
    print('total time cost:', t1 - t0)


if __name__ == '__main__':
    main()