1. Overview of RNN
Recurrent Neural Networks (RNN) Recurrent Neural Networks.
Traditional neural networks cannot remember past interactions, RNN solves this problem. They are networks with loops that allow information to persist. As shown in the figure below, Xt is input to node A of the neural network, and a value Ht is output. Loops in a network allow information to pass from one step of the network to the next.
2. Simple RNN application
1、Simple RNN
Use a simple RNN implementation of Keras to predict sales from historical datasets.
The Simple RNN function prototype is as follows:
A Simple RNN is a fully connected RNN.
tensorflow.python.keras.layers.SimpleRNN(
units,
activation='tanh',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
return_sequences=False,
return_state=False,
go_backwards=False,
stateful=False,
unroll=False)2. Dataset
The first column is the month and the second column is the sales data for each month. Sample data:
| Month | Sales of shampoo over a three year period |
| January 1 | 266 |
| January 2 | 145.9 |
| January 3 | 183.1 |
| January 4th | 119.3 |
| January 5th | 180.3 |
Looking at the chart, it can be seen that the sales are very volatile, but there is an upward trend line.
3. Reference code
# load and plot dataset
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
import pandas as pd
# 解析时间
def parser(x):
return datetime.strptime('200' + x, '%Y-%m')
series = read_csv('sales-of-shampoo-over-a-three-ye.csv', header=0, parse_dates=[0], index_col=0, squeeze=True,
date_parser=parser)
# summarize first few rows
print(series.head())
# line plot
series.plot()
pyplot.show()
from pandas import DataFrame
from pandas import Series
from pandas import concat
from pandas import read_csv
from pandas import datetime
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import SimpleRNN
#from keras.regularizers import L1L2
from math import sqrt
import matplotlib
# be able to save images on server
matplotlib.use('Agg')
from matplotlib import pyplot
import numpy
# frame a sequence as a supervised learning problem
def timeseries_to_supervised(data, lag=1):
df = DataFrame(data)
columns = [df.shift(i) for i in range(1, lag + 1)]
columns.append(df)
df = concat(columns, axis=1)
return df
# create a differenced series
def difference(dataset, interval=1):
diff = list()
for i in range(interval, len(dataset)):
value = dataset[i] - dataset[i - interval]
diff.append(value)
return Series(diff)
# invert differenced value
def inverse_difference(history, yhat, interval=1):
return yhat + history[-interval]
# scale train and test data to [-1, 1]
def scale(train, test):
# fit scaler
scaler = MinMaxScaler(feature_range=(-1, 1))
scaler = scaler.fit(train)
# transform train
train = train.reshape(train.shape[0], train.shape[1])
train_scaled = scaler.transform(train)
# transform test
test = test.reshape(test.shape[0], test.shape[1])
test_scaled = scaler.transform(test)
return scaler, train_scaled, test_scaled
# inverse scaling for a forecasted value
def invert_scale(scaler, X, yhat):
new_row = [x for x in X] + [yhat]
array = numpy.array(new_row)
array = array.reshape(1, len(array))
inverted = scaler.inverse_transform(array)
return inverted[0, -1]
# fit an LSTM network to training data
def fit_rnn(train, n_batch, nb_epoch, n_neurons):
X, y = train[:, 0:-1], train[:, -1]
X = X.reshape(X.shape[0], 1, X.shape[1])
model = Sequential()
model.add(SimpleRNN(n_neurons, batch_input_shape=(n_batch, X.shape[1], X.shape[2]), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(nb_epoch):
model.fit(X, y, epochs=1, batch_size=n_batch, verbose=1, shuffle=False)
model.reset_states()
return model
# run a repeated experiment
def run_rnn(series, n_lag, n_repeats, n_epochs, n_batch, n_neurons):
# transform data to be stationary
raw_values = series.values
diff_values = difference(raw_values, 1)
# transform data to be supervised learning
supervised = timeseries_to_supervised(diff_values, n_lag)
supervised_values = supervised.values[n_lag:, :]
# split data into train and test-sets
train, test = supervised_values[0:-12], supervised_values[-12:]
# transform the scale of the data
scaler, train_scaled, test_scaled = scale(train, test)
# run experiment
error_scores = list()
for r in range(n_repeats):
# fit the model
train_trimmed = train_scaled[2:, :]
rnn_model = fit_rnn(train_trimmed, n_batch, n_epochs, n_neurons)
# forecast test dataset
test_reshaped = test_scaled[:, 0:-1]
test_reshaped = test_reshaped.reshape(len(test_reshaped), 1, 1)
output = rnn_model.predict(test_reshaped, batch_size=n_batch)
predictions = list()
for i in range(len(output)):
yhat = output[i, 0]
X = test_scaled[i, 0:-1]
# invert scaling
yhat = invert_scale(scaler, X, yhat)
# invert differencing
yhat = inverse_difference(raw_values, yhat, len(test_scaled) + 1 - i)
# store forecast
predictions.append(yhat)
# report performance
rmse = sqrt(mean_squared_error(raw_values[-12:], predictions))
print('%d) Test RMSE: %.3f' % (r + 1, rmse))
error_scores.append(rmse)
return error_scores
# configure the experiment
def run():
# load dataset
series = read_csv('sales-of-shampoo-over-a-three-ye.csv', header=0, parse_dates=[0], index_col=0, squeeze=True,
date_parser=parser)
# configure the experiment
n_lag = 1
n_repeats = 30
n_epochs = 1000
n_batch = 4
n_neurons = 3
# run the experiment
results = DataFrame()
results['results'] = run_rnn(series, n_lag, n_repeats, n_epochs, n_batch, n_neurons)
results.plot(title="RNN RMSE Iteration")
pyplot.show()
# summarize results
print(results.describe())
# save boxplot
pyplot.savefig('plot_rnn_rmse.png')
# entry point
run()The epoch cycle needs to be performed manually, so that at the end of each epoch, reset_states and delete the internal state. If you use epochs=nb_epoch directly without manual looping, the network will reset_states at the end of each batch.
The training results are as follows
1) Test RMSE: 98.509
2) Test RMSE: 96.719
3) Test RMSE: 97.397
4) Test RMSE: 82.224
5) Test RMSE: 77.399
6) Test RMSE: 87.999
7) Test RMSE: 98.929
8) Test RMSE: 105.284
9) Test RMSE: 103.274
10) Test RMSE: 74.841
11) Test RMSE: 95.774
12) Test RMSE: 101.559
13) Test RMSE: 120.763
14) Test RMSE: 95.495
15) Test RMSE: 107.296
16) Test RMSE: 86.461
17) Test RMSE: 76.648
18) Test RMSE: 83.285
19) Test RMSE: 112.248
20) Test RMSE: 77.225
21) Test RMSE: 96.166
22) Test RMSE: 84.767
23) Test RMSE: 84.564
24) Test RMSE: 92.602
25) Test RMSE: 75.713
26) Test RMSE: 86.159
27) Test RMSE: 101.537
28) Test RMSE: 109.397
29) Test RMSE: 93.453
30) Test RMSE: 99.279
results
count 30.000000
mean 93.432153
std 11.782779
min 74.840867
25% 84.614619
50% 95.634082
75% 100.972315
max 120.763087
Multiple training results, as you can see from the graph below, the RMSE varies between 74 and 120 during iterations, but becomes more stable over time.
