MachineLP的Github(欢迎follow):https://github.com/MachineLP
本小节使用torch搭建RNN模型,训练和测试:
(1)数据处理模块,根据现有样本的所有数据,创建词典,之后对应词典找到每个词的索引。
(2)定义模型超参数:embedding的大小、rnn隐含单元、rnn层数、迭代次数、批量大小、序列长度、学习率。
(3)定义训练数据
(4)定义模型(定义需要的RNN结构)。
(5)定义损失函数,选用适合的损失函数。
(6)定义优化算法(SGD、Adam等)。
(7)保存模型。
---------------------------------我是可爱的分割线---------------------------------
训练样本下载: https://download.csdn.net/download/u014365862/10455691
数据处理代码部分:data_utils.py
# coding=utf-8 import torch import os class Dictionary(object): def __init__(self): self.word2idx = {} self.idx2word = {} self.idx = 0 def add_word(self, word): if not word in self.word2idx: self.word2idx[word] = self.idx self.idx2word[self.idx] = word self.idx += 1 def __len__(self): return len(self.word2idx) class Corpus(object): def __init__(self): self.dictionary = Dictionary() def get_data(self, path, batch_size=20): # Add words to the dictionary with open(path, 'r') as f: tokens = 0 for line in f: words = line.split() + ['<eos>'] tokens += len(words) for word in words: self.dictionary.add_word(word) # Tokenize the file content ids = torch.LongTensor(tokens) token = 0 with open(path, 'r') as f: for line in f: words = line.split() + ['<eos>'] for word in words: ids[token] = self.dictionary.word2idx[word] token += 1 num_batches = ids.size(0) // batch_size ids = ids[:num_batches*batch_size] return ids.view(batch_size, -1)
训练代码部分:train.py
import torch import torch.nn as nn import numpy as np from torch.nn.utils import clip_grad_norm from data_utils import Dictionary, Corpus # 判定GPU是否存在 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 定义超参数 embed_size = 128 hidden_size = 1024 num_layers = 1 num_epochs = 5 num_samples = 1000 # number of words to be sampled batch_size = 20 seq_length = 30 learning_rate = 0.002 # 加载 "Penn Treebank" 数据集 corpus = Corpus() ids = corpus.get_data('data/train.txt', batch_size) vocab_size = len(corpus.dictionary) num_batches = ids.size(1) // seq_length # 定义RNN(LSTM) class RNNLM(nn.Module): def __init__(self, vocab_size, embed_size, hidden_size, num_layers): super(RNNLM, self).__init__() self.embed = nn.Embedding(vocab_size, embed_size) self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True) self.linear = nn.Linear(hidden_size, vocab_size) def forward(self, x, h): # word embedding x = self.embed(x) # 前向计算 out, (h, c) = self.lstm(x, h) # Reshape output to (batch_size*sequence_length, hidden_size) # 这里说明使用序列的所有输出 out = out.reshape(out.size(0)*out.size(1), out.size(2)) # Decode hidden states of all time steps out = self.linear(out) return out, (h, c) model = RNNLM(vocab_size, embed_size, hidden_size, num_layers).to(device) # 定义损失函数和优化算法 criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # Truncated backpropagation def detach(states): return [state.detach() for state in states] # 训练模型 for epoch in range(num_epochs): # Set initial hidden and cell states states = (torch.zeros(num_layers, batch_size, hidden_size).to(device), torch.zeros(num_layers, batch_size, hidden_size).to(device)) for i in range(0, ids.size(1) - seq_length, seq_length): # Get mini-batch inputs and targets inputs = ids[:, i:i+seq_length].to(device) targets = ids[:, (i+1):(i+1)+seq_length].to(device) # Forward pass states = detach(states) outputs, states = model(inputs, states) loss = criterion(outputs, targets.reshape(-1)) # Backward and optimize model.zero_grad() loss.backward() clip_grad_norm(model.parameters(), 0.5) optimizer.step() step = (i+1) // seq_length if step % 100 == 0: print ('Epoch [{}/{}], Step[{}/{}], Loss: {:.4f}, Perplexity: {:5.2f}' .format(epoch+1, num_epochs, step, num_batches, loss.item(), np.exp(loss.item()))) # Test the model with torch.no_grad(): with open('sample.txt', 'w') as f: # Set intial hidden ane cell states state = (torch.zeros(num_layers, 1, hidden_size).to(device), torch.zeros(num_layers, 1, hidden_size).to(device)) # Select one word id randomly prob = torch.ones(vocab_size) input = torch.multinomial(prob, num_samples=1).unsqueeze(1).to(device) for i in range(num_samples): # Forward propagate RNN output, state = model(input, state) # Sample a word id prob = output.exp() word_id = torch.multinomial(prob, num_samples=1).item() # Fill input with sampled word id for the next time step input.fill_(word_id) # File write word = corpus.dictionary.idx2word[word_id] word = '\n' if word == '<eos>' else word + ' ' f.write(word) if (i+1) % 100 == 0: print('Sampled [{}/{}] words and save to {}'.format(i+1, num_samples, 'sample.txt')) # Save the model checkpoints torch.save(model.state_dict(), 'model.ckpt')
---------------------------------我是可爱的分割线---------------------------------
总结:
本节使用RNN训练语言模型。
torch系列:
2. torch02:logistic regression--MNIST识别
4. torch04:全连接神经网络--MNIST识别和自己数据集
6. torch06:ResNet--Cifar识别和自己数据集