实战电影评论情感分析
情感分析是一段文字表达的情绪状态。其中，一段文本可以使一个句子、一个段落或者一个文档。主要涉及两个问题：文本表达和文本分类。在深度学习出现之前，主流的表示方法有BOW(词袋模型)和topic model(主题模型)，分类模型主要有SVM和LR。
载入数据：IMDB情感分析数据集，训练集和测试集分别包含了25000条已标注的电影评论，满分了10分，小于等于4为负面评论。
# -*- coding: utf-8 -*-
import numpy as np
# 加载已训练好的词典向量模型，包含400000的文本向量，每行有50维的数据
words_list = np.load('wordsList.npy')
print('载入word列表')
words_list = words_list.tolist() # 转化为list
words_list = [word.decode('UTF-8') for word in words_list]
word_vectors = np.load('wordVectors.npy')
print('载入文本向量')
print(len(words_list))
print(word_vectors.shape)
Home_index = words_list.index("home")
print(word_vectors[Home_index])
# 加载电影数据
import os
from os.path import isfile, join
pos_files = ['pos/' + f for f in os.listdir('pos/') if isfile(join('pos/', f))]
neg_files = ['neg/' + f for f in os.listdir('neg/') if isfile(join('neg/', f))]
num_words = []
for pf in pos_files:
with open(pf, "r", encoding='utf-8') as f:
line = f.readline()
counter = len(line.split())
num_words.append(counter)
print('正面评价完结')
for pf in neg_files:
with open(pf, "r", encoding='utf-8') as f:
line = f.readline()
counter = len(line.split())
num_words.append(counter)
print('负面评价完结')
num_files = len(num_words)
print('文件总数', num_files)
print('所有的词的数量', sum(num_words))
print('平均文件词的长度', sum(num_words)/len(num_words))
'''
# 可视化
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('qt4agg')
# 指定默认字体
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family'] = 'sans-serif'
#%matplotlib inline
plt.hist(num_words, 50, facecolor='g')
plt.xlabel('文本长度')
plt.ylabel('频次')
plt.axis([0, 1200, 0, 8000])
plt.show()
'''
# 大部分文本都在230之内
max_seg_len = 300
# 将文本生成一个索引矩阵，得到一个25000x300矩阵
import re
strip_special_chars = re.compile("[^A-Za-z0-9 ]+")
def cleanSentence(string):
string = string.lower().replace("<br />", " ")
return re.sub(strip_special_chars, "", string.lower())
print('保存idxMatrix...')
max_seg_num = 300
ids = np.zeros((num_files, max_seg_num), dtype="int32")
file_count = 0
'''
for pf in pos_files:
with open(pf, "r", encoding="utf-8") as f:
indexCounter = 0
line = f.readline()
cleanedLine = cleanSentence(line)
split = cleanedLine.split()
for word in split:
try:
ids[file_count][indexCounter] = words_list.index(word)
except ValueError:
ids[file_count][indexCounter] = 399999 # 未知的词
indexCounter = indexCounter + 1
if indexCounter >= max_seg_num:
break
file_count = file_count + 1
print(file_count)
print('保存完成1')
for nf in neg_files:
with open(nf, "r", encoding="utf-8") as f:
indexCounter = 0
line = f.readline()
cleanedLine = cleanSentence(line)
split = cleanedLine.split()
for word in split:
try:
ids[file_count][indexCounter] = words_list.index(word)
except ValueError:
ids[file_count][indexCounter] = 399999 # 未知的词
indexCounter = indexCounter + 1
if indexCounter >= max_seg_num:
break
file_count = file_count + 1
# 保存到文件
np.save('idxMatrix', ids)
print('保存完成2')
'''
# 模型设置
batch_size = 24
lstm_units = 64
num_labels = 2
iterations = 200000
max_seg_num = 250
ids = np.load('idsMatrix.npy')
# 返回一个数据集的迭代器， 返回一批训练集合
from random import randint
def get_train_batch():
labels = []
arr = np.zeros([batch_size, max_seg_num])
for i in range(batch_size):
if (i % 2 == 0):
num = randint(1, 11499)
labels.append([1, 0])
else:
num = randint(13499, 24999)
labels.append([0, 1])
arr[i] = ids[num-1: num]
return arr, labels
def get_test_batch():
labels = []
arr = np.zeros([batch_size, max_seg_num])
for i in range(batch_size):
num = randint(11499, 13499)
if (num <= 12499):
labels.append([1, 0])
else:
labels.append([0, 1])
arr[i] = ids[num-1:num]
return arr, labels
num_dimensions = 300 # Dimensions for each word vector
import tensorflow as tf
tf.reset_default_graph()
labels = tf.placeholder(tf.float32, [batch_size, num_labels])
input_data = tf.placeholder(tf.int32, [batch_size, max_seg_num])
data = tf.Variable(tf.zeros([batch_size, max_seg_num, num_dimensions]), dtype=tf.float32)
data = tf.nn.embedding_lookup(word_vectors, input_data)
# 配置LSTM网络
lstmCell = tf.contrib.rnn.BasicLSTMCell(lstm_units)
lstmCell = tf.contrib.rnn.DropoutWrapper(cell=lstmCell, output_keep_prob=0.75) # 避免一些过拟合
value, _ = tf.nn.dynamic_rnn(lstmCell, data, dtype=tf.float32)
# 第一个输出可以被认为是最后的隐藏状态，该向量将重新确定维度，然后乘以一个权重加上偏置，获得最终的label
weight = tf.Variable(tf.truncated_normal([lstm_units, num_labels]))
bias = tf.Variable(tf.constant(0.1, shape=[num_labels]))
value = tf.transpose(value, [1, 0, 2])
last = tf.gather(value, int(value.get_shape()[0]) - 1)
prediction = (tf.matmul(last, weight) + bias)
# 预测函数以及正确率评估参数
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 将标准的交叉熵损失函数定义为损失值，选择Adam作为优化函数
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels))
optimizer = tf.train.AdamOptimizer().minimize(loss)
#sess = tf.InteractiveSession(config=tf.ConfigProto(allow_soft_placement, log_device_placement))
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False))
#saver = tf.train.Saver()
#saver.restore(sess, tf.train.latest_checkpoint('models'))
iterations = 10
for i in range(iterations):
next_batch, next_batch_labels = get_test_batch()
print("正确率:", (sess.run(
accuracy, {input_data: next_batch, labels: next_batch_labels})) * 100)
'''
# 使用tensorboard可视化损失值和正确值
import datetime
sess = tf.InteractiveSession()
#tf.device("/cpu:0")
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
tf.summary.scalar('Loss', loss)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
writer = tf.summary.FileWriter(logdir, sess.graph)
for i in range(iterations):
# 下个批次的数据
nextBatch, nextBatchLabels = get_train_batch();
sess.run(optimizer, {input_data: nextBatch, labels: nextBatchLabels})
# 每50次写入一次leadboard
if (i % 50 == 0):
summary = sess.run(merged, {input_data: nextBatch, labels: nextBatchLabels})
writer.add_summary(summary, i)
# 每10,000次保存一个模型
if (i % 10000 == 0 and i != 0):
save_path = saver.save(sess, "models/pretrained_lstm.ckpt", global_step=i)
print("saved to %s" % save_path)
writer.close()
'''