文本相似在问答系统中有很重要的应用,如基于知识的问答系统(Knowledge-based QA),基于文档的问答系统(Documen-based QA),以及基于FAQ的问答系统(Community-QA)等。像 对于问题的内容,需要进行相似度匹配,从而选择出与问题最接近,同时最合理的答案。本节介绍 基于bert的余弦距离计算相似度。
学习bert可以看这里:https://blog.csdn.net/u014365862/article/details/104412737
训练/预测:
# 绘图案例 an example of matplotlib
import numpy as np
import matplotlib.pyplot as plt
from scipy.special import jn
from IPython.display import display, clear_output
import time
from sklearn.model_selection import KFold, train_test_split, GridSearchCV
'''
x = np.linspace(0,5)
f, ax = plt.subplots()
ax.set_title("Bessel functions")
for n in range(1,10):
time.sleep(1)
ax.plot(x, jn(x,n))
clear_output(wait=True)
display(f)
# close the figure at the end, so we don't get a duplicate
# of the last plot
plt.close()
'''
from keras.layers import *
from bert4keras.backend import keras, set_gelu
from bert4keras.bert import build_bert_model
from bert4keras.optimizers import Adam
from bert4keras.snippets import sequence_padding, DataGenerator
from bert4keras.tokenizer import Tokenizer
import pandas as pd
import numpy as np
set_gelu('tanh') # 切换gelu版本
maxlen = 32
batch_size = 16
num_classes = 2
epochs = 20
learning_rate = 2e-5
# sim roeberta_zh
config_path = 'albert_tiny_google_zh_489k/albert_config.json'
checkpoint_path = 'albert_tiny_google_zh_489k/albert_model.ckpt'
dict_path = 'albert_tiny_google_zh_489k/vocab.txt'
def load_data(filename):
D = []
data = pd.read_csv(filename)
data.dropna(axis=0, how='any', inplace=True)
data = data.values.tolist()
for per_data in data:
D.append( (per_data[0],per_data[1],int(per_data[2])) )
return D
# 加载数据集
train_val_data = load_data('train_data.csv')
# test_data = load_data('dev.csv')
# 查看一下数据
print ( 'train>>>>', train_val_data[0] )
print ( '训练验证集数量:', len(train_val_data) )
random_order = range(len(train_val_data))
np.random.shuffle(list(random_order))
train_data = [train_val_data[j] for i, j in enumerate(random_order) if i % 5 != 1 ]
valid_data = [train_val_data[j] for i, j in enumerate(random_order) if i % 5 == 1 ]
test_data = valid_data
print ( '训练集数量:', len(train_data) )
print ( '验证集数量:', len(valid_data) )
print ( '测试集数量:', len(test_data) )
# 建立分词器
tokenizer = Tokenizer(dict_path, do_lower_case=True)
class data_generator(DataGenerator):
"""数据生成器
"""
def __iter__(self, random=False):
idxs = list(range(len(self.data)))
if random:
np.random.shuffle(idxs)
batch_token_ids, batch_segment_ids, batch_labels = [], [], []
for i in idxs:
text1, text2, label = self.data[i]
# print(text1, text2, label)
token_ids, segment_ids = tokenizer.encode(text1, text2, max_length=maxlen)
batch_token_ids.append(token_ids)
batch_segment_ids.append(segment_ids)
batch_labels.append([label])
if len(batch_token_ids) == self.batch_size or i == idxs[-1]:
batch_token_ids = sequence_padding(batch_token_ids)
batch_segment_ids = sequence_padding(batch_segment_ids)
batch_labels = sequence_padding(batch_labels)
yield [batch_token_ids, batch_segment_ids], batch_labels
batch_token_ids, batch_segment_ids, batch_labels = [], [], []
# 加载预训练模型
bert = build_bert_model(
model='albert',
config_path=config_path,
checkpoint_path=checkpoint_path,
with_pool=True,
return_keras_model=False,
)
output = Dropout(rate=0.1)(bert.model.output)
output = Dense(units=num_classes,
activation='softmax',
kernel_initializer=bert.initializer)(output)
model = keras.models.Model(bert.model.input, output)
model.summary()
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=Adam(learning_rate), # 用足够小的学习率
# optimizer=PiecewiseLinearLearningRate(Adam(5e-5), {10000: 1, 30000: 0.1}),
metrics=['accuracy'],
)
# 转换数据集
train_generator = data_generator(train_data, batch_size)
valid_generator = data_generator(valid_data, batch_size)
test_generator = data_generator(test_data, batch_size)
def evaluate(data):
total, right = 0., 0.
for x_true, y_true in data:
y_pred = model.predict(x_true).argmax(axis=1)
y_true = y_true[:, 0]
total += len(y_true)
right += (y_true == y_pred).sum()
return right / total
class Evaluator(keras.callbacks.Callback):
def __init__(self):
self.best_val_acc = 0.
def on_epoch_end(self, epoch, logs=None):
val_acc = evaluate(valid_generator)
if val_acc > self.best_val_acc:
self.best_val_acc = val_acc
model.save_weights('best_model.weights')
test_acc = evaluate(test_generator)
print(u'val_acc: %.5f, best_val_acc: %.5f, test_acc: %.5f\n'
% (val_acc, self.best_val_acc, test_acc))
evaluator = Evaluator()
model.fit_generator(train_generator.forfit(),
steps_per_epoch=len(train_generator),
epochs=epochs,
callbacks=[evaluator])
model.load_weights('best_model.weights')
print(u'final test acc: %05f\n' % (evaluate(test_generator)))
import numpy as np
import tensorflow as tf
from holmes.models.model_base.model_base import ModelBase
from bert4keras.backend import keras, set_gelu
from bert4keras.bert import build_bert_model
from bert4keras.tokenizer import Tokenizer
from bert4keras.snippets import sequence_padding
set_gelu('tanh')
class ALBertEmbedding():
'''通过ALBert计算句向量
'''
def __init__(self, words_list=None,
config_path=albert_config_path,
checkpoint_path = albert_checkpoint_path,
dict_path = albert_dict_path,
albert_checkpoint_path = albert_checkpoint_path ):
self.session = tf.Session()
keras.backend.set_session(self.session)
self.bert = build_bert_model(
model='albert',
config_path=config_path,
# checkpoint_path=checkpoint_path,
with_pool=True,
return_keras_model=False,)
self.encoder = keras.models.Model(self.bert.model.inputs, self.bert.model.outputs[0])
self.encoder.load_weights(albert_checkpoint_path, by_name=True)
self.tokenizer = Tokenizer(dict_path, do_lower_case=True)
def init(self, words_list=None, update=True):
token_ids_list, segment_ids_list = [], []
for words in words_list:
token_ids, segment_ids = self.tokenizer.encode(words)
token_ids_list.append(token_ids)
segment_ids_list.append(segment_ids)
token_ids_list = sequence_padding(token_ids_list)
segment_ids_list = sequence_padding(segment_ids_list)
self.words_list_pre = self.encoder.predict([token_ids_list, segment_ids_list])
self.words_list_pre = self._normalize(self.words_list_pre)
return self
def _normalize(self, x):
x /= (np.array(x)**2).sum(axis=1, keepdims=True)**0.5
return x
def _predict(self, words):
with self.session.as_default():
with self.session.graph.as_default():
token_ids, segment_ids = self.tokenizer.encode( words )
pre = self.encoder.predict([np.array([token_ids]), np.array([segment_ids])])
pre = self._normalize(pre)
return pre
# 句向量
def predict(self, words):
with self.session.as_default():
with self.session.graph.as_default():
token_ids, segment_ids = self.tokenizer.encode( words )
pre = self.encoder.predict([np.array([token_ids]), np.array([segment_ids])])
pre = self._normalize(pre)
return np.dot( self.words_list_pre[:], pre[0] )