Python 实现Jaccard相似度计算，判断英文新闻标题相似度

相似文档检测

Mission

data.csv中包含了一个新闻标题列表，试通过近似检测方法，通过Jaccard相似度，检测相似文章，将结果保存到csv文件中，不同文章间用空行隔开。

Work

思路：

• 两个词作为一段来计算，末尾不够截掉
• Jaccard相关系数大于0.5则认为两个新闻标题相似
• 利用并查集将相似的合并在一起

Code

import pandas as pd
import nltk
import numpy as np
class Jaccard:
    def __init__(self, _len): # _len 为步长值，语句切分的步长值
        self._len = _len
        return

    def cut2list(self, paragraph):
        words = []
        sents = nltk.sent_tokenize(paragraph)
        for sent in sents:
           words.append(nltk.word_tokenize(sent)) # 得到每句中的分词

        word_set = []
        for sent in words: # 遍历每个句子
            _len = len(sent) # 句子长度
            for i in range(0, _len, self._len): # 按照步长值划分句子
                if (i+self._len) > _len: # 溢出跳出
                    break
                tmp = ""
                for j in range(0, self._len): # 包括步长值内单词
                    tmp += sent[i+j]
                word_set.append(tmp)
        return word_set

    def jaccard(self, str1, str2):
        str1, str2 = self.cut2list(str1), self.cut2list(str2)
        str1, str2 = set(str1), set(str2) # 去重
        cnt = 0
        for i in str1:
            if i in str2:
                cnt = cnt + 1 # 计算交集

        fenmu = len(str1) + len(str2) - cnt
        jaccard_coefficient = float(cnt / fenmu)
        return jaccard_coefficient

class Merge: # 并查集
    def __init__(self, _len): # _len 为元素个数
        self._len = _len
        self.pre = []
        for i in range(0,self._len): # 初始化指向自己
            self.pre.append(i)
        pass

    def find(self, x):
        if self.pre[x] == x:
            return x
        else:
            self.pre[x] = self.find(self.pre[x])
            return self.pre[x]

    def merge(self, x, y):
        x,y  = self.pre[x], self.pre[y]
        if x != y:
            self.pre[x] = self.pre[y]

if __name__ == '__main__':
    data = pd.read_csv("data.csv",sep='\\t', engine='python')
    jaccard = Jaccard(2)
    jaccard_coefficient = [] # 用于记录两两之间的coefficient相关系数

    num = data.shape[0] # 多少条新闻
    for i in range(0, num): #得到相关系数矩阵
        tmp = []
        for j in range(0, num):
            if i == j:
                tmp.append(0)
            else:
                tmp.append(jaccard.jaccard(data.iloc[i]['title'], data.iloc[j]['title']))
        jaccard_coefficient.append(tmp)

    print(np.array(jaccard_coefficient).shape)
    print(jaccard_coefficient)
    merge = Merge(num)
    for i in range(0, num):
        for j in range(0, num):
            if jaccard_coefficient[i][j] > 0.5: # 关系大的加到一个并查集里
                merge.merge(i,j)

    classify = [[] for i in range(num)]  # 同一类放在一起

    # print('classify: ', classify)
    for i in range(num): # 和父节点放在一起
        classify[merge.pre[i]].append(i)

    classify_tmp = []
    for i in classify: #去除已合并结点
        if i != []:
            classify_tmp.append(i)
    classify = classify_tmp

    new_data = pd.DataFrame(columns=("title",'publisher', 'E')) # 将新的数据添加到空DataFrame中
    null_row = pd.DataFrame({'title':['na'],'publisher':['na'],'E':['na']}) # 空行，用于分割

    for i in classify: #加入到新的DataFrame中
        for j in i:
            new_data = new_data.append(data.iloc[j])
        new_data = new_data.append(null_row)

    print(new_data)

    new_data.to_csv('result.csv',sep=',')