文本表示方法--词频-逆文件频率(TF-IDF)

一.TF-IDF相关概念

1.1词频(TF)

1.2逆文件频率(IDF)

1.3词频-逆文件频率(TF-IDF)

二.代码实现

import numpy as np

# 1.1分词过的语料
corpus = [
    '这 是 第一个 文档',
    '这是 第二个 文档',
    '这是 最后 一个 文档',
    '现在 没有 文档 了 文档'
]

# 词袋统计
words_list = []

for corpu in corpus:
    words_list.append(corpu.split())
print(words_list)

## 1.手动实现
# 1.2统计词的数量
from collections import Counter

count_list = []
for words in words_list:
    count = Counter(words)
    count_list.append(count)
print(count_list)
print(count_list[0].values())

# 1.3定义函数
import math


def tf(word, count):
    return count[word] / sum(count.values())


def idf(word, count_list):
    n_contain = sum([1 for count in count_list if word in count])
    return math.log(len(count_list) / (1 + n_contain))


def tf_idf(word, count, count_list):
    return tf(word, count) * idf(word, count_list)


# 1.4 输出结果
for index, count in enumerate(count_list):
    print('第{}个文档的TF IDF的信息'.format(index + 1))
    # 分值
    scores = {
    
    word: tf_idf(word, count, count_list) for word in count}
    sorted_word = sorted(scores.items(), key=lambda x: x[1], reverse=True)
    for word, score in sorted_word:
        print('word:{}, TF IDF :{}'.format(word, round(score, 5)))

## 2.Gensim
print(words_list)
# 2.1 获取词频
from gensim import corpora

dic = corpora.Dictionary(words_list)

# 第一个元素是词在词典中的id,第二个是词在文档中出现的次数
new_corpus = [dic.doc2bow(words) for words in words_list]
print(new_corpus)
# 2.2 每一个词对应的id
print(dic.token2id)
from gensim import models

tfidf = models.TfidfModel(new_corpus)
tfidf.save('tfidf.model')
# 加载模型
models.TfidfModel.load('tfidf.model')

tf_idf_vec = []
for corpu in corpus:
    # 词带
    doc_bow = dic.doc2bow(corpu.lower().split())
    vec = tfidf[doc_bow]
    tf_idf_vec.append(vec)
print(tf_idf_vec)

## 3.Sklearn
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vec = TfidfVectorizer()
tf_idf_matrix = tfidf_vec.fit_transform(corpus)
# 得到语料库所有不重复的词
print(tfidf_vec.get_feature_names())
# 得到所有的单词对应的id
print(tfidf_vec.vocabulary_)
# 获得tf idf矩阵的输出
print(tf_idf_matrix.toarray())

三.缺点