Construya un vector de frecuencia de palabras para describir el documento. En Python, esto se puede lograr usando una lista, que generalmente es un conjunto o matriz ordenados.
from collections import Counter
from nltk.tokenize import TreebankWordTokenizer
from nlpia.data.loaders import kite_text
import nltk
tokenizer = TreebankWordTokenizer()
# Treebank 分词器假定文档已经被分割成独立的句子，因此它只会忽略字符串最末端的标点符号。
tokens = tokenizer.tokenize(kite_text.lower())
token_counts = Counter(tokens)
print(token_counts)
# 去掉停用词
stopwords = nltk.corpus.stopwords.words('english')
tokens = [x for x in tokens if x not in stopwords]
kite_counts = Counter(tokens)
print(kite_counts)
# 构建词频向量
document_vector = []
doc_length = len(tokens)
for key, value in kite_counts.most_common():
document_vector.append(value / doc_length)
print(document_vector)