第十二次作业——朴素贝叶斯应用：垃圾邮件分类

text = "Everybody knows waste paper and used coke cans are discarded everywhere. You might have seen plastic bags flying in the sky and getting caught in the trees when the wind blows or maybe you have seen old cans floating in the rivers and polluting the water. Our environment is the place in which we live, but it is being ruined by us. "

#导包
import csv
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

#预处理
def preprocessing(text):
    #text=text.decode("utf-8)
    tokens=[word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    stops=stopwords.words('english') #停用词
    tokens=[token for token in tokens if token not in stops] #去掉停用词
    
    tokens=[token.lower() for token in tokens if len(token)>=3] #去掉短语3的词
    lmtzr=WordNetLemmatizer() #还原词性
    tokens=[lmtzr.lemmatize(token) for token in tokens]
    preprocessed_text=' '.join(tokens) #剩下的词重新连接成字符串
    return preprocessed_text
preprocessing (text)

#读取数据集
import csv
file_path=r'D:\SMSSpamCollectionjs.txt'
sms=open(file_path,'r',encoding='utf-8')
sms_data=[]
sms_label=[]
csv_rreader=csv.reader(sms,delimiter='\t')

#将数据分别存放数据列表和目标分类列表
for line in csv_reader:
    sms_label.append(line[0])
    sms_data.append(preprocessing(line[1]))
sms.close()
print("邮件总数为：",len(sms_target))
sms_target

#按0.7,0.3比例分为训练集和测试集
import numpy as np
sms_data=np.array(sms_data)
sms_label=np.array(sms_label)
from sklearn.model_selection import train_text_split
x_train, x_test, y_train, y_test = train_text_split(sms_data, sms_label, test_size=0.3, random_state=0, stratify=sms_label)

#将其向量化
from sklearn.feature_extraction.text import TfidfVectorizer   ##建立数据的特征向量
vectorizer=TfidfVectorizer(min_df=2,ngram_range=(1,2),stop_words='english',strip_accents='unicode',norm='12')

X_train=vectorizer.fit_transform(x_train)
X_test=vectorizer.transform(x_test)

import numpy as np      #观察向量
a = X_train.toarray()
#X_test = X_test.toarray()
#X_train.shape
#X_train
 
for i in range(1000):    #输出不为0的列
    for j in range(5984):
        if a[i,j]!=0:
            print(i,j,a[i,j])

        
#朴素贝叶斯分类器
from sklearn.navie_bayes import MultionmialNB
clf= MultionmialNB().fit(X_train,y_train)
y_nb_pred=clf.predict(X_test)

#分类结果显示
from sklearn.metrics import confusion_matrix
from sjlearn.metrics import classification_report

print(y_nb_pred.shape,y_nb_pred)#x_test预测结果
print('nb_confusion_matrix:')
cm=confusion_matrix(y_test,y_nb_pred)#混淆矩阵
print(cm)
print('nb_classification_report:')
cr=classification_report(y_test,y_nb_pred)#主要分类指标的文本报告
print(cr)