利用朴素贝叶斯可以对文档进行分类，比如说进行垃圾邮件的过滤等接下来的案例是对评分进行分类的，经过学习判断一句话应该属于几分，0-5分之间。
先利用爬虫爬取样本数据，该数据来自公开课的评论。
# coding=utf-8
import urllib2
from sgmllib import SGMLParser
import jieba
class CommentParser(SGMLParser):
def __init__(self):
SGMLParser.__init__(self)
self.__start_table = False
self.__start_p = False
self.__value_p = ''
self.__value_div = ''
self.__p_state = 0
self.data = []
def start_table(self, attr):
for k, v in attr:
if k == 'class' and v == 'table table-hover':
self.__start_table = True
def end_table(self):
if self.__start_table:
self.data.append([self.__value_p, self.__value_div])
self.__value_p = ''
self.__value_div = ''
self.__p_state = 0
self.__start_table = False
def start_div(self, attr):
if self.__start_table:
for k, v in attr:
if k == 'data-score':
self.__value_div = v
def end_div(self):
pass
def start_p(self, attrs):
if self.__start_table:
self.__p_state += 1
self.__start_p = True
def end_p(self):
if self.__start_table:
self.__start_p = False
def handle_data(self, data):
if self.__start_table and self.__start_p and self.__p_state == 3:
self.__value_p += data
def get_page(url):
page = urllib2.urlopen(url).read()
paraser = CommentParser()
paraser.feed(page)
value = paraser.data
return value
def download():
url = 'http://coursegraph.com/reviews/'
for i in range(1, 9):
value = get_page(url + str(i))
with open('result.txt', 'a+') as f:
for row in value:
f.write('[' + row[1] + ']' + row[0].strip().replace('\n', '').replace('\r', '') + '\n')
def jieba_chn():
all_value = open('result.txt', 'r+').readlines()
with open('result1.txt', 'w+') as f:
for row in all_value:
value = row[:5][1:4]
jb = jieba.cut_for_search(row[5:])
for row in jb:
if len(row) > 1:
value += ',' + row
f.write(value.encode('utf-8') + '\n')
#下载数据
# download()
#对文档数据进行分词操作
jieba_chn()
很简单的一个网络爬虫，然后利用**结巴**分词吧文档分割成词组，并去除掉标点符号等操作。具体的结果可以下载本案例进行查看。
下面开始进行利用朴素贝叶斯进行分类操作
先读取文档数据
def load_data_set():
dataSet = []
labels = []
with open('result1.txt', 'r+') as f:
for row in f.readlines():
t = row.strip().replace('\n', '').split(',')
labels.append(round(float(t[0]), 1))
dataSet.append(t[1:])
return dataSet, labels创建一个单词向量和对应的标签
def create_vocab_list(dataSet, labels):
vocabSet = []
labelSet = []
for index, document in enumerate(dataSet):
vocabSet.extend(list(set(document)))
labelSet.extend([labels[index] for i in range(len(set(document)))])
return vocabSet, labelSet根据单词出现修改相应的标签
def set_of_words2_vec(vocabList, label, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
for index, r in enumerate(vocabList):
if r == word:
returnVec[index] += label
return returnVec对数据惊醒学习操作，计算出词组出现的概率分布
def trainNB0(trainMatrix, trainCategory):
numTrainDocs = len(trainMatrix)
numWords = len(trainMatrix[0])
labelSet = list(set(trainCategory))
pAbusive = {}
for r in labelSet:
pAbusive[str(r)] = len([row for row in trainCategory if row == r]) \
/ float(numTrainDocs)
pNumber = {}
pDenom = {}
for row in labelSet:
pNumber[str(row)] = ones(numWords)
pDenom[str(row)] = 2.0
for i in range(numTrainDocs):
pNumber[str(trainCategory[i])] += [row / trainCategory[i] for row in trainMatrix[i]]
pDenom[str(trainCategory[i])] += sum(trainMatrix[i]) / trainCategory[i]
ret = {}
for i in range(len(labelSet)):
ret[str(labelSet[i])] = pNumber[str(labelSet[i])] / pDenom[str(labelSet[i])]
return ret, pAbusive判断测试词组的出现的概率，选择出出现概率最高的一项，就是该词组的评分了。
def classifyNB(vec2Classify, pVec, pClass, trainCategory):
labelSet = list(set(trainCategory))
p = {}
for row in labelSet:
p[str(row)] = sum(vec2Classify * pVec[str(row)]) + log(pClass[str(row)])
m = sorted(p.items(), key=lambda k: k[1], reverse=True)
return float(m[0][0])一下是对文档进行测试的操作，
def testingNB():
dataSet, labels = load_data_set()
vocabSet, labelSet = create_vocab_list(dataSet, labels)
trainMatrix = []
for index, row in enumerate(dataSet):
trainMatrix.append(set_of_words2_vec(vocabSet, labels[index], row))
pV, pAb = trainNB0(trainMatrix, labels)
testEntry = ['学习', '很棒', '真不错']
testEntry = list(set(testEntry))
thisDoc = array(set_of_words2_vec(vocabSet, 1, testEntry))
print testEntry, 'classified as: ', classifyNB(thisDoc, pV, pAb, labels)
def test(number):
'''
验证算法的正确性
:param number: 当成测试样本的额百分比
:return:
'''
dataSet, labels = load_data_set()
test_number = int(len(dataSet) * number)
testSet = []
for i in range(test_number):
randIndex = int(random.uniform(0, len(dataSet)))
testSet.append([dataSet[randIndex], labels[randIndex]])
del (dataSet[randIndex])
del (labels[randIndex])
# 进行学习
vocabSet, labelSet = create_vocab_list(dataSet, labels)
trainMatrix = []
for index, row in enumerate(dataSet):
trainMatrix.append(set_of_words2_vec(vocabSet, labels[index], row))
pV, pAb = trainNB0(trainMatrix, labels)
# 进行测试
errorCount = 0
for row in testSet:
testEntry = row[0]
testEntry = list(set(testEntry))
thisDoc = array(set_of_words2_vec(vocabSet, 1, testEntry))
ret = classifyNB(thisDoc, pV, pAb, labels)
if ret != row[1]:
print "classification error", row[1], ret
errorCount += 1
print 'the error rate is: ', float(errorCount) / len(testSet)
test(0.1)
# testingNB()悲剧的是测试的结果很不理想，难道中文不能这样分词，还是那一个细节出现问题，还请大神指导下，权当学习一下吧！
[案例现在地址](http://download.csdn.net/detail/u010154424/9602826)