概述
- 优点:在数据较少的情况下仍然有效,可以处理多类别问题。
- 缺点:对于输入数据的准备方式较为敏感。
- 使用数据类型:标称型数据。
贝叶斯决策理论的核心思想:选择具有最高概率的决策。
使用条件概率来分类
对于某个数据点x,y:
- 如果,那么属于类别
- 如果,那么属于类别
使用贝叶斯准则(公式:),可以通过已知的三个概率值来计算未知的概率值。
使用朴素贝叶斯进行文档分类
朴素贝叶斯的两个假设分别是:
1.一个特征或者单词出现的可能性和它和其他单词相邻没有关系。
由统计学知,如果每个特征需要N个样本,那么对于包含1000个特征的词汇表需要N^1000个样本。如果特征之间相互独立,那么样本数就可以从N^1000减少到1000xN。即一个特征或者单词出现的可能性与它和其他单词相邻没有关系。
2.每个特征同等重要。
文档和单词
getwords函数的代码通过将单词转换为小写形式,从而减少了特征的总数,这意味着,程序会将位于句首以大写字母开头的单词与位于句中全小写形式的单词视为相同——这样做非常好,因为具有不同大小写形式的同一单词往往代表的含义是相同的。而这一点可能对区分垃圾邮件和非垃圾邮件是至关重要的。除此之外,如果超过半数以上的单词都是大写时,那就说明必定有其他的特征存在。
对分类器进行训练
classifier类中的变量fc将记录位于各分类中的不同特征的数量,例如:
{'python' : {'bad' : 0, 'good' : 6}, 'the' : {'bad' : 3, 'good' : 3}
变量cc是一个记录各分类被使用次数的字典。
扫描二维码关注公众号,回复:
3531650 查看本文章
代码:
from pysqlite2 import dbapi2 as sqlite
import re
import math
def getwords(doc):
splitter=re.compile('\\W*')
print doc
# Split the words by non-alpha characters
words=[s.lower() for s in splitter.split(doc)
if len(s)>2 and len(s)<20]
# Return the unique set of words only
return dict([(w,1) for w in words])
class classifier:
def __init__(self,getfeatures,filename=None):
# Counts of feature/category combinations
self.fc={}
# Counts of documents in each category
self.cc={}
self.getfeatures=getfeatures
def setdb(self,dbfile):
self.con=sqlite.connect(dbfile)
self.con.execute('create table if not exists fc(feature,category,count)')
self.con.execute('create table if not exists cc(category,count)')
def incf(self,f,cat):
count=self.fcount(f,cat)
if count==0:
self.con.execute("insert into fc values ('%s','%s',1)"
% (f,cat))
else:
self.con.execute(
"update fc set count=%d where feature='%s' and category='%s'"
% (count+1,f,cat))
def fcount(self,f,cat):
res=self.con.execute(
'select count from fc where feature="%s" and category="%s"'
%(f,cat)).fetchone()
if res==None: return 0
else: return float(res[0])
def incc(self,cat):
count=self.catcount(cat)
if count==0:
self.con.execute("insert into cc values ('%s',1)" % (cat))
else:
self.con.execute("update cc set count=%d where category='%s'"
% (count+1,cat))
def catcount(self,cat):
res=self.con.execute('select count from cc where category="%s"'
%(cat)).fetchone()
if res==None: return 0
else: return float(res[0])
def categories(self):
cur=self.con.execute('select category from cc');
return [d[0] for d in cur]
def totalcount(self):
res=self.con.execute('select sum(count) from cc').fetchone();
if res==None: return 0
return res[0]
def train(self,item,cat):
features=self.getfeatures(item)
# Increment the count for every feature with this category
for f in features:
self.incf(f,cat)
# Increment the count for this category
self.incc(cat)
self.con.commit()
def fprob(self,f,cat):
if self.catcount(cat)==0: return 0
# The total number of times this feature appeared in this
# category divided by the total number of items in this category
return self.fcount(f,cat)/self.catcount(cat)
def weightedprob(self,f,cat,prf,weight=1.0,ap=0.5):
# Calculate current probability
basicprob=prf(f,cat)
# Count the number of times this feature has appeared in
# all categories
totals=sum([self.fcount(f,c) for c in self.categories()])
# Calculate the weighted average
bp=((weight*ap)+(totals*basicprob))/(weight+totals)
return bp
class naivebayes(classifier):
def __init__(self,getfeatures):
classifier.__init__(self,getfeatures)
self.thresholds={}
def docprob(self,item,cat):
features=self.getfeatures(item)
# Multiply the probabilities of all the features together
p=1
for f in features: p*=self.weightedprob(f,cat,self.fprob)
return p
def prob(self,item,cat):
catprob=self.catcount(cat)/self.totalcount()
docprob=self.docprob(item,cat)
return docprob*catprob
def setthreshold(self,cat,t):
self.thresholds[cat]=t
def getthreshold(self,cat):
if cat not in self.thresholds: return 1.0
return self.thresholds[cat]
def classify(self,item,default=None):
probs={}
# Find the category with the highest probability
max=0.0
for cat in self.categories():
probs[cat]=self.prob(item,cat)
if probs[cat]>max:
max=probs[cat]
best=cat
# Make sure the probability exceeds threshold*next best
for cat in probs:
if cat==best: continue
if probs[cat]*self.getthreshold(best)>probs[best]: return default
return best
class fisherclassifier(classifier):
def cprob(self,f,cat):
# The frequency of this feature in this category
clf=self.fprob(f,cat)
if clf==0: return 0
# The frequency of this feature in all the categories
freqsum=sum([self.fprob(f,c) for c in self.categories()])
# The probability is the frequency in this category divided by
# the overall frequency
p=clf/(freqsum)
return p
def fisherprob(self,item,cat):
# Multiply all the probabilities together
p=1
features=self.getfeatures(item)
for f in features:
p*=(self.weightedprob(f,cat,self.cprob))
# Take the natural log and multiply by -2
fscore=-2*math.log(p)
# Use the inverse chi2 function to get a probability
return self.invchi2(fscore,len(features)*2)
def invchi2(self,chi, df):
m = chi / 2.0
sum = term = math.exp(-m)
for i in range(1, df//2):
term *= m / i
sum += term
return min(sum, 1.0)
def __init__(self,getfeatures):
classifier.__init__(self,getfeatures)
self.minimums={}
def setminimum(self,cat,min):
self.minimums[cat]=min
def getminimum(self,cat):
if cat not in self.minimums: return 0
return self.minimums[cat]
def classify(self,item,default=None):
# Loop through looking for the best result
best=default
max=0.0
for c in self.categories():
p=self.fisherprob(item,c)
# Make sure it exceeds its minimum
if p>self.getminimum(c) and p>max:
best=c
max=p
return best
def sampletrain(cl):
cl.train('Nobody owns the water.','good')
cl.train('the quick rabbit jumps fences','good')
cl.train('buy pharmaceuticals now','bad')
cl.train('make quick money at the online casino','bad')
cl.train('the quick brown fox jumps','good')