版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/u011659379/article/details/48296063
写了一个增量式的爬虫，但是并不完美，希望大牛们可以指正指正！
爬虫以爬4567.tv这个网站的电影为例。把保存过得电影链接以set的形式保存到本地，然后下次运行的时候就会读取这些链接，然后用set的运算将去除以解析过得链接。
比如:
<span style="font-size:24px;">a=set([1,2,3])#假设这是上次解析过得
b=set([3,4,5])#这是本次要解析的链接，但是3是上次解析过的
c=(a|b)-a # c=set([4,5]),这次就只解析4、5</span>下面是爬虫的代码。
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 07 21:46:15 2015
@author: Administrator
"""
import re,time
import requests as req
from multiprocessing.dummy import Pool
import socket
import cPickle
homeurl='http://www.4567.tv'
page='http://www.4567.tv/search.asp?page=%d&searchword=&searchtype=-1'
page_list=[page%d for d in xrange(1,896)]
#<a class="play-img" href="/film/id21425.html" title="黄河在咆哮" target="_blank"><img src="http://tu.joy3g.com/20150820213138526.jpg" alt="黄河在咆哮"><i></i><em>历史战争</em></a>
movie_find=re.compile('<a class="play-img" href="(.*?)"')#找电影链接
#<title>花千骨-在线观看-迅雷下载-网盘下载-4567高清视界</title>
title_find=re.compile('<title>(.*?)-')#找电影标题
#download_list
#href="javascript:;" onclick="start('gvod://222.187.220.236:8089/8AFB9BA57CDF62FDA5E687729AFC5DB5A42D2AD8/571834140/[迅雷下载www.2tu.cc]花Q骨01.HDTV.mp4')">
dlist=re.compile('href="javascript:;" onclick="start\(\'(.*?)\'\)')#找电影下载链接
class tv():
socket.setdefaulttimeout(20)#设置超时
def __init__(self,trytimes,index_list):
self.file_=open('tv.csv','a')#这个是保存电影下载链接的
self.page_list=index_list#每一页的链接
self.trytimes=trytimes
self.movie_list=[]
self.movie_info=[]
self.index_try=[]
self.info_try=[]
"""
self.info_finish是用来保存已解析过的电影链接的
"""
try:
info=open('info.txt','r')
self.info_finish=cPickle.load(info)
except:
self.info_finish=set()
#如果之前已经运行过代码，这个函数就好执行，没有运行过则不运行这个函数
def preprocess(self):
if len(self.page_list)-895>=1:
self.page_list=self.page_list[:len(self.page_list)-895]
elif len(self.page_list)<=895:
self.page_list=[self.page_list[0]]
#从每一个找到电影链接的函数
def get_movie_list(self,url):
global movie_list,index_try
try:
s=req.get(url)
list_=movie_find.findall(s.content)
list_=[homeurl+ss for ss in list_]
s.close()
self.movie_list.extend(list_)
self.index_finish.add(url)
except:
print 'error %s\n'%url
self.index_try.append(url)
#从电影链接页面解析出电影标题和下载链接
def get_movie_info(self,murl):
global movie_info,info_try
try:
s=req.get(murl)
title=title_find.findall(s.content)[0]
dl=dlist.findall(s.content)
s.close()
self.movie_info.append([title,dl])
self.info_finish.add(murl)
except:
print 'error %s\n'%murl
self.info_try.append(murl)
#写出
def writeout(self,info):
global file_
title,dlink=info
for i in xrange(len(dlink)):
self.file_.write('%d,%s,%s\n'%(i,title,dlink[i]))
#执行函数
def run(self):
pool=Pool(20)
st=time.time()
#pre process
#self.info_finish为空，则说明没有执行过代码，则不执行pretry函数
if len(self.info_finish)!=0:
self.preprocess()
#step 1
pool.map(self.get_movie_list,self.page_list)
print 'step 1 finished!\ncost time:%fs'%(time.time()-st)
st1=time.time()
#step 2
if len(self.info_finish)!=0:
#self.info_finish不为空，取没有解析过的电影
self.movie_list=(set(self.movie_list)|self.info_finish)-set(self.movie_list)
pool.map(self.get_movie_info,list(self.movie_list))
print 'step 2 finished!\ncost time:%fs'%(time.time()-st1)
st2=time.time()
#step 3
pool.map(self.writeout,self.movie_info)
print 'step 3 finished!\ntotal time:%fs'%(time.time()-st2)
pool.close()
pool.join()
if len(self.index_try)!=0:
return 1
elif len(self.index_try)==0&len(self.info_try)!=0:
return 2
else:
return 0
def start(self):
num=self.run()
pool=Pool(20)
#如果有解析失败的页面，则最多重试3次。
if num!=0:
i=1
while i<=self.trytimes:
index=self.index_try
info=self.info_try
self.movie_list=[]
self.movie_info=[]
self.index_try=[]
self.info_try=[]
if len(index)!=0:
pool.map(self.get_movie_list,index)
pool.map(self.get_movie_info,self.movie_list)
pool.map(self.writeout,self.movie_info)
if len(info)!=0:
pool.map(self.get_movie_info,info)
pool.map(self.writeout,self.movie_info)
i+=1
print 'retry %d times!'%self.trytimes
info=open('info.txt','w')
cPickle.dump(self.info_finish,info)
self.file_.close()
pool.close()
pool.join()
if __name__=='__main__':
d=tv(3,page_list)
d.start()