一、整体结构
二、代理IP类的构建（domain.py文件）
''' 实现_ init_ 方法, 负责初始化,包含如下字段: ip: 代理的IP地址 port:代理IP的端口号 protocol: 代理IP支持的协议类型,http是0, https是1, https和http都支持是2 nick_ type: 代理IP的匿名程度,高匿:0,匿名: 1,透明:2 speed:代理IP的响应速度，单位s area:代理IP所在地区 score:代理IP的评分，用于衡量代理的可用性;默认分值可以通过配置文件进行配置.在进行代理可用性检查的时候， 每遇到一次请求失败就减1份,减到0的时候从池中删除.如果检查代理可用,就恢复默认分值 disable_domains:不可用域名列表,有些代理IP在某些域名下不可用，但是在其他域名下可用在配置文件: settings.py中定义MAX_ _SCORE = 50,表示代理IP的默认最高分数提供_ _str__方法， 返回数据字符串 ''' from settings import MAX_SCORE #从settings模块中导入MAX_SCORE变量，这个变量的意思就是给每一个IP一个分数（分数高代表这个IP可用性强，初始化为MAX_SCORE） class Proxy(object): def __init__(self,ip,port,protocol=-1,nick_type=-1,speed=-1,area=None,score=MAX_SCORE,disable_ip=[]): self.ip=ip #代理ip self.port=port #代理ip端口 self.protocol=protocol #代表代理ip的协议类型 self.nick_type=nick_type #匿名程度 self.speed=speed #代理ip速度 self.area=area #代理ip地址 self.score=score #ip分数 self.disable_ip=disable_ip def __str__(self): return str(self.__dict__) #__dict__ : 类的属性（包含一个字典，由类的数据属性组成）
三、配置文件settings.py
可能有人会疑惑为什么要用配置文件保存变量，这样的话还要在其他模块引入，这不是麻烦吗？
因为后面我们可能还要对它们的某些初始值进行改变，这个时候如果如果不用配置文件方式引入，而是直接写上数字。这个时候改代码太麻烦了
import logging MAX_SCORE=50 #代理ip初始分数 #下面这一部分是log.py的内容，是用来控制日志文件的 LOG_LEVEL = logging.INFO #控制日志文件报错级别 ''' 报错级别一共5种（从上到下级别依次递增） logger.debug("") logger.info("") logger.warning("") logger.error("") logger.critical("") 如果设置报错级别为INFO，那么就不会报debug的错误 ''' LOG_FMT = '%(asctime)s %(filename)s [line:%(lineno)d] %(levelname)s: %(message)s' #日志内容格式 LOG_DATEFMT = '%Y-%m-%d %H:%M:%S' #日志内容的时间部分格式（也就是”年-月-日 时:分:秒“） LOG_FILENAME = 'log.log' #要把日志文件放在哪，就放在本目录下log.log文件 TEST_TIMEOUT = 5 #这个是requess访问链接设置的超时时间 #因为我把数据放入了mongodb数据库，所以这里配置的是mongodb数据库连接信息 MONGO_URL = 'mongodb:#127.0.0.1:27017' ''' 这几个是分别爬取几个网站上代理ip的具体爬虫路径 ''' SPIDERS=[ 'IPProxyPool.core.proxy_spider.proxy_spiders.ip66Spider', 'IPProxyPool.core.proxy_spider.proxy_spiders.KuaidailiSpider', 'IPProxyPool.core.proxy_spider.proxy_spiders.ProxylistplusSpider', 'IPProxyPool.core.proxy_spider.proxy_spiders.XiciSpider' ] #这个是多长时间爬取一次代理ip来使用 RUN_SPIDERS_INTERVEL = 12 #这个是最多开多少协程来判断ip是否可用 TEST_PROXIES_ASYNC_COUNT = 10 #这个是多少时间更新一次数据库内ip信息，因为代理ip存活时间有限 TEST_PROXIES_INTERVAL = 2
四、日志信息模块log.py
import logging import sys class Logger(object): def __init__(self): self._logger = logging.getLogger() //得到一个日志处理对象 #传参数，告诉它写入日志内容格式 self.formatter = logging.Formatter(fmt=settings.LOG_FMT,datefmt=settings.LOG_DATEFMT) #把要写入的日志加入句柄里面，到时候会输出到文件内保存 self._logger.addHandler(self._get_file_handler(settings.LOG_FILENAME)) #把要写入的日志加入句柄里面，到时候会输出到控制台查看 self._logger.addHandler(self._get_console_handler()) #设置写入日志级别 self._logger.setLevel(settings.LOG_LEVEL) def _get_file_handler(self,filename): #设置一下要往那个文件里面写，和编码问题 filehandler = logging.FileHandler(filename=filename,encoding="utf-8") filehandler.setFormatter(self.formatter) #日志格式 return filehandler def _get_console_handler(self): console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(self.formatter) return console_handler @property def logger(self): return self._logger logger = Logger()._logger if __name__ == '__main__': #下面的是用来测试这个模块 logger.debug("1") logger.info("2") logger.warning("3") logger.error("4") logger.critical("5")
可能有些一些关键字不懂，可以点下面链接：
Python中“*”和“**”的用法 || yield的用法 || ‘$in’和'$nin' || python @property的含义
五、requests请求头
import random USER_AGENTS =[ #requests请求头 "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Avant Browser/1.2.789rel1 (http:#www.avantbrowser.com)", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5", "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7", "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)", "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5", "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)", "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110622 Firefox/6.0a2", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0 )", "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90)", "Mozilla/5.0 (Windows; U; Windows XP) Gecko MultiZilla/1.6.1.0a", "Mozilla/2.02E (Win95; U)", "Mozilla/3.01Gold (Win95; I)", "Mozilla/4.8 [en] (Windows NT 5.1; U)", "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.4) Gecko Netscape/7.1 (ax)", "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3", "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1", "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3", "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13", "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", "Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1"] def get_requests_headers(): #这个方法就是随机获取一个请求头 headers = { #除了user-Agent变一下，其他大多都不用变 'User-Agent' : random.choice(USER_AGENTS), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip,deflate' } return headers if __name__ == '__main__': #模块检查 print(get_requests_headers())