1.打开middlewares.py文件
2.添加代理类
import base64
""" 阿布云代理配置"""
proxy_server = "http://http-dyn.abuyun.com:9020"
proxy_user = "HEWGR9329K68Z11D"
proxy_pass = "ACEBFFCDFE62615F"
proxy_auth = "Basic " + base64.urlsafe_b64encode(bytes((proxy_user + ":" + proxy_pass), "ascii")).decode("utf8")
class ABYProxyMiddleware(object):
""" 阿布云代理中间件 """
def process_request(self, request, spider):
request.meta["proxy"] = proxy_server
request.headers["Proxy-Authorization"] = proxy_auth
3.打开settings.py文件开启中间件:
DOWNLOADER_MIDDLEWARES = {
'Securities.middlewares.ABYProxyMiddleware': 1,
}
4.阿布云每秒请求次数有限制,可以在settings.py中对scrapy请求速度进行设置
""" 启用限速设置 """
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 0.2 # 初始下载延迟
DOWNLOAD_DELAY = 0.2 # 每次请求间隔时间