scrapy-下载器中间件

from faker import Faker
class MySpiderMiddleware(object):
    def __init__(self):
        self.fake = Faker()

    def process_request(self,request,spider):
        request.headers.setdefault('User-Agent',self.fake.user_agent())

DOWNLOADER_MIDDLEWARES = {
   #'middle.middlewares.MyCustomDownloaderMiddleware': 543,
   'middle.middlewares.MySpiderMiddleware': 100,
   'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}