scrapy Downloader Middlewares 中间件

class UserAgentMiddleware(object):
    """ Change User-Agent """

    def process_request(self, request, spider):
        agent = random.choice(agents)
        request.headers["User-Agent"] = agent


class CookiesMiddleware(object):
    """ Change Cookie """

    def process_request(self, request, spider):
        cookie = random.choice(cookies)
        request.cookies = cookie


class ProxyMiddleware(object):
    """Everyone Request change proxy"""

    def process_request(self, request, spider):
        try:
            #  get ip address
            PROXY_POOL_URL = 'xxxxxx'
            response = requests.get(url=PROXY_POOL_URL)
            if response.status_code == 200:
                proxy = response.text.strip()
                print('-------use---proxy------: http://',proxy)
                request.meta['proxy'] = "http://" + proxy

        except Exception as e :
            print("-----Error:---Proxy---ERROR-----", e)

url : https://scrapy-chs.readthedocs.io/zh_CN/0.24/topics/downloader-middleware.html