Requests 代理池

Requests 本身不提供代理池，然而爬数据又要用，所以只能自己搞。其实还挺简单的。我也不知道为什么这么有用的 feature 一直没有被加入。

import requests


class Client:

    def __init__(self): self._session = requests.Session() self.proxies = None def set_proxy_pool(self, proxies, auth=None, https=True): """Randomly choose a proxy for every GET/POST request :param proxies: list of proxies, like ["ip1:port1", "ip2:port2"] :param auth: if proxy needs auth :param https: default is True, pass False if you don't need https proxy """ from random import choice if https: self.proxies = [{'http': 'http://' + p, 'https': 'https://' + p} for p in proxies] else: self.proxies = [{'http': 'http://' + p} for p in proxies] def get_with_random_proxy(url, **kwargs): proxy = choice(self.proxies) kwargs['proxies'] = proxy if auth: kwargs['auth'] = auth return self._session.original_get(url, **kwargs) def post_with_random_proxy(url, *args, **kwargs): proxy = choice(self.proxies) kwargs['proxies'] = proxy if auth: kwargs['auth'] = auth return self._session.original_post(url, *args, **kwargs) self._session.original_get = self._session.get self._session.get = get_with_random_proxy self._session.original_post = self._session.post self._session.post = post_with_random_proxy def remove_proxy_pool(self): self.proxies = None self._session.get = self._session.original_get self._session.post = self._session.original_post del self._session.original_get del self._session.original_post # You can define whatever operations using self._session 

替换掉 Session 原本的 get 和 post 方法就行了，不会有什么副作用。class Client 并不必需，直接操作 Session 是一样的。

可以用 httpbin 来做验证

def test_proxy():
    # visit http://cn-proxy.com/ to get available proxies if test failed proxy_ips = ['112.25.41.136', '180.97.29.57'] client = Client() client.set_proxy_pool(proxy_ips) for _ in range(5): result = client._session.get('http://httpbin.org/ip').json() assert result['origin'] in proxy_ips result = client._session.post('http://httpbin.org/post', data={'m':'1'}).json() assert result['form'] == {'m': '1'} print(result['origin']) assert result['origin'] in proxy_ips client.remove_proxy_pool() client.set_proxy_pool(proxy_ips, https=False) for _ in range(5): result = client._session.get('http://httpbin.org/ip').json() print(result['origin']) assert result['origin'] in proxy_ips

转载自 ：https://laike9m.com/blog/requests-dai-li-chi,92/