NO.2_python_scrapy_反爬虫（随机请求头&IP代理）&取消链接去重

1.随机请求头

# -*- coding: utf-8 -*-
"""
所有请求头的USER_AGENTS网址
http://www.useragentstring.com/pages/useragentstring.php?name=All
"""
import json
import random
import requests


USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36 Chrome 41.0.2227.0',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36',
]

#从列表中随机选择一个
**User_Agent = random.choice(USER_AGENTS)**

headers = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent': User_Agent
}


def testUserAgent():
    resp = requests.get("http://httpbin.org/user-agent", headers=headers)
    user_agent = json.loads(resp.text)['user-agent']#将返回的字符串loads为字典
    print(user_agent)

def testProxy():
    proxies = {"https": "223.199.25.183:9999"}
    resp = requests.get("http://httpbin.org/ip", proxies=proxies)
    origin = json.loads(resp.text)['origin']
    print(origin)


# https://www.xicidaili.com/nn/ 西刺代理
if __name__ == "__main__":
    testUserAgent()

 testProxy()

在middleware.py文件进行修改，

#将多个请求头放在一个列表中
User_Agent = [
	'请求头1'，
	'请求头2'，
	'请求头3'，
	...
	]
#需要使用的时候从列表中随机抽取一个请求头，代码如下
```python
class user_agentdownloadmiddleware(Object):
	def process_request(self,request,spider):
		User_Agent = random.chioce(self.User_Agent)
		request.headers['User_Agent'] = User_Agent

在setting.py中进行修改

将配置文件中的downloadmiddleware取消注销即可

取消链接去重

def parse():
	...
	#此处的dont_filter即为去重的函数，将其设置为True则表示不要去重复的链接进行去重，即可以对同一个链接进行反复地请求并且获得数据
	yield scrapy.Request(self.start_urls[0],dont_filter = True)
	...