python 简单的并发爬虫

import requests
from multiprocessing import Pool
import time

headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
}
def get_url():
    url_list=[]
    for i in range(10):
        url='https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start={}'.format(str((i+1)*20))
        url_list.append(url)
    return url_list

def get_data(url):
    res=requests.get(url,headers=headers)
    print(res)
    time.sleep(1)

if __name__=='__main__':
    urls=get_url()
    pool=Pool(processes=4)#设置进程数目
    pool.map(get_data,urls)

multiprocessing包可以使用pip install mutiprocess下载