python 3 线程/进程池concurrent.futures模块使用

一、Executor和Future

      concurrent.futures模块的基础是Exectuor，Executor是一个抽象类，ThreadPoolExecutor和ProcessPoolExecutor是其非常有用的两个子类。Future可以把它理解为一个在未来完成的操作，有异步编程的概念。

二、线程池和进程池

1.使用submit来操作线程池/进程池：

线程池：

# 线程池：
from concurrent.futures import ThreadPoolExecutor
import urllib.request
URLS = ['http://www.163.com', 'https://www.baidu.com/', 'https://github.com/']
def load_url(url):
    with urllib.request.urlopen(url, timeout=60) as conn:
        print('%r page is %d bytes' % (url, len(conn.read())))

executor = ThreadPoolExecutor(max_workers=3)

for url in URLS:
    future = executor.submit(load_url,url)
    print(future.done())

print('主线程')

# 运行结果：
False
False
False
主线程
'https://www.baidu.com/' page is 227 bytes
'http://www.163.com' page is 662047 bytes
'https://github.com/' page is 54629 bytes

进程池：

# 进程池：同上
from concurrent.futures import ProcessPoolExecutor
import urllib.request
URLS = ['http://www.163.com', 'https://www.baidu.com/', 'https://github.com/']
def load_url(url):
    with urllib.request.urlopen(url, timeout=60) as conn:
        print('%r page is %d bytes' % (url, len(conn.read())))

executor = ProcessPoolExecutor(max_workers=3)
if __name__ == '__main__': # 要加main

    for url in URLS:
        future = executor.submit(load_url,url)
        print(future.done())
    print('主线程')

#运行结果：
False  #　子进程只完成创建，并没有执行完成
False　
False
主线程　＃　子进程创建完成就会向下执行主线程，并不会等待子进程执行完毕
'http://www.163.com' page is 662049 bytes
'https://www.baidu.com/' page is 227 bytes
'https://github.com/' page is 54629 bytes

2.使用map来操作线程池/进程池：

from concurrent.futures import ThreadPoolExecutor
import urllib.request
URLS = ['http://www.163.com', 'https://www.baidu.com/', 'https://github.com/']
def load_url(url):
    with urllib.request.urlopen(url, timeout=60) as conn:
        print('%r page is %d bytes' % (url, len(conn.read())))

executor = ThreadPoolExecutor(max_workers=3)

executor.map(load_url,URLS)

print('主线程')

# 运行结果：
主线程
'http://www.163.com' page is 662047 bytes
'https://www.baidu.com/' page is 227 bytes
'https://github.com/' page is 54629 bytes

　　从运行结果可以看出，map是按照URLS列表元素的顺序返回的，并且写出的代码更加简洁直观，我们可以根据具体的需求任选一种。

3.wait：

　　wait方法接会返回一个tuple(元组)，tuple中包含两个set(集合)，一个是completed(已完成的)另外一个是uncompleted(未完成的)。使用wait方法的一个优势就是获得更大的自由度，它接收三个参数FIRST_COMPLETED, FIRST_EXCEPTION 和ALL_COMPLETE，默认设置为ALL_COMPLETED。

　　如果采用默认的ALL_COMPLETED，程序会阻塞直到线程池里面的所有任务都完成，再执行主线程：

from concurrent.futures import ThreadPoolExecutor,wait,as_completed
import urllib.request
URLS = ['http://www.163.com', 'https://www.baidu.com/', 'https://github.com/']
def load_url(url):
    with urllib.request.urlopen(url, timeout=60) as conn:
        print('%r page is %d bytes' % (url, len(conn.read())))

executor = ThreadPoolExecutor(max_workers=3)

f_list = []
for url in URLS:
    future = executor.submit(load_url,url)
    f_list.append(future)
print(wait(f_list))

print('主线程')

# 运行结果：
'http://www.163.com' page is 662047 bytes
'https://www.baidu.com/' page is 227 bytes
'https://github.com/' page is 54629 bytes
DoneAndNotDoneFutures(done={<Future at 0x2d0f898 state=finished returned NoneType>, <Future at 0x2bd0630 state=finished returned NoneType>, <Future at 0x2d27470 state=finished returned NoneType>}, not_done=set())
主线程

　　如果采用FIRST_COMPLETED参数，程序并不会等到线程池里面所有的任务都完成。

from concurrent.futures import ThreadPoolExecutor,wait,as_completed
import urllib.request
URLS = ['http://www.163.com', 'https://www.baidu.com/', 'https://github.com/']
def load_url(url):
    with urllib.request.urlopen(url, timeout=60) as conn:
        print('%r page is %d bytes' % (url, len(conn.read())))

executor = ThreadPoolExecutor(max_workers=3)

f_list = []
for url in URLS:
    future = executor.submit(load_url,url)
    f_list.append(future)
print(wait(f_list,return_when='FIRST_COMPLETED'))

print('主线程')

# 运行结果：
'http://www.163.com' page is 662047 bytes
DoneAndNotDoneFutures(done={<Future at 0x2bd15c0 state=finished returned NoneType>}, not_done={<Future at 0x2d0d828 state=running>, <Future at 0x2d27358 state=running>})
主线程
'https://www.baidu.com/' page is 227 bytes
'https://github.com/' page is 54629 bytes

 应用线程池：

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import requests
import time,os
def get_page(url):
    print('<%s> is getting [%s]'%(os.getpid(),url))
    response = requests.get(url)
    if response.status_code==200:  #200代表状态：下载成功了
        return {'url':url,'text':response.text}
def parse_page(res):
    res = res.result()
    print('<%s> is getting [%s]'%(os.getpid(),res['url']))
    with open('db.txt','a') as f:
        parse_res = 'url:%s size:%s\n'%(res['url'],len(res['text']))
        f.write(parse_res)
if __name__ == '__main__':
    # p = ThreadPoolExecutor()
    p = ProcessPoolExecutor()
    l = [
        'http://www.baidu.com',
        'http://www.baidu.com',
        'http://www.baidu.com',
        'http://www.baidu.com',
    ]
    for url in l:
        res = p.submit(get_page,url).add_done_callback(parse_page) #这里的回调函数拿到的是一个对象。得
        #  先把返回的res得到一个结果。即在前面加上一个res.result() #谁好了谁去掉回调函数
                                # 回调函数也是一种编程思想。不仅开线程池用，开线程池也用
    p.shutdown()  #相当于进程池里的close和join
    print('主',os.getpid())