python学习笔记(60) 回调和爬虫

from multiprocessing import Pool
import requests

def get(url):
    response = requests.get(url)
    if response.status_code == 200:
        return url,response.content.decode('utf-8')


def call_back(args):
    url,content = args
    print(url,len(content))


if __name__ == '__main__':
    url_lst = [
        'http://www.sohu.com/',
        'https://www.sogou.com/',
        'https://www.baidu.com/',
        'https://www.cnblogs.com/',
    ]
    p = Pool(5)
    for i in url_lst:
        p.apply_async(get,args=(i,),callback=call_back)
    p.close()
    p.join()