aiohttp的模板

 1 import aiohttp
 2 import asyncio
 3 import async_timeout
 4 from urllib.parse import urljoin,urldefrag
 5 
 6 root_url = 'http://python/org/'  # 开始的url
 7 crawled_urls,url_hub = [], [root_url]
 8 headers = {'user-agent': 'Opera/9.80 (X11; Linux x86_64; U; en) Presto/2.2.15 Version/10.10'}
 9 
10 async def get_body(url):
11     async with aiohttp.ClientSession() as session:
12         try:
13             with async_timeout.timeout(10): #超时时间的设定
14                 async with session.get(url,headers=headers) as response:
15                     if response.status == 200:
16                         html = await response.text()
17                         return {'error':'','html':html,'url':url}
18                     else:
19                         return {'error':response.status,'html':'','url':url}
20         except Exception as err:
21             return {'error':response.status,'html':'','url':url}
22 
23 async def handle_task(task_id,work_queue):
24     while not work_queue.empty():
25         queue_url = await work_queue.get()
26         if not queue_url in crawled_urls:
27 
28             body = await get_body(queue_url)
29             if not body['error']:
30                 crawled_urls.append(queue_url)
31                 parse(body)
32             else:
33                 print('{}爬取失败'.format(queue_url))
34 
35 
36 #解析返回的数据
37 def parse(body):
38     pass
39 
40 
41 
42 def remove_fragment(url):
43     pure_url, frag = urldefrag(url)
44     return pure_url
45 
46 #解析html，拼接新的url
47 def get_urls(html):
48     new_urls = [url.split('"')[0] for url in str(html).replace("'", '"').split('href="')[1:]]
49     return [urljoin(root_url, remove_fragment(new_url)) for new_url in new_urls]
50 
51 if __name__ == '__main__':
52     q = asyncio.Queue()  #初始化一个异步的队列
53     [q.put_nowait(url) for url in url_hub]  #从初始的url队列中遍历，把url放入到队列中
54     loop = asyncio.get_event_loop()
55     tasks = [handle_task(task_id, q) for task_id in range(3)]  #3个并发
56     loop.run_until_complete(asyncio.wait(tasks))
57     loop.close()
58     for u in crawled_urls:
59         print(u)
60     print('-' * 30)
61     print(len(crawled_urls))

View Code