python aiohttp与aiohttp

初步使用

# -*- coding: utf-8 -*-
import asyncio 
import aiohttp 


#async以及await关键字将函数异步化,首先异步获取相应，然后异步读取响应的内容
async def fetch(): 
    #Aiohttp使用ClientSession作为主要的接口发起请求，Session(会话)在使用完毕之后需要关闭，关闭Session是另一个异步操作，所以每次你都需要使用async with关键字，with语句可以保证在处理session的时候，总是能正确的关闭它。
    async with aiohttp.ClientSession() as session:         
        async with session.get("http://xxx") as response:                
            response = await response.read()                         
            print(response) 

#创建一个asyncio loop的实例， 然后将任务加入其中
loop = asyncio.get_event_loop() 
loop.run_until_complete(fetch())

指定session

# -*- coding: utf-8 -*-
import asyncio 
import aiohttp 



async def fetch(jsessionId): 
    async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:        
        async with session.get("http://xxx") as response:                
            response = await response.read()                         
            print(response) 

loop = asyncio.get_event_loop() 
loop.run_until_complete(fetch('xxxxxxxxxxxxxxxxxxxxxx'))

多条url请求

import asyncio 
import aiohttp  

async def fetch(url, jsessionId):      
async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:           
    async with session.get(url) as response:             
        return await response.read()             

async def run(r):     
url = "http://xxx/{}"
tasks = []         
for i in range(r):             
    task = fetch(url.format(i), 'xxxxxxxxxxxxxxxxxxxxxx')               
    tasks.append(task)         
    responses = await asyncio.gather(*tasks)         
    # you now have all response bodies in this variable         
    print(responses)     


loop = asyncio.get_event_loop() 
loop.run_until_complete(run(4))

从文件中读取第一列作为url参数,进行多条url请求

# -*- coding: utf-8 -*-
import aiohttp
import asyncio
import sys, os
from datetime import datetime 
import json

async def fetch(url, session, sem):
    async with sem:
        async with session.get(url) as resp:
            #不加下面这一行的话，不会实现协程异步
            await asyncio.sleep(0)
            return await resp.read()


async def run():
    tasks = []
    numbers = []
    url = "http://xxx/msisdn={}"
    jsessionId = 'A3FB6AA9CE57ACB5CF105E9428CE0AB2'
    #脚本名：sys.argv[0],参数1：sys.argv[1],参数2：sys.argv[2]
    file = sys.argv[1]
    #会报：concurrent.futures._base.TimeoutError原因为：Once it's a big number such as 30,000 it can't be physically done within 10 seconds due to networks/ram/cpu capacity.所以需要限制携程的信号量
    sem = asyncio.Semaphore(1000)
    #将session以参数传入fetch中，让所有请求只使用一个session，而不用每个请求都创建一个session
    async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:
        if os.path.isfile(file):
            with open(file, mode = 'r') as f:
                for line in f:
                    #split()默认以空格分隔
                    phoneNumber = line.split()[0]
                    numbers.append(phoneNumber)
                    #此次没有使用asyncio.ensure_future，当前自己使用与不使用暂未发现区别
                    task = fetch(url.format(phoneNumber), session, sem)
                    tasks.append(task)

            #它搜集所有的Future对象，然后等待他们返回
            responses = await asyncio.gather(*tasks)
            for j in range(len(responses)):
                #由于返回的json信息为unicode编码，中文显示需要转换一下
                resJson = json.loads(responses[j].decode('utf-8'))
                if resJson["reDesc"] != "操作成功":
                    print('{}:{}'.format(numbers[j],responses[j].decode('utf-8')))
        else:
            print("{} not found".format(file))


a = datetime.now()
loop = asyncio.get_event_loop()
loop.run_until_complete(run())
loop.close()
b = datetime.now()
#通过开始与结束时间之差来确定花费了多少时间
print('Cost {} seconds'.format((b - a).seconds))

使用docopt获取session参数、读取文件、输出文件

# -*- coding: utf-8 -*-


"""xx查询

Usage:
miguQuery <session> <inputfile> <outputfile>


"""

from docopt import docopt
import aiohttp
import asyncio
import sys, os
from datetime import datetime
import json

#使用async以及await关键字将函数异步化。在hello()中实际上有两个异步操作：首先异步获取相应，然后异步读取响应的内容。
async def fetchTest(url, jsessionId):
    async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:
        async with session.get(url) as resp:
            return await resp.read()

async def fetch(url, session, sem):
    #conn = aiohttp.TCPConnector(limit=30)
    #async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:
    async with sem:
        async with session.get(url) as resp:
            #不加下面这一行的话，不会实现协程异步
            await asyncio.sleep(0)
            #read()是一个异步操作，这意味着它不会立即返回结果，仅仅返回生成器，所以添加上了await
            return await resp.read()

async def run(url, jsessionId, inputfile, outputfile):
    tasks = []
    numbers = []
    count = 0
    #脚本名：sys.argv[0],参数1：sys.argv[1],参数2：sys.argv[2]
    #file = sys.argv[1]
    file = inputfile 
    #会报：concurrent.futures._base.TimeoutError原因为：Once it's a big number such as 30,000 it can't be physically done within 10 seconds due to networks/ram/cpu capacity.所以需要限制携程的信号量
    sem = asyncio.Semaphore(1000)
    #将session以参数传入fetch中，让所有请求只使用一个session，而不用每个请求都创建一个session
    async with aiohttp.ClientSession(headers={"Cookie":"JSESSIONID={}".format(jsessionId)}) as session:
    with open(file, mode = 'r') as f:
        for line in f:
            #split()默认以空格分隔
            phoneNumber = line.split()[0]
            numbers.append(phoneNumber)
            #print(url.format(phoneNumber))
            #task = asyncio.ensure_future(fetch(url.format(phoneNumber), jssessionId))
            #包装在asyncio的Future对象中，然后将Future对象列表作为任务传递给事件循环。
            task = fetch(url.format(phoneNumber), session, sem)
            tasks.append(task)

    ##它搜集所有的Future对象，然后等待他们返回
    responses = await asyncio.gather(*tasks)
    file = open(outputfile, 'w')
    for j in range(len(responses)):
        resJson = json.loads(responses[j].decode('utf-8'))
        if resJson["reDesc"] != "[FCMG]操作成功":
            #print('{}:{}'.format(numbers[j],responses[j].decode('utf-8')))
            file.write('{}:{}\n'.format(numbers[j],responses[j].decode('utf-8')))
            count += 1
    file.close()
    print("总计查询到包月号码:{}个".format(count))
    print("查询结果输出到:{}".format(outputfile))





if __name__ == '__main__':
    # 将绑定交互参数
    arguments = docopt(__doc__)
    session = arguments['<session>']
    inputfile = arguments['<inputfile>']
    outputfile = arguments['<outputfile>']
    url = "http://xxx?msisdn={}"

    if os.path.isfile(inputfile) == False:
        print("找不到此文件:{}".format(inputfile))
        exit(0)
    if os.path.isfile(outputfile) == True:
        print("文件: {} 已经存在，请保存为另一个名字".format(outputfile))
        exit(0)



    #创建一个asyncio loop的实例， 然后将任务加入其中
    loop = asyncio.get_event_loop()
    resp = loop.run_until_complete(fetchTest(url.format('13666198249'), session)).decode('utf-8')
    if 'login.jsp' in resp:
        print("当前session:{} 不正确或已过期，请重新传入session参数".format(session))
        loop.close()
        exit(0)

    a = datetime.now()
    loop.run_until_complete(run(url, session, inputfile, outputfile))
    loop.close()
    b = datetime.now()
    print('Cost {} seconds'.format((b - a).seconds))