python基础--性能相关、自定义异步io

性能相关

循环操作

import requests

def fetch_async(url):
    response = requests.get(url)
    return response


url_list = ['http://www.baidu.com', 'http://www.cnblogs.com']

for url in url_list:
    res = fetch_async(url)
    print(res)

线程池

from concurrent.futures import ThreadPoolExecutor
import requests


def fetch_async(url):
    response = requests.get(url)
    return response


url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ThreadPoolExecutor(5)
for url in url_list:
    pool.submit(fetch_async, url)
pool.shutdown(wait=True)

多线程+回调函数执行

from concurrent.futures import ThreadPoolExecutor
import requests

def fetch_async(url):
    response = requests.get(url)
    return response


def callback(future):
    print(future.result())


url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ThreadPoolExecutor(5)
for url in url_list:
    v = pool.submit(fetch_async, url)
    v.add_done_callback(callback)
pool.shutdown(wait=True)

多进程

from concurrent.futures import ProcessPoolExecutor
import requests


def fetch_async(url):
    response = requests.get(url)
    return response


def callback(future):
    print(future.result())


url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ProcessPoolExecutor(5)
for url in url_list:
    v = pool.submit(fetch_async, url)
    v.add_done_callback(callback)
pool.shutdown(wait=True)

异步io

setblocking非阻塞

socket通信默认是阻塞的，当我们设置sk.setblocking(False)时候，socket通信就变成非阻塞的了，如下的代码运行会报错，因为sk.connect((‘www.cnblogs.com’, 80))连接服务端时候，是非阻塞的，就继续执行之后的代码，当sendall发送时候，可能还没有连接到服务端，所以会报错

import socket

sk = socket.socket()
sk.setblocking(False)

# 连接的过程是:阻塞
sk.connect(('www.cnblogs.com', 80))

# HTTP协议
sk.sendall(
    b'GET /safly HTTP/1.1\r\nHost: www.cnblogs.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36\r\n\r\n')

# 等待服务端返回内容:阻塞
data = sk.recv(8096)
print(data)

sk.close()

输出如下：

Traceback (most recent call last):
  File "1. 高性能本质/本质.py", line 7, in <module>
    sk.connect(('www.cnblogs.com', 80))
BlockingIOError: [WinError 10035] 无法立即完成一个非阻止性套接字操作。

setblocking非阻塞

我们可以通过socket的非阻塞功能，来自定义异步非阻塞，从而实现单线程下的并发操作，其实利用了io多路复用的特点，IO多路复用：监听多个socket是否发生变化

思路如下：
1、封装(socket\回掉函数)类
2、构造可读、可写列表，循环监听
3、回调函数

import socket
import select


class Request(object):
    def __init__(self, sk, callback):
        self.sk = sk
        self.callback = callback

    def fileno(self):
        return self.sk.fileno()


class AsyncHttp(object):

    def __init__(self):
        self.fds = []
        self.conn = []

    def add(self, url, callback):
        sk = socket.socket()
        sk.setblocking(False)
        try:
            sk.connect((url, 80))
        except BlockingIOError as e:
            pass
        req = Request(sk, callback)
        self.fds.append(req)
        self.conn.append(req)

    def run(self):
        """
        监听socket是否发生变化
        :return:
        """
        while True:
            r, w, e = select.select(self.fds, self.conn, [], 0.05)  # sk.fileno() = req.fileno()

            # w=已经连接成功的socket列表 w=[sk1,sk2]
            for req in w:
                print("w", req)
                req.sk.sendall(
                    b'GET /wupeiqi HTTP/1.1\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36\r\n\r\n')
                # 已经连接成功的socket，无需再继续监听
                self.conn.remove(req)

            # r=服务端给用户返回数据了 r=[sk1,]
            for req in r:
                print("r", req)
                data = req.sk.recv(8096)
                req.callback(data)

                req.sk.close()  # 断开连接：短连接、无状态
                self.fds.remove(req)  # 不再监听

            if not self.fds:
                break


ah = AsyncHttp()


def callback1(data):
    print(11111, data)


def callback2(data):
    print(22222, data)


ah.add('www.cnblogs.com', callback1)  # sk1
ah.add('www.baidu.com', callback2)  # sk2

ah.run()

输出如下：

E:\python\python_sdk\python.exe "1. 高性能本质/s1.py"
w <__main__.Request object at 0x04CA6AF0>
w <__main__.Request object at 0x04CA6B70>
r <__main__.Request object at 0x04CA6AF0>
11111 b'HTTP/1.1 400 Bad Request\r\nDate: Thu, 10 May 2018 12:34:29 GMT\r\nContent-Type: text/html\r\nContent-Length: 242\r\nConnection: close\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\r\n<html>\r\n<head><title>400 Bad Request</title></head>\r\n<body bgcolor="white">\r\n<h1>400 Bad Request</h1>\r\n<p>Your browser sent a request that this server could not understand.</body>\r\n</html>\r\n'
r <__main__.Request object at 0x04CA6B70>
22222 b'HTTP/1.1 400 Bad Request\r\nDate: Thu, 10 May 2018 12:34:29 GMT\r\nServer: Apache\r\nContent-Length: 226\r\nConnection: Keep-Alive\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\n<html><head>\n<title>400 Bad Request</title>\n</head><body>\n<h1>Bad Request</h1>\n<p>Your browser sent a request that this server could not understand.<br />\n</p>\n</body></html>\n'

Process finished with exit code 0