[Xiao Mu learns Python] Python implements web server (aiohttp)

1 Introduction

https://github.com/aio-libs/aiohttp
https://docs.aiohttp.org/en/stable/index.html

Asynchronous HTTP client/server framework for asyncio and Python
asynchronous http client/server framework

main feature:

• Supports client and server sides of HTTP protocol.
• Support client and server web sockets out of the box and avoid callback hell.
• Provides middleware and pluggable routing for web servers.

2. Download and install

Installation library:

pip3 install aiohttp
# pip install aiodns
# pip install aiohttp[speedups]

3. Code testing

3.1 Client

• Client: Wants to get something from the network.

import aiohttp
import asyncio

async def main():

    async with aiohttp.ClientSession() as session:
        async with session.get('https://www.baidu.com/') as response:

            print("Status:", response.status)
            print("Content-type:", response.headers['content-type'])

            html = await response.text()
            print("Body:", html[:15], "...")

asyncio.run(main())

After running:

error reported.
Modify the code as follows:

import aiohttp
import asyncio

async def main():

    async with aiohttp.ClientSession() as session:
        async with session.get('https://www.baidu.com/') as response:

            print("Status:", response.status)
            print("Content-type:", response.headers['content-type'])

            html = await response.text()
            print("Body:", html[:15], "...")

# asyncio.run(main())
loop = asyncio.get_event_loop()
loop.run_until_complete(main())

After running it again, no error was reported.

3.2 Server

• Server: Example using a simple server.

# examples/server_simple.py
from aiohttp import web

async def handle(request):
    name = request.match_info.get('name', "Anonymous")
    text = "Hello, " + name
    return web.Response(text=text)

async def wshandle(request):
    ws = web.WebSocketResponse()
    await ws.prepare(request)

    async for msg in ws:
        if msg.type == web.WSMsgType.text:
            await ws.send_str("Hello, {}".format(msg.data))
        elif msg.type == web.WSMsgType.binary:
            await ws.send_bytes(msg.data)
        elif msg.type == web.WSMsgType.close:
            break

    return ws


app = web.Application()
app.add_routes([web.get('/', handle),
                web.get('/echo', wshandle),
                web.get('/{name}', handle)])

if __name__ == '__main__':
    web.run_app(app)

After running:

Browser access URL:

http://127.0.0.1:8080/

4. More tests

4.1 asyncio

asyncio is a standard library introduced in Python version 3.4, with built-in support for asynchronous IO.

The programming model of asyncio is a message loop. We directly obtain an EventLoop reference from the asyncio module, and then throw the coroutine that needs to be executed into the EventLoop for execution, thus achieving asynchronous IO.

import asyncio

@asyncio.coroutine
def hello():
    print("Hello world!111")
    print("Hello world!22")
    # 异步调用asyncio.sleep(1):
    r = yield from asyncio.sleep(1)
    print("Hello again!333")
    print("Hello again!444")

# 获取EventLoop:
loop = asyncio.get_event_loop()
# 执行coroutine
loop.run_until_complete(hello())
loop.close()

import threading
import asyncio

@asyncio.coroutine
def hello():
    print('Hello world! (%s)' % threading.currentThread())
    yield from asyncio.sleep(1)
    print('Hello again! (%s)' % threading.currentThread())

loop = asyncio.get_event_loop()
tasks = [hello(), hello()]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

import asyncio

@asyncio.coroutine
def wget(host):
    print('wget %s...' % host)
    connect = asyncio.open_connection(host, 80)
    reader, writer = yield from connect
    header = 'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % host
    writer.write(header.encode('utf-8'))
    yield from writer.drain()
    while True:
        line = yield from reader.readline()
        if line == b'\r\n':
            break
        print('%s header > %s' % (host, line.decode('utf-8').rstrip()))
    # Ignore the body, close the socket
    writer.close()

loop = asyncio.get_event_loop()
tasks = [wget(host) for host in ['www.sina.com.cn', 'www.sohu.com', 'www.163.com']]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

asyncio provides complete asynchronous IO support;
asynchronous operations need to be completed through yield from in coroutine;

4.2 aiohttp+HTTP server

Write an HTTP server:

import asyncio

from aiohttp import web

async def index(request):
    await asyncio.sleep(0.5)
    return web.Response(text='<h1>Index</h1>', content_type= 'text/html')

async def hello(request):
    await asyncio.sleep(0.5)
    text = '<h1>hello, %s!</h1>' % request.match_info['name']
    return web.Response(text=text, content_type= 'text/html')

async def init(loop):
    app = web.Application(loop=loop)
    app.router.add_route('GET', '/', index)
    app.router.add_route('GET', '/hello/{name}', hello)
    srv = await loop.create_server(app.make_handler(), '127.0.0.1', 8000)
    print('Server started at http://127.0.0.1:8000...')
    return srv

loop = asyncio.get_event_loop()
loop.run_until_complete(init(loop))
loop.run_forever()

4.3 aiohttp+crawler example

pip install bs4 

Write a crawler example:

import asyncio
import aiohttp
from bs4 import BeautifulSoup
import logging

class AsnycSpider(object):

    def __init__(self, url_list, max_threads):

        self.urls = url_list
        self.results = {
    
    }
        self.max_threads = max_threads

    def __parse_results(self, url, html):
        try:
            soup = BeautifulSoup(html, 'html.parser')
            title = soup.find('title').get_text()
        except Exception as e:
            raise e

        if title:
            self.results[url] = title

    async def get_body(self, url):
        async with aiohttp.ClientSession() as session:
            async with session.get(url, timeout=30) as response:
                assert response.status == 200
                html = await response.read()
                return response.url, html

    async def get_results(self, url):
        url, html = await self.get_body(url)
        self.__parse_results(url, html)
        return 'Completed'

    async def handle_tasks(self, task_id, work_queue):
        while not work_queue.empty():
            current_url = await work_queue.get()
            try:
                task_status = await self.get_results(current_url)
            except Exception as e:
                logging.exception('Error for {}'.format(current_url), exc_info=True)

    def eventloop(self):
        q = asyncio.Queue()
        [q.put_nowait(url) for url in self.urls]
        loop = asyncio.get_event_loop()
        tasks = [self.handle_tasks(task_id, q, ) for task_id in range(self.max_threads)]
        loop.run_until_complete(asyncio.wait(tasks))
        # loop.close()

if __name__ == '__main__':
    async_example = AsnycSpider([
        'https://www.qq.com/',
        'https://www.163.com/',
        'https://news.baidu.com/',
        'https://blog.csdn.net/'], 5)
    async_example.eventloop()
    print(async_example.results)

4.4 aiohttp+requests comparison

Among the many HTTP clients in Python, there are these: requests, aiohttp and httpx. Without the help of other third-party libraries, requests can only send synchronous requests; aiohttp can only send asynchronous requests; httpx can send both synchronous requests and asynchronous requests.

pip install requests

• test_requests.py

import random
import time
import datetime
import requests


def make_request(session):
    resp = session.get('http://httpbin.org/get')
    # result = resp.text
    # print(result)
    pass


def main():
    session = requests.Session()
    start = time.time()
    for _ in range(100):
        make_request(session)
    end = time.time()
    print(f'发送100次请求，耗时：{
      
      end - start}')


if __name__ == '__main__':
    main()

• test_aiohttp.py

import aiohttp
import random
import datetime
import asyncio
import time

async def request(client):
    async with client.get('http://httpbin.org/get') as resp:
        # print(resp.status)
        # print(await resp.text())
        pass

async def main():
    async with aiohttp.ClientSession() as client:
        start = time.time()
        task_list = []
        for _ in range(100):
            req = request(client)
            task = asyncio.create_task(req)
            task_list.append(task)
        await asyncio.gather(*task_list)
        end = time.time()
    print(f'发送100次请求，耗时：{
      
      end - start}')

asyncio.run(main())

Conclusion

如果您觉得该方法或代码有一点点用处，可以给作者点个赞，或打赏杯咖啡；╮(￣▽￣)╭
如果您感觉方法或代码不咋地//(ㄒoㄒ)// ，就在评论处留言，作者继续改进；o_O???
如果您需要相关功能的代码定制化开发，可以留言私信作者；(✿◡‿◡)
感谢各位大佬童鞋们的支持！( ´ ▽´ )ﾉ ( ´ ▽´)! ! !