Python进程池的简单使用

Python中有进程池的概念，相比线程池，更加稳定。在此基础上，进行一个简单封装。

首先写一个工具类ProcessUtil.py

#!/user/bin/env python3
# -*- coding: utf-8 -*-
from multiprocessing.pool import Pool


class TaskAction(object):
    def do_in_action(self):
        pass


class TaskProcess(object):
    def __init__(self, core_size=None):
        if core_size is None:
            self.pool = Pool()
        else:
            if type(core_size) is not int:
                raise ValueError('core_size can\'t handle type  %s ,only int support ' % type(core_size))
            self.pool = Pool(core_size)

    # 检查task参数有效性
    @staticmethod
    def _check_task(tasks):
        # check tasks type is list ?
        if type(tasks) is not list:
            raise ValueError('tasks can\'t handle type %s , only list support' % type(tasks))
        # check tasks' subset is TaskAction ?
        for child_task in tasks:
            if not isinstance(child_task, TaskAction):
                raise ValueError(
                    'tasks\' subset can\'t handle type %s , only TaskAction support' % type(child_task))

    # 异步执行进程
    def execute_task(self, tasks):
        apply_results = []
        # check params
        self._check_task(tasks)
        # 添加任务至进程池
        for child_task in tasks:
            apply_results.append(self.pool.apply_async(child_task.do_in_action))
        # 拒绝添加新的Process
        self.pool.close()
        # 等待所有子进程执行完毕
        self.pool.join()
        # 封装请求结果
        results = []
        for res in apply_results:
            results.append(res.get())
        return results

然后是一个测试case：

#!/user/bin/env python3
# -*- coding: utf-8 -*-
import json
import os
import random
# 进程的试炼
import time
import unittest

from util import DateUtil
from util import ProcessUtil


class ProcessTest(unittest.TestCase):
    def setUp(self):
        print('============================= %s is ready to start...' % self._testMethodName)

    def tearDown(self):
        print('============================= %s had finished...' % self._testMethodName)

    def test_04(self):
        list = []
        # 构造任务列
        for i in range(100):
            user_id = i + 1
            access_token = DateUtil.DateParser.parse_stamp_second(DateUtil.DateParser.now()) + user_id + random.randint(
                10, 99)
            list.append(DemoTest(user_id, access_token))
        # 执行任务列
        result = ProcessUtil.TaskProcess(3).execute_task(list)
        print('All sub processes done.')
        print(json.dumps(result))


class DemoTest(ProcessUtil.TaskAction):
    def __init__(self, user_id, access_token):
        self.user_id = user_id
        self.access_token = access_token

    def do_in_action(self):
        second = random.random() * 3
        time.sleep(second)
        print('UserId : %s , AccessToken : %s , (%s) ' % (self.user_id, self.access_token, os.getpid()))
        return second


if __name__ == '__main__':
    unittest.main()

控制台部分打印信息：

UserId : 90 , AccessToken : 1504516412.0 , (15272) 
UserId : 88 , AccessToken : 1504516457.0 , (7456) 
UserId : 91 , AccessToken : 1504516405.0 , (15272) 
UserId : 92 , AccessToken : 1504516425.0 , (7456) 
UserId : 89 , AccessToken : 1504516452.0 , (7096) 
UserId : 93 , AccessToken : 1504516471.0 , (15272) 
UserId : 96 , AccessToken : 1504516435.0 , (15272) 
UserId : 97 , AccessToken : 1504516460.0 , (15272) 
UserId : 98 , AccessToken : 1504516439.0 , (15272) 
UserId : 94 , AccessToken : 1504516455.0 , (7456) 
UserId : 95 , AccessToken : 1504516454.0 , (7096) 
UserId : 100 , AccessToken : 1504516402.0 , (7456) 
UserId : 99 , AccessToken : 1504516426.0 , (15272) 
.
----------------------------------------------------------------------
Ran 1 test in 48.824s

从结果上看进程设置了3个进程分别在 7456 ，7096，15272 三个pid上运行，提高了效率

最后吐槽下，java中实现线程池好累，要关心池的配置，资源释放，类型校验等等

Python进程池的简单使用

猜你喜欢