这里通过一个实例,说明多线程适合IO密集型任务,多进程适合计算密集型任务。
一、IO密集型任务:
import multiprocessing
import time
import threading
# 定义全局变量Queue
g_queue = multiprocessing.Queue()
# 首先定义一个队列,并定义初始化队列的函数:
def init_queue():
print("init g_queue start")
while not g_queue.empty():
g_queue.get()
for _index in range(10):
g_queue.put(_index)
print("init g_queue end")
return
# 定义一个IO密集型任务:利用time.sleep(),分别从队列中获取任务数据
def task_io(task_id):
print("IOTask[%s] start" % task_id)
while not g_queue.empty():
time.sleep(1)
try:
data = g_queue.get(block=True, timeout=1)
print("IOTask[%s] get data: %s" % (task_id, data))
except Exception as excep:
print("IOTask[%s] error: %s" % (task_id, str(excep)))
print("IOTask[%s] end" % task_id)
return
if __name__ == '__main__':
print("cpu count:", multiprocessing.cpu_count(), "\n")
print("========== 直接执行IO密集型任务 ==========")
init_queue()
time_0 = time.time()
task_io(0)
print("结束:", time.time() - time_0, "\n")
print("========== 多线程执行IO密集型任务 ==========")
init_queue()
time_0 = time.time()
thread_list = [threading.Thread(target=task_io, args=(i,)) for i in range(5)]
for t in thread_list:
t.start()
for t in thread_list:
if t.is_alive():
t.join()
print("结束:", time.time() - time_0, "\n")
print("========== 多进程执行IO密集型任务 ==========")
init_queue()
time_0 = time.time()
process_list = [multiprocessing.Process(target=task_io, args=(i,)) for i in range(multiprocessing.cpu_count())]
for p in process_list:
p.start()
for p in process_list:
if p.is_alive():
p.join()
print("结束:", time.time() - time_0, "\n")
打印:
[root@localhost demo01]# python3 text1109.py
cpu count: 4
========== 直接执行IO密集型任务 ==========
init g_queue start
init g_queue end
IOTask[0] start
IOTask[0] get data: 0
IOTask[0] get data: 1
IOTask[0] get data: 2
IOTask[0] get data: 3
IOTask[0] get data: 4
IOTask[0] get data: 5
IOTask[0] get data: 6
IOTask[0] get data: 7
IOTask[0] get data: 8
IOTask[0] get data: 9
IOTask[0] end
结束: 10.016204833984375
========== 多线程执行IO密集型任务 ==========
init g_queue start
init g_queue end
IOTask[0] start
IOTask[0] end
IOTask[1] start
IOTask[3] start
IOTask[2] start
IOTask[4] start
IOTask[1] get data: 0
IOTask[2] get data: 1
IOTask[3] get data: 2
IOTask[4] get data: 3
IOTask[1] get data: 4
IOTask[2] get data: 5
IOTask[4] get data: 6
IOTask[3] get data: 7
IOTask[1] get data: 8
IOTask[2] get data: 9
IOTask[2] end
IOTask[3] error:
IOTask[3] end
IOTask[4] error:
IOTask[4] end
IOTask[1] error:
IOTask[1] end
结束: 5.012259006500244
========== 多进程执行IO密集型任务 ==========
init g_queue start
init g_queue end
IOTask[0] start
IOTask[1] start
IOTask[2] start
IOTask[3] start
IOTask[0] get data: 0
IOTask[1] get data: 1
IOTask[2] get data: 2
IOTask[3] get data: 3
IOTask[0] get data: 4
IOTask[1] get data: 5
IOTask[2] get data: 6
IOTask[3] get data: 7
IOTask[0] get data: 8
IOTask[1] get data: 9
IOTask[1] end
IOTask[3] error:
IOTask[3] end
IOTask[2] error:
IOTask[2] end
IOTask[0] error:
IOTask[0] end
结束: 5.0164711475372314
结果说明:
对于IO密集型任务:
- 直接执行用时:10.016204833984375秒
- 多线程执行用时:5.012259006500244秒
- 多进程执行用时:5.0164711475372314秒
说明多线程适合IO密集型任务。
二、计算密集型任务:
import multiprocessing
import time
import threading
# 定义全局变量Queue
g_queue = multiprocessing.Queue()
def init_queue():
print("init g_queue start")
while not g_queue.empty():
g_queue.get()
for _index in range(10):
g_queue.put(_index)
print("init g_queue end")
return
g_search_list = list(range(10000))
# 定义一个计算密集型任务:利用一些复杂加减乘除、列表查找等
def task_cpu(task_id):
print("CPUTask[%s] start" % task_id)
while not g_queue.empty():
count = 0
for i in range(10000):
count += pow(3*2, 3*2) if i in g_search_list else 0
try:
data = g_queue.get(block=True, timeout=1)
print("CPUTask[%s] get data: %s" % (task_id, data))
except Exception as excep:
print("CPUTask[%s] error: %s" % (task_id, str(excep)))
print("CPUTask[%s] end" % task_id)
return task_id
if __name__ == '__main__':
print("cpu count:", multiprocessing.cpu_count(), "\n")
print("========== 直接执行CPU密集型任务 ==========")
init_queue()
time_0 = time.time()
task_cpu(0)
print("结束:", time.time() - time_0, "\n")
print("========== 多线程执行CPU密集型任务 ==========")
init_queue()
time_0 = time.time()
thread_list = [threading.Thread(target=task_cpu, args=(i,)) for i in range(5)]
for t in thread_list:
t.start()
for t in thread_list:
if t.is_alive():
t.join()
print("结束:", time.time() - time_0, "\n")
print("========== 多进程执行cpu密集型任务 ==========")
init_queue()
time_0 = time.time()
process_list = [multiprocessing.Process(target=task_cpu, args=(i,)) for i in range(multiprocessing.cpu_count())]
for p in process_list:
p.start()
for p in process_list:
if p.is_alive():
p.join()
print("结束:", time.time() - time_0, "\n")
打印:
[root@localhost demo01]# python3 text1109.py
cpu count: 4
========== 直接执行CPU密集型任务 ==========
init g_queue start
init g_queue end
CPUTask[0] start
CPUTask[0] get data: 0
CPUTask[0] get data: 1
CPUTask[0] get data: 2
CPUTask[0] get data: 3
CPUTask[0] get data: 4
CPUTask[0] get data: 5
CPUTask[0] get data: 6
CPUTask[0] get data: 7
CPUTask[0] get data: 8
CPUTask[0] get data: 9
CPUTask[0] end
结束: 5.545855522155762
========== 多线程执行CPU密集型任务 ==========
init g_queue start
init g_queue end
CPUTask[0] start
CPUTask[1] start
CPUTask[2] start
CPUTask[4] start
CPUTask[3] start
CPUTask[0] get data: 0
CPUTask[2] get data: 1
CPUTask[1] get data: 2
CPUTask[4] get data: 3
CPUTask[3] get data: 4
CPUTask[0] get data: 5
CPUTask[2] get data: 6
CPUTask[1] get data: 7
CPUTask[3] get data: 8
CPUTask[4] get data: 9
CPUTask[4] end
CPUTask[2] error:
CPUTask[2] end
CPUTask[1] error:
CPUTask[1] end
CPUTask[0] error:
CPUTask[0] end
CPUTask[3] error:
CPUTask[3] end
结束: 9.969066858291626
========== 多进程执行cpu密集型任务 ==========
init g_queue start
init g_queue end
CPUTask[0] start
CPUTask[1] start
CPUTask[3] start
CPUTask[2] start
CPUTask[0] get data: 0
CPUTask[1] get data: 1
CPUTask[3] get data: 2
CPUTask[2] get data: 3
CPUTask[0] get data: 4
CPUTask[1] get data: 5
CPUTask[3] get data: 6
CPUTask[2] get data: 7
CPUTask[0] get data: 8
CPUTask[1] get data: 9
CPUTask[1] end
CPUTask[3] error:
CPUTask[3] end
CPUTask[2] error:
CPUTask[2] end
CPUTask[0] error:
CPUTask[0] end
结束: 3.3789467811584473
结果说明:
对于计算密集型任务:
- 直接执行用时:5.545855522155762秒
- 多线程执行用时:9.969066858291626秒
- 多进程执行用时:3.3789467811584473秒
说明多进程适合计算密集型任务。
参考链接:
Python进阶:聊聊IO密集型任务、计算密集型任务,以及多线程、多进程
进程通信(multiprocessing.Queue)