Python示例代码之多进程

多进程使用multiprocessing模块，使用多进程的目的，是提高cpu的利用率。

Queue  \ Pipe 可以实现进程间数据的传递，Manager 实现了进程间数据的共享，即多个进程可以修改同一份数据。

多进程使用Lock示例

Lock用来锁定进程间共享的资源，防止争夺。

from multiprocessing import Process, Lock

def f(l, i):
    l.acquire()
    print('hello world', i)
    l.release()

if __name__ == '__main__':
    lock = Lock()
    ps = []
    for num in range(10):
        p = Process(target = f, args = (lock, num))
        p.start()
        ps .append(p)
    for p in ps:
       p.join()

多进程使用Pipe示例

多进程使用Pipe消息传送的形式共享数据。

import time
from multiprocessing import Process, Pipe

def f(conn):
    while True:
        cmd = conn.recv()
        print("child process received:", cmd)
        if cmd == "quit" or cmd == "exit":
            break
        time.sleep(1)
        conn.send(["child process cmd complete:", cmd])
    conn.close()

if __name__ == '__main__':
    parent_conn, child_conn = Pipe()
    p = Process(target = f, args = (child_conn,))
    p.start()
    while True:
        cmd = input()
        print("you input:", cmd)
        parent_conn.send(cmd)
        if cmd == "quit" or cmd == "exit":
            break
        print(parent_conn.recv())
    p.join()

多进程使用Queue示例

from multiprocessing import Process, Queue
import threading

def f(qq):
    print("in child:",qq.qsize())
    qq.put([42, None, 'hello'])

if __name__ == '__main__':
    q = Queue()
    q.put("test123")

    p = Process(target = f, args=(q,))
    p.start()
    p.join()
    print(q.get_nowait())
    print(q.get_nowait())

多进程使用示例Manager

from multiprocessing import Process, Manager
import os

def f(i, d, l):
    d[i] = os.getpid()
    l.append(os.getpid())

if __name__ == '__main__':
    with Manager() as manager:
        d = manager.dict()
        l = manager.list(range(5))
        p_list = []
        for i in range(10):
            p = Process(target=f, args=(i, d, l))
            p.start()
            p_list.append(p)
        for res in p_list:
            res.join()
        print(d)
        print(l)

进程池使用示例

from  multiprocessing import Process, Pool
import os
import time

def Foo(i):
    time.sleep(2)
    print("in process",os.getpid())
    return i + 100

def Bar(arg):
    print('-->exec done:', arg, os.getpid())

if __name__ == '__main__':
    pool = Pool(processes = 3)
    print("main process:", os.getpid())
    for i in range(10):
        pool.apply_async(func = Foo, args = (i, ), callback = Bar)
    pool.close()
    pool.join()

多进程多线程综合使用示例

import multiprocessing
import time,threading, os

def thread_run(parent_id):
    print("parent_id", parent_id, " thread:", threading.get_ident())

def run(name):
    time.sleep(2)
    print('hello', name)
    t = threading.Thread(target = thread_run, args = (os.getpid(),))
    t.start()
    t.join()

if __name__ == '__main__':
    for i in range(10):
        p = multiprocessing.Process(target = run, args = ('Shixuan%s' %i,))
        p.start()
    p.join()