Introduction
text and pictures of this article from the network, only to learn, exchange, not for any commercial purposes, belongs to original author, if any questions, please contact us for treatment.
Author: Jiang dog 
When using multi-process, the method in main (), otherwise there will be abnormal warning.
Process() Basic use: and Thread()similar.
Pool() Basic use:
Where the map method and the built-in map function the same with them, but how to support the process.
from multiprocessing import Pool
pool = Pool(2)
pool.map(fib, [35] * 2)multiprocessing.dummy Modules:
multiprocessing.dummy replicates the API of multiprocessing but is no more than a wrapper around the threading module.
For more than part of the knowledge point, I had no practical use, but simply to understand and write Demo carried out exercises to understand not very thorough.
# -*- coding: utf-8 -*-
from multiprocessing import Process, Pool
from multiprocessing.dummy import Pool as DummyPool import time import datetime def log_time(methond_name): def decorator(f): def wrapper(*args, **kwargs): start_time = time.time() res = f(*args, **kwargs) end_time = time.time() print('%s cost %ss' % (methond_name, (end_time - start_time))) return res return wrapper return decorator def fib(n): if n <=2 : return 1 return fib(n-1) + fib(n-2) @log_time('single_process') def single_process(): fib(33) fib(33) @log_time('multi_process') def multi_process(): jobs = [] for _ in range(2): p = Process(target=fib, args=(33, )) p.start() jobs.append(p) for j in jobs: j.join() @log_time('pool_process') def pool_process(): pool = Pool(2) pool.map(fib, [33]*2) @log_time('dummy_pool') def dummy_pool(): pool = DummyPool(2) pool.map(fib, [33]*2) if __name__ == '__main__': single_process() multi_process() pool_process() dummy_pool()
Based on the parmap Pipe
A little difficult to understand. Note: If you Python foundation solid enough, you can point me into the dress to see my latest entry to the combat tutorial review
queue
Consumers realize production model, a task store queue, a queue to store the results. multiprocessingThe module also has Queue, but is not available task_done()and join()methods. Therefore, the use of Queuestored results, JoinableQueue() to store task.
Modeled on the Demo, a consumer and a producer process process:
# -*- coding: utf-8 -*-
from multiprocessing import Process, Queue, JoinableQueue
import time
import random def double(n): return n * 2 def producer(name, task_q): while 1: n = random.random() if n > 0.8: # 大于0.8时跳出 task_q.put(None) print('%s break.' % name) break print('%s produce %s.' % (name, n)) task_q.put((double, n)) def consumer(name, task_q, result_q): while 1: task = task_q.get() if task is None: print('%s break.' % name) break func, arg = task res = func(arg) time.sleep(0.5) # 阻塞 task_q.task_done() result_q.put(res) print('%s consume %s, result %s' % (name, arg, res)) def run(): task_q = JoinableQueue() result_q = Queue() processes = [] p1 = Process(name='p1', target=producer, args=('p1', task_q)) c1 = Process(name='c1', target=consumer, args=('c1', task_q, result_q)) p1.start() c1.start() processes.append(p1) processes.append(c1) # join()阻塞主进程 for p in processes: p.join() # 子进程结束后，输出result中的值 while 1: if result_q.empty(): break result = result_q.get() print('result is: %s' % result) if __name__ == '__main__': run()If there is more than consumer()a process, there will only be a consumer()process can be taken out Noneand break, others will be in task_q.get()has been suspended, try to consumer()add a timeout to exit the method.
import queue
def consumer(name, task_q, result_q): while 1: try: task = task_q.get(1) # 1s except queue.Empty: print('%s time out, break.' % name) if task is None: print('%s break.' % name) break func, arg = task res = func(arg) time.sleep(0.5) # 阻塞 task_q.task_done() result_q.put(res) print('%s consume %s, result %s' % (name, arg, res))Shared memory
Use sharedctypesof Array, Valueto shared memory.
The following example is modeled.
# -*- coding: utf-8 -*-
from pprint import pprint
# 共享内存
from multiprocessing import sharedctypes, Process, Lock from ctypes import Structure, c_bool, c_double pprint(sharedctypes.typecode_to_type) lock = Lock() class Point(Structure): _fields_ = [('x', c_double), ('y', c_double)] # _fields_ def modify(n, b, s, arr, A): n.value **= 2 b.value = True s.value = s.value.upper() arr[0] = 10 for a in A: a.x **= 2 a.y **= 2 if __name__ == '__main__': n = sharedctypes.Value('i', 7) b = sharedctypes.Value(c_bool, False, lock=False) s = sharedctypes.Array('c', b'hello world', lock=lock) # bytes arr = sharedctypes.Array('i', range(5), lock=True) A = sharedctypes.Array(Point, [(1.875, -6.25), (-5.75, 2.0)], lock=lock) p = Process(target=modify, args=(n, b, s, arr, A)) p.start() p.join() print(n.value) print(b.value) print(s.value) print(arr[:]) print([(a.x, a.y) for a in A])The actual project using the Valuetask to monitor the status of the child process, update and delete stored by memcached.
# -*- coding: utf-8 -*-
from multiprocessing import Process, Value
import time
import datetime import random FINISHED = 3 FAILED = 4 INPROCESS = 2 WAITING = 1 def execute_method(status, process): time.sleep(1) status.value = INPROCESS # test time.sleep(1) status.value = FINISHED # test time.sleep(0.5) def run(execute_code): status = Value('i', WAITING ) process = Value('f', 0.0) # mem_cache.set('%s_status' % execute_code, status.value, 0) # mem_cache.set('%s_process' % execute_code, process .value, 0) p = Process(target=execute_method, args=(status, process)) p.start() start_time = datetime.datetime.now() while True: print(status.value) now_time = datetime.datetime.now() if (now_time - start_time).seconds > 30: # 超过30sbreak # mem_cache.delete('%s_status' % execute_code) # mem_cache.delete('%s_process' % execute_code) print('execute failed') p.terminate() break if status.value == 3: # mem_cache.delete('%s_status' % execute_code) # mem_cache.delete('%s_process' % execute_code) print('end execute') break else: # mem_cache.set('%s_status' % execute_code, status.value, 0) # mem_cache.set('%s_process' % execute_code, process .value, 0) print('waiting or executing') time.sleep(0.5) p.join()Service process
The following example is modeled on the example of the service process blog, simply shows Managera common way to share.
A multiprocessing.Manager objects will control a server process, other processes can access the server process by proxy. Common share in the following ways:
1. the Namespace. Create a shared namespace.
2. Value / Array. And ways to share the above objects, like ctypes.
dict / list. Create a shareable
3. dict / list, the corresponding data structure to support the method.
4. Condition / Event / Lock / Queue / Semaphore. Primitive objects to create a shareable correspond synchronization.
# -*- coding: utf-8 -*-
from multiprocessing import Manager, Process
def modify(ns, lproxy, dproxy): ns.name = 'new_name' lproxy.append('new_value') dproxy['new'] = 'new_value' def run(): # 数据准备 manager = Manager() ns = manager.Namespace() ns.name = 'origin_name' lproxy = manager.list() lproxy.append('origin_value') dproxy = manager.dict() dproxy['origin'] = 'origin_value' # 子进程 p = Process(target=modify, args=(ns, lproxy, dproxy)) p.start() print(p.pid) p.join() print('ns.name: %s' % ns.name) print('lproxy: %s' % lproxy) print('dproxy: %s' % dproxy) if __name__ == '__main__': run()Mainly showing the example Managerof the type of shared objects and agents are known to by View source register()method.
multiprocessing/managers.py:
#
# Definition of SyncManager
#
class SyncManager(BaseManager): ''' Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. ''' SyncManager.register('Queue', queue.Queue) SyncManager.register('JoinableQueue', queue.Queue) SyncManager.register('Event', threading.Event, EventProxy) SyncManager.register('Lock', threading.Lock, AcquirerProxy) SyncManager.register('RLock', threading.RLock, AcquirerProxy) SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) SyncManager.register('Barrier', threading.Barrier, BarrierProxy) SyncManager.register('Pool', pool.Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) SyncManager.register('Value', Value, ValueProxy) SyncManager.register('Array', Array, ArrayProxy) SyncManager.register('Namespace', Namespace, NamespaceProxy) # types returned by methods of PoolProxy SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False) SyncManager.register('AsyncResult', create_method=False)In addition to the child process, it may also be utilized Manager()for communication between different processes, such as the following simple implementation of distributed processes.
Distribution process
And the example of the main difference is between non-child communication process.
manager_server.py:
# -*- coding: utf-8 -*-
from multiprocessing.managers import BaseManager
host = '127.0.0.1'
port = 8080 authkey = b'python' shared_list = [] class ServerManager(BaseManager): pass ServerManager.register('get_list', callable=lambda: shared_list) server_manager = ServerManager(address=(host, port), authkey=authkey) server = server_manager.get_server() server.serve_forever()manager_client.py
# -*- coding: utf-8 -*-
from multiprocessing.managers import BaseManager
host = '127.0.0.1'
port = 8080 authkey = b'python' class ClientManager(BaseManager): pass ClientManager.register('get_list') client_manager = ClientManager(address=(host, port), authkey=authkey) client_manager.connect() l = client_manager.get_list() print(l) l.append('new_value') print(l)After running several times, shared_listwe will continue to be added new_value.
Distributed process modeled on the Liao Xuefeng tutorial with appropriate modifications.
manager_server.py:
# -*- coding: utf-8 -*-
from multiprocessing.managers import BaseManager
from multiprocessing import Condition, Value import queue host = '127.0.0.1' port = 8080 authkey = b'python' task_q = queue.Queue(10) result_q = queue.Queue(20) cond = Condition() done = Value('i', 0) def double(n): return n * 2 class ServerManager(BaseManager): pass ServerManager.register('get_task_queue', callable=lambda: task_q) ServerManager.register('get_result_queue', callable=lambda: result_q) ServerManager.register('get_cond', callable=lambda: cond) ServerManager.register('get_done', callable=lambda: done) ServerManager.register('get_double', callable=double) server_manager = ServerManager(address=(host, port), authkey=authkey) server = server_manager.get_server() print('start server') server.serve_forever( manager_producer.py:
# -*- coding: utf-8 -*-
from multiprocessing.managers import BaseManager
import random
import time host = '127.0.0.1' port = 8080 authkey = b'python' class ProducerManager(BaseManager): pass ProducerManager.register('get_task_queue') ProducerManager.register('get_cond') ProducerManager.register('get_done') producer_manager = ProducerManager(address=(host, port), authkey=authkey) producer_manager.connect() task_q = producer_manager.get_task_queue() cond = producer_manager.get_cond() # done = producer_manager.get_done() count = 20 # 最多有20个任务 while count > 0: if cond.acquire(): if not task_q.full(): n = random.randint(0, 10) task_q.put(n) print("Producer:deliver one, now tasks:%s" % task_q.qsize()) cond.notify() count -= 1 time.sleep(0.5) else: print("Producer:already full, stop deliver, now tasks:%s" % task_q.qsize()) cond.wait() cond.release() # done.value = 1 print('Producer break')manager_consumer.py:
# -*- coding: utf-8 -*-
from multiprocessing.managers import BaseManager
host = '127.0.0.1'
port = 8080 authkey = b'python' class ConsumerManager(BaseManager): pass ConsumerManager.register('get_task_queue') ConsumerManager.register('get_result_queue') ConsumerManager.register('get_cond') # ConsumerManager.register('get_done') ConsumerManager.register('get_double') consumer_manager = ConsumerManager(address=(host, port), authkey=authkey) consumer_manager.connect() task_q = consumer_manager.get_task_queue() result_q = consumer_manager.get_result_queue() cond = consumer_manager.get_cond() # done = consumer_manager.get_done() while 1: if result_q.full(): print('result queue is full') break if cond.acquire(): if not task_q.empty(): arg = task_q.get() res = consumer_manager.get_double(arg) print("Consumer:consume one, now tasks:%s" % task_q.qsize()) result_q.put(res) cond.notify() else: print("Consumer:only 0, stop consume, products") cond.wait() cond.release() while 1: if result_q.empty(): break result = result_q.get() print('result is: %s' % result)