Python multi-process programming, process communication, use of process pool, process exit

python multiprocessing

Mainly used library: multiprocessing

Test code: (In order to save space, some blank lines are deleted, for the sake of specification, do not delete in specific occasions)

basic use

import them
import time
from multiprocessing import Process

def task(n):
    while n:
        n -= 1
        print "task: pid %s %d" % (str(os.getpid()), n)
        time.sleep(1)

def main():
    p1 = Process(target=task, args=(5,))
    p2 = Process(target=task, args=(5,))

    p1.start()
    p2.start()
    p1.join() # Join the current main thread, so that the main thread waits for the execution of the current thread to complete before proceeding
    p2.join()

    print "End"

if __name__ == '__main__':
    main()

multi-process communication

Use: Queue
example in mulitprocessing: The producer-consumer mode is used for communication. In some cases, attention needs to be paid to the synchronization of data, and it is also necessary to pay attention to programming. It is not considered for the time being.

# coding=utf8
__author__ = 'Administrator'

import them
import sys
import time
import signal
import random
from multiprocessing import Process
from multiprocessing import Queue

def produce(message_queue):
    while True:
        if message_queue.qsize() < 5:
            tmp = random.randint(0, 0xfff)
            message_queue.put(tmp)
            print "produce: %s " % str(tmp)
        time.sleep(1)


def consume(message_queue):
    while True:
        if not message_queue.empty():
            print "consume: %s " % str(message_queue.get())
        time.sleep(1)

def produce_consume():
    message_queue = Queue(maxsize=100) # Use the Queue class provided by the process to ensure that data can be shared between processes
    for item in range(0, 10): # Queue initialization
        message_queue.put(random.randint(0, 0xfff))

    producer = Process(target=produce, args=(message_queue,)) # Start the producer process
    consumer = Process(target=consume, args=(message_queue,)) # start the consumer process

    producer.start() # start the process
    consumer.start()
    producer.join() # join the main process
    consumer.join()

    print "end"


if __name__ == '__main__':
    produce_consume()

Use of many processes

Use: process pool, Pool in multiprocessing

import them
import sys
import time
import signal
import random
from multiprocessing import Process
from multiprocessing import Queue
from multiprocessing import Pool

def pool_task(message_list):
    while len(message_list):
        print "task: %s, %s" % (str(os.getpid()), str(message_list.pop()))
        time.sleep(1)

def process_pool():
    p = Pool()
    x = ["a", "b", "c"]
    for i in xrange(0, 3):
        tmp = [str(x_i) + str(item) for x_i, item in enumerate(x)]
        p.apply_async(pool_task, args=(tmp,)) # Asynchronous execution, when apply() is used, apply_async is also called internally
    p.close()
    p.join()

if __name__ == '__main__':
    process_pool()

process exit

There are usually several ways to end a process:

1. sys.exit(0), exit the current process, where 0 is the exit code
2. os._exit() , limited by the operating system, but not affected on win32 and unix platforms
3. os.kill(pid, singnal), to end a process, you need to provide the process ID
4. os.popen(“taskkill.exe /pid:”+ str(pid)) end the process of the window platform

Commonly used are sys.exit(0) and os.kill