Study Notes--Python Multiprocessing

os.fork

This method only supports Linux/Unix

1  import os
 2  
3  print ( ' Current Process %s start ... ' % os.getpid())
 4 pid = os.fork() #Copy the main process to spawn a child process
 5  if pid < 0:
 6      print ( ' error in fork ' )
 7  elif pid == 0:
 8      print ( ' I am child process %s and my parent process is %s ' % (os.getpid(), os.getppid()))
 9  else :
 10      print ( 'I %s created a child process %s' % (os.getppid(), os.getpid()))

 

multiprocessing

process pool

 1 from multiprocessing import Pool
 2 import os, time, random
 3 
 4 def run_task(name):
 5     print('Task %s (pid = %s) is running...' % (name, os.getpid()))
 6     time.sleep(random.random() * 3)
 7     print('Task %s end.' % name)
 8 
 9 if __name__ == '__main__':
10     print('Curret process %s.' % os.getpid())
11     p = Pool(processes = 3)  # 创建进程池
12     for i in range(5):
13         p.apply_async(run_task, args = (i, ))  # 启动进程
14     print('Waiting for all subprocesses done...')
15     p.close()
16     p.join()
17     print('All subprocesses done.')

 

interprocess communication

Queue

 1 from multiprocessing import Queue, Process
 2 import os, time, random
 3 
 4 def proc_write(q, urls):
 5     print('Process (%s) is writig...' % os.getpid())
 6     for url in urls:
 7         q.put(url) # 往队列中添加
 8         print('Put %s to queue...' % url)
 9         time.sleep(random.random())
10 
11 def proc_read(q):
12      print ( ' Process (%s) is writig... ' % os.getpid())
 13      while True:
 14          url ​​= q.get(True)   #Get out of the queue 
15          print ( ' Get %s from queue. ' % url)
 16  
17  if  __name__ == ' __main__ ' :
 18      q = Queue()   #Create message queue 
19      #Create process 
20      proc_writer1 = Process(target = proc_write, args = (q, [ ' url_1 ', ' url_2 ' , ' url_3 ' , ' url_4 ' ]))
 21      proc_writer2 = Process(target = proc_write, args = (q, [ ' url_5 ' , ' url_6 ' , ' url_7 ' , ' url_8 ' ]))
 22      proc_reader = Process(target = proc_read, args = (q, ))
 23      #Start the process, add 
24 to the message queue      proc_writer1.start()
 25      proc_writer2.start()
 26      #Start the read queue process
27      proc_reader.start()
 28      #Wait for the end of proc_writer 
29      proc_writer1.join()
 30      proc_writer2.join()
 31      #Because there is an infinite loop in the proc_reader process, here is forcibly terminated 
32      proc_reader.terminate()

 

Pipe