Queue & producer consumer model

queue

ipc mechanism: Process Communications

Pipeline: pipe-based shared memory space

Queue: pipe + lock queue

from multiprocessing import Process,Queue

### 案例一
q = Queue()
q.put('hyc')
q.put([1,2,4])
q.put(2)
print(q.get())
print(q.get())
print(q.get())
# q.put(5)
# q.put(5)
print(q.get()) # 默认就会一直等着拿值

At this point the program to run will block here

from multiprocessing import Process,Queue

## 案例2
q = Queue(4)
q.put('我妻由乃')
q.put([1,2,4])
q.put('我妻善逸')
q.put(2)

q.put('乔碧萝')  #队列满了的情况再放值,会阻塞

Is the same reason, have set the four values, when placed in the fifth value, it will be blocked

from multiprocessing import Process,Queue

q = Queue(3)
q.put('zhao',block=True,timeout=2) #
q.put('zhao',block=True,timeout=2) #
q.put('zhao',block=True,timeout=2) #

q.put('zhao',block=True,timeout=5) # put里的  block=True 如果满了会等待,timeout最多等待n s,如果ns还是队列还是满的就报错了

After waiting 5 seconds will complain

from multiprocessing import Process,Queue

q = Queue()
q.put('yyyy')
q.get()
q.get(block=True,timeout=5) # block=True 阻塞等待,timeout最多等5s, 剩下同上

When the value is the same reason

q = Queue(3)
q.put('qwe')
q.put('qwe')
q.put('qwe')

q.put('qwe',block=False) # 对于put来说block=False 如果队列满了就直接报错

q = Queue(3)
q.put('qwe')
q.get()


q.get(block=False)
# block = Flase 拿不到不阻塞,直接报错

When the block = Flase time, timeout just does not make sense, because it will direct error

q = Queue(1)
q.put('123')
# q.get()
q.put_nowait('666') # block = False
q.get_nowait() # block = False

Will direct error and in fact the same block = False, not blocking, direct error when there is a problem

Producer and consumer model

Manufacturer: production data task

Consumers: the task of processing data

The relationship between producers and consumers: producers - queue (pots) -> Consumers

Producers can not stop production, to reach their maximum productivity, consumers can keep spending, also reached their maximum consumption efficiency.
Producer-consumer model greatly improves the efficiency of production and consumer spending Producer s efficiency.

from multiprocessing import Process, Queue

def producer(q,name,food):
    '''生产者'''
    for i in range(10):
        print(f'{name}生产了{food}{i}')
        res = f'{food}{i}'
        q.put(res)
    q.put(None)

def consumer(q,name):
    '''消费者'''
    while True:
        res = q.get(timeout=5)
        if res is None:break
        print(f'{name}吃了{res}')

if __name__ == '__main__':
    q = Queue()
    p1 = Process(target=producer,args=(q,'rocky','包子'))
    c1 = Process(target=consumer,args=(q,'成哥'))
    p1.start()
    c1.start()

This is the simplest model of producers and consumers, and his role is that each producer will produce goods, when the producers of finished production of 10 buns, out of the for loop, and finally into a None. Then when the consumer receives None, it will be break out.

In a producer to a consumer when this method is feasible. But when multiple producers time to a consumer, one of the first producers of finished food production and put None, but the producer has not produced the second place finish, then the consumer has already received the first producer of None. In this case there will be a problem, so we can change the thinking about, put out to None

from multiprocessing import Process,Queue
import time,random

def producer(q,name,food):
    '''生产者'''
    for i in range(3):
        print(f'{name}生产了{food}{i}')
        time.sleep(random.randint(1, 3))
        res = f'{food}{i}'
        q.put(res)
    # q.put(None)

def consumer(q,name):
    '''消费者'''
    while True:
        res = q.get(timeout=5)
        if res is None:break
        time.sleep(random.randint(1,3))
        print(f'{name}吃了{res}')

if __name__ == '__main__':
    q = Queue()
    p1 = Process(target=producer,args=(q,'rocky','包子'))
    p2 = Process(target=producer,args=(q,'mac','韭菜'))
    p3 = Process(target=producer,args=(q,'nick','蒜泥'))
    c1 = Process(target=consumer,args=(q,'成哥'))
    c2 = Process(target=consumer,args=(q,'浩南哥'))
    p1.start()
    p2.start()
    p3.start()
    c1.start()
    c2.start()
    p1.join()
    p2.join()
    p3.join() # 生产者生产完毕
    q.put(None)# 几个消费者put几次
    q.put(None)

But there will be new problems with this method, the method used is because consumers while True, he will not stop. So we have to use the daemon learned before:

from multiprocessing import Process,Queue,JoinableQueue
import time,random

def producer(q,name,food):
    '''生产者'''
    for i in range(3):
        print(f'{name}生产了{food}{i}')
        time.sleep(random.randint(1, 3))
        res = f'{food}{i}'
        q.put(res)
    # q.put(None)

def consumer(q,name):
    '''消费者'''
    while True:
        res = q.get()
        # if res is None:break
        time.sleep(random.randint(1,3))
        print(f'{name}吃了{res}')
        q.task_done() #

if __name__ == '__main__':
    q = JoinableQueue()
    p1 = Process(target=producer,args=(q,'rocky','包子'))
    p2 = Process(target=producer,args=(q,'mac','韭菜'))
    p3 = Process(target=producer,args=(q,'nick','蒜泥'))
    c1 = Process(target=consumer,args=(q,'成哥'))
    c2 = Process(target=consumer,args=(q,'浩南哥'))
    p1.start()
    p2.start()
    p3.start()
    c1.daemon = True
    c2.daemon = True
    c1.start()
    c2.start()
    p1.join()
    p2.join()
    p3.join() # 生产者生产完毕
    # q.put(None)# 几个消费者put几次
    # q.put(None)
    q.join() # 分析
    # 生产者生产完毕--这是主进程最后一行代码结束--q.join()消费者已经取干净了,没有存在的意义了.
    #这是主进程最后一行代码结束,消费者已经取干净了,没有存在的意义了.守护进程的概念.

joinableQueue

from multiprocessing import Process,Queue,JoinableQueue


q = JoinableQueue()

q.put('zhao') # 放队列里一个任务
q.put('qian')

print(q.get())
q.task_done() # 完成了一次任务
print(q.get())
q.task_done() # 完成了一次任务
q.join() #计数器不为0的时候 阻塞等待计数器为0后通过

# 想象成一个计数器 :put +1   task_done -1