介绍
Python的multiprocessing模块不但支持多进程，其中managers子模块还支持把多进程分布到多台机器上。一个服务进程可以作为调度者，将任务分布到其他多个机器的多个进程中，依靠网络通信。
想到这，就在想是不是可以使用此模块来实现一个简单的作业调度系统。
实现
Job
首先创建一个Job类，为了测试简单，只包含一个job id属性，将来可以封装一些作业状态，作业命令，执行用户等属性。
job.py
1 |
#!/usr/bin/env python |
2 |
# -*- coding: utf-8 -*- |
3 |
4 |
class Job: |
5 |
def __init__(self, job_id): |
6 |
self.job_id = job_id |
Master
Master用来派发作业和显示运行完成的作业信息
master.py
01 |
#!/usr/bin/env python |
02 |
# -*- coding: utf-8 -*- |
03 |
04 |
from Queue import Queue |
05 |
from multiprocessing.managers import BaseManager |
06 |
from job import Job |
07 |
08 |
09 |
class Master: |
10 |
11 |
def __init__(self): |
12 |
# 派发出去的作业队列 |
13 |
self.dispatched_job_queue = Queue() |
14 |
# 完成的作业队列 |
15 |
self.finished_job_queue = Queue() |
16 |
17 |
def get_dispatched_job_queue(self): |
18 |
return self.dispatched_job_queue |
19 |
20 |
def get_finished_job_queue(self): |
21 |
return self.finished_job_queue |
22 |
23 |
def start(self): |
24 |
# 把派发作业队列和完成作业队列注册到网络上 |
25 |
BaseManager.register('get_dispatched_job_queue',callable=self.get_dispatched_job_queue) |
26 |
BaseManager.register('get_finished_job_queue',callable=self.get_finished_job_queue) |
27 |
28 |
# 监听端口和启动服务 |
29 |
manager = BaseManager(address=('0.0.0.0', 8888), authkey='jobs') |
30 |
manager.start() |
31 |
32 |
# 使用上面注册的方法获取队列 |
33 |
dispatched_jobs = manager.get_dispatched_job_queue() |
34 |
finished_jobs = manager.get_finished_job_queue() |
35 |
36 |
# 这里一次派发10个作业，等到10个作业都运行完后，继续再派发10个作业 |
37 |
job_id = 0 |
38 |
while True: |
39 |
for i in range(0, 10): |
40 |
job_id = job_id + 1 |
41 |
job = Job(job_id) |
42 |
print('Dispatch job: %s' % job.job_id) |
43 |
dispatched_jobs.put(job) |
44 |
45 |
while not dispatched_jobs.empty(): |
46 |
job = finished_jobs.get(60) |
47 |
print('Finished Job: %s' % job.job_id) |
48 |
49 |
manager.shutdown() |
50 |
51 |
if __name__ == "__main__": |
52 |
master = Master() |
53 |
master.start() |
Slave
Slave用来运行master派发的作业并将结果返回
slave.py
01 |
#!/usr/bin/env python |
02 |
# -*- coding: utf-8 -*- |
03 |
04 |
import time |
05 |
from Queue import Queue |
06 |
from multiprocessing.managers import BaseManager |
07 |
from job import Job |
08 |
09 |
10 |
class Slave: |
11 |
12 |
def __init__(self): |
13 |
# 派发出去的作业队列 |
14 |
self.dispatched_job_queue = Queue() |
15 |
# 完成的作业队列 |
16 |
self.finished_job_queue = Queue() |
17 |
18 |
def start(self): |
19 |
# 把派发作业队列和完成作业队列注册到网络上 |
20 |
BaseManager.register('get_dispatched_job_queue') |
21 |
BaseManager.register('get_finished_job_queue') |
22 |
23 |
# 连接master |
24 |
server = '127.0.0.1' |
25 |
print('Connect to server %s...' % server) |
26 |
manager = BaseManager(address=(server, 8888), authkey='jobs') |
27 |
manager.connect() |
28 |
29 |
# 使用上面注册的方法获取队列 |
30 |
dispatched_jobs = manager.get_dispatched_job_queue() |
31 |
finished_jobs = manager.get_finished_job_queue() |
32 |
33 |
# 运行作业并返回结果，这里只是模拟作业运行，所以返回的是接收到的作业 |
34 |
while True: |
35 |
job = dispatched_jobs.get(timeout=1) |
36 |
print('Run job: %s ' % job.job_id) |
37 |
time.sleep(1) |
38 |
finished_jobs.put(job) |
39 |
40 |
if __name__ == "__main__": |
41 |
slave = Slave() |
42 |
slave.start() |
测试
分别打开三个linux终端，第一个终端运行master，第二个和第三个终端用了运行slave，运行结果如下
master
01 |
$ python master.py |
02 |
Dispatch job: 1 |
03 |
Dispatch job: 2 |
04 |
Dispatch job: 3 |
05 |
Dispatch job: 4 |
06 |
Dispatch job: 5 |
07 |
Dispatch job: 6 |
08 |
Dispatch job: 7 |
09 |
Dispatch job: 8 |
10 |
Dispatch job: 9 |
11 |
Dispatch job: 10 |
12 |
Finished Job: 1 |
13 |
Finished Job: 2 |
14 |
Finished Job: 3 |
15 |
Finished Job: 4 |
16 |
Finished Job: 5 |
17 |
Finished Job: 6 |
18 |
Finished Job: 7 |
19 |
Finished Job: 8 |
20 |
Finished Job: 9 |
21 |
Dispatch job: 11 |
22 |
Dispatch job: 12 |
23 |
Dispatch job: 13 |
24 |
Dispatch job: 14 |
25 |
Dispatch job: 15 |
26 |
Dispatch job: 16 |
27 |
Dispatch job: 17 |
28 |
Dispatch job: 18 |
29 |
Dispatch job: 19 |
30 |
Dispatch job: 20 |
31 |
Finished Job: 10 |
32 |
Finished Job: 11 |
33 |
Finished Job: 12 |
34 |
Finished Job: 13 |
35 |
Finished Job: 14 |
36 |
Finished Job: 15 |
37 |
Finished Job: 16 |
38 |
Finished Job: 17 |
39 |
Finished Job: 18 |
40 |
Dispatch job: 21 |
41 |
Dispatch job: 22 |
42 |
Dispatch job: 23 |
43 |
Dispatch job: 24 |
44 |
Dispatch job: 25 |
45 |
Dispatch job: 26 |
46 |
Dispatch job: 27 |
47 |
Dispatch job: 28 |
48 |
Dispatch job: 29 |
49 |
Dispatch job: 30 |
slave1
01 |
$ python slave.py |
02 |
Connect to server 127.0.0.1... |
03 |
Run job: 1 |
04 |
Run job: 2 |
05 |
Run job: 3 |
06 |
Run job: 5 |
07 |
Run job: 7 |
08 |
Run job: 9 |
09 |
Run job: 11 |
10 |
Run job: 13 |
11 |
Run job: 15 |
12 |
Run job: 17 |
13 |
Run job: 19 |
14 |
Run job: 21 |
15 |
Run job: 23 |
slave2
01 |
$ python slave.py |
02 |
Connect to server 127.0.0.1... |
03 |
Run job: 4 |
04 |
Run job: 6 |
05 |
Run job: 8 |
06 |
Run job: 10 |
07 |
Run job: 12 |
08 |
Run job: 14 |
09 |
Run job: 16 |
10 |
Run job: 18 |
11 |
Run job: 20 |
12 |
Run job: 22 |
13 |
Run job: 24 |