python celery组件使用

         

Prepare

install:

 

1. pip install celery

选择broker，安装，这里假设使用Redis：

 

1. apt-get install redis-server

 

Configure

首先认真阅读官方celery文档的get start部分，如果有时间的话，最好全部看一边…

然后参考阅读别人的best practices，基本就可以干活了。

 

几个要点

1. task相关的文件，最好都是用绝对导入；否则，应该在task function上面指定name；
2. 如果需要root权限执行，需要在相关文件中加入platforms.C_FORCE_ROOT=True，但是最好别用root；
3. 可以根据需要消除pickle的警告，设置CELERY_ACCEPT_CONTENT=['pickle',]；
4. 默认不发心跳，需要加上BROKER_HEARTBEAT=10，来消除心跳相关警告；
5.

 

Router

router是不支持通配符的，如果需要，可以自己写一个自定义Router类。下面是一个celery.py的例子：

 

1. from __future__ import absolute_import
3. from celery importCelery, platforms
4. from settings import CELERY_BROKER
5. from kombu importQueue,Exchange
8. classMyRouter(object):
10. '''router for tasks using wildcard'''
12. def route_for_task(self, task,*args,**kwargs):
13. if task.startswith('writer'):
14. return{'queue':'async_writer','routing_key':'async_writer'}
15. elif task.startswith('caller'):
16. return{'queue':'async_caller','routing_key':'async_caller'}
17. else:
18. return{'queue':'default','routing_key':'default'}
21. QUEUES =(
22. Queue('default',Exchange('default'), routing_key='default'),
23. Queue('async_writer',Exchange('async_writer'),
24. routing_key='async_writer'),
25. Queue('async_caller',Exchange('async_caller'),
26. routing_key='async_caller'),
27. )
29. platforms.C_FORCE_ROOT =True
31. app =Celery('async',
32. broker=CELERY_BROKER,
33. include=['async.writer','async.caller','async.checker',])
35. app.conf.update(CELERY_ACCEPT_CONTENT=['pickle',],
36. CELERY_IGNORE_RESULT=True,
37. CELERY_DISABLE_RATE_LIMITS=True,
38. CELERY_DEFAULT_EXCHANGE='default',
39. CELERY_DEFAULT_QUEUE='default',
40. CELERY_DEFAULT_ROUTING_KEY='default',
41. CELERY_DEFAULT_EXCHANGE_TYPE='topic',
42. CELERY_TASK_SERIALIZER='pickle',
43. CELERY_RESULT_SERIALIZER='pickle',
44. BROKER_HEARTBEAT=10,
45. CELERY_QUEUES=QUEUES,
46. CELERY_ROUTES=(MyRouter(),),
47. )
49. if __name__ =="__main__":
50. app.start()

 

Start

官方给出的init.d脚本不是很好用，下面是一个自己写的参考：

 

1. #!/bin/bash
2. #
3. # PushserverCore uWSGI Web Server init script
4. #
5. ### BEGIN INIT INFO
6. # Provides: PushserverCore
7. # Required-Start: $remote_fs $remote_fs $network $syslog
8. # Required-Stop: $remote_fs $remote_fs $network $syslog
9. # Default-Start: 2 3 4 5
10. # Default-Stop: 0 1 6
11. # Short-Description: Start PushserverCore Service for generic init daemon
12. # Description: PushserverCore Service thrift Server backend.
13. ### END INIT INFO
15. NAME="Core Thrift Server"
16. PROJECT=PushserverCore
17. PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/var/app/enabled/$PROJECT
18. DESC="PushserverCore"
19. APP_DIR=/var/app/enabled/$PROJECT/Core
20. APP_PATH=$APP_DIR/CoreServer.py
21. CELERY_LOG_PATH=/var/app/log/PushserverCore/celery.log
23. print_succ()
24. {
25. echo "$(tput setaf 2)$(tput bold)DONE$(tput sgr0)"
26. }
28. print_fail()
29. {
30. echo "$(tput setaf 1)$(tput bold)FAILED$(tput sgr0)"
31. }
33. stop_service()
34. {
35. echo "stoping $NAME..."
36. if pgrep -f $APP_PATH >/dev/null 2>&1;then
37. pkill -f $APP_PATH
38. fi
39. print_succ
40. }
42. start_service()
43. {
44. if pgrep -f $APP_PATH >/dev/null 2>&1;then
45. echo "$NAME service is already running."
46. return
47. else
48. echo "starting $NAME service..."
49. nohup python $APP_PATH >/dev/null 2>&1&
50. fi
51. sleep 3
52. if pgrep -f $APP_PATH >/dev/null 2>&1;then
53. print_succ
54. else
55. print_fail
56. fi
57. }
59. stop_worker()
60. {
61. echo "stoping celery worker..."
62. if pgrep -f celery >/dev/null 2>&1;then
63. pkill -f celery
64. fi
65. print_succ
66. }
68. start_worker()
69. {
70. if pgrep -f celery >/dev/null 2>&1;then
71. echo "celery is already running"
72. return
73. else
74. echo "starting celery worker..."
75. celery -A async multi start writer caller default -Q:writer async_writer -Q:caller async_caller -Q:default default \
76. -c 7-l INFO --workdir=$APP_DIR --logfile=$CELERY_LOG_PATH
77. fi
78. sleep 3
79. if pgrep -f celery >/dev/null 2>&1;then
80. print_succ
81. else
82. print_fail
83. fi
84. }
86. check_status()
87. {
88. if pgrep -f $APP_PATH >/dev/null 2>&1;then
89. echo "$NAME is running"
90. else
91. echo "$NAME is not running"
92. fi
94. if pgrep -f celery >/dev/null 2>&1;then
95. echo "celery worker is running"
96. else
97. echo "celery worker is not running"
98. fi
100. }
102. set-e
104. ./lib/lsb/init-functions
106. case"$1"in
107. start)
108. echo "Starting $DESC..."
109. start_service
110. start_worker
111. ;;
112. stop)
113. echo "Stopping $DESC..."
114. stop_service
115. stop_worker
116. ;;
118. restart)
119. echo "Restarting $DESC..."
120. stop_service
121. stop_worker
122. sleep 3
123. start_service
124. start_worker
125. echo "Checking..."
126. check_status
127. ;;
129. status)
130. check_status
131. ;;
132. *)
133. echo "Usage: $NAME {start|stop|restart|status}">&2
134. exit 1
135. ;;
136. esac
138. exit 0

重点需要关注的是celery multi start的用法，注意start后面跟的是worker的名字（取数据的worker），也可以简单的写3，然后-Q:worker_name queue_name，最后-c是实际的worker（干活的worker）的数目，-Q是给队列指定worker。例子中的语句，意思是启动3个worker，分别命名为writer, caller和default，然后启动3个队列，名字分别是async_writer, async_caller和default，每个worker分配7个进程用来干活。

 

转自：

https://www.zybuluo.com/delight/note/24865