本文档描述了PS生产环境nginx+keepalived全套搭建过程,包括如下产品:
- Peoplesoft HCM 9.2.027
- Nginx:nginx-1.16.0
- Keepalived:keepalived-2.0.6
节点环境信息:
虚拟机序号 |
服务器名称 |
虚拟机主机名 |
CPU核数 |
内存 |
OS版本 |
IP地址 |
1 |
PS应用服务器01 |
EHR-APP01 |
16 |
64 |
Oracle Linux 7.4 |
10.160.144.59 |
2 |
PS应用服务器02 |
EHR-APP02 |
16 |
64 |
Oracle Linux 7.4 |
10.160.144.60 |
3 |
Keepalived虚拟IP |
|
|
|
|
10.160.144.68 |
2.1 全局环境
配置hosts:
vi /etc/hosts
10.160.144.68 hrms.companyname.cn
10.160.144.59 EHR-APP01.companyname.cn EHR-APP01
10.160.144.60 EHR-APP02.companyname.cn EHR-APP02
-
-
- 修改linux内核参数
-
修改linux内核参数:
vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536
* soft stack 65536
* hard stack 65536
vi /etc/security/limits.d/20-nproc.conf
* soft nproc unlimited
vi /etc/sysctl.conf
fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
net.core.netdev_max_backlog = 102400
net.core.somaxconn = 65535
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500
net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.tcp_max_orphans = 102400
net.ipv4.tcp_max_syn_backlog = 102400
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
/sbin/sysctl –p
临时关闭SELinux:即时生效
setenforce 0
关闭SELinux:需重启操作系统生效
vim /etc/selinux/config
[root@EHR-APP01 Packages]# more /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
2.2 nginx环境安装
安装依赖包:
yum install gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel patch
1. 获取nginx安装包:
wget http://nginx.org/download/nginx-1.6.2.tar.gz
2. 解压安装包:
tar -zxvf nginx-1.6.2.tar.gz
3. 获取upstream模块插件:
nginx_upstream_check_module-0.3.0.tar.gz
4. 解压插件:
tar -zxvf nginx_upstream_check_module-0.3.0.tar.gz
5. 获取sticky模块插件:
nginx-sticky-module-ng-1.2.5.tar.gz
6. 解压插件:
tar -zxvf nginx-sticky-module-ng-1.2.5.tar.gz
7. 进入nginx源码解压后目录:
cd /usr/local/src/nginx-1.6.2
8. 编译nginx:
./configure --prefix=/usr/local/nginx --with-pcre --with-http_stub_status_module --with-http_ssl_module --with-http_gzip_static_module --with-http_realip_module --add-module=/usr/local/src/nginx_upstream_check_module-0.3.0 --add-module=/usr/local/src/nginx-sticky-module-ng-1.2.5
9. 编译安装:
make && make install
10. 添加组和用户
/usr/sbin/groupadd www
/usr/sbin/useradd -g www www
启动:/usr/local/nginx/sbin/nginx
停止:/usr/local/nginx/sbin/nginx -s stop
重启:/usr/local/nginx/sbin/nginx -s reload
日志文件目录:/usr/local/nginx/logs
测试:访问http://EHR-APP01:80,见如下图表示安装成功:
注:需关闭服务器防火墙:
systemctl stop firewalld.service
-
-
- Nginx参数调整
-
#user nobody; user www www; worker_processes 8; error_log logs/error.log; #error_log logs/error.log notice; #error_log logs/error.log info; pid logs/nginx.pid; #Specifies the value for maximum file descriptors that can be opened by this process. worker_rlimit_nofile 65535; events { use epoll; worker_connections 65535; } http { include mime.types; default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log logs/access.log main; #charset gb2312; server_names_hash_bucket_size 128; client_header_buffer_size 32k; large_client_header_buffers 4 32k; client_max_body_size 8m; sendfile on; tcp_nopush on; keepalive_timeout 65; tcp_nodelay on; fastcgi_connect_timeout 300; fastcgi_send_timeout 300; fastcgi_read_timeout 300; fastcgi_buffer_size 64k; fastcgi_buffers 4 64k; fastcgi_busy_buffers_size 128k; fastcgi_temp_file_write_size 128k; gzip on; gzip_min_length 1k; gzip_buffers 4 16k; gzip_http_version 1.0; gzip_comp_level 2; gzip_types text/plain application/x-javascript text/css application/xml; gzip_vary on; #limit_zone crawler $binary_remote_addr 10m; #\u4e0b\u9762\u662fserver\u865a\u62df\u4e3b\u673a\u7684\u914d\u7f6e upstream hrms.companyname.cn { # ip_hash; sticky; server 10.160.144.59:8000; server 10.160.144.60:8000; } server { listen 80; server_name localhost; location / { proxy_pass http://hrms.companyname.cn;
} } } |
2.3 keepalived环境安装
yum install libnl libnl-devel libnfnetlink-devel
1. 获取安装包:
wget http://www.keepalived.org/software/keepalived-2.0.6.tar.gz
2. 解压安装包:
tar -zxvf keepalived-2.0.6.tar.gz
3. 编译:
./configure --prefix=/usr/local/keepalived
4. 安装:
make && make install
1. 将keepalived 安装成 Linux 系统服务:
a. 创建文件夹:mkdir /etc/keepalived
b. 拷贝配置文件:cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
c. 拷贝命令文件:
cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
ln -s /usr/local/sbin/keepalived /usr/sbin/
2.修改keepalived配置文件:
vi /etc/keepalived/keepalived.conf
主节点配置如下:
! Configuration File for keepalived
global_defs { ## keepalived 自带的邮件提醒需要开启 sendmail 服务。 建议用独立的监控或第三方 SMTP router_id EHR-APP01 ## 标识本节点的字条串,通常为 hostname } ## keepalived 会定时执行脚本并对脚本执行的结果进行分析,动态调整 vrrp_instance 的优先级。如果脚本执行结果为 0,并且 weight 配置的值大于 0,则优先级相应的增加。如果脚本执行结果非 0,并且 weight配置的值小于 0,则优先级相应的减少。其他情况,维持原本配置的优先级,即配置文件中 priority 对应的值。 vrrp_script chk_nginx { script "/etc/keepalived/nginx_check.sh" ## 检测 nginx 状态的脚本路径 interval 2 ## 检测时间间隔 weight -20 ## 如果条件成立,权重-20 } ## 定义虚拟路由, VI_1 为虚拟路由的标示符,自己定义名称 vrrp_instance VI_1 { state BACKUP #BACKUP备状态,防止资源抢占,主备节点均采用BACKUP interface ens192 ## 绑定虚拟 IP 的网络接口,与本机 IP 地址所在的网络接口相同, 我的是 eth0 virtual_router_id 33 ## 虚拟路由的 ID 号, 两个节点设置必须一样, 可选 IP 最后一段使用, 相同的 VRID 为一个组,他将决定多播的 MAC 地址 mcast_src_ip 10.160.144.59 ## 本机 IP 地址 priority 100 ## 节点优先级, 值范围 0-254, MASTER 要比 BACKUP 高 nopreempt ## 优先级高的设置 nopreempt 解决异常恢复后再次抢占的问题 advert_int 1 ## 组播信息发送间隔,两个节点设置必须一样, 默认 1s ## 设置验证信息,两个节点必须一致 authentication { auth_type PASS auth_pass Zgjmadmin0817 ## 真实生产,按需求对应该过来 } ## 将 track_script 块加入 instance 配置块 track_script { chk_nginx ## 执行 Nginx 监控的服务 } # # 虚拟 IP 池, 两个节点设置必须一样 virtual_ipaddress { 10.160.144.68 ## 虚拟 ip,可以定义多个 } } |
备节点配置如下:
! Configuration File for keepalived
global_defs { ## keepalived 自带的邮件提醒需要开启 sendmail 服务。 建议用独立的监控或第三方 SMTP,本次忽略邮件提醒 router_id EHR-APP02 ## 标识本节点的字条串,通常为 hostname } ## keepalived 会定时执行脚本并对脚本执行的结果进行分析,动态调整 vrrp_instance 的优先级。如果脚本执行结果为 0,并且 weight 配置的值大于 0,则优先级相应的增加。如果脚本执行结果非 0,并且 weight配置的值小于 0,则优先级相应的减少。其他情况,维持原本配置的优先级,即配置文件中 priority 对应的值。 vrrp_script chk_nginx { script "/etc/keepalived/nginx_check.sh" ## 检测 nginx 状态的脚本路径 interval 2 ## 检测时间间隔 weight -20 ## 如果条件成立,权重-20 } ## 定义虚拟路由, VI_1 为虚拟路由的标示符,自己定义名称 vrrp_instance VI_1 { state BACKUP #BACKUP备状态,防止资源抢占,主备节点均采用BACKUP interface ens192 ## 绑定虚拟 IP 的网络接口,与本机 IP 地址所在的网络接口相同 virtual_router_id 33 ## 虚拟路由的 ID 号, 两个节点设置必须一样, 可选 IP 最后一段使用, 相同的 VRID 为一个组,他将决定多播的 MAC 地址 mcast_src_ip 10.160.144.60 ## 本机 IP 地址 priority 100 ## 节点优先级, 值范围 0-254, MASTER 要比 BACKUP 高 nopreempt ## 优先级高的设置 nopreempt 解决异常恢复后再次抢占的问题 advert_int 1 ## 组播信息发送间隔,两个节点设置必须一样, 默认 1s ## 设置验证信息,两个节点必须一致 authentication { auth_type PASS auth_pass Zgjmadmin0817 } ## 将 track_script 块加入 instance 配置块 track_script { chk_nginx ## 执行 Nginx 监控的服务 } # # 虚拟 IP 池, 两个节点设置必须一样 virtual_ipaddress { 10.160.144.68 ## 虚拟 ip,可以定义多个 } }
|
3.编写nginx状态检测脚本:(已在keepalived.conf文件中配置)
vi /etc/keepalived/nginx_check.sh
#!/bin/bash if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then /usr/local/nginx/sbin/nginx sleep 2 if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then killall keepalived fi fi |
注:如果 nginx 停止运行,尝试启动,如果启动失败则强制kill本机的 keepalived 进程, keepalied将虚拟 ip 绑定另一台机器上
保存后,赋予执行权限:
chmod 755 /etc/keepalived/nginx_check.sh
4.关闭防火墙:
systemctl stop firewalld.service
启动:service keepalived start
停止:service keepalived stop
重启:systemctl restart keepalived
日志文件目录:tail –f /var/log/messages