PEOPLESOFT项目生产环境安装配置手册-NGINX+KEEPALIVED

本文档描述了PS生产环境nginx+keepalived全套搭建过程，包括如下产品：

Peoplesoft HCM 9.2.027
Nginx：nginx-1.16.0
Keepalived：keepalived-2.0.6

节点环境信息：

虚拟机序号	服务器名称	虚拟机主机名	CPU核数（虚拟机）	内存（虚拟机）	OS版本（虚拟机）	IP地址
1	PS应用服务器01	EHR-APP01	16	64	Oracle Linux 7.4	10.160.144.59
2	PS应用服务器02	EHR-APP02	16	64	Oracle Linux 7.4	10.160.144.60
3	Keepalived虚拟IP					10.160.144.68

安装配置

2.1 全局环境

1. 1. 配置hosts

配置hosts：

vi /etc/hosts

10.160.144.68 hrms.companyname.cn

10.160.144.59 EHR-APP01.companyname.cn EHR-APP01

10.160.144.60 EHR-APP02.companyname.cn EHR-APP02

1. 1. 修改linux内核参数

修改linux内核参数：

vi /etc/security/limits.conf

* soft nofile 65536

* hard nofile 65536

* soft nproc 65536

* hard nproc 65536

* soft stack 65536

* hard stack 65536

vi /etc/security/limits.d/20-nproc.conf

* soft nproc unlimited

vi /etc/sysctl.conf

fs.file-max = 6815744

kernel.sem = 250 32000 100 128

kernel.shmmni = 4096

kernel.shmall = 1073741824

kernel.shmmax = 4398046511104

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

net.core.netdev_max_backlog = 102400

net.core.somaxconn = 65535

fs.aio-max-nr = 1048576

net.ipv4.ip_local_port_range = 9000 65500

net.ipv4.ip_forward = 1

net.ipv4.ip_nonlocal_bind = 1

net.ipv4.conf.lo.arp_ignore = 1

net.ipv4.conf.lo.arp_announce = 2

net.ipv4.conf.all.arp_ignore = 1

net.ipv4.conf.all.arp_announce = 2

net.ipv4.tcp_max_orphans = 102400

net.ipv4.tcp_max_syn_backlog = 102400

net.ipv4.tcp_timestamps = 0

net.ipv4.tcp_synack_retries = 1

net.ipv4.tcp_syn_retries = 1

/sbin/sysctl –p

1. 1. 关闭SELinux

临时关闭SELinux：即时生效

setenforce 0

关闭SELinux：需重启操作系统生效

vim /etc/selinux/config

[root@EHR-APP01 Packages]# more /etc/selinux/config

# This file controls the state of SELinux on the system.

# SELINUX= can take one of these three values:

# enforcing - SELinux security policy is enforced.

# permissive - SELinux prints warnings instead of enforcing.

# disabled - No SELinux policy is loaded.

SELINUX=disabled

# SELINUXTYPE= can take one of three two values:

# targeted - Targeted processes are protected,

# minimum - Modification of targeted policy. Only selected processes are protected.

# mls - Multi Level Security protection.

SELINUXTYPE=targeted

2.2 nginx环境安装

1. 1. 安装依赖包

安装依赖包：

yum install gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel patch

1. 1. 安装nginx

1. 获取nginx安装包：

wget http://nginx.org/download/nginx-1.6.2.tar.gz

2. 解压安装包：

tar -zxvf nginx-1.6.2.tar.gz

3. 获取upstream模块插件：

nginx_upstream_check_module-0.3.0.tar.gz

4. 解压插件：

tar -zxvf nginx_upstream_check_module-0.3.0.tar.gz

5. 获取sticky模块插件：

nginx-sticky-module-ng-1.2.5.tar.gz

6. 解压插件：

tar -zxvf nginx-sticky-module-ng-1.2.5.tar.gz

7. 进入nginx源码解压后目录：

cd /usr/local/src/nginx-1.6.2

8. 编译nginx：

./configure --prefix=/usr/local/nginx --with-pcre --with-http_stub_status_module --with-http_ssl_module --with-http_gzip_static_module --with-http_realip_module --add-module=/usr/local/src/nginx_upstream_check_module-0.3.0 --add-module=/usr/local/src/nginx-sticky-module-ng-1.2.5

9. 编译安装：

make && make install

10. 添加组和用户

/usr/sbin/groupadd www

/usr/sbin/useradd -g www www

1. 1. 启停nginx

启动：/usr/local/nginx/sbin/nginx

停止：/usr/local/nginx/sbin/nginx -s stop

重启：/usr/local/nginx/sbin/nginx -s reload

日志文件目录：/usr/local/nginx/logs

测试：访问http://EHR-APP01:80，见如下图表示安装成功：

注：需关闭服务器防火墙：

systemctl stop firewalld.service

1. 1. Nginx参数调整

#user nobody;

user www www;

worker_processes 8;

error_log logs/error.log;

#error_log logs/error.log notice;

#error_log logs/error.log info;

pid logs/nginx.pid;

#Specifies the value for maximum file descriptors that can be opened by this process.

worker_rlimit_nofile 65535;

events {

use epoll;

worker_connections 65535;

}

http {

include mime.types;

default_type application/octet-stream;

log_format main '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for"';

access_log logs/access.log main;

#charset gb2312;

server_names_hash_bucket_size 128;

client_header_buffer_size 32k;

large_client_header_buffers 4 32k;

client_max_body_size 8m;

sendfile on;

tcp_nopush on;

keepalive_timeout 65;

tcp_nodelay on;

fastcgi_connect_timeout 300;

fastcgi_send_timeout 300;

fastcgi_read_timeout 300;

fastcgi_buffer_size 64k;

fastcgi_buffers 4 64k;

fastcgi_busy_buffers_size 128k;

fastcgi_temp_file_write_size 128k;

gzip on;

gzip_min_length 1k;

gzip_buffers 4 16k;

gzip_http_version 1.0;

gzip_comp_level 2;

gzip_types text/plain application/x-javascript text/css application/xml;

gzip_vary on;

#limit_zone crawler $binary_remote_addr 10m;

#\u4e0b\u9762\u662fserver\u865a\u62df\u4e3b\u673a\u7684\u914d\u7f6e

upstream hrms.companyname.cn {

# ip_hash;

sticky;

server 10.160.144.59:8000;

server 10.160.144.60:8000;

}

server {

listen 80;

server_name localhost;

location / {

proxy_pass http://hrms.companyname.cn;

}

2.3 keepalived环境安装

1. 1. 安装依赖包

yum install libnl libnl-devel libnfnetlink-devel

1. 1. 安装keepalived

1. 获取安装包：

wget http://www.keepalived.org/software/keepalived-2.0.6.tar.gz

2. 解压安装包：

tar -zxvf keepalived-2.0.6.tar.gz

3. 编译：

./configure --prefix=/usr/local/keepalived

4. 安装：

make && make install

1. 1. 配置keepalived

1. 将keepalived 安装成 Linux 系统服务：

a. 创建文件夹：mkdir /etc/keepalived

b. 拷贝配置文件：cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/

c. 拷贝命令文件：

cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/

ln -s /usr/local/sbin/keepalived /usr/sbin/

2.修改keepalived配置文件：

vi /etc/keepalived/keepalived.conf

主节点配置如下：

! Configuration File for keepalived

global_defs {

## keepalived 自带的邮件提醒需要开启 sendmail 服务。建议用独立的监控或第三方 SMTP

router_id EHR-APP01 ## 标识本节点的字条串，通常为 hostname

}

## keepalived 会定时执行脚本并对脚本执行的结果进行分析，动态调整 vrrp_instance 的优先级。如果脚本执行结果为 0，并且 weight 配置的值大于 0，则优先级相应的增加。如果脚本执行结果非 0，并且 weight配置的值小于 0，则优先级相应的减少。其他情况，维持原本配置的优先级，即配置文件中 priority 对应的值。

vrrp_script chk_nginx {

script "/etc/keepalived/nginx_check.sh" ## 检测 nginx 状态的脚本路径

interval 2 ## 检测时间间隔

weight -20 ## 如果条件成立，权重-20

}

## 定义虚拟路由， VI_1 为虚拟路由的标示符，自己定义名称

vrrp_instance VI_1 {

state BACKUP #BACKUP备状态，防止资源抢占，主备节点均采用BACKUP

interface ens192 ## 绑定虚拟 IP 的网络接口，与本机 IP 地址所在的网络接口相同，我的是 eth0

virtual_router_id 33 ## 虚拟路由的 ID 号，两个节点设置必须一样，可选 IP 最后一段使用, 相同的 VRID 为一个组，他将决定多播的 MAC 地址

mcast_src_ip 10.160.144.59 ## 本机 IP 地址

priority 100 ## 节点优先级，值范围 0-254， MASTER 要比 BACKUP 高

nopreempt ## 优先级高的设置 nopreempt 解决异常恢复后再次抢占的问题

advert_int 1 ## 组播信息发送间隔，两个节点设置必须一样，默认 1s

## 设置验证信息，两个节点必须一致

authentication {

auth_type PASS

auth_pass Zgjmadmin0817 ## 真实生产，按需求对应该过来

}

## 将 track_script 块加入 instance 配置块

track_script {

chk_nginx ## 执行 Nginx 监控的服务

} #

# 虚拟 IP 池, 两个节点设置必须一样

virtual_ipaddress {

10.160.144.68 ## 虚拟 ip，可以定义多个

}

备节点配置如下：

! Configuration File for keepalived

global_defs {

## keepalived 自带的邮件提醒需要开启 sendmail 服务。建议用独立的监控或第三方 SMTP，本次忽略邮件提醒

router_id EHR-APP02 ## 标识本节点的字条串，通常为 hostname

}

vrrp_script chk_nginx {

script "/etc/keepalived/nginx_check.sh" ## 检测 nginx 状态的脚本路径

interval 2 ## 检测时间间隔

weight -20 ## 如果条件成立，权重-20

}

## 定义虚拟路由， VI_1 为虚拟路由的标示符，自己定义名称

vrrp_instance VI_1 {

state BACKUP #BACKUP备状态，防止资源抢占，主备节点均采用BACKUP

interface ens192 ## 绑定虚拟 IP 的网络接口，与本机 IP 地址所在的网络接口相同

virtual_router_id 33 ## 虚拟路由的 ID 号，两个节点设置必须一样，可选 IP 最后一段使用, 相同的 VRID 为一个组，他将决定多播的 MAC 地址

mcast_src_ip 10.160.144.60 ## 本机 IP 地址

priority 100 ## 节点优先级，值范围 0-254， MASTER 要比 BACKUP 高

nopreempt ## 优先级高的设置 nopreempt 解决异常恢复后再次抢占的问题

advert_int 1 ## 组播信息发送间隔，两个节点设置必须一样，默认 1s

## 设置验证信息，两个节点必须一致

authentication {

auth_type PASS

auth_pass Zgjmadmin0817

}

## 将 track_script 块加入 instance 配置块

track_script {

chk_nginx ## 执行 Nginx 监控的服务

} #

# 虚拟 IP 池, 两个节点设置必须一样

virtual_ipaddress {

10.160.144.68 ## 虚拟 ip，可以定义多个

}

3.编写nginx状态检测脚本：(已在keepalived.conf文件中配置)

vi /etc/keepalived/nginx_check.sh

#!/bin/bash

if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

/usr/local/nginx/sbin/nginx

sleep 2

if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

killall keepalived

注：如果 nginx 停止运行，尝试启动，如果启动失败则强制kill本机的 keepalived 进程， keepalied将虚拟 ip 绑定另一台机器上

保存后，赋予执行权限：

chmod 755 /etc/keepalived/nginx_check.sh

4.关闭防火墙：

systemctl stop firewalld.service

1. 1. 启停keepalived

启动：service keepalived start

停止：service keepalived stop

重启：systemctl restart keepalived

日志文件目录：tail –f /var/log/messages