Keepalived implements haproxy load balancing and high availability

Keepalived implements haproxy load balancing and high availability

1. What is keepalived

keepalived is a service software in cluster management to ensure the high availability of the cluster. Its function is similar to heartbeat to prevent single point of failure.

2. How keepalived implements failover

When the keepalived service is working, the main master node will continuously send heartbeat information to the backup node, telling the backup node that it is still alive. When the master node fails, it cannot send heartbeats, so it will call its own takeover program to take over the ip resources and services of the master node.

3. Important functions of keepalived

keepalived has three important functions, namely:

• Manage LVS load balancing software
• Realize the health check of LVS cluster nodes
• High availability (failover) as a system network service

4. Keepalived implements haproxy load balancing and high availability deployment

Environment description :

CPU name	IP address	installed services	system
master	192.168.183.135	haproxy、keepalived	i tried8
backup	192.168.183.136	haproxy、keepalived	i tried8
RS1	192.168.183.137	httpd	i tried8
RS2	192.168.183.138	nginx	i tried8

1. Deploy the web interface

Configure RS1

//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname RS1
[root@localhost ~]# bash
[root@RS1 ~]# setenforce 0
[root@RS1 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@RS1 ~]# systemctl disable --now firewalld.service
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

//配置yum源
[root@RS1 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2495  100  2495    0     0   9415      0 --:--:-- --:--:-- --:--:--  9379
[root@RS1 ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//安装httpd
[root@RS1 ~]# dnf -y install httpd
[root@RS1 ~]# echo 'httpd' > /var/www/html/index.html
[root@RS1 ~]# systemctl enable --now httpd
Created symlink /etc/systemd/system/multi-user.target.wants/httpd.service → /usr/lib/systemd/system/httpd.service.
[root@RS1 ~]# curl 192.168.183.137
httpd
[root@RS1 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port     Peer Address:Port  Process  
LISTEN   0        128              0.0.0.0:22            0.0.0.0:*              
LISTEN   0        128                    *:80                  *:*              
LISTEN   0        128                 [::]:22               [::]:*         

Configure RS2

//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname RS2
[root@localhost ~]# bash
[root@RS2 ~]# setenforce 0
[root@RS2 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@RS2 ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

//配置yum源
[root@RS2 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2495  100  2495    0     0  12729      0 --:--:-- --:--:-- --:--:-- 12729
[root@RS2 ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//安装nginx
[root@RS2 ~]# dnf install -y nginx
[root@RS2 ~]# echo 'nginx' > /usr/share/nginx/html/index.html
[root@RS2 ~]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@RS2 ~]# curl 192.168.183.138
nginx
[root@RS2 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port     Peer Address:Port  Process  
LISTEN   0        128              0.0.0.0:22            0.0.0.0:*              
LISTEN   0        128              0.0.0.0:80            0.0.0.0:*              
LISTEN   0        128                 [::]:22               [::]:*              
LISTEN   0        128                 [::]:80               [::]:*        

2. Deploy haproxy load balancing

configure master

//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname master
[root@localhost ~]# bash
[root@master ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@master ~]# setenforce 0
[root@master ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config

//下载haproxy软件包及所需依赖包
[root@master ~]# ls
anaconda-ks.cfg  haproxy-2.1.3.tar.gz
[root@master ~]# dnf -y install make gcc pcre-devel bzip2-devel openssl-devel systemd-devel wget vim

//创建用户
[root@master ~]# useradd -rMs /sbin/nologin haproxy

//解压软件包进行安装
[root@master ~]# tar -xf haproxy-2.1.3.tar.gz 
[root@master ~]# cd haproxy-2.1.3
[root@master haproxy-2.1.3]# make -j $(grep 'processor' /proc/cpuinfo |wc -l)  \
> TARGET=linux-glibc  \
> USE_OPENSSL=1  \
> USE_ZLIB=1  \
> USE_PCRE=1  \
> USE_SYSTEMD=1

[root@master haproxy-2.1.3]# make install PREFIX=/usr/local/haproxy

//复制命令到/usr/sbin目录下
[root@master haproxy-2.1.3]# cp haproxy /usr/sbin/

//修改内核参数
[root@master haproxy-2.1.3]# cd
[root@master ~]# vim /etc/sysctl.conf
[root@master ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1

//修改配置文件
[root@master ~]# cat /etc/haproxy/haproxy.cfg
global
    daemon
    maxconn 256
 
defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
 
frontend http-in
    bind *:80
    default_backend servers 

backend servers
    server web01 192.168.183.137:80
    server web02 192.168.183.138:80

//写service文件启动服务
[root@master ~]# cat >> /usr/lib/systemd/system/haproxy.service <<EOF
> [Unit]
> Description=HAProxy Load Balancer
> After=syslog.target network.target
>  
> [Service]
> ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /etc/haproxy/haproxy.cfg   -c -q
> ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg  -p /var/run/haproxy.pid
> ExecReload=/bin/kill -USR2 $MAINPID
>  
> [Install]
> WantedBy=multi-user.target
> EOF

[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl start haproxy

//查看效果
[root@master ~]# curl 192.168.183.135
httpd
[root@master ~]# curl 192.168.183.135
nginx
[root@master ~]# curl 192.168.183.135
httpd
[root@master ~]# curl 192.168.183.135
nginx

configure backup

//修改主机名关闭防火墙和selinux
[root@localhost ~]# hostnamectl set-hostname backup
[root@localhost ~]# bash
[root@backup ~]# systemctl disable --now firewalld
Removed /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@backup ~]# setenforce 0
[root@backup ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config

//下载haproxy软件包及所需依赖包
[root@backup ~]# ls
anaconda-ks.cfg  haproxy-2.1.3.tar.gz
[root@backup ~]# dnf -y install make gcc pcre-devel bzip2-devel openssl-devel systemd-devel wget vim

//创建用户
[root@backup ~]# useradd -rMs /sbin/nologin haproxy

//解压软件包进行安装
[root@backup ~]# tar -xf haproxy-2.1.3.tar.gz 
[root@backup ~]# cd haproxy-2.1.3
[root@backup haproxy-2.1.3]# make -j $(grep 'processor' /proc/cpuinfo |wc -l)  \
> TARGET=linux-glibc  \
> USE_OPENSSL=1  \
> USE_ZLIB=1  \
> USE_PCRE=1  \
> USE_SYSTEMD=1

[root@backup haproxy-2.1.3]# make install PREFIX=/usr/local/haproxy

//复制命令到/usr/sbin/目录下
[root@backup haproxy-2.1.3]# cp haproxy /usr/sbin/

//修改内核参数
[root@backup haproxy-2.1.3]# cd
[root@backup ~]# vim /etc/sysctl.conf
[root@backup ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1

//修改配置文件
[root@backup ~]# mkdir /etc/haproxy
[root@backup ~]# cat /etc/haproxy/haproxy.cfg
global
    daemon
    maxconn 256
 
defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
 
frontend http-in
    bind *:80
    default_backend servers
 
backend servers
    server web01 192.168.183.137:80
    server web02 192.168.183.138:80

//写service文件启动服务
[root@backup ~]# cat >> /usr/lib/systemd/system/haproxy.service <<EOF
> [Unit]
> Description=HAProxy Load Balancer
> After=syslog.target network.target
>  
> [Service]
> ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /etc/haproxy/haproxy.cfg   -c -q
> ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg  -p /var/run/haproxy.pid
> ExecReload=/bin/kill -USR2 $MAINPID
>  
> [Install]
> WantedBy=multi-user.target
> EOF

[root@backup ~]# systemctl daemon-reload
[root@backup ~]# systemctl start haproxy

//查看效果
[root@backup ~]# curl 192.168.183.136
httpd
[root@backup ~]# curl 192.168.183.136
nginx
[root@backup ~]# curl 192.168.183.136
httpd
[root@backup ~]# curl 192.168.183.136
nginx

//停掉backup端的haproxy
[root@backup ~]# systemctl stop haproxy

3. Deploy keepalived high availability

configure master

//安装keepalived
[root@master ~]# dnf -y install keepalived

//编辑配置文件启动服务
[root@master ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
    
    
   router_id lb01
}
 
vrrp_instance VI_1 {
    
    
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
    
    
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
    
    
        192.168.183.250
    }
}
 
virtual_server 192.168.183.250 80 {
    
    
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.183.135 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.183.136 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@master ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.

//查看vip
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
       valid_lft 1034sec preferred_lft 1034sec
    inet 192.168.183.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

//vip访问web界面
[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx
[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx

configure backup

//安装keepalived
[root@backup ~]# dnf -y install keepalived

//编辑配置文件启动服务
[root@backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
    
    
   router_id lb02
}
 
vrrp_instance VI_1 {
    
    
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
    
    
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
    
    
        192.168.183.250
    }
}
 
virtual_server 192.168.183.250 80 {
    
    
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.183.135 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.183.136 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@backup ~]# systemctl start keepalived

4. Write scripts

configure master

[root@master ~]# mkdir /scripts
[root@master ~]# cd /scripts/
[root@master scripts]# cat check_hp.sh
#!/bin/bash
haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
if [ $haproxy_status -lt 1 ];then
    systemctl stop keepalived
fi

[root@master scripts]# cat notify.sh
#!/bin/bash
VIP=$2
case "$1" in
  master)
        haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
        if [ $haproxy_status -lt 1 ];then
            systemctl start haproxy
        fi
  ;;
  backup)
        haproxy_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bhaproxy\b'|wc -l)
        if [ $haproxy_status -gt 0 ];then
            systemctl stop haproxy
        fi
  ;;
  *)
        echo "Usage:$0 master|backup VIP"
  ;;
esac
[root@master scripts]# chmod +x check_hp.sh notify.sh 
[root@master scripts]# ll
total 8
-rwxr-xr-x. 1 root root 148 Oct  9 20:57 check_hp.sh
-rwxr-xr-x. 1 root root 443 Oct  9 21:00 notify.sh

configure backup

[root@backup ~]# mkdir /scripts
[root@backup ~]# cd /scripts/
[root@backup scripts]# scp [email protected]:/scripts/notify.sh .
The authenticity of host '192.168.183.135 (192.168.183.135)' can't be established.
ECDSA key fingerprint is SHA256:c/bKicNnB6SvIpxi/x93PuBCTI8v7FuwiL4pI+1R16w.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.183.135' (ECDSA) to the list of known hosts.
[email protected]'s password: 
notify.sh                                     100%  443   528.7KB/s   00:00    
[root@backup scripts]# ll
total 4
-rwxr-xr-x. 1 root root 443 Oct  9 21:03 notify.sh

5. Configure keepalived to join the monitoring script

configure master

[root@master ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
    
    
   router_id lb01
}

vrrp_script haproxy_check {
    
    		//添加这部分
    script "/scripts/check_hp.sh"
    interval 1
    weight -20
}
 
vrrp_instance VI_1 {
    
    
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
    
    
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
    
    
        192.168.183.250
    }

    track_script {
    
    			//及这部分
        haproxy_check
    }
    notify_master "/scripts/notify.sh master 192.168.183.250"
}
 
virtual_server 192.168.183.250 80 {
    
    
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.183.135 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.183.136 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@master ~]# systemctl restart keepalived

configure backup

backup does not need to detect whether nginx is normal, it starts nginx when it is upgraded to MASTER, and shuts down when it is downgraded to BACKUP

[root@backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
    
    
   router_id lb02
}
 
vrrp_instance VI_1 {
    
    
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
    
    
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
    
    
        192.168.183.250
    }

    notify_master "/scripts/notify.sh master 192.168.183.250"
    notify_backup "/scripts/notify.sh backup 192.168.183.250"		//增加着两条
}
 
virtual_server 192.168.183.250 80 {
    
    
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.183.135 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.183.136 80 {
    
    
        weight 1
        TCP_CHECK {
    
    
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@backup ~]# systemctl restart keepalived

6. Inspection

master end

[root@master ~]# curl 192.168.183.250
httpd
[root@master ~]# curl 192.168.183.250
nginx
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
       valid_lft 1609sec preferred_lft 1609sec
    inet 192.168.183.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@master ~]# systemctl stop haproxy
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
       valid_lft 1593sec preferred_lft 1593sec
    inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

backup side

[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:07:de:9b brd ff:ff:ff:ff:ff:ff
    inet 192.168.183.136/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
       valid_lft 1541sec preferred_lft 1541sec
    inet 192.168.183.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe07:de9b/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@backup ~]# curl 192.168.183.250
httpd
[root@backup ~]# curl 192.168.183.250
nginx
[root@backup ~]# curl 192.168.183.250
httpd
[root@backup ~]# curl 192.168.183.250
nginx