Build a semi-synchronous master-slave replication MySQL cluster based on keepalived+GTID

4 MySQL servers: Set up a master-slave replication cluster, one master server (MySQL-master), two slave servers (MySQL-slave1, MySQL-slave2), and a delayed backup server (delay-backup) - perform backup operations on Executed within a certain delay time to resolve the impact of backup on the performance of the source database server

2 mysqlrouter servers: Install keepalived software to achieve high-availability read-write separation service

1 ansible central control server: realizes batch management of servers in the entire MySQL cluster

The IP address of the server required for the project

mysql-master 192.168.15.159

mysql-slave-1 192.168.15.157

mysql-slave-2 192.168.15.160

delay-backup 192.168.15.161

ansible central control machine 192.168.15.162

mysql-router-1 192.168.15.169 vip：192.168.15.200

mysql-router-2 192.168.15.170 vip：192.168.15.210

Project architecture diagram

Project steps:

1. Set up 4 MySQL database servers, install the MySQL system, and install the semi-synchronization plug-in

Download the mysql binary installation package from the official mysql: mysql-5.7.41-linux-glibc2.12-x86_64.tar.gz

One-click installation of mysql script in Linux system:

#解决软件的依赖关系并且安装需要的工具
yum  install cmake ncurses-devel gcc  gcc-c++  vim  lsof bzip2 openssl-devel ncurses-compat-libs net-tools -y

#解压mysql二进制安装包
tar xf mysql-5.7.41-linux-glibc2.12-x86_64.tar.gz

#移动mysql解压后的文件到/usr/local下改名叫mysql
#/usr/local/mysql是mysql的安装目录
mv mysql-5.7.41-linux-glibc2.12-x86_64 /usr/local/mysql

#新建组和用户 mysql
groupadd mysql
#mysql这个用户的shell是/bin/false 属于mysql组，-r表示为系统用户
useradd -r -g mysql -s /bin/false mysql

#关闭firewalld防火墙服务，并且设置开机不要启动
service firewalld stop
systemctl  disable  firewalld

#临时关闭selinux
setenforce 0
#永久关闭selinux
sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config

#新建存放数据的目录
mkdir /data/mysql -p

#修改/data/mysql目录的权限归mysql用户和mysql组所有，这样mysql用户启动的mysql进程可以对这个文件夹进行读写了
chown mysql:mysql /data/mysql/
#只是允许mysql这个用户和mysql组可以访问，其他人都不能访问
chmod 750 /data/mysql/

#进入/usr/local/mysql/bin目录
cd /usr/local/mysql/bin/

#初始化mysql
./mysqld  --initialize --user=mysql --basedir=/usr/local/mysql/  --datadir=/data/mysql 

#让mysql支持ssl方式登录的设置
./mysql_ssl_rsa_setup --datadir=/data/mysql/

#修改PATH变量，加入mysql bin目录的路径
#临时修改PATH变量的值
export PATH=/usr/local/mysql/bin/:$PATH
#重新启动linux系统后也生效，永久修改
echo  'PATH=/usr/local/mysql/bin:$PATH' >>/root/.bashrc

#复制support-files里的mysql.server文件到/etc/init.d/目录下叫mysqld
cp  ../support-files/mysql.server   /etc/init.d/mysqld

#修改/etc/init.d/mysqld脚本文件里的datadir目录的值
sed  -i '70c  datadir=/data/mysql'  /etc/init.d/mysqld

#生成/etc/my.cnf配置文件
cat  >/etc/my.cnf  <<EOF
[mysqld_safe]

[client]
socket=/data/mysql/mysql.sock

[mysqld]
socket=/data/mysql/mysql.sock
port = 3306
open_files_limit = 8192
innodb_buffer_pool_size = 512M
character-set-server=utf8

[mysql]
auto-rehash
prompt=\\u@\\d \\R:\\m  mysql>
EOF

#临时修改：修改内核的open file(允许1个进程打开的文件)的数量 默认情况下是1024
ulimit -n 1000000
#永久修改：设置开机启动的时候也配置生效
echo "ulimit -n 1000000" >>/etc/rc.local
chmod +x /etc/rc.d/rc.local

#老式的设置mysql服务开机启动
#将mysqld添加到linux系统里服务管理名单里
/sbin/chkconfig --add mysqld
#设置mysqld服务开机启动
/sbin/chkconfig mysqld on

#启动mysqld进程
service mysqld start

#初次修改密码-p后面接的是系统自动生成的临时密码
#初次修改密码需要使用--connect-expired-password 选项
#-e 后面接的表示是在mysql里需要执行命令  execute 执行
#set password='Sanchuang123#';  修改root用户的密码为Sanchuang123#
mysql -uroot -p'FJjh7(li2sdu#' --connect-expired-password   -e  "set password='Sanchuang123#';"

#检验上一步修改密码是否成功，如果有输出能看到mysql里的数据库，说明成功。
mysql -uroot -p'Sanchuang123#'  -e "show databases;"

Enable binary logging on the server

Add the following configuration to the configuration file:

log_bin
server_id = 1 	#服务器的编号

Configure semi-synchronization on the master

root@(none) 20:04  mysql>install plugin rpl_semi_sync_master soname 'semisync_master.so';
root@(none) 20:07  mysql>set global rpl_semi_sync_master_timeout = 1; 	#临时修改变量
root@(none) 20:10  mysql>set global rpl_semi_sync_master_enabled = 1;

Modify configuration file

[root@sumeng-master ~]# vim /etc/my.cnf
[mysqld]
...
rpl_semi_sync_master_enabled=1
rpl_semi_sync_master_timeout=1000  #1 second 超时时间设置为1s

Refresh the master's mysqld service after modifying the configuration file

[root@sumeng-master ~]# service mysqld restart
Shutting down MySQL............. SUCCESS! 
Starting MySQL.. SUCCESS!

Configure semi-sync on the slave server

root@(none) 20:06  mysql>install plugin rpl_semi_sync_slave soname 'semisync_slave.so';
root@(none) 20:16  mysql> set global rpl_semi_sync_slave_enabled=1;

Modify the configuration file on the slave server

[root@sumeng-slave ~]# vim /etc/my.cnf
[mysqld]
...
rpl_semi_sync_slave_enabled=1

Refresh the mysqld service on the slave

[root@sumeng-slave ~]# service mysqld restart
Shutting down MySQL.. SUCCESS! 
Starting MySQL.. SUCCESS!

2. Set up ansible server

Install ansible

[root@localhost ~]# yum install epel-release -y
[root@localhost ~]# yum install ansible -y

Edit the configuration and create a host list:

master server ip: 192.168.15.159

slave server ip: 192.168.15.157, 192.168.15.160, 192.168.15.161

[root@localhost ~]# vim /etc/ansible/hosts
[db]
192.168.15.159
192.168.15.157
192.168.15.160
192.168.15.161
[dbslaves]
192.168.15.161
192.168.15.157
192.168.15.160

Let ansible establish a secret-free channel between all MySQL node servers

[root@localhost ~]# ssh-keygen -t rsa 	#生成秘钥
[root@localhost ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected] 	#将本地生成的ssh公钥信息复制到远程主机上
[root@localhost ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected]
[root@localhost ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected]
[root@localhost ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected]

Test in turn whether the password-free channel is successfully established

[root@localhost ~]# ssh '[email protected]'
[root@localhost ~]# ssh '[email protected]'
[root@localhost ~]# ssh '[email protected]'
[root@localhost ~]# ssh '[email protected]'

Copy the binary log on the master to the local. After getting the basic data, transfer it to each slave.

[root@localhost ~]# scp [email protected]:/root/all_db.SQL .
[root@localhost ~]# ansible -m copy -a "src=/root/all_db.SQL dest=/root" dbslaves

Import basic data on all slave machines

mysql -uroot -p'Sanchuang123#' <all_db.SQL

3. Enable GTID function and start master-slave replication

Add configuration in the configuration files of slave and master:

gtid-mode=ON 	#开启GTID功能
enforce-gtid-consistency=ON

Create a new authorized user on the master to copy the binary log to the slave

grant replication slave on *.* to 'zhouzhou'@'192.168.15.%' identified by 'Sanchuang123#';

Configure master info information on slave

In order to prevent the environment from being impure, perform the following operations before starting the slave:

root@(none) 16:32  mysql>reset master;
Query OK, 0 rows affected (0.01 sec)

root@(none) 16:39  mysql>show master status;
+--------------------------+----------+--------------+------------------+-------------------+
| File                     | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
+--------------------------+----------+--------------+------------------+-------------------+
| sumeng-master-bin.000001 |      154 |              |                  |                   |
+--------------------------+----------+--------------+------------------+-------------------+
1 row in set (0.00 sec)

root@(none) 16:39  mysql>stop slave;
Query OK, 0 rows affected, 1 warning (0.00 sec)

root@(none) 16:40  mysql>reset slave all;
Query OK, 0 rows affected (0.01 sec)

Configure master information on slaves

root@(none) 16:47  mysql>CHANGE MASTER TO MASTER_HOST='192.168.15.159',
    -> MASTER_USER='zhouzhou',
    -> MASTER_PASSWORD='Sanchuang123#',
    -> MASTER_PORT=3306,
    -> MASTER_AUTO_POSITION=1;
Query OK, 0 rows affected, 2 warnings (0.01 sec)

root@(none) 16:47  mysql>start slave;
Query OK, 0 rows affected (0.01 sec)

Check whether the slave is configured successfully. Both Slave_IO_Running and Slave_SQL_Running are yes, indicating that both the IO thread and the SQL thread are started.

root@(none) 16:57  mysql>show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 192.168.15.159
                  Master_User: zhouzhou
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: sumeng-master-bin.000002
          Read_Master_Log_Pos: 154
               Relay_Log_File: localhost-relay-bin.000002
                Relay_Log_Pos: 383
        Relay_Master_Log_File: sumeng-master-bin.000002
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
              Replicate_Do_DB: 
          Replicate_Ignore_DB: 
           Replicate_Do_Table: 
       Replicate_Ignore_Table: 
      Replicate_Wild_Do_Table: 
  Replicate_Wild_Ignore_Table: 
                   Last_Errno: 0
                   Last_Error: 
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 154
              Relay_Log_Space: 594
              Until_Condition: None
               Until_Log_File: 
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File: 
           Master_SSL_CA_Path: 
              Master_SSL_Cert: 
            Master_SSL_Cipher: 
               Master_SSL_Key: 
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error: 
               Last_SQL_Errno: 0
               Last_SQL_Error: 
  Replicate_Ignore_Server_Ids: 
             Master_Server_Id: 1
                  Master_UUID: b53f58ce-2a25-11ee-b708-000c29cf54b0
             Master_Info_File: /data/mysql/master.info
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
           Master_Retry_Count: 86400
                  Master_Bind: 
      Last_IO_Error_Timestamp: 
     Last_SQL_Error_Timestamp: 
               Master_SSL_Crl: 
           Master_SSL_Crlpath: 
           Retrieved_Gtid_Set: 
            Executed_Gtid_Set: 
                Auto_Position: 1
         Replicate_Rewrite_DB: 
                 Channel_Name: 
           Master_TLS_Version: 
1 row in set (0.00 sec)

Possible problems during configuration: The slave's io thread cannot start. The reason is that the GTID number on the slave is larger than that on the master. This means that the data on the slave is newer than that on the master, causing the io thread to fail to start. Success, the solution is to clear the master information and salve information on all slaves, and then reconfigure and start.

4. Configure a delayed backup server to read binary files from MySQL-slave1

root@(none) 16:54  mysql>stop slave;
Query OK, 0 rows affected, 1 warning (0.00 sec)

root@(none) 17:10  mysql>reset master;
Query OK, 0 rows affected (0.01 sec)

root@(none) 17:10  mysql>reset slave all;
Query OK, 0 rows affected (0.01 sec)

root@(none) 17:10  mysql>CHANGE MASTER TO MASTER_HOST='192.168.15.157',
    -> MASTER_USER='zhouzhou',
    -> MASTER_PASSWORD='Sanchuang123#',
    -> MASTER_PORT=3306,
    -> MASTER_AUTO_POSITION=1;
Query OK, 0 rows affected, 2 warnings (0.01 sec)

root@(none) 17:10  mysql>change master to master_delay = 10;
Query OK, 0 rows affected (0.01 sec)

root@(none) 17:11  mysql>start slave;
Query OK, 0 rows affected (0.00 sec)

When setting delayed backup, the default unit is seconds.

5. Use rsync+sersync to achieve real-time file synchronization and remotely synchronize the data on the master to the ansible server (equivalent to an off-site backup server)

Configure rsync on the backup server-ansible server. rsync on the backup server is a daemon listening on port 873.

1. Permanently turn off selinux and firewall

[root@localhost ~]# service firewalld stop
Redirecting to /bin/systemctl stop firewalld.service
[root@localhost ~]# setenforce 0     #临时关闭
[root@localhost ~]# vim /etc/selinux/config    #永久关闭，修改SELINUX=disabled
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected. 
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted

2. Install rsync server software

[root@localhost ~]# yum install rsync xinetd -y

3. Set rsync to start at boot, and start the xinetd process - the process that provides nanny services. rsync is the process it takes care of.

[root@localhost ~]# vim /etc/rc.d/rc.local
...
/usr/bin/rsync --daemon --config=/etc/rsyncd.conf
[root@localhost ~]# chmod +x /etc/rc.d/rc.local
[root@localhost ~]# systemctl start xinetd

4. Create rsync.conf configuration file

[root@localhost ~]# vim /etc/rsyncd.conf
uid = root
gid = root
use chroot = yes
max connections = 0
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid 
lock file = /var/run/rsync.lock 
secrets file = /etc/rsync.pass  
motd file = /etc/rsyncd.Motd
[back_data]
    path = /backup
    comment = A directory in which data is stored
    ignore errors = yes
    read only = no
    hosts allow = 192.168.15.159

5. Create user authentication file

Add users and passwords allowed for transmission, format: username:password, you can set multiple

[root@localhost ~]# vim /etc/rsync.pass
sumeng:sumeng123456

6. Set file permissions--Set file owner read and write permissions

[root@localhost ~]# chmod 600 /etc/rsyncd.conf 
[root@localhost ~]# chmod 600 /etc/rsync.pass

7. Start rsync and xinetd

[root@localhost ~]# /usr/bin/rsync --daemon --config=/etc/rsyncd.conf
[root@localhost ~]# systemctl start xinetd

8. Check the port number that rsync listens to

[root@localhost ~]# netstat -anplut|grep rsync
tcp        0      0 0.0.0.0:873             0.0.0.0:*               LISTEN      20313/rsync         
tcp6       0      0 :::873                  :::*                    LISTEN      20313/rsync

Configure rsync on the data source server

1. Permanently turn off selinux and firewall

[root@sumeng-master ~]# service firewalld stop
Redirecting to /bin/systemctl stop firewalld.service
[root@sumeng-master ~]# setenforce 0
[root@sumeng-master ~]# vim /etc/selinux/config
...
SELINUX=disabled
...

2. Install rsync server software

[root@sumeng-master ~]# yum install rsync xinetd -y
[root@sumeng-master ~]# vim /etc/rc.d/rc.local
...
/usr/bin/rsync --daemon     #添加该行
[root@sumeng-master ~]# chmod +x /etc/rc.d/rc.local
[root@sumeng-master ~]# vim /etc/rsyncd.conf
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
motd file = /etc/rsyncd.Motd
[Sync]
comment = Sync
uid = root
gid = root
port= 873
[root@sumeng-master ~]# systemctl start xinetd

3. Create authentication password file

[root@sumeng-master ~]# vim /etc/passwd.txt
[root@sumeng-master ~]# cat /etc/passwd.txt 
sumeng123456
[root@sumeng-master ~]# chmod 600 /etc/passwd.txt     #设置文件权限

4. Test data synchronization

Data synchronization between data source server 192.168.15.159 and backup server 192.168.15.162

[root@sumeng-master ~]# rsync -avH --port=873 --progress --delete /backup [email protected]::back_data --password-file=/etc/passwd.txt

sending incremental file list
backup/

sent 60 bytes  received 16 bytes  152.00 bytes/sec
total size is 0  speedup is 0.00

Note: If the servers on both sides do not create a new /backup directory, an error will be reported, resulting in the inability to back up the past.

Install the sersync tool to trigger rsync in real time for synchronization - installed on the data source server

1. Modify the default parameters of inotify--the default kernel parameters are too small

[root@sumeng-master ~]# sysctl -w fs.inotify.max_queued_events="99999999"
fs.inotify.max_queued_events = 99999999
[root@sumeng-master ~]# sysctl -w fs.inotify.max_user_watches="99999999"
fs.inotify.max_user_watches = 99999999
[root@sumeng-master ~]# sysctl -w fs.inotify.max_user_instances="65535"
fs.inotify.max_user_instances = 65535
[root@sumeng-master ~]# vim /etc/sysctl.conf    #永久修改参数
fs.inotify.max_queued_events=99999999
fs.inotify.max_user_watches=99999999
fs.inotify.max_user_instances=65535

2. Install sersync

#wget用于从指定网址下载网络文件
[root@sumeng-master ~]# wget http://down.whsir.com/downloads/sersync2.5.4_64bit_binary_stable_final.tar.gz
[root@sumeng-master ~]# tar xf sersync2.5.4_64bit_binary_stable_final.tar.gz
[root@sumeng-master ~]# mv GNU-Linux-x86/ /usr/local/sersync

3. Create rsync

[root@sumeng-master ~]# cd /usr/local/sersync/
[root@sumeng-master sersync]# cp confxml.xml confxml.xml-bak    #备份配置文件，防止修改出错，便于还原
[root@sumeng-master sersync]# cp confxml.xml data_configxml.xml     #复制用于同步data目录的文件，data_configxml.xml是后面需要使用的配置文件

4. Modify the configuration file data_configxml.xml

[root@sumeng-master sersync]# vim data_configxml.xml
<sersync>
        <localpath watch="/backup">
            <remote ip="192.168.15.162" name="back_data"/>
            <!--<remote ip="192.168.8.39" name="tongbu"/>-->
            <!--<remote ip="192.168.8.40" name="tongbu"/>-->
        </localpath>
        <rsync>
            <commonParams params="-artuz"/>
            <auth start="false" users="root" passwordfile="/etc/passwd.txt"/>
            <userDefinedPort start="false" port="874"/><!-- port=874 -->
            <timeout start="false" time="100"/><!-- timeout=100 -->
            <ssh start="false"/>
        </rsync>
        <failLog path="/tmp/rsync_fail_log.sh" timeToExecute="60"/><!--default every 60mins execute once-->
        <crontab start="false" schedule="600"><!--600mins-->
            <crontabfilter start="false">
                <exclude expression="*.php"></exclude>
                <exclude expression="info/*"></exclude>
            </crontabfilter>
        </crontab>
        <plugin start="false" name="command"/>
</sersync>

5. Start the service

#修改环境变量，可以直接使用文件名执行
[root@sumeng-master sersync]# PATH=/usr/local/sersync:$PATH
[root@sumeng-master sersync]# echo 'PATH=/usr/local/sersync:$PATH' >>/root/.bashrc
[root@sumeng-master sersync]# sersync2 -d -r -o  /usr/local/sersync/data_configxml.xml
set the system param
execute：echo 50000000 > /proc/sys/fs/inotify/max_user_watches
execute：echo 327679 > /proc/sys/fs/inotify/max_queued_events
parse the command param
option: -d 	run as a daemon
option: -r 	rsync all the local files to the remote servers before the sersync work
option: -o 	config xml name：  /usr/local/sersync/data_configxml.xml
daemon thread num: 10
parse xml config file
host ip : localhost	host port: 8008
daemon start，sersync run behind the console 
config xml parse success
please set /etc/rsyncd.conf max connections=0 Manually
sersync working thread 12  = 1(primary thread) + 1(fail retry thread) + 10(daemon sub threads) 
Max threads numbers is: 22 = 12(Thread pool nums) + 10(Sub threads)
please according your cpu ，use -n param to adjust the cpu rate
------------------------------------------
rsync the directory recursivly to the remote servers once
working please wait...
execute command: cd /backup && rsync -artuz -R --delete ./ 192.168.15.162::back_data >/dev/null 2>&1 
run the sersync: 
watch path is: /backup

6. Set up sersync monitoring to automatically execute on boot

[root@sumeng-master sersync]# vim /etc/rc.d/rc.local
...
/usr/local/sersync/sersync2 -d -r -o  /usr/local/sersync/data_configxml.xml    #添加此行

Verification: Go to the /backup directory and create some new files or folders to test whether they can be seen on the backup server.

Write a backup script on the master and set up scheduled tasks for scheduled backup

[root@sumeng-master backup]# vim backup_alldb.sh
[root@sumeng-master backup]# cat backup_alldb.sh 
#!/bin/bash

mkdir -p /backup
mysqldump -uroot -p"Sanchuang123#" --all-databases --triggers --routines --events >/backup/$(date +%Y%m%d%H%M%S)_all_db.SQL
[root@sumeng-master backup]# crontab -e
no crontab for root - using an empty one
crontab: installing new crontab
[root@sumeng-master backup]# crontab -l
30 2 * * * bash /backup/backup_alldb.sh

6. Install and deploy the mysqlrouter middleware software to achieve separation of reading and writing.

Start two servers installed and deployed with mysqlrouter middleware software to achieve high availability.

Separation of reading and writing: Operations such as additions, deletions, modifications, and queries of the database will be separated. Read operations are performed on the slave, and both read and write operations can be performed on the master, which essentially plays a role in load balancing.

mysqlrouter is a read-write separation software officially provided to us by MySQL. It is a lightweight MySQL middleware that can only achieve simple read/write separation through different ports. The premise of mysqlrouter is that the backend implements the main MySQL server. Copied from.

1. Download mysqlrouter

https://dev.mysql.com/get/Downloads/MySQL-Router/mysql-router-community-8.0.23-1.el7.x86_64.rpm

2. Remotely transfer the downloaded rpm file to the virtual machine, and then install it directly

[root@mysql-router-1 ~]# rpm -ivh mysql-router-community-8.0.23-1.el7.x86_64.rpm 
警告：mysql-router-community-8.0.23-1.el7.x86_64.rpm: 头V3 DSA/SHA1 Signature, 密钥 ID 5072e1f5: NOKEY
准备中...                          ################################# [100%]
正在升级/安装...
   1:mysql-router-community-8.0.23-1.e################################# [100%]

3. Modify the configuration file /etc/mysqlrouter.conf

cd /etc/mysqlrouter #Enter the directory where the configuration file is stored

Add the following code

#read
[routing:slaves]
bind_address = 192.168.15.169:7001
destinations = 192.168.15.157:3306,192.168.15.160:3306
mode = read-only        #只读模式
connect_timeout = 1

#write and read
[routing:masters]
bind_address = 192.168.15.169:7002
destinations = 192.168.15.159:3306
mode = read-write         #可读可写
connect_timeout = 1

4. Start the mysqlrouter service

[root@mysql-router-1 mysqlrouter]# service mysqlrouter restart
Redirecting to /bin/systemctl restart mysqlrouter.service
[root@mysql-router-1 mysqlrouter]# netstat -anplut|grep mysqlrouter     #检查端口
tcp        0      0 192.168.15.169:7001     0.0.0.0:*               LISTEN      8808/mysqlrouter    
tcp        0      0 192.168.15.169:7002     0.0.0.0:*               LISTEN      8808/mysqlrouter

5. Create 2 test accounts on the master, one is readable and the other is writable.

root@(none) 09:56  mysql>grant all on *.* to 'scwrite'@'%' identified by 'Sanchuang123#';
Query OK, 0 rows affected, 1 warning (0.00 sec)

root@(none) 09:56  mysql>grant select on *.* to 'scread'@'%' identified by 'Sanchuang123#';
Query OK, 0 rows affected, 1 warning (0.01 sec)

6. Test the read-write separation effect on the client machine

#只读用户无法对数据库进行更改
[root@sc-docker ~]# mysql -h 192.168.15.169 -P 7001 -uscread -p"Sanchuang123#"
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 9
Server version: 5.7.41-log MySQL Community Server (GPL)

Copyright (c) 2000, 2023, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

scread@(none) 10:30  mysql>create database sumeng;
ERROR 1044 (42000): Access denied for user 'scread'@'%' to database 'sumeng'

[root@sc-docker ~]# mysql -h 192.168.15.169 -P 7002 -uscwrite -p"Sanchuang123#"
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 9
Server version: 5.7.41-log MySQL Community Server (GPL)

Copyright (c) 2000, 2023, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

scwrite@(none) 10:35  mysql>create database Tuesday;
Query OK, 1 row affected (0.00 sec)

The key point of separation of reading and writing is user permissions, allowing different users to connect to different ports. In the end, they still have to go to the back-end mysql server to verify whether they have read and write permissions. Mysqlrouter only splits reading and writing, allowing the application to Connect to different ports

7. Install keepalived to achieve high availability, and configure 2 vrrp instances to achieve the high availability function of dual VIP

Highly available HA: There will be no single point of failure. If one is broken, the other one can take over. Work will not be affected. There is backup.

keepalived is a routing software written in C language. Its main goal is to provide simple and powerful load balancing and high availability functions for Linux systems and Linux-based infrastructure. vip: virtual IP address, which will be bound to the machine with high priority

1. Modify the mysqlrouter machine configuration file

Modify the bind_address configuration in the configuration file /etc/mysqlrouter/mysqlrouter.conf of the two mysqlrouter machines to 0.0.0.0:[port], so that the machine can bind VIP. After changing the configuration file, remember to refresh the mysqlrouter service

2. Install keepalived

[root@mysql-router-1 ~]# yum install keepalived -y

3. Modify configuration file

The storage path of the keepalived configuration file is: /etc/keepalived/keepalived.conf

The idea of implementing dual VIP: Create 2 vrrp instances, 2 VIPs, and the 2 instances are active and backup for each other

The meaning of each configuration in the configuration file:

mysql-router-1 configuration:

[root@mysql-router-1 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     [email protected]
     [email protected]
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
  #vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 99
    priority 200
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.15.200
    }
}

vrrp_instance VI_2 {
    state backup
    interface ens33
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.15.210
    }
}

mysql-router-2 configuration:

[root@mysql-router-2 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     [email protected]
     [email protected]
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
  #vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state backup
    interface ens33
    virtual_router_id 99
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.15.200
    }
}

vrrp_instance VI_2 {
    state MASTER
    interface ens33
    virtual_router_id 66
    priority 200
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.15.210
    }
}

4. Check if vip appears

mysql-router-1, vip is 192.168.15.200:

[root@mysql-router-1 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:74:8a:2a brd ff:ff:ff:ff:ff:ff
    inet 192.168.15.169/24 brd 192.168.15.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.15.200/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe74:8a2a/64 scope link 
       valid_lft forever preferred_lft forever

mysql-router-2, vip is 192.168.15.210:

[root@mysql-router-2 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f4:23:bc brd ff:ff:ff:ff:ff:ff
    inet 192.168.15.170/24 brd 192.168.15.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.15.210/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fef4:23bc/64 scope link 
       valid_lft forever preferred_lft forever

By adding the same domain name corresponding to 2 VIPs in the dns domain name, you can achieve dns load balancing

8. Use the sysbench stress testing tool to test the performance of the entire MySQL cluster

1. Install sysbench on the client machine

[root@sc-docker ~]# yum install epel-release -y
[root@sc-docker ~]# yum install sysbench -y

2. Use the library on the master for testing

root@mysql 14:56  mysql>create database test_db;
Query OK, 1 row affected (0.01 sec)

3. Construct test tables and test data

When using a client to build test tables and test data remotely, remember to turn off the firewall to avoid unsuccessful connections.

[root@localhost ~]# sysbench --db-driver=mysql --time=300 --threads=10 --report-interval=1 --mysql-host=192.168.15.200 --mysql-port=7002 --mysql-user=scwrite --mysql-password=Sanchuang123# --mysql-db=test_db --tables=10 --table_size=1000 oltp_read_write prepare
sysbench 1.0.17 (using system LuaJIT 2.0.4)

Initializing worker threads...

Creating table 'sbtest1'...
Creating table 'sbtest10'...
Creating table 'sbtest7'...
Creating table 'sbtest3'...
Creating table 'sbtest4'...
Creating table 'sbtest6'...
Creating table 'sbtest8'...
Creating table 'sbtest5'...
Creating table 'sbtest9'...
Creating table 'sbtest2'...
Inserting 1000 records into 'sbtest4'
Inserting 1000 records into 'sbtest7'
Inserting 1000 records into 'sbtest5'
Inserting 1000 records into 'sbtest10'
Inserting 1000 records into 'sbtest6'
Inserting 1000 records into 'sbtest9'
Inserting 1000 records into 'sbtest8'
Inserting 1000 records into 'sbtest1'
Inserting 1000 records into 'sbtest2'
Inserting 1000 records into 'sbtest3'
Creating a secondary index on 'sbtest8'...
Creating a secondary index on 'sbtest7'...
Creating a secondary index on 'sbtest5'...
Creating a secondary index on 'sbtest4'...
Creating a secondary index on 'sbtest1'...
Creating a secondary index on 'sbtest10'...
Creating a secondary index on 'sbtest9'...
Creating a secondary index on 'sbtest6'...
Creating a secondary index on 'sbtest2'...
Creating a secondary index on 'sbtest3'...

Parameter description used when building:

--db-driver=mysql: database driver

--time=300: Continuous access for 300 seconds

--threads=10: Use 10 threads to simulate concurrent access

--report-interval=1: Output a test progress report every second, 0 means only output the final report results, the default is 0

--mysql-host=192.168.15.200 --mysql-port=7002 --mysql-user=scwrite --mysql-password=Sanchuang123# --mysql-db=test_db: Indicates information about connecting to the database. Mysqlrouter is connected here. Machine configuration VIP and port

--tables=10 --table_size=1000: Construct 10 test tables in the test library, and construct 1000 test data in each test table. If the test data is too large, the disk of the MySQL cluster may be exhausted and the cluster may collapse.

oltp_read_write: indicates reading and writing tests

prepare: In the preparation stage, create a table for testing and fill in the data.

4. Test database read and write performance

Switch the prepare phase to the run phase for stress testing

Test results per second:

[root@localhost ~]# sysbench --db-driver=mysql --time=300 --threads=10 --report-interval=1 --mysql-host=192.168.15.200 --mysql-port=7002 --mysql-user=scwrite --mysql-password=Sanchuang123# --mysql-db=test_db --tables=10 --table_size=1000 oltp_read_write run
sysbench 1.0.17 (using system LuaJIT 2.0.4)

Running the test with following options:
Number of threads: 10
Report intermediate results every 1 second(s)
Initializing random number generator from current time


Initializing worker threads...

Threads started!

[ 1s ] thds: 10 tps: 286.27 qps: 5828.84 (r/w/o: 4095.29/1151.06/582.49) lat (ms,95%): 43.39 err/s: 0.00 reconn/s: 0.00
[ 2s ] thds: 10 tps: 279.17 qps: 5609.51 (r/w/o: 3932.48/1118.69/558.34) lat (ms,95%): 52.89 err/s: 0.00 reconn/s: 0.00
[ 3s ] thds: 10 tps: 234.86 qps: 4661.30 (r/w/o: 3253.11/939.46/468.73) lat (ms,95%): 59.99 err/s: 0.00 reconn/s: 0.00
[ 4s ] thds: 10 tps: 178.46 qps: 3574.26 (r/w/o: 2506.47/710.86/356.93) lat (ms,95%): 68.05 err/s: 0.00 reconn/s: 0.00
[ 5s ] thds: 10 tps: 174.15 qps: 3470.99 (r/w/o: 2425.09/696.60/349.30) lat (ms,95%): 71.83 err/s: 0.00 reconn/s: 0.00
[ 6s ] thds: 10 tps: 164.90 qps: 3334.96 (r/w/o: 2334.57/671.59/328.80) lat (ms,95%): 81.48 err/s: 0.00 reconn/s: 0.00
[ 7s ] thds: 10 tps: 118.21 qps: 2336.07 (r/w/o: 1637.85/461.80/236.41) lat (ms,95%): 112.67 err/s: 0.00 reconn/s: 0.00
[ 8s ] thds: 10 tps: 135.99 qps: 2729.87 (r/w/o: 1914.91/541.97/272.99) lat (ms,95%): 97.55 err/s: 0.00 reconn/s: 0.00
[ 9s ] thds: 10 tps: 156.04 qps: 3147.90 (r/w/o: 2201.63/634.18/312.09) lat (ms,95%): 104.84 err/s: 0.00 reconn/s: 0.00
[ 10s ] thds: 10 tps: 179.90 qps: 3597.09 (r/w/o: 2522.66/713.62/360.81) lat (ms,95%): 69.29 err/s: 1.00 reconn/s: 0.00

Test result parameter description:

thds: 10: Indicates that there are 10 threads performing stress testing

tps: Indicates how many transactions are performed per second

qps: Indicates how many requests are performed per second

(r/w/o: 4095.29/1151.06/582.49): There are 4095.29 read requests per second, 1151.06 write requests, and 582.49 other requests. In fact, the essence is to disassemble the qps requests.

lat (ms,95%): 43.39: Indicates that 95% of request delays are below 43.39 milliseconds

err/s: 0.00 reconn/s: 0.00: Indicates that 0 requests failed per second and 0 network reconnections occurred.

Overall test results:

[root@localhost ~]# sysbench --db-driver=mysql --time=60 --threads=10 --mysql-host=192.168.15.200 --mysql-port=7002 --mysql-user=scwrite --mysql-password=Sanchuang123# --mysql-db=test_db --tables=10 --table_size=1000 oltp_read_write run
sysbench 1.0.17 (using system LuaJIT 2.0.4)

Running the test with following options:
Number of threads: 10
Initializing random number generator from current time


Initializing worker threads...

Threads started!

SQL statistics:
    queries performed:
        read:                            126266
        write:                           36073
        other:                           18037
        total:                           180376
    transactions:                        9018   (150.03 per sec.)
    queries:                             180376 (3000.88 per sec.)
    ignored errors:                      1      (0.02 per sec.)
    reconnects:                          0      (0.00 per sec.)

General statistics:
    total time:                          60.1034s
    total number of events:              9018

Latency (ms):
         min:                                   21.10
         avg:                                   66.58
         max:                                  262.32
         95th percentile:                      108.68
         sum:                               600417.67

Threads fairness:
    events (avg/stddev):           901.8000/13.27
    execution time (avg/stddev):   60.0418/0.03

Test result analysis:

SQL statistics database statement statistics

General statistics According to comprehensive statistics, a total of 9018 executions were performed, and the actual execution time was 60.1034s.

Latency (ms) counts the actual execution time of the thread, in milliseconds. The minimum execution time for a single time is 21.10ms, the maximum is 262.32ms, the average is 66.58ms, 95% of the execution time is 108.68ms, and the total execution time is 600417.67ms.

Threads fairness summarizes statistics of execution transactions and execution time

5. After completing the stress test, clear the data

cleanup phase: delete test data after testing

[root@localhost ~]# sysbench --db-driver=mysql --time=300 --threads=10 --report-interval=1 --mysql-host=192.168.15.200 --mysql-port=7002 --mysql-user=scwrite --mysql-password=Sanchuang123# --mysql-db=test_db --tables=10 --table_size=1000 oltp_read_write cleanup
sysbench 1.0.17 (using system LuaJIT 2.0.4)

Dropping table 'sbtest1'...
Dropping table 'sbtest2'...
Dropping table 'sbtest3'...
Dropping table 'sbtest4'...
Dropping table 'sbtest5'...
Dropping table 'sbtest6'...
Dropping table 'sbtest7'...
Dropping table 'sbtest8'...
Dropping table 'sbtest9'...
Dropping table 'sbtest10'...

9. Build a monitoring system based on prometheus+grafana to monitor the database cluster

Install and deploy prometheus+grafana on a new server to monitor the performance of master, slave1, slave2, and delay_backup servers.

1. Install and deploy the mysqld_exporter component

In the Prometheus architecture, exporter is the component responsible for collecting data and reporting information to Prometheus Server. mysqld_exporter is a commonly used database monitoring tool.

download

Download address: https://prometheus.io/download/

Upload the mysqld_exporter software package to each mysql node

First upload the software package to the ansible server, and then distribute it to the master and slave servers in batches

[root@localhost ~]# ls
2_all_db.SQL  3_all_db.SQL  all_db.SQL  anaconda-ks.cfg  mysqld_exporter-0.15.0.linux-amd64.tar.gz
[root@localhost ~]# ansible -m copy -a "src=/root/mysqld_exporter-0.15.0.linux-amd64.tar.gz dest=/root" db

Unzip it simultaneously on each mysql server and move it to the /usr/local/mysqld_exporter directory

[root@sumeng-master ~]# tar -zxvf mysqld_exporter-0.15.0.linux-amd64.tar.gz 
mysqld_exporter-0.15.0.linux-amd64/
mysqld_exporter-0.15.0.linux-amd64/mysqld_exporter
mysqld_exporter-0.15.0.linux-amd64/NOTICE
mysqld_exporter-0.15.0.linux-amd64/LICENSE
[root@sumeng-master ~]# mv mysqld_exporter-0.15.0.linux-amd64 /usr/local/mysqld_exporter
[root@sumeng-master ~]# cd /usr/local/mysqld_exporter/
[root@sumeng-master mysqld_exporter]# ls
LICENSE  mysqld_exporter  NOTICE

Edit configuration file

First create a user in the database to connect to the local database.

root@mysql 10:37  mysql>create user 'sumeng'@'%' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

root@mysql 10:38  mysql>grant all on *.* to 'sumeng'@'%';
Query OK, 0 rows affected (0.00 sec)

Then go to each server to edit the configuration file synchronously - edit it in the /usr/local/mysqld_exporter directory

[root@sumeng-master mysqld_exporter]# vim my.cnf
[root@sumeng-master mysqld_exporter]# cat my.cnf 
[client]
user=sumeng
password=123456

Add the mysqld_exporter command to the environment variable and set the background startup

Synchronized on each mysql server

[root@sumeng-master mysqld_exporter]# PATH=/usr/local/mysqld_exporter/:$PATH
[root@sumeng-master mysqld_exporter]# echo "PATH=/usr/local/mysqld_exporter/:$PATH" >>/root/.bashrc
[root@sumeng-master mysqld_exporter]# nohup mysqld_exporter --config.my-cnf=/usr/local/mysqld_exporter/my.cnf &
[1] 5877

Check the process and port to see if they are open

[root@sumeng-master mysqld_exporter]# ps aux|grep exporter
root       5877  0.0  0.4 719816  7984 pts/0    Sl   10:48   0:00 mysqld_exporter --config.my-cnf=/usr/local/mysqld_exporter/my.cnf
root       5883  0.0  0.0 112824   984 pts/0    S+   10:50   0:00 grep --color=auto exporter
[root@sumeng-master mysqld_exporter]# netstat -anplut|grep mysqld
tcp6       0      0 :::3306                 :::*                    LISTEN      4501/mysqld         
tcp6       0      0 :::9104                 :::*                    LISTEN      5877/mysqld_exporte 
tcp6       0      0 192.168.15.159:3306     192.168.15.157:44850    ESTABLISHED 4501/mysqld         
tcp6       0      0 192.168.15.159:3306     192.168.15.160:60524    ESTABLISHED 4501/mysqld

Web access test: ip + 9140 (port number)

2. Install and deploy prometheus

Create a new virtual machine and install and deploy prometheus+grafana on it

Upload the installation package, decompress it and move it to /prometheus

[root@prometheus_grafana ~]# ls
anaconda-ks.cfg  prometheus-2.43.0.linux-amd64.tar.gz
[root@prometheus_grafana ~]# tar -zvxf prometheus-2.43.0.linux-amd64.tar.gz 
prometheus-2.43.0.linux-amd64/
prometheus-2.43.0.linux-amd64/LICENSE
prometheus-2.43.0.linux-amd64/consoles/
prometheus-2.43.0.linux-amd64/consoles/prometheus.html
prometheus-2.43.0.linux-amd64/consoles/node-disk.html
prometheus-2.43.0.linux-amd64/consoles/node-overview.html
prometheus-2.43.0.linux-amd64/consoles/prometheus-overview.html
prometheus-2.43.0.linux-amd64/consoles/index.html.example
prometheus-2.43.0.linux-amd64/consoles/node-cpu.html
prometheus-2.43.0.linux-amd64/consoles/node.html
prometheus-2.43.0.linux-amd64/prometheus
prometheus-2.43.0.linux-amd64/promtool
prometheus-2.43.0.linux-amd64/NOTICE
prometheus-2.43.0.linux-amd64/console_libraries/
prometheus-2.43.0.linux-amd64/console_libraries/prom.lib
prometheus-2.43.0.linux-amd64/console_libraries/menu.lib
prometheus-2.43.0.linux-amd64/prometheus.yml
[root@prometheus_grafana ~]# mv prometheus-2.43.0.linux-amd64 /prometheus
[root@prometheus_grafana ~]# cd /prometheus/
[root@prometheus_grafana prometheus]# ls
console_libraries  consoles  LICENSE  NOTICE  prometheus  prometheus.yml  promtool

Configure prometheus as a service service

[root@prometheus_grafana prometheus]# vim /usr/lib/systemd/system/prometheus.service
[root@prometheus_grafana prometheus]# cat /usr/lib/systemd/system/prometheus.service 
[Unit]
Description=prometheus
 
[Service]
ExecStart=/prometheus/prometheus --config.file=/prometheus/prometheus.yml
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
 
[Install]
WantedBy=multi-user.target
[root@prometheus_grafana prometheus]# systemctl daemon-reload     #重新加载systemd相关服务
[root@prometheus_grafana prometheus]# service prometheus restart
Redirecting to /bin/systemctl restart prometheus.service
[root@prometheus_grafana prometheus]# ps aux|grep prometheus     #检查prometheus进程是否存在
root      22447  1.0  2.0 798956 38004 ?        Ssl  11:50   0:00 /prometheus/prometheus --config.file=/prometheus/prometheus.yml
root      22456  0.0  0.0 112824   992 pts/1    S+   11:50   0:00 grep --color=auto prometheus
[root@prometheus_grafana prometheus]# netstat -anplut|grep prometheus     #查看端口
tcp6       0      0 :::9090                 :::*                    LISTEN      22447/prometheus    
tcp6       0      0 ::1:58730               ::1:9090                ESTABLISHED 22447/prometheus    
tcp6       0      0 ::1:9090                ::1:58730               ESTABLISHED 22447/prometheus

Visit the prometheus service page

Access ip + port number (9090), you can see the following page

Add prometheus node node and refresh the service

[root@prometheus_grafana ~]# cd /prometheus/
[root@prometheus_grafana prometheus]# ls
console_libraries  consoles  LICENSE  NOTICE  prometheus  prometheus.yml  promtool
[root@prometheus_grafana prometheus]# vim prometheus.yml
[root@prometheus_grafana prometheus]# cat prometheus.yml
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

 - job_name: "master"
   static_configs:
      - targets: ["192.168.15.159:9104"]
 - job_name: "slave1"
   static_configs:
      - targets: ["192.168.15.157:9104"]
 - job_name: "slave2"
   static_configs:
      - targets: ["192.168.15.160:9104"]
 - job_name: "backup"
   static_configs:
      - targets: ["192.168.15.161:9104"]
[root@prometheus_grafana prometheus]# service prometheus restart
Redirecting to /bin/systemctl restart prometheus.service

Visit the prometheus service page to check whether the node is added successfully.

3. Install grafana

Upload the grafana installation package and install it

[root@prometheus_grafana ~]# ls
anaconda-ks.cfg    prometheus-2.43.0.linux-amd64.tar.gz
grafana-enterprise-9.4.7-1.x86_64.rpm
[root@prometheus_grafana ~]# yum install grafana-enterprise-9.4.7-1.x86_64.rpm -y

Start grafana

[root@prometheus_grafana ~]# service grafana-server start
Starting grafana-server (via systemctl):                   [  确定  ]
[root@prometheus_grafana ~]# netstat -anplut|grep grafana     #查看端口
tcp        0      0 192.168.15.178:49718    34.120.177.193:443      ESTABLISHED 23204/grafana       
tcp6       0      0 :::3000                 :::*                    LISTEN      23204/grafana