MySQL高可用方案MGR+consul组合测试

要完整的模拟一套consul+MGR的完整环境，我们可能需要配置如下的服务器：

3台作为consul server,另外3台作为MySQL服务器，做MGR单主模式。

一、Mysql MGR 的搭建：

下载新版的mysql 8.0.13

wget https://cdn.mysql.com/archives/mysql-8.0/mysql-8.0.13-linux-glibc2.12-x86_64.tar.xz

使用uuidgen命令生成个uuid：

uuidgen
b71ed720-3219-4cb9-8349-0ecc04be979b

修改hosts

10.99.35.211 opsys-vm1-211
10.99.35.212 opsys-vm2-212
10.99.35.213 opsys-vm3-213

安装mysql-8.0.13

cd /apps
tar xvf mysql-8.0.13-linux-glibc2.12-x86_64.tar
tar xvf mysql-8.0.13-linux-glibc2.12-x86_64.tar.xz
mv mysql-8.0.13-linux-glibc2.12-x86_64 mysql-8.0.13
mkdir -p /data/mysql/3307
chown -R mysql:mysql /data/mysql/3307
cd /apps/mysql-8.0.13
vim my_3307.cnf    --> my_3307.cnf在最后有一个例子
bin/mysqld --defaults-file=/apps/mysql-8.0.13/my_3307.cnf --initialize --lc_messages_dir=/apps/mysql-8.0.13/share
nohup bin/mysqld_safe --defaults-file=/apps/mysql-8.0.13/my_3307.cnf &

/apps/mysql-8.0.13/bin/mysql -uroot -p -S /data/mysql/3307/mysql.sock
alter user root@localhost identified by 'aspire@123-';
install plugin group_replication soname 'group_replication.so';

RESET MASTER

配置启动MGR
登陆opsys-vm1-211，配置启动mgr

set sql_log_bin=0;
CREATE USER repluser@'%' IDENTIFIED WITH 'mysql_native_password' BY '123qwe';
GRANT REPLICATION SLAVE ON *.* TO 'repluser'@'%';
FLUSH PRIVILEGES;
set sql_log_bin=1;

CHANGE MASTER TO MASTER_USER='repluser',MASTER_PASSWORD='123qwe'  FOR CHANNEL 'group_replication_recovery';

#group_replication_bootstrap_group参数仅在第一台启动的MGR实例上开启，其他实例不要操作
set global group_replication_bootstrap_group=ON;
START GROUP_REPLICATION;
set global group_replication_bootstrap_group=OFF;

SELECT * FROM performance_schema.replication_group_members;

登陆opsys-vm1-212、opsys-vm1-213，配置启动mgr

set sql_log_bin=0;
CREATE USER repluser@'%' IDENTIFIED WITH 'mysql_native_password' BY '123qwe';
GRANT REPLICATION SLAVE ON *.* TO 'repluser'@'%';
FLUSH PRIVILEGES;
set sql_log_bin=1;

CHANGE MASTER TO MASTER_USER='repluser',MASTER_PASSWORD='123qwe'  FOR CHANNEL 'group_replication_recovery';

START GROUP_REPLICATION;

SELECT * FROM performance_schema.replication_group_members;

至此，完成mgr

mysql [(none)]> SELECT * FROM performance_schema.replication_group_members;
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+
| CHANNEL_NAME              | MEMBER_ID                            | MEMBER_HOST   | MEMBER_PORT | MEMBER_STATE | MEMBER_ROLE | MEMBER_VERSION |
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+
| group_replication_applier | 98297c82-7b91-11e9-8774-000c292741d1 | opsys-vm1-211 |        3307 | ONLINE       | PRIMARY     | 8.0.13         |
| group_replication_applier | 9b1373e9-7b91-11e9-a067-000c2985a8a2 | opsys-vm3-213 |        3307 | ONLINE       | SECONDARY   | 8.0.13         |
| group_replication_applier | 9b9fa56c-7b91-11e9-b8a6-000c29f9538d | opsys-vm2-212 |        3307 | ONLINE       | SECONDARY   | 8.0.13         |
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+

二、consul服务部署
consul server是这三台服务器：

10.99.35.214
10.99.35.215
10.99.35.216

consul的安装非常容易，从 https://www.consul.io/downloads.html 这里下载以后，解压即可使用，就是一个二进制文件，其他的都没有了。我这里使用的是consul_1.4.4版本。文件下载以后解压放到/usr/local/bin。就可以使用了。不依赖任何东西。

unzip consul_1.4.4_linux_amd64.zip
mv consul /usr/local/bin/

上面的3台服务器都安装。3台机器都创建目录，分别是放配置文件，以及存放数据的。

mkdir -p /etc/consul.d/
mkdir -p /data/consul_cluster/
mkdir -p /data/consul_cluster/scripts

主要的server的配置就需要一个server.json,这里有一个重要概念就是domain,这里是tk，也就是我们所属的一个域，通过域的方式来提供访问。对于每台机器来说，advertise_addr是根据每台的实际IP来的，这里是10.99.35.214的服务器的配置，10.99.35.215,10.99.35.216的配置就可以按照类似的方式来做。

vim /etc/consul.d/server.json
{
  "addresses": {
    "http": "0.0.0.0",
    "dns": "0.0.0.0"
  },
  "bind_addr": "0.0.0.0",
  "advertise_addr": "10.99.35.214",
  "bootstrap_expect": 3,
  "datacenter": "syk-mgrdb",
  "data_dir": "/data/consul_cluster",
  "dns_config": {
    "allow_stale": true,
    "max_stale": "87600h",
    "node_ttl": "0s",
    "service_ttl": {
      "*": "0s"
    }
  },
  "domain": "tk",
  "enable_syslog": false,
  "leave_on_terminate": false,
  "log_level": "info",
  "node_name": "opsys-vm4-214",
  "node_meta": {
    "location": "B1 in test"
  },
  "performance": {
    "raft_multiplier": 1
  },
  "ports": {
    "http": 8500,
    "dns": 53
  },
  "reconnect_timeout": "72h",
  "retry_join": [
    "10.99.35.215",
    "10.99.35.216"
  ],
  "retry_interval": "10s",
  "server": true,
  "skip_leave_on_interrupt": true,
  "ui": true
}

nohup consul agent -config-dir=/etc/consul.d > /data/consul_cluster/consul.log &

对于客户端来说，需要的是一个client.json,这里就需要配置所有的consul server,因此同样需要在三台mysql服务器上创建目录，分别是放配置文件，以及存放数据的。

mkdir -p /etc/consul.d/
mkdir -p /data/consul_cluster/
mkdir -p /data/consul_cluster/scripts

对于每台机器来说，advertise_addr是根据每台的实际IP来的，这里是10.99.35.211的服务器的配置，10.99.35.212,10.99.35.213的配置就可以按照类似的方式来做。

vim /etc/consul.d/client.json
{
  "addresses": {
    "http": "0.0.0.0",
    "dns": "0.0.0.0"
  },
  "bind_addr": "0.0.0.0",
  "advertise_addr": "10.99.35.211",
  "datacenter": "syk-mgrdb",
  "data_dir": "/data/consul_cluster",
  "enable_script_checks": true,
  "enable_syslog": false,
  "leave_on_terminate": true,
  "log_level": "info",
  "node_name": "opsys-vm1-211",
  "node_meta": {
    "location": "B1 in test"
  },
  "ports": {
    "dns": 53,
    "http": 8500
  },
  "rejoin_after_leave": true,
  "retry_join": [
    "10.99.35.214",
    "10.99.35.215",
    "10.99.35.216"
  ],
  "retry_interval": "10s",
  "skip_leave_on_interrupt": false
}

nohup consul agent -config-dir=/etc/consul.d > /data/consul_cluster/consul.log &

[root@opsys-vm4-214 consul.d]# consul members -http-addr='10.99.35.214:8500'
Node           Address            Status  Type    Build  Protocol  DC         Segment
opsys-vm4-214  10.99.35.214:8301  alive   server  1.4.4  2         syk-mgrdb  <all>
opsys-vm5-215  10.99.35.215:8301  alive   server  1.4.4  2         syk-mgrdb  <all>
opsys-vm6-216  10.99.35.216:8301  alive   server  1.4.4  2         syk-mgrdb  <all>
opsys-vm1-211  10.99.35.211:8301  alive   client  1.4.4  2         syk-mgrdb  <default>
opsys-vm2-212  10.99.35.212:8301  alive   client  1.4.4  2         syk-mgrdb  <default>
opsys-vm3-213  10.99.35.213:8301  alive   client  1.4.4  2         syk-mgrdb  <default>

在consul client 10.99.35.211、10.99.35.212、10.99.35.213上编写检测primay脚本和检测slave脚本

[root@opsys-vm2-212 consul_cluster]# cat /data/consul_cluster/scripts/check_mgr_primary.sh
#!/bin/bash
port=$1
user="root"
passwod="aspire@123-"

#comm="/usr/local/mysql_5.7.23/bin/mysql -u$user -h 127.0.0.1 -P $port "
comm="/apps/mysql-8.0.13/bin/mysql -u$user -P $port -p$passwod -S /data/mysql/3307/mysql.sock"
value=`$comm -Nse "select 1"`
primary_member=`$comm -Nse "select variable_value from performance_schema.global_status WHERE VARIABLE_NAME= 'group_replication_primary_member'"`
server_uuid=`$comm -Nse "select variable_value from performance_schema.global_variables where VARIABLE_NAME='server_uuid';"`
if [ -z $value ]
then
    echo "mysql $port is down....."
    exit 2
fi
# 判断节点状态
node_state=`$comm -Nse "select MEMBER_STATE from performance_schema.replication_group_members where MEMBER_ID='$server_uuid'"`
if [ $node_state != "ONLINE" ]
then
    echo "MySQL $port state is not online...."
    exit 2
fi
# 判断是不是主节点
if [[ $server_uuid == $primary_member ]]
then
    echo "MySQL $port Instance is master ........"
    exit 0
else
    echo "MySQL $port Instance is slave ........"
    exit 2
fi
#end

[root@opsys-vm2-212 consul_cluster]# cat /data/consul_cluster/scripts/check_mgr_secondary.sh
#!/bin/bash
port=$1
user="root"
passwod="aspire@123-"

#comm="/usr/local/mysql_5.7.23/bin/mysql -u$user -h 127.0.0.1 -P $port "
comm="/apps/mysql-8.0.13/bin/mysql -u$user -P $port -p$passwod -S /data/mysql/3307/mysql.sock"
value=`$comm -Nse "select 1"`
primary_member=`$comm -Nse "select variable_value from performance_schema.global_status WHERE VARIABLE_NAME= 'group_replication_primary_member'"`
server_uuid=`$comm -Nse "select variable_value from performance_schema.global_variables where VARIABLE_NAME='server_uuid';"`
# 判断mysql是否存活
if [ -z $value ]
then
    echo "mysql $port is down....."
    exit 2
fi
# 判断节点状态
node_state=`$comm -Nse "select MEMBER_STATE from performance_schema.replication_group_members where MEMBER_ID='$server_uuid'"`
if [ $node_state != "ONLINE" ]
then
    echo "MySQL $port state is not online...."
    exit 2
fi
# 判断是不是主节点
if [[ $server_uuid != $primary_member ]]
then
    echo "MySQL $port Instance is slave ........"
    exit 0
else
    node_num=`$comm -Nse "select count(*) from performance_schema.replication_group_members"`
    # 判断如果没有任何从节点，主节点也注册从角色服务。
    if [ $node_num -eq 1 ]
    then
         echo "MySQL $port Instance is slave ........"
         exit 0
    else
         echo "MySQL $port Instance is master ........"
         exit 2
    fi
fi
#end

访问consulserver的web页面 http://10.99.35.214:8500/ui/

三、MGR+Consul高可用实现

检测MGR集群状态，查看哪个是主节点

mysql [(none)]> SELECT * FROM performance_schema.replication_group_members;
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+
| CHANNEL_NAME              | MEMBER_ID                            | MEMBER_HOST   | MEMBER_PORT | MEMBER_STATE | MEMBER_ROLE | MEMBER_VERSION |
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+
| group_replication_applier | 98297c82-7b91-11e9-8774-000c292741d1 | opsys-vm1-211 |        3307 | ONLINE       | PRIMARY     | 8.0.13         |
| group_replication_applier | 9b1373e9-7b91-11e9-a067-000c2985a8a2 | opsys-vm3-213 |        3307 | ONLINE       | SECONDARY   | 8.0.13         |
| group_replication_applier | 9b9fa56c-7b91-11e9-b8a6-000c29f9538d | opsys-vm2-212 |        3307 | ONLINE       | SECONDARY   | 8.0.13         |
+---------------------------+--------------------------------------+---------------+-------------+--------------+-------------+----------------+
3 rows in set (0.00 sec)

在consul client 10.99.35.211、10.99.35.212、10.99.35.213上接服务发现的json脚本，并reload。三台机器都需要相应的json脚本检测mysql是否为主或从，配置文件修改相应的IP即可使用

检测master

[root@opsys-vm1-211 consul.d]# cat test_mgr_write.db.json
{
  "services": [
    {
      "id": "syk3307_mgr_rw",
      "name": "syk3307-mysql-rw",
      "address": "",
      "port": 3307,
      "enable_tag_override": false,
      "checks": [
        {
          "id": "syk3307_mgr_rw-check-01",
          "name": "MySQL Write Check",
          "args": ["/data/consul_cluster/scripts/check_mgr_primary.sh","3307"],
          "interval": "15s",
          "timeout": "1s",
          "service_id": "syk3307_mgr_rw"
        }
      ]
    }
  ]
}

检测slave

[root@opsys-vm1-211 consul.d]# cat test_mgr_read.db.json
{
  "services": [
    {
      "id": "syk3307_mgr_ro",
      "name": "syk3307-mysql-ro",
      "address": "",
      "port": 3307,
      "enable_tag_override": false,
      "checks": [
        {
          "id": "syk3307_mgr_ro-check-02",
          "name": "MySQL Write Check",
          "args": ["/data/consul_cluster/scripts/check_mgr_secondary.sh","3307"],
          "interval": "15s",
          "timeout": "1s",
          "service_id": "syk3307_mgr_ro"
        }
      ]
    }
  ]
}

reload consul

[root@opsys-vm1-211 shell]# consul reload
Configuration reload triggered

查看consul server ui界面即可看到三台mysql的mgr集群已经注册到consul服务上了

注意：由于每台mysql server 上都有master、slave 检测脚本、而mysql server 只能是master 或者slave、所以存在失败的检测，master检测只有一个成功，slave检测只有一个失败

连接测试

[root@opsys-35-209 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 10.99.35.214
nameserver 10.99.35.215
nameserver 10.99.35.216

[root@opsys-35-209 ~]# dig @10.99.35.214 -p 53 syk3307-mysql-rw.service.tk

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.68.rc1.el6_10.3 <<>> @10.99.35.214 -p 53 syk3307-mysql-rw.service.tk
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 47308
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 2
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;syk3307-mysql-rw.service.tk.    IN    A

;; ANSWER SECTION:
syk3307-mysql-rw.service.tk. 0    IN    A    10.99.35.211

;; ADDITIONAL SECTION:
syk3307-mysql-rw.service.tk. 0    IN    TXT    "location=B1 in test"
syk3307-mysql-rw.service.tk. 0    IN    TXT    "consul-network-segment="

;; Query time: 1 msec
;; SERVER: 10.99.35.214#53(10.99.35.214)
;; WHEN: Thu Apr 30 10:53:06 2020
;; MSG SIZE  rcvd: 129

[root@opsys-35-209 ~]# dig @10.99.35.214 -p 53 syk3307-mysql-ro.service.tk

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.68.rc1.el6_10.3 <<>> @10.99.35.214 -p 53 syk3307-mysql-ro.service.tk
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 34863
;; flags: qr aa rd; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 4
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;syk3307-mysql-ro.service.tk.    IN    A

;; ANSWER SECTION:
syk3307-mysql-ro.service.tk. 0    IN    A    10.99.35.212
syk3307-mysql-ro.service.tk. 0    IN    A    10.99.35.213

;; ADDITIONAL SECTION:
syk3307-mysql-ro.service.tk. 0    IN    TXT    "location=B1 in test"
syk3307-mysql-ro.service.tk. 0    IN    TXT    "consul-network-segment="
syk3307-mysql-ro.service.tk. 0    IN    TXT    "consul-network-segment="
syk3307-mysql-ro.service.tk. 0    IN    TXT    "location=B1 in test"

;; Query time: 1 msec
;; SERVER: 10.99.35.214#53(10.99.35.214)
;; WHEN: Thu Apr 30 10:53:32 2020
;; MSG SIZE  rcvd: 213

到此为止：mysql mgr集群+consul实现高可用集群方案已尽全部完成

mysql配置文件如下：

[root@opsys-vm1-211 mysql-8.0.13]# cat my_3307.cnf
[mysql]
port            = 3307
socket          = /data/mysql/3307/mysql.sock
default-character-set=utf8mb4

[mysqld]
user=mysql
server-id = 352113307
datadir=/data/mysql/3307
log-error = /data/mysql/3307/mysql_error.log
default_storage_engine=InnoDB   
character_set_server=utf8mb4
log_bin_trust_function_creators=OFF
federated
open_files_limit    = 10240
port            = 3307
socket          = /data/mysql/3307/mysql.sock
back_log = 600
max_connections = 5000
max_connect_errors = 1000
lower_case_table_names=1

table_open_cache = 2048
thread_cache_size = 8
thread_stack = 192K
sort_buffer_size = 8M
join_buffer_size = 8M
read_buffer_size = 2M
read_rnd_buffer_size = 16M
max_allowed_packet = 32M
bulk_insert_buffer_size = 128M
max_heap_table_size = 64M
tmp_table_size = 64M

binlog_cache_size = 1M
log-slave-updates
log-bin = binlog
log-bin-index=binlog.index
binlog_format = row
sync_binlog             = 1
max_binlog_size         = 128M

key_buffer_size = 128M
myisam_sort_buffer_size = 64M
myisam_max_sort_file_size = 64M
myisam_repair_threads = 1

innodb_flush_method=O_DIRECT
innodb_file_per_table=1
innodb_strict_mode=1
innodb_buffer_pool_size = 4G
innodb_data_home_dir = /data/mysql/3307
innodb_data_file_path = ibdata1:100M:autoextend
innodb_max_dirty_pages_pct = 90
innodb_lock_wait_timeout = 2

innodb_log_buffer_size = 8M
innodb_log_group_home_dir = /data/mysql/3307
innodb_log_file_size = 64M
innodb_log_files_in_group = 3
innodb_flush_log_at_trx_commit = 1

enforce-gtid-consistency=true
gtid-mode=on
master_info_repository="TABLE"
relay_log_info_repository="TABLE"
relay_log_recovery=1
skip_slave_start
log_slave_updates=1
relay_log=mysql-relay-bin
binlog_checksum=NONE
transaction_write_set_extraction=XXHASH64
loose-group_replication_group_name="b71ed720-3219-4cb9-8349-0ecc04be979b"
loose-group_replication_start_on_boot=off
loose-group_replication_local_address= "10.99.35.211:33071"
loose-group_replication_group_seeds= "10.99.35.211:33071,10.99.35.212:33071,10.99.35.213:33071"
loose-group_replication_bootstrap_group= off

[mysqld_safe]
open-files-limit = 10240

[client]
default-character-set= utf8mb4

#end