mha 搭建文档
1，三台服务器开启相对应端口防火墙或者关闭防火墙 &&selinux
systemctl stop firewalld && setenforce 0
2,三台服务器配置免密登陆
1.77
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
1.78
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
1.81
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
1，三台服务器开启相对应端口防火墙或者关闭防火墙 &&selinux
systemctl stop firewalld && setenforce 0
2,三台服务器配置免密登陆
1.77
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
1.78
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
1.81
[root@localhost ~]# ssh-keygen -t rsa
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.77
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.78
[root@localhost ~]# ssh-copy-id /root/.ssh/id_rsa.pub 192.168.1.81
3,安装MySQL服务 附带自动化MySQL安装脚本，分享至百度云网盘上了
install.py ###使用脚本安装请注意安装提醒
配置主从
CHANGE MASTER TO
MASTER_HOST='192.168.1.77',
MASTER_USER='repl',
MASTER_PASSWORD='123',
MASTER_PORT=3306,
MASTER_LOG_FILE='mysqlbin.000001',
MASTER_LOG_POS=4,
MASTER_CONNECT_RETRY=10; start slave;show slave status \G;
配置主从
CHANGE MASTER TO
MASTER_HOST='192.168.1.77',
MASTER_USER='repl',
MASTER_PASSWORD='123',
MASTER_PORT=3306,
MASTER_LOG_FILE='mysqlbin.000001',
MASTER_LOG_POS=4,
MASTER_CONNECT_RETRY=10; start slave;show slave status \G;
4,各节点安装DBD::mysql(此处直接使用第三方epel源yum安装，Cpanm安装贼麻烦)
#安装epel源
yum install epel-release -y && yum clean all && yum makecache
#安装DBD::mysql
yum -y install perl-DBD-MySQL
5,各节点安装mha-node
yum install mha4mysql-node-0.56-0.el6.noarch.rpm(资源包共享云盘密码)
安装完会在/usr/bin 下生成4个文件
apply_diff_relay_logs 识别差异日志并应用于其他slave
save_binary_logs 保存和复制二进制日志
filter_mysqlbinlog
purge_relay_logs 清除中继日志
#
cp /usr/bin/apply_diff_relay_logs /usr/local/bin/
cp /usr/bin/save_binary_logs /usr/local/bin/
cp /usr/bin/filter_mysqlbinlog /usr/local/bin/
cp /usr/bin/purge_relay_logs /usr/local/bin/
yum install mha4mysql-node-0.56-0.el6.noarch.rpm(资源包共享云盘密码)
安装完会在/usr/bin 下生成4个文件
apply_diff_relay_logs 识别差异日志并应用于其他slave
save_binary_logs 保存和复制二进制日志
filter_mysqlbinlog
purge_relay_logs 清除中继日志
#
cp /usr/bin/apply_diff_relay_logs /usr/local/bin/
cp /usr/bin/save_binary_logs /usr/local/bin/
cp /usr/bin/filter_mysqlbinlog /usr/local/bin/
cp /usr/bin/purge_relay_logs /usr/local/bin/
6，从节点安装manager （为节约成本 manager放在从服务器上）
#安装manager需要的perl模块
yum install perl-DBD-MySQL perl-Config-Tiny perl-Log-Dispatch perl-Parallel-ForkManager perl-Time-HiRes -y
tar -xf mha4mysql-manager-0.56.tar.gz && cd mha4mysql-manager-0.56
perl Makefile.PL #该过程可能会报错，大都是因为perl模块不全导致的 yum install perl* #安装所有的perl模块（centos7可能会安装mariadb 执行yum remove mysql -y）
make && make install ##安装完成会在/usr/local/bin 下面有关于masterha的脚本
perl Makefile.PL #该过程可能会报错，大都是因为perl模块不全导致的 yum install perl* #安装所有的perl模块（centos7可能会安装mariadb 执行yum remove mysql -y）
make && make install ##安装完成会在/usr/local/bin 下面有关于masterha的脚本
[root@localhost qingke]# ll /usr/local/bin/
total 40
-r-xr-xr-x. 1 root root 1995 Jan 19 13:41 masterha_check_repl
-r-xr-xr-x. 1 root root 1779 Jan 19 13:41 masterha_check_ssh
-r-xr-xr-x. 1 root root 1865 Jan 19 13:41 masterha_check_status
-r-xr-xr-x. 1 root root 3201 Jan 19 13:41 masterha_conf_host
-r-xr-xr-x. 1 root root 2517 Jan 19 13:41 masterha_manager
-r-xr-xr-x. 1 root root 2165 Jan 19 13:41 masterha_master_monitor
-r-xr-xr-x. 1 root root 2373 Jan 19 13:41 masterha_master_switch
-r-xr-xr-x. 1 root root 5171 Jan 19 13:41 masterha_secondary_check
-r-xr-xr-x. 1 root root 1739 Jan 19 13:41 masterha_stop
total 40
-r-xr-xr-x. 1 root root 1995 Jan 19 13:41 masterha_check_repl
-r-xr-xr-x. 1 root root 1779 Jan 19 13:41 masterha_check_ssh
-r-xr-xr-x. 1 root root 1865 Jan 19 13:41 masterha_check_status
-r-xr-xr-x. 1 root root 3201 Jan 19 13:41 masterha_conf_host
-r-xr-xr-x. 1 root root 2517 Jan 19 13:41 masterha_manager
-r-xr-xr-x. 1 root root 2165 Jan 19 13:41 masterha_master_monitor
-r-xr-xr-x. 1 root root 2373 Jan 19 13:41 masterha_master_switch
-r-xr-xr-x. 1 root root 5171 Jan 19 13:41 masterha_secondary_check
-r-xr-xr-x. 1 root root 1739 Jan 19 13:41 masterha_stop
配置masterha
[root@localhost masterha]# cat app1.cnf ########
[server default]
manager_workdir=/var/log/masterha/app1
manager_log=/var/log/masterha/app1/manager.log
remote_workdir = /var/log/masterha/app1
[root@localhost masterha]# cat app1.cnf ########
[server default]
manager_workdir=/var/log/masterha/app1
manager_log=/var/log/masterha/app1/manager.log
remote_workdir = /var/log/masterha/app1
user=root
password=123
ssh_user=root
repl_user=repl
repl_password=123
ping_interval=1
password=123
ssh_user=root
repl_user=repl
repl_password=123
ping_interval=1
secondary_check_script= masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78
master_ip_failover_script= /etc/masterha/master_ip_failover ###自动切换IP脚本
master_ip_failover_script= /etc/masterha/master_ip_failover ###自动切换IP脚本
#shutdown_script= /etc/masterha/power_manager
# report_script= /etc/masterha/send_report
master_ip_online_change_script=/etc/masterha/master_ip_online_change ###手动切换IP脚本
# report_script= /etc/masterha/send_report
master_ip_online_change_script=/etc/masterha/master_ip_online_change ###手动切换IP脚本
[server1] #主
hostname=192.168.1.77
master_binlog_dir=/qingke/mysql/binlog
candidate_master=1
check_repl_delay=0
hostname=192.168.1.77
master_binlog_dir=/qingke/mysql/binlog
candidate_master=1
check_repl_delay=0
[server2] #备主
hostname=192.168.1.78
master_binlog_dir=/qingke/mysql/binlog
candidate_master=1
check_repl_delay=0
hostname=192.168.1.78
master_binlog_dir=/qingke/mysql/binlog
candidate_master=1
check_repl_delay=0
[server3] #从
hostname=192.168.1.81
master_binlog_dir=/qingke/mysql/binlog
no_master=1 #永不为主
check_repl_delay=0
hostname=192.168.1.81
master_binlog_dir=/qingke/mysql/binlog
no_master=1 #永不为主
check_repl_delay=0
#vi master_ip_failover #######脚本太长就不贴了
百度网盘分享 id
#vi master_ip_online_change ######
手动切换脚本，我也附加到百度网盘上了 不建议这样使用 除非想把当前数据库脱离group
##从节点加入定期purge relay脚本
[root@localhost scripts]# cat purge_relay_log.sh
#!/bin/bash
user=root
passwd=123
port=3306
work_dir='/qingke/mysql/relaylog'
log_dir='/qingke/purge_relay'
purge='/usr/local/bin/purge_relay_logs'
if [ ! -d $log_dir ]
then
mkdir $log_dir -p
fi
$purge --user=$user --password=$passwd --disable_relay_log_purge --port=$port --workdir=$work_dir >> $log_dir/purge_relay_logs.log 2>&1
[root@localhost scripts]# crontab -l
1 1 * * 0 /bin/bash /root/scripts/purge_relay_log.sh
#手动检测试下purge脚本是否ok
[root@localhost masterha]# purge_relay_logs --user=root --password=123 --port=3306 -disable_relay_log_purge --workdir=/qingke/mysql/relaylog/
2018-01-22 15:50:42: purge_relay_logs script started.
Found relay_log.info: /qingke/mysql/data/relay-log.info
Opening /qingke/mysql/relaylog/mysqlrelay.000004 ..
Opening /qingke/mysql/relaylog/mysqlrelay.000005 ..
Executing SET GLOBAL relay_log_purge=1; FLUSH LOGS; sleeping a few seconds so that SQL thread can delete older relay log files (if it keeps up); SET GLOBAL relay_log_purge=0; .. ok.
2018-01-22 15:50:45: All relay log purging operations succeeded.
[root@localhost masterha]#
1 1 * * 0 /bin/bash /root/scripts/purge_relay_log.sh
#手动检测试下purge脚本是否ok
[root@localhost masterha]# purge_relay_logs --user=root --password=123 --port=3306 -disable_relay_log_purge --workdir=/qingke/mysql/relaylog/
2018-01-22 15:50:42: purge_relay_logs script started.
Found relay_log.info: /qingke/mysql/data/relay-log.info
Opening /qingke/mysql/relaylog/mysqlrelay.000004 ..
Opening /qingke/mysql/relaylog/mysqlrelay.000005 ..
Executing SET GLOBAL relay_log_purge=1; FLUSH LOGS; sleeping a few seconds so that SQL thread can delete older relay log files (if it keeps up); SET GLOBAL relay_log_purge=0; .. ok.
2018-01-22 15:50:45: All relay log purging operations succeeded.
[root@localhost masterha]#
### 开始检测
[root@localhost ~]# masterha_check_ssh --conf=/etc/masterha/app1.cnf ###检测各MySQL服务器免密登陆连通是否有问题
Fri Jan 19 16:02:14 2018 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Fri Jan 19 16:02:14 2018 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Fri Jan 19 16:02:14 2018 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Fri Jan 19 16:02:14 2018 - [info] Starting SSH connection tests..
Fri Jan 19 16:02:15 2018 - [debug]
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.77:22) to [email protected](192.168.1.78:22)..
Fri Jan 19 16:02:14 2018 - [debug] ok.
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.77:22) to [email protected](192.168.1.81:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug]
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.78:22) to [email protected](192.168.1.77:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.78:22) to [email protected](192.168.1.81:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:16 2018 - [debug]
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.81:22) to [email protected](192.168.1.77:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.81:22) to [email protected](192.168.1.78:22)..
Fri Jan 19 16:02:16 2018 - [debug] ok.
Fri Jan 19 16:02:16 2018 - [info] All SSH connection tests passed successfully.
[root@localhost ~]#
Fri Jan 19 16:02:14 2018 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Fri Jan 19 16:02:14 2018 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Fri Jan 19 16:02:14 2018 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Fri Jan 19 16:02:14 2018 - [info] Starting SSH connection tests..
Fri Jan 19 16:02:15 2018 - [debug]
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.77:22) to [email protected](192.168.1.78:22)..
Fri Jan 19 16:02:14 2018 - [debug] ok.
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.77:22) to [email protected](192.168.1.81:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug]
Fri Jan 19 16:02:14 2018 - [debug] Connecting via SSH from [email protected](192.168.1.78:22) to [email protected](192.168.1.77:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.78:22) to [email protected](192.168.1.81:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:16 2018 - [debug]
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.81:22) to [email protected](192.168.1.77:22)..
Fri Jan 19 16:02:15 2018 - [debug] ok.
Fri Jan 19 16:02:15 2018 - [debug] Connecting via SSH from [email protected](192.168.1.81:22) to [email protected](192.168.1.78:22)..
Fri Jan 19 16:02:16 2018 - [debug] ok.
Fri Jan 19 16:02:16 2018 - [info] All SSH connection tests passed successfully.
[root@localhost ~]#
[root@localhost app1]# masterha_check_repl --conf=/etc/masterha/app1.cnf ####检测MySQL主从连接状态
Mon Jan 22 14:59:24 2018 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 22 14:59:24 2018 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Mon Jan 22 14:59:24 2018 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Mon Jan 22 14:59:24 2018 - [info] MHA::MasterMonitor version 0.56.
Mon Jan 22 14:59:25 2018 - [info] GTID failover mode = 0
Mon Jan 22 14:59:25 2018 - [info] Dead Servers:
Mon Jan 22 14:59:25 2018 - [info] Alive Servers:
Mon Jan 22 14:59:25 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 14:59:25 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 14:59:25 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 14:59:25 2018 - [info] Alive Slaves:
Mon Jan 22 14:59:25 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 14:59:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 14:59:25 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 14:59:25 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 14:59:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 14:59:25 2018 - [info] Current Alive Master: 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 14:59:25 2018 - [info] Checking slave configurations..
Mon Jan 22 14:59:25 2018 - [info] read_only=1 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 14:59:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 14:59:25 2018 - [info] read_only=1 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 14:59:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 14:59:25 2018 - [info] Checking replication filtering settings..
Mon Jan 22 14:59:25 2018 - [info] binlog_do_db= , binlog_ignore_db=
Mon Jan 22 14:59:25 2018 - [info] Replication filtering check ok.
Mon Jan 22 14:59:25 2018 - [info] GTID (with auto-pos) is not supported
Mon Jan 22 14:59:25 2018 - [info] Starting SSH connection tests..
Mon Jan 22 14:59:28 2018 - [info] All SSH connection tests passed successfully.
Mon Jan 22 14:59:28 2018 - [info] Checking MHA Node version..
Mon Jan 22 14:59:28 2018 - [info] Version check ok.
Mon Jan 22 14:59:28 2018 - [info] Checking SSH publickey authentication settings on the current master..
Mon Jan 22 14:59:29 2018 - [info] HealthCheck: SSH to 192.168.1.77 is reachable.
Mon Jan 22 14:59:29 2018 - [info] Master MHA Node version is 0.56.
Mon Jan 22 14:59:29 2018 - [info] Checking recovery script configurations on 192.168.1.77(192.168.1.77:3306)..
Mon Jan 22 14:59:29 2018 - [info] Executing command: save_binary_logs --command=test --start_pos=4 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/save_binary_logs_test --manager_version=0.56 --start_file=mysqlbin.000011
Mon Jan 22 14:59:29 2018 - [info] Connecting to [email protected](192.168.1.77:22)..
Creating /var/log/masterha/app1 if not exists.. ok.
Checking output directory is accessible or not..
ok.
Binlog found at /qingke/mysql/binlog, up to mysqlbin.000011
Mon Jan 22 14:59:29 2018 - [info] Binlog setting check done.
Mon Jan 22 14:59:29 2018 - [info] Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers..
Mon Jan 22 14:59:29 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.78 --slave_ip=192.168.1.78 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 14:59:29 2018 - [info] Connecting to [email protected](192.168.1.78:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 14:59:29 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.81 --slave_ip=192.168.1.81 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 14:59:29 2018 - [info] Connecting to [email protected](192.168.1.81:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 14:59:30 2018 - [info] Slaves settings check done.
Mon Jan 22 14:59:30 2018 - [info]
192.168.1.77(192.168.1.77:3306) (current master)
+--192.168.1.78(192.168.1.78:3306)
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 14:59:30 2018 - [info] Checking replication health on 192.168.1.78..
Mon Jan 22 14:59:30 2018 - [info] ok.
Mon Jan 22 14:59:30 2018 - [info] Checking replication health on 192.168.1.81..
Mon Jan 22 14:59:30 2018 - [info] ok.
Mon Jan 22 14:59:30 2018 - [info] Checking master_ip_failover_script status:
Mon Jan 22 14:59:30 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306
Mon Jan 22 14:59:30 2018 - [info] ok.
Mon Jan 22 14:59:30 2018 - [info] Checking replication health on 192.168.1.81..
Mon Jan 22 14:59:30 2018 - [info] ok.
Mon Jan 22 14:59:30 2018 - [info] Checking master_ip_failover_script status:
Mon Jan 22 14:59:30 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306
IN SCRIPT TEST====/sbin/ifconfig eno16777736:1 down==/sbin/ifconfig eno16777736:1 192.168.1.82/24===
Checking the Status of the script.. OK
Mon Jan 22 14:59:30 2018 - [info] OK.
Mon Jan 22 14:59:30 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 14:59:30 2018 - [info] Got exit code 0 (Not master dead).
Mon Jan 22 14:59:30 2018 - [info] OK.
Mon Jan 22 14:59:30 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 14:59:30 2018 - [info] Got exit code 0 (Not master dead).
MySQL Replication Health is OK.
[root@localhost app1]#
启动MHA
[root@localhost app1]# nohup masterha_manager --conf=/etc/masterha/app1.cnf &> /tmp/mha_manager.log &
[root@localhost app1]# cat /var/log/masterha/app1/manager.log
Mon Jan 22 15:04:53 2018 - [info] MHA::MasterMonitor version 0.56.
Mon Jan 22 15:04:54 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:04:54 2018 - [info] Dead Servers:
Mon Jan 22 15:04:54 2018 - [info] Alive Servers:
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:04:54 2018 - [info] Alive Slaves:
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:04:54 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:04:54 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Current Alive Master: 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Checking slave configurations..
Mon Jan 22 15:04:54 2018 - [info] read_only=1 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:04:54 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:04:54 2018 - [info] read_only=1 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:04:54 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:04:54 2018 - [info] Checking replication filtering settings..
Mon Jan 22 15:04:54 2018 - [info] binlog_do_db= , binlog_ignore_db=
Mon Jan 22 15:04:54 2018 - [info] Replication filtering check ok.
Mon Jan 22 15:04:54 2018 - [info] GTID (with auto-pos) is not supported
Mon Jan 22 15:04:54 2018 - [info] Starting SSH connection tests..
Mon Jan 22 15:04:56 2018 - [info] All SSH connection tests passed successfully.
Mon Jan 22 15:04:56 2018 - [info] Checking MHA Node version..
Mon Jan 22 15:04:57 2018 - [info] Version check ok.
Mon Jan 22 15:04:57 2018 - [info] Checking SSH publickey authentication settings on the current master..
Mon Jan 22 15:04:57 2018 - [info] HealthCheck: SSH to 192.168.1.77 is reachable.
Mon Jan 22 15:04:57 2018 - [info] Master MHA Node version is 0.56.
Mon Jan 22 15:04:57 2018 - [info] Checking recovery script configurations on 192.168.1.77(192.168.1.77:3306)..
Mon Jan 22 15:04:57 2018 - [info] Executing command: save_binary_logs --command=test --start_pos=4 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/save_binary_logs_test --manager_version=0.56 --start_file=mysqlbin.000011
Mon Jan 22 15:04:57 2018 - [info] Connecting to [email protected](192.168.1.77:22)..
Creating /var/log/masterha/app1 if not exists.. ok.
Checking output directory is accessible or not..
ok.
Binlog found at /qingke/mysql/binlog, up to mysqlbin.000011
Mon Jan 22 15:04:57 2018 - [info] Binlog setting check done.
Mon Jan 22 15:04:57 2018 - [info] Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers..
Mon Jan 22 15:04:57 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.78 --slave_ip=192.168.1.78 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 15:04:57 2018 - [info] Connecting to [email protected](192.168.1.78:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 15:04:58 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.81 --slave_ip=192.168.1.81 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 15:04:58 2018 - [info] Connecting to [email protected](192.168.1.81:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 15:04:58 2018 - [info] Slaves settings check done.
Mon Jan 22 15:04:58 2018 - [info]
192.168.1.77(192.168.1.77:3306) (current master)
+--192.168.1.78(192.168.1.78:3306)
+--192.168.1.81(192.168.1.81:3306)
[root@localhost app1]#
启动MHA
[root@localhost app1]# nohup masterha_manager --conf=/etc/masterha/app1.cnf &> /tmp/mha_manager.log &
[root@localhost app1]# cat /var/log/masterha/app1/manager.log
Mon Jan 22 15:04:53 2018 - [info] MHA::MasterMonitor version 0.56.
Mon Jan 22 15:04:54 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:04:54 2018 - [info] Dead Servers:
Mon Jan 22 15:04:54 2018 - [info] Alive Servers:
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:04:54 2018 - [info] Alive Slaves:
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:04:54 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:04:54 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:04:54 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Current Alive Master: 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:04:54 2018 - [info] Checking slave configurations..
Mon Jan 22 15:04:54 2018 - [info] read_only=1 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:04:54 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:04:54 2018 - [info] read_only=1 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:04:54 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:04:54 2018 - [info] Checking replication filtering settings..
Mon Jan 22 15:04:54 2018 - [info] binlog_do_db= , binlog_ignore_db=
Mon Jan 22 15:04:54 2018 - [info] Replication filtering check ok.
Mon Jan 22 15:04:54 2018 - [info] GTID (with auto-pos) is not supported
Mon Jan 22 15:04:54 2018 - [info] Starting SSH connection tests..
Mon Jan 22 15:04:56 2018 - [info] All SSH connection tests passed successfully.
Mon Jan 22 15:04:56 2018 - [info] Checking MHA Node version..
Mon Jan 22 15:04:57 2018 - [info] Version check ok.
Mon Jan 22 15:04:57 2018 - [info] Checking SSH publickey authentication settings on the current master..
Mon Jan 22 15:04:57 2018 - [info] HealthCheck: SSH to 192.168.1.77 is reachable.
Mon Jan 22 15:04:57 2018 - [info] Master MHA Node version is 0.56.
Mon Jan 22 15:04:57 2018 - [info] Checking recovery script configurations on 192.168.1.77(192.168.1.77:3306)..
Mon Jan 22 15:04:57 2018 - [info] Executing command: save_binary_logs --command=test --start_pos=4 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/save_binary_logs_test --manager_version=0.56 --start_file=mysqlbin.000011
Mon Jan 22 15:04:57 2018 - [info] Connecting to [email protected](192.168.1.77:22)..
Creating /var/log/masterha/app1 if not exists.. ok.
Checking output directory is accessible or not..
ok.
Binlog found at /qingke/mysql/binlog, up to mysqlbin.000011
Mon Jan 22 15:04:57 2018 - [info] Binlog setting check done.
Mon Jan 22 15:04:57 2018 - [info] Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers..
Mon Jan 22 15:04:57 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.78 --slave_ip=192.168.1.78 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 15:04:57 2018 - [info] Connecting to [email protected](192.168.1.78:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 15:04:58 2018 - [info] Executing command : apply_diff_relay_logs --command=test --slave_user='root' --slave_host=192.168.1.81 --slave_ip=192.168.1.81 --slave_port=3306 --workdir=/var/log/masterha/app1 --target_version=5.7.20-log --manager_version=0.56 --relay_log_info=/qingke/mysql/data/relay-log.info --relay_dir=/qingke/mysql/data/ --slave_pass=xxx
Mon Jan 22 15:04:58 2018 - [info] Connecting to [email protected](192.168.1.81:22)..
Checking slave recovery environment settings..
Opening /qingke/mysql/data/relay-log.info ... ok.
Relay log found at /qingke/mysql/relaylog, up to mysqlrelay.000002
Temporary relay log file is /qingke/mysql/relaylog/mysqlrelay.000002
Testing mysql connection and privileges..mysql: [Warning] Using a password on the command line interface can be insecure.
done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Mon Jan 22 15:04:58 2018 - [info] Slaves settings check done.
Mon Jan 22 15:04:58 2018 - [info]
192.168.1.77(192.168.1.77:3306) (current master)
+--192.168.1.78(192.168.1.78:3306)
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:04:58 2018 - [info] Checking master_ip_failover_script status:
Mon Jan 22 15:04:58 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306
Mon Jan 22 15:04:58 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306
IN SCRIPT TEST====/sbin/ifconfig eno16777736:1 down==/sbin/ifconfig eno16777736:1 192.168.1.82/24===
Checking the Status of the script.. OK
Mon Jan 22 15:04:58 2018 - [info] OK.
Mon Jan 22 15:04:58 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 15:04:58 2018 - [info] Set master ping interval 1 seconds.
Mon Jan 22 15:04:58 2018 - [info] Set secondary check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78
Mon Jan 22 15:04:58 2018 - [info] Starting ping health check on 192.168.1.77(192.168.1.77:3306)..
Mon Jan 22 15:04:58 2018 - [info] Ping(SELECT) succeeded, waiting until MySQL doesn't respond..
[root@localhost app1]#
###check MHA status
[root@localhost app1]# masterha_check_status --conf=/etc/masterha/app1.cnf
app1 (pid:59745) is running(0:PING_OK), master:192.168.1.77
Mon Jan 22 15:04:58 2018 - [info] OK.
Mon Jan 22 15:04:58 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 15:04:58 2018 - [info] Set master ping interval 1 seconds.
Mon Jan 22 15:04:58 2018 - [info] Set secondary check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78
Mon Jan 22 15:04:58 2018 - [info] Starting ping health check on 192.168.1.77(192.168.1.77:3306)..
Mon Jan 22 15:04:58 2018 - [info] Ping(SELECT) succeeded, waiting until MySQL doesn't respond..
[root@localhost app1]#
###check MHA status
[root@localhost app1]# masterha_check_status --conf=/etc/masterha/app1.cnf
app1 (pid:59745) is running(0:PING_OK), master:192.168.1.77
[root@localhost app1]# ip a 查看master服务器 ip 状态 1.82 已经附加到eno上
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eno16777736: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:d4:40:49 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.77/24 brd 192.168.1.255 scope global eno16777736
valid_lft forever preferred_lft forever
inet 192.168.1.82/24 brd 192.168.1.255 scope global secondary eno16777736:1
valid_lft forever preferred_lft forever
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eno16777736: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:d4:40:49 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.77/24 brd 192.168.1.255 scope global eno16777736
valid_lft forever preferred_lft forever
inet 192.168.1.82/24 brd 192.168.1.255 scope global secondary eno16777736:1
valid_lft forever preferred_lft forever
###故障切换模拟
[root@localhost app1]# mysqladmin -uroot -p123 shutdown 或者执行pkill mysql 直接杀掉MySQL进程
[root@localhost app1]# cat /var/log/masterha/app1/manager.log
[root@localhost app1]# mysqladmin -uroot -p123 shutdown 或者执行pkill mysql 直接杀掉MySQL进程
[root@localhost app1]# cat /var/log/masterha/app1/manager.log
Mon Jan 22 15:10:21 2018 - [warning] Got error on MySQL select ping: 2006 (MySQL server has gone away)
Mon Jan 22 15:10:21 2018 - [info] Executing SSH check script: save_binary_logs --command=test --start_pos=4 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/save_binary_logs_test --manager_version=0.56 --binlog_prefix=mysqlbin
Mon Jan 22 15:10:21 2018 - [info] Executing secondary network check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78 --user=root --master_host=192.168.1.77 --master_ip=192.168.1.77 --master_port=3306 --master_user=root --master_password=123 --ping_type=SELECT
Mon Jan 22 15:10:22 2018 - [info] HealthCheck: SSH to 192.168.1.77 is reachable.
Monitoring server 192.168.1.77 is reachable, Master is not reachable from 192.168.1.77. OK.
Monitoring server 192.168.1.78 is reachable, Master is not reachable from 192.168.1.78. OK.
Mon Jan 22 15:10:22 2018 - [info] Master is not reachable from all other monitoring servers. Failover should start.
Mon Jan 22 15:10:22 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:22 2018 - [warning] Connection failed 2 time(s).. #############3次连接MySQL不通
Mon Jan 22 15:10:23 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:23 2018 - [warning] Connection failed 3 time(s)..
Mon Jan 22 15:10:24 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:24 2018 - [warning] Connection failed 4 time(s)..
Mon Jan 22 15:10:24 2018 - [warning] Master is not reachable from health checker!
Mon Jan 22 15:10:24 2018 - [warning] Master 192.168.1.77(192.168.1.77:3306) is not reachable!
Mon Jan 22 15:10:24 2018 - [warning] SSH is reachable.
Mon Jan 22 15:10:24 2018 - [info] Connecting to a master server failed. Reading configuration file /etc/masterha_default.cnf and /etc/masterha/app1.cnf again, and trying to connect to all servers to check server status..
Mon Jan 22 15:10:24 2018 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 22 15:10:24 2018 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Mon Jan 22 15:10:24 2018 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Mon Jan 22 15:10:25 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:10:25 2018 - [info] Dead Servers:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Alive Servers:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:25 2018 - [info] Alive Slaves:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Checking slave configurations..
Mon Jan 22 15:10:25 2018 - [info] read_only=1 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:10:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.78(192.168.1.78:3306). ###此处warning是从库开启了日志回收
Mon Jan 22 15:10:25 2018 - [info] read_only=1 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:10:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:10:25 2018 - [info] Checking replication filtering settings..
Mon Jan 22 15:10:25 2018 - [info] Replication filtering check ok.
Mon Jan 22 15:10:25 2018 - [info] Master is down!
Mon Jan 22 15:10:25 2018 - [info] Terminating monitoring script.
Mon Jan 22 15:10:25 2018 - [info] Got exit code 20 (Master dead).
Mon Jan 22 15:10:25 2018 - [info] MHA::MasterFailover version 0.56.
Mon Jan 22 15:10:25 2018 - [info] Starting master failover.
Mon Jan 22 15:10:25 2018 - [info]
Mon Jan 22 15:10:25 2018 - [info] * Phase 1: Configuration Check Phase..
Mon Jan 22 15:10:25 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:10:27 2018 - [info] Dead Servers:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Checking master reachability via MySQL(double check)...
Mon Jan 22 15:10:27 2018 - [info] ok.
Mon Jan 22 15:10:27 2018 - [info] Alive Servers:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:27 2018 - [info] Alive Slaves:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Starting Non-GTID based failover.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] ** Phase 1: Configuration Check Phase completed.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 2: Dead Master Shutdown Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Forcing shutdown so that applications never connect to the current master..
Mon Jan 22 15:10:27 2018 - [info] Executing master IP deactivation script:
Mon Jan 22 15:10:27 2018 - [info] /etc/masterha/master_ip_failover --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306 --command=stopssh --ssh_user=root
Mon Jan 22 15:10:21 2018 - [info] Executing SSH check script: save_binary_logs --command=test --start_pos=4 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/save_binary_logs_test --manager_version=0.56 --binlog_prefix=mysqlbin
Mon Jan 22 15:10:21 2018 - [info] Executing secondary network check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78 --user=root --master_host=192.168.1.77 --master_ip=192.168.1.77 --master_port=3306 --master_user=root --master_password=123 --ping_type=SELECT
Mon Jan 22 15:10:22 2018 - [info] HealthCheck: SSH to 192.168.1.77 is reachable.
Monitoring server 192.168.1.77 is reachable, Master is not reachable from 192.168.1.77. OK.
Monitoring server 192.168.1.78 is reachable, Master is not reachable from 192.168.1.78. OK.
Mon Jan 22 15:10:22 2018 - [info] Master is not reachable from all other monitoring servers. Failover should start.
Mon Jan 22 15:10:22 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:22 2018 - [warning] Connection failed 2 time(s).. #############3次连接MySQL不通
Mon Jan 22 15:10:23 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:23 2018 - [warning] Connection failed 3 time(s)..
Mon Jan 22 15:10:24 2018 - [warning] Got error on MySQL connect: 2003 (Can't connect to MySQL server on '192.168.1.77' (111))
Mon Jan 22 15:10:24 2018 - [warning] Connection failed 4 time(s)..
Mon Jan 22 15:10:24 2018 - [warning] Master is not reachable from health checker!
Mon Jan 22 15:10:24 2018 - [warning] Master 192.168.1.77(192.168.1.77:3306) is not reachable!
Mon Jan 22 15:10:24 2018 - [warning] SSH is reachable.
Mon Jan 22 15:10:24 2018 - [info] Connecting to a master server failed. Reading configuration file /etc/masterha_default.cnf and /etc/masterha/app1.cnf again, and trying to connect to all servers to check server status..
Mon Jan 22 15:10:24 2018 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Mon Jan 22 15:10:24 2018 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Mon Jan 22 15:10:24 2018 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Mon Jan 22 15:10:25 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:10:25 2018 - [info] Dead Servers:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Alive Servers:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:25 2018 - [info] Alive Slaves:
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:25 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:25 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:25 2018 - [info] Checking slave configurations..
Mon Jan 22 15:10:25 2018 - [info] read_only=1 is not set on slave 192.168.1.78(192.168.1.78:3306).
Mon Jan 22 15:10:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.78(192.168.1.78:3306). ###此处warning是从库开启了日志回收
Mon Jan 22 15:10:25 2018 - [info] read_only=1 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:10:25 2018 - [warning] relay_log_purge=0 is not set on slave 192.168.1.81(192.168.1.81:3306).
Mon Jan 22 15:10:25 2018 - [info] Checking replication filtering settings..
Mon Jan 22 15:10:25 2018 - [info] Replication filtering check ok.
Mon Jan 22 15:10:25 2018 - [info] Master is down!
Mon Jan 22 15:10:25 2018 - [info] Terminating monitoring script.
Mon Jan 22 15:10:25 2018 - [info] Got exit code 20 (Master dead).
Mon Jan 22 15:10:25 2018 - [info] MHA::MasterFailover version 0.56.
Mon Jan 22 15:10:25 2018 - [info] Starting master failover.
Mon Jan 22 15:10:25 2018 - [info]
Mon Jan 22 15:10:25 2018 - [info] * Phase 1: Configuration Check Phase..
Mon Jan 22 15:10:25 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] GTID failover mode = 0
Mon Jan 22 15:10:27 2018 - [info] Dead Servers:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Checking master reachability via MySQL(double check)...
Mon Jan 22 15:10:27 2018 - [info] ok.
Mon Jan 22 15:10:27 2018 - [info] Alive Servers:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:27 2018 - [info] Alive Slaves:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Starting Non-GTID based failover.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] ** Phase 1: Configuration Check Phase completed.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 2: Dead Master Shutdown Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Forcing shutdown so that applications never connect to the current master..
Mon Jan 22 15:10:27 2018 - [info] Executing master IP deactivation script:
Mon Jan 22 15:10:27 2018 - [info] /etc/masterha/master_ip_failover --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306 --command=stopssh --ssh_user=root
IN SCRIPT TEST====/sbin/ifconfig eno16777736:1 down==/sbin/ifconfig eno16777736:1 192.168.1.82/24===
Disabling the VIP on old master: 192.168.1.77
Mon Jan 22 15:10:27 2018 - [info] done.
Mon Jan 22 15:10:27 2018 - [warning] shutdown_script is not set. Skipping explicit shutting down of the dead master. shutdown脚本未开启
Mon Jan 22 15:10:27 2018 - [info] * Phase 2: Dead Master Shutdown Phase completed.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3: Master Recovery Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.1: Getting Latest Slaves Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] The latest binary log file/position on all slaves is mysqlbin.000011:319
Mon Jan 22 15:10:27 2018 - [info] Latest slaves (Slaves that received relay log files to the latest):
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] The oldest binary log file/position on all slaves is mysqlbin.000011:319
Mon Jan 22 15:10:27 2018 - [info] Oldest slaves:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.2: Saving Dead Master's Binlog Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Fetching dead master's binary logs..
Mon Jan 22 15:10:27 2018 - [info] Executing command on the dead master 192.168.1.77(192.168.1.77:3306): save_binary_logs --command=save --start_file=mysqlbin.000011 --start_pos=319 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog --handle_raw_binlog=1 --disable_log_bin=0 --manager_version=0.56
Creating /var/log/masterha/app1 if not exists.. ok.
Concat binary/relay logs from mysqlbin.000011 pos 319 to mysqlbin.000011 EOF into /var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog ..
Binlog Checksum enabled
Dumping binlog format description event, from position 0 to 154.. ok.
No need to dump effective binlog data from /qingke/mysql/binlog/mysqlbin.000011 (pos starts 319, filesize 319). Skipping.
Binlog Checksum enabled
/var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog has no effective data events.
Event not exists.
Mon Jan 22 15:10:27 2018 - [info] Additional events were not found from the orig master. No need to save.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.3: Determining New Master Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Finding the latest slave that has all relay logs for recovering other slaves..
Mon Jan 22 15:10:27 2018 - [info] All slaves received relay logs to the same position. No need to resync each other.
Mon Jan 22 15:10:27 2018 - [info] Searching new master from slaves..
Mon Jan 22 15:10:27 2018 - [info] Candidate masters from the configuration file:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] Non-candidate masters:
Mon Jan 22 15:10:27 2018 - [info] Searching from candidate_master slaves which have received the latest relay log events..
Mon Jan 22 15:10:27 2018 - [info] New master is 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:27 2018 - [info] Starting master failover..
Mon Jan 22 15:10:27 2018 - [info]
From:
192.168.1.77(192.168.1.77:3306) (current master)
+--192.168.1.78(192.168.1.78:3306)
+--192.168.1.81(192.168.1.81:3306)
#####新老交替
To:
192.168.1.78(192.168.1.78:3306) (new master)
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.3: New Master Diff Log Generation Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.4: Master Log Apply Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] *NOTICE: If any error happens from this phase, manual recovery is needed.
Mon Jan 22 15:10:27 2018 - [info] Starting recovery on 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:10:27 2018 - [info] This server has all relay logs. Waiting all logs to be applied..
Mon Jan 22 15:10:27 2018 - [info] done.
Mon Jan 22 15:10:27 2018 - [info] All relay logs were successfully applied.
Mon Jan 22 15:10:27 2018 - [info] Getting new master's binlog name and position..
Mon Jan 22 15:10:27 2018 - [info] mysqlbin.000016:319
Mon Jan 22 15:10:27 2018 - [info] All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.1.78', MASTER_PORT=3306, MASTER_LOG_FILE='mysqlbin.000016', MASTER_LOG_POS=319, MASTER_USER='repl', MASTER_PASSWORD='xxx'; ### 此处记录主从切换后新主binlog位置
Mon Jan 22 15:10:27 2018 - [info] Executing master IP activate script:
Mon Jan 22 15:10:27 2018 - [info] /etc/masterha/master_ip_failover --command=start --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306 --new_master_host=192.168.1.78 --new_master_ip=192.168.1.78 --new_master_port=3306 --new_master_user='root' --new_master_password='123'
Unknown option: new_master_user
Unknown option: new_master_password
Mon Jan 22 15:10:27 2018 - [info] done.
Mon Jan 22 15:10:27 2018 - [warning] shutdown_script is not set. Skipping explicit shutting down of the dead master. shutdown脚本未开启
Mon Jan 22 15:10:27 2018 - [info] * Phase 2: Dead Master Shutdown Phase completed.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3: Master Recovery Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.1: Getting Latest Slaves Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] The latest binary log file/position on all slaves is mysqlbin.000011:319
Mon Jan 22 15:10:27 2018 - [info] Latest slaves (Slaves that received relay log files to the latest):
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] The oldest binary log file/position on all slaves is mysqlbin.000011:319
Mon Jan 22 15:10:27 2018 - [info] Oldest slaves:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.81(192.168.1.81:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.2: Saving Dead Master's Binlog Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Fetching dead master's binary logs..
Mon Jan 22 15:10:27 2018 - [info] Executing command on the dead master 192.168.1.77(192.168.1.77:3306): save_binary_logs --command=save --start_file=mysqlbin.000011 --start_pos=319 --binlog_dir=/qingke/mysql/binlog --output_file=/var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog --handle_raw_binlog=1 --disable_log_bin=0 --manager_version=0.56
Creating /var/log/masterha/app1 if not exists.. ok.
Concat binary/relay logs from mysqlbin.000011 pos 319 to mysqlbin.000011 EOF into /var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog ..
Binlog Checksum enabled
Dumping binlog format description event, from position 0 to 154.. ok.
No need to dump effective binlog data from /qingke/mysql/binlog/mysqlbin.000011 (pos starts 319, filesize 319). Skipping.
Binlog Checksum enabled
/var/log/masterha/app1/saved_master_binlog_from_192.168.1.77_3306_20180122151025.binlog has no effective data events.
Event not exists.
Mon Jan 22 15:10:27 2018 - [info] Additional events were not found from the orig master. No need to save.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.3: Determining New Master Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] Finding the latest slave that has all relay logs for recovering other slaves..
Mon Jan 22 15:10:27 2018 - [info] All slaves received relay logs to the same position. No need to resync each other.
Mon Jan 22 15:10:27 2018 - [info] Searching new master from slaves..
Mon Jan 22 15:10:27 2018 - [info] Candidate masters from the configuration file:
Mon Jan 22 15:10:27 2018 - [info] 192.168.1.78(192.168.1.78:3306) Version=5.7.20-log (oldest major version between slaves) log-bin:enabled
Mon Jan 22 15:10:27 2018 - [info] Replicating from 192.168.1.77(192.168.1.77:3306)
Mon Jan 22 15:10:27 2018 - [info] Primary candidate for the new Master (candidate_master is set)
Mon Jan 22 15:10:27 2018 - [info] Non-candidate masters:
Mon Jan 22 15:10:27 2018 - [info] Searching from candidate_master slaves which have received the latest relay log events..
Mon Jan 22 15:10:27 2018 - [info] New master is 192.168.1.78(192.168.1.78:3306)
Mon Jan 22 15:10:27 2018 - [info] Starting master failover..
Mon Jan 22 15:10:27 2018 - [info]
From:
192.168.1.77(192.168.1.77:3306) (current master)
+--192.168.1.78(192.168.1.78:3306)
+--192.168.1.81(192.168.1.81:3306)
#####新老交替
To:
192.168.1.78(192.168.1.78:3306) (new master)
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.3: New Master Diff Log Generation Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] * Phase 3.4: Master Log Apply Phase..
Mon Jan 22 15:10:27 2018 - [info]
Mon Jan 22 15:10:27 2018 - [info] *NOTICE: If any error happens from this phase, manual recovery is needed.
Mon Jan 22 15:10:27 2018 - [info] Starting recovery on 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:10:27 2018 - [info] This server has all relay logs. Waiting all logs to be applied..
Mon Jan 22 15:10:27 2018 - [info] done.
Mon Jan 22 15:10:27 2018 - [info] All relay logs were successfully applied.
Mon Jan 22 15:10:27 2018 - [info] Getting new master's binlog name and position..
Mon Jan 22 15:10:27 2018 - [info] mysqlbin.000016:319
Mon Jan 22 15:10:27 2018 - [info] All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.1.78', MASTER_PORT=3306, MASTER_LOG_FILE='mysqlbin.000016', MASTER_LOG_POS=319, MASTER_USER='repl', MASTER_PASSWORD='xxx'; ### 此处记录主从切换后新主binlog位置
Mon Jan 22 15:10:27 2018 - [info] Executing master IP activate script:
Mon Jan 22 15:10:27 2018 - [info] /etc/masterha/master_ip_failover --command=start --ssh_user=root --orig_master_host=192.168.1.77 --orig_master_ip=192.168.1.77 --orig_master_port=3306 --new_master_host=192.168.1.78 --new_master_ip=192.168.1.78 --new_master_port=3306 --new_master_user='root' --new_master_password='123'
Unknown option: new_master_user
Unknown option: new_master_password
IN SCRIPT TEST====/sbin/ifconfig eno16777736:1 down==/sbin/ifconfig eno16777736:1 192.168.1.82/24===
Enabling the VIP - 192.168.1.82/24 on the new master - 192.168.1.78
Mon Jan 22 15:10:28 2018 - [info] OK.
Mon Jan 22 15:10:28 2018 - [info] ** Finished master recovery successfully.
Mon Jan 22 15:10:28 2018 - [info] * Phase 3: Master Recovery Phase completed.
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] * Phase 4: Slaves Recovery Phase..
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] * Phase 4.1: Starting Parallel Slave Diff Log Generation Phase..
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] -- Slave diff file generation on host 192.168.1.81(192.168.1.81:3306) started, pid: 60296. Check tmp log /var/log/masterha/app1/192.168.1.81_3306_20180122151025.log if it takes time..
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] Log messages from 192.168.1.81 ...
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 22 15:10:29 2018 - [info] End of log messages from 192.168.1.81.
Mon Jan 22 15:10:29 2018 - [info] -- 192.168.1.81(192.168.1.81:3306) has the latest relay log events.
Mon Jan 22 15:10:29 2018 - [info] Generating relay diff files from the latest slave succeeded.
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] * Phase 4.2: Starting Parallel Slave Log Apply Phase..
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] -- Slave recovery on host 192.168.1.81(192.168.1.81:3306) started, pid: 60298. Check tmp log /var/log/masterha/app1/192.168.1.81_3306_20180122151025.log if it takes time..
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] Log messages from 192.168.1.81 ...
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] Starting recovery on 192.168.1.81(192.168.1.81:3306)..
Mon Jan 22 15:10:29 2018 - [info] This server has all relay logs. Waiting all logs to be applied..
Mon Jan 22 15:10:29 2018 - [info] done.
Mon Jan 22 15:10:29 2018 - [info] All relay logs were successfully applied.
Mon Jan 22 15:10:29 2018 - [info] Resetting slave 192.168.1.81(192.168.1.81:3306) and starting replication from the new master 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:10:29 2018 - [info] Executed CHANGE MASTER.
Mon Jan 22 15:10:29 2018 - [info] Slave started.
Mon Jan 22 15:10:30 2018 - [info] End of log messages from 192.168.1.81.
Mon Jan 22 15:10:30 2018 - [info] -- Slave recovery on host 192.168.1.81(192.168.1.81:3306) succeeded.
Mon Jan 22 15:10:30 2018 - [info] All new slave servers recovered successfully.
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] * Phase 5: New master cleanup phase..
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] Resetting slave info on the new master..
Mon Jan 22 15:10:30 2018 - [info] 192.168.1.78: Resetting slave info succeeded.
Mon Jan 22 15:10:30 2018 - [info] Master failover to 192.168.1.78(192.168.1.78:3306) completed successfully.
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] OK.
Mon Jan 22 15:10:28 2018 - [info] ** Finished master recovery successfully.
Mon Jan 22 15:10:28 2018 - [info] * Phase 3: Master Recovery Phase completed.
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] * Phase 4: Slaves Recovery Phase..
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] * Phase 4.1: Starting Parallel Slave Diff Log Generation Phase..
Mon Jan 22 15:10:28 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] -- Slave diff file generation on host 192.168.1.81(192.168.1.81:3306) started, pid: 60296. Check tmp log /var/log/masterha/app1/192.168.1.81_3306_20180122151025.log if it takes time..
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] Log messages from 192.168.1.81 ...
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:28 2018 - [info] This server has all relay logs. No need to generate diff files from the latest slave.
Mon Jan 22 15:10:29 2018 - [info] End of log messages from 192.168.1.81.
Mon Jan 22 15:10:29 2018 - [info] -- 192.168.1.81(192.168.1.81:3306) has the latest relay log events.
Mon Jan 22 15:10:29 2018 - [info] Generating relay diff files from the latest slave succeeded.
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] * Phase 4.2: Starting Parallel Slave Log Apply Phase..
Mon Jan 22 15:10:29 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] -- Slave recovery on host 192.168.1.81(192.168.1.81:3306) started, pid: 60298. Check tmp log /var/log/masterha/app1/192.168.1.81_3306_20180122151025.log if it takes time..
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] Log messages from 192.168.1.81 ...
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:29 2018 - [info] Starting recovery on 192.168.1.81(192.168.1.81:3306)..
Mon Jan 22 15:10:29 2018 - [info] This server has all relay logs. Waiting all logs to be applied..
Mon Jan 22 15:10:29 2018 - [info] done.
Mon Jan 22 15:10:29 2018 - [info] All relay logs were successfully applied.
Mon Jan 22 15:10:29 2018 - [info] Resetting slave 192.168.1.81(192.168.1.81:3306) and starting replication from the new master 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:10:29 2018 - [info] Executed CHANGE MASTER.
Mon Jan 22 15:10:29 2018 - [info] Slave started.
Mon Jan 22 15:10:30 2018 - [info] End of log messages from 192.168.1.81.
Mon Jan 22 15:10:30 2018 - [info] -- Slave recovery on host 192.168.1.81(192.168.1.81:3306) succeeded.
Mon Jan 22 15:10:30 2018 - [info] All new slave servers recovered successfully.
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] * Phase 5: New master cleanup phase..
Mon Jan 22 15:10:30 2018 - [info]
Mon Jan 22 15:10:30 2018 - [info] Resetting slave info on the new master..
Mon Jan 22 15:10:30 2018 - [info] 192.168.1.78: Resetting slave info succeeded.
Mon Jan 22 15:10:30 2018 - [info] Master failover to 192.168.1.78(192.168.1.78:3306) completed successfully.
Mon Jan 22 15:10:30 2018 - [info]
----- Failover Report -----
app1: MySQL Master failover 192.168.1.77(192.168.1.77:3306) to 192.168.1.78(192.168.1.78:3306) succeeded
Master 192.168.1.77(192.168.1.77:3306) is down!
Check MHA Manager logs at localhost.localdomain:/var/log/masterha/app1/manager.log for details.
Started automated(non-interactive) failover.
Invalidated master IP address on 192.168.1.77(192.168.1.77:3306)
The latest slave 192.168.1.78(192.168.1.78:3306) has all relay logs for recovery.
Selected 192.168.1.78(192.168.1.78:3306) as a new master.
192.168.1.78(192.168.1.78:3306): OK: Applying all logs succeeded.
192.168.1.78(192.168.1.78:3306): OK: Activated master IP address.
192.168.1.81(192.168.1.81:3306): This host has the latest relay log events.
Generating relay diff files from the latest slave succeeded.
192.168.1.81(192.168.1.81:3306): OK: Applying all logs succeeded. Slave started, replicating from 192.168.1.78(192.168.1.78:3306)
192.168.1.78(192.168.1.78:3306): Resetting slave info succeeded.
Master failover to 192.168.1.78(192.168.1.78:3306) completed successfully. ###切换成功
Invalidated master IP address on 192.168.1.77(192.168.1.77:3306)
The latest slave 192.168.1.78(192.168.1.78:3306) has all relay logs for recovery.
Selected 192.168.1.78(192.168.1.78:3306) as a new master.
192.168.1.78(192.168.1.78:3306): OK: Applying all logs succeeded.
192.168.1.78(192.168.1.78:3306): OK: Activated master IP address.
192.168.1.81(192.168.1.81:3306): This host has the latest relay log events.
Generating relay diff files from the latest slave succeeded.
192.168.1.81(192.168.1.81:3306): OK: Applying all logs succeeded. Slave started, replicating from 192.168.1.78(192.168.1.78:3306)
192.168.1.78(192.168.1.78:3306): Resetting slave info succeeded.
Master failover to 192.168.1.78(192.168.1.78:3306) completed successfully. ###切换成功
old——master 起服务
CHANGE MASTER TO
MASTER_HOST='192.168.1.78',
MASTER_USER='repl',
MASTER_PASSWORD='123',
MASTER_PORT=3306,
MASTER_LOG_FILE='mysqlbin.000016',
MASTER_LOG_POS=319,
MASTER_CONNECT_RETRY=10; start slave;show slave status \G;
CHANGE MASTER TO
MASTER_HOST='192.168.1.78',
MASTER_USER='repl',
MASTER_PASSWORD='123',
MASTER_PORT=3306,
MASTER_LOG_FILE='mysqlbin.000016',
MASTER_LOG_POS=319,
MASTER_CONNECT_RETRY=10; start slave;show slave status \G;
[root@localhost app1]# nohup masterha_manager --conf=/etc/masterha/app1.cnf &> /tmp/mha_manager.log & mha重新起服务
Mon Jan 22 15:22:44 2018 - [info] Slaves settings check done.
Mon Jan 22 15:22:44 2018 - [info]
192.168.1.78(192.168.1.78:3306) (current master)
+--192.168.1.77(192.168.1.77:3306) #老的主现在已经是从库了
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:22:44 2018 - [info]
192.168.1.78(192.168.1.78:3306) (current master)
+--192.168.1.77(192.168.1.77:3306) #老的主现在已经是从库了
+--192.168.1.81(192.168.1.81:3306)
Mon Jan 22 15:22:44 2018 - [info] Checking master_ip_failover_script status:
Mon Jan 22 15:22:44 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.78 --orig_master_ip=192.168.1.78 --orig_master_port=3306
Mon Jan 22 15:22:44 2018 - [info] /etc/masterha/master_ip_failover --command=status --ssh_user=root --orig_master_host=192.168.1.78 --orig_master_ip=192.168.1.78 --orig_master_port=3306
IN SCRIPT TEST====/sbin/ifconfig eno16777736:1 down==/sbin/ifconfig eno16777736:1 192.168.1.82/24===
Checking the Status of the script.. OK
Mon Jan 22 15:22:44 2018 - [info] OK.
Mon Jan 22 15:22:44 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 15:22:44 2018 - [info] Set master ping interval 1 seconds.
Mon Jan 22 15:22:44 2018 - [info] Set secondary check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78
Mon Jan 22 15:22:44 2018 - [info] Starting ping health check on 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:22:44 2018 - [info] Ping(SELECT) succeeded, waiting until MySQL doesn't respond..
[root@localhost app1]#
Mon Jan 22 15:22:44 2018 - [info] OK.
Mon Jan 22 15:22:44 2018 - [warning] shutdown_script is not defined.
Mon Jan 22 15:22:44 2018 - [info] Set master ping interval 1 seconds.
Mon Jan 22 15:22:44 2018 - [info] Set secondary check script: masterha_secondary_check -s 192.168.1.77 -s 192.168.1.78
Mon Jan 22 15:22:44 2018 - [info] Starting ping health check on 192.168.1.78(192.168.1.78:3306)..
Mon Jan 22 15:22:44 2018 - [info] Ping(SELECT) succeeded, waiting until MySQL doesn't respond..
[root@localhost app1]#
总结：
1，若果有二级从库要开启log_slave_update=1参数，否则在切换时可能会造成数据丢失
2，从库开启read-only参数设为只读，在切换时会自动调节read-only状态
3，切换时需要 rm -rf /var/log/masterha/app1/app1.failover.complete 可以写个脚本检测这个文件 若果存在就删除
Mon Jan 22 15:24:56 2018 - [error][/usr/local/share/perl5/MHA/MasterFailover.pm, ln309] Last failover was done at 2018/01/22 15:00:30. Current time is too early to do failover again. If you want to do failover, manually remove /var/log/masterha/app1/app1.failover.complete and run this script again.
4，一旦发生切换，管理进程将会退出，无法再次测试，需要将故障数据库重新加入MHA中来，然后开启管理进程
5，原主节点重新加入后只能做从库 在检测日志
[root@localhost qingke]# cat /var/log/masterha/app1/manager.log|grep 'Statement should be:'|tail -n 1 #新主切换后master binlog位置记录
Mon Jan 22 15:10:27 2018 - [info] All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.1.78', MASTER_PORT=3306, MASTER_LOG_FILE='mysqlbin.000016', MASTER_LOG_POS=319, MASTER_USER='repl', MASTER_PASSWORD='xxx';
6，在切换时要注意主机版本看下网卡名称 否则vip搞不定的
7，手动切换数据库，会使当前数据库停用，在执行手动切换时，manager必须要关闭
[root@localhost app1]# masterha_master_switch --conf=/etc/masterha/app1.cnf --master_state=alive
1，若果有二级从库要开启log_slave_update=1参数，否则在切换时可能会造成数据丢失
2，从库开启read-only参数设为只读，在切换时会自动调节read-only状态
3，切换时需要 rm -rf /var/log/masterha/app1/app1.failover.complete 可以写个脚本检测这个文件 若果存在就删除
Mon Jan 22 15:24:56 2018 - [error][/usr/local/share/perl5/MHA/MasterFailover.pm, ln309] Last failover was done at 2018/01/22 15:00:30. Current time is too early to do failover again. If you want to do failover, manually remove /var/log/masterha/app1/app1.failover.complete and run this script again.
4，一旦发生切换，管理进程将会退出，无法再次测试，需要将故障数据库重新加入MHA中来，然后开启管理进程
5，原主节点重新加入后只能做从库 在检测日志
[root@localhost qingke]# cat /var/log/masterha/app1/manager.log|grep 'Statement should be:'|tail -n 1 #新主切换后master binlog位置记录
Mon Jan 22 15:10:27 2018 - [info] All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.1.78', MASTER_PORT=3306, MASTER_LOG_FILE='mysqlbin.000016', MASTER_LOG_POS=319, MASTER_USER='repl', MASTER_PASSWORD='xxx';
6，在切换时要注意主机版本看下网卡名称 否则vip搞不定的
7，手动切换数据库，会使当前数据库停用，在执行手动切换时，manager必须要关闭
[root@localhost app1]# masterha_master_switch --conf=/etc/masterha/app1.cnf --master_state=alive
百度云盘address
https://pan.baidu.com/s/1dHevv2p
mseo