docker+lvm部署mgr(m_shell)

安装Docker

宿主机建议关闭SeLinux功能：

[root@linux ~]# sed -i s/SELINUX=enforcing/SELINUX=disabled/g /etc/selinux/config

卸载掉旧版本的 Docker

yum remove -y docker docker-client docker-client-latest docker-common \
  docker-latest docker-latest-logrotate docker-logrotate \
  docker-selinux docker-engine-selinux docker-engine

执行以下安装命令去安装依赖包

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum -y install docker-ce docker-ce-cli containerd.io

启动Docker

systemctl enable docker
systemctl start docker
systemctl status docker

开放所有端口

firewall-cmd --add-port=0-65535/tcp --permanent
firewall-cmd --reload
firewall-cmd --list-ports

查看版本

docker version
docker info

修改Docker本地镜像与容器的存储位置的方法

默认情况下Docker的存放位置为：/var/lib/docker 
可以通过命令查看具体位置：
docker info | grep "Docker Root Dir"

#使用逻辑卷

[root@test ~]# lsblk
NAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda       8:0    0   20G  0 disk 
├─sda1    8:1    0   18G  0 part /
└─sda2    8:2    0    2G  0 part [SWAP]
sdb       8:16   0   20G  0 disk 
├─sdb1    8:17   0    2G  0 part 
├─sdb2    8:18   0    2G  0 part 
├─sdb3    8:19   0    2G  0 part 
├─sdb4    8:20   0    1K  0 part 
├─sdb5    8:21   0    2G  0 part 
├─sdb6    8:22   0    2G  0 part 
├─sdb7    8:23   0    2G  0 part 
├─sdb8    8:24   0    2G  0 part 
├─sdb9    8:25   0    2G  0 part 
├─sdb10   8:26   0    2G  0 part 
└─sdb11   8:27   0    2G  0 part 

[root@test ~]# vgcreate vg_docker /dev/sdb1 /dev/sdb2 /dev/sdb3 /dev/sdb5 /dev/sdb6 /dev/sdb7 /dev/sdb8 /dev/sdb9 /dev/sdb10 /dev/sdb11
  Physical volume "/dev/sdb1" successfully created.
  Physical volume "/dev/sdb2" successfully created.
  Physical volume "/dev/sdb3" successfully created.
  Physical volume "/dev/sdb5" successfully created.
  Physical volume "/dev/sdb6" successfully created.
  Physical volume "/dev/sdb7" successfully created.
  Physical volume "/dev/sdb8" successfully created.
  Physical volume "/dev/sdb9" successfully created.
  Physical volume "/dev/sdb10" successfully created.
  Physical volume "/dev/sdb11" successfully created.

[root@test lib]#  lvcreate -n lv_docker -L 19G vg_docker
  Logical volume "lv_docker" created.


[root@test lib]# mkfs.ext4 /dev/vg_docker/lv_docker
mke2fs 1.42.9 (28-Dec-2013)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
1245184 inodes, 4980736 blocks
249036 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2153775104
152 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks: 
	32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 
	4096000

Allocating group tables: done                            
Writing inode tables: done                            
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done   

 mv /var/lib/docker{
    
    ,.bak}
 mkdir /var/lib/docker

[root@test lib]#  mount /dev/vg_docker/lv_docker /var/lib/docker
[root@test lib]#  cp -r /var/lib/docker.bak/* /var/lib/docker/
[root@test lib]# echo "/dev/vg_docker/lv_docker /var/lib/docker ext4 defaults 0 0" >> /etc/fstab

[root@test lib]# rm -rf /var/lib/docker.bak 
[root@test lib]#  systemctl start docker

容器部署

docker run -d --name mysql_m -h mysql_m  \
 -p 13306:3306 \
 -v /sys/fs/cgroup:/sys/fs/cgroup \
 --privileged=true centos:latest \
 /usr/sbin/init


docker run -d --name mysql_s1 -h mysql_s1  \
 -p 23306:3306 \
 -v /sys/fs/cgroup:/sys/fs/cgroup \
 --privileged=true centos:latest \
 /usr/sbin/init


docker run -d --name mysql_s2 -h mysql_s2  \
 -p 33306:3306 \
 -v /sys/fs/cgroup:/sys/fs/cgroup \
 --privileged=true centos:latest \
 /usr/sbin/init

进入容器更新yum源.

[root@test ~]# docker exec -it mysql_m /bin/bash
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
yum clean all && yum makecache

[root@mysql_m /]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2495  100  2495    0     0  10060      0 --:--:-- --:--:-- --:--:-- 10060

[root@mysql_m /]# 
[root@mysql_m /]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*

数据库部署（三套

一.环境配置

#1. 修改主机名

[root@mysql_m ~]# cat /etc/hosts
127.0.0.1	localhost
::1	localhost ip6-localhost ip6-loopback
fe00::0	ip6-localnet
ff00::0	ip6-mcastprefix
ff02::1	ip6-allnodes
ff02::2	ip6-allrouters
172.17.0.4	mysql_m
172.17.0.2	mysql_s1
172.17.0.3	mysql_s2

2. 调整系统资源限制

[root@mysql_m ~]# cat>>/etc/security/limits.conf <<EOF
> mysql soft nproc 65536
> mysql hard nproc 65536
> mysql soft nofile 65536
> mysql hard nofile 65536
> EOF

[root@mysql_m ~]#  cat>>/etc/pam.d/login <<EOF
> session required /lib/security/pam_limits.so
> session required pam_limits.so
> EOF

[root@mysql_m ~]# cat>>/etc/profile<<EOF 
> if [ $USER = "mysql" ]; 
> then
> ulimit -u 16384 -n 65536
> fi
> EOF

#3. 修改系统内核参数

cat>>/etc/sysctl.conf <<EOF
fs.aio-max-nr = 1048576
fs.file-max = 681574400
kernel.shmmax = 137438953472
kernel.shmmni = 4096
kernel.sem = 250 32000 100 200
net.ipv4.ip_local_port_range = 9000 65000
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586
EOF

#4. 关闭防火墙和selinux

systemctl stop firewalld
systemctl disable firewalld
[root@mysql_s2 /]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/gp' /etc/selinux/config 

二.数据库安装路径规划

#2.1. 创建MySQL系统用户、软件目录、权限

[root@mysql_s1 ~]# useradd  mysql
[root@mysql_s2 ~]#  passwd mysql
Changing password for user mysql.
New password: 
BAD PASSWORD: The password is shorter than 8 characters
Retype new password: 
passwd: all authentication tokens updated successfully.

解压软件、创建目录、权限修改 
[root@mysql_m MGR]# tar -xvf mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz -C /usr/local/
[root@mysql_m MGR]# tar -xvf mysql-shell-8.0.32-linux-glibc2.12-x86-64bit.tar.gz -C /usr/local/
[root@mysql_m MGR]# cd /usr/local/
[root@mysql_m local]# mv mysql-8.0.32-linux-glibc2.12-x86_64/ mysql
[root@mysql_m local]# mv mysql-shell-8.0.32-linux-glibc2.12-x86-64bit/ mysql-shell

[root@mysql_m local]# mkdir -p /usr/local/mysql/data
[root@mysql_m local]# chown mysql.mysql /usr/local/mysql/ -R
[root@mysql_m local]#  chown mysql.mysql /usr/local/mysql-shell/ -R
[root@mysql_m local]# ll 
total 56
drwxr-xr-x. 10 mysql mysql 4096 Feb  9 13:16 mysql
drwxr-xr-x.  6 mysql mysql 4096 Dec  9 06:18 mysql-shell

#2. 2配置系统环境变量

cat>>/etc/profile <<EOF
export PATH=$PATH:/usr/local/mysql-shell/bin/:/usr/local/mysql/bin/
EOF
source /etc/profile

#3.3 配置MySQL数据库参数文件

cat>> /etc/my.cnf <<EOF
[mysqld]  
character-set-server=utf8
port=3306   
socket=/tmp/mysql.sock
basedir=/usr/local/mysql
datadir=/usr/local/mysql/data
log-error=/usr/local/mysql/data/mysqld.log
pid-file=/usr/local/mysql/data/mysql.pid
user = mysql
tmpdir = /tmp
default-storage-engine=INNODB

#复制框架
server_id=1
gtid_mode=ON
enforce_gtid_consistency=ON
master_info_repository=TABLE
relay_log_info_repository=TABLE
binlog_checksum=NONE
log_slave_updates=ON
log_bin=binlog
binlog_format=ROW
max_connections=200
max_allowed_packet=16M

#组复制设置
#server必须为每个事务收集写集合，并使用XXHASH64哈希算法将其编码为散列
transaction_write_set_extraction=XXHASH64
#告知插件加入或创建组命名，UUID
loose-group_replication_group_name="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
#server启动时不自启组复制,为了避免每次启动自动引导具有相同名称的第二个组,所以设置为OFF。
loose-group_replication_start_on_boot=off
#告诉插件使用IP地址，端口33061用于接收组中其他成员转入连接
loose-group_replication_local_address="master:33061"
#启动组server，种子server，加入组应该连接这些的ip和端口；其他server要加入组得由组成员同意
loose-group_replication_group_seeds="master:33061,slave1:33061,slave2:33061"
loose-group_replication_ip_whitelist="master,slave1,slave2,manager"
loose-group_replication_bootstrap_group=off
# 使用MGR的单主模式
loose-group_replication_single_primary_mode=on
loose-group_replication_enforce_update_everywhere_checks=off
disabled_storage_engines = MyISAM,BLACKHOLE,FEDERATED,CSV,ARCHIVE
EOF

解决报错

[root@mysql_m data]# /usr/local/mysql/bin/mysqld --initialize-insecure --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data
/usr/local/mysql/bin/mysqld: error while loading shared libraries: libaio.so.1: cannot open shared object file: No such file or directory
[root@mysql_m data]#   yum install -y libaio
[root@mysql_m data]# yum -y install numactl.x86_64

3.4初始化master

[root@mysql_s1 local]# cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
[root@mysql_s1 local]# chmod +x /etc/init.d/mysqld

#加入到开机自启动列表
[root@mysql_m mysql]# chkconfig --add mysqld
[root@mysql_m mysql]# chkconfig --list

#3.5 启动MySQL实例

[root@mysql_m mysql]# ps -ef|grep mysqld
root         526     407  0 01:55 pts/0    00:00:00 grep --color=auto mysqld
[root@mysql_m mysql]# systemctl status mysqld
● mysqld.service - LSB: start and stop MySQL
   Loaded: loaded (/etc/rc.d/init.d/mysqld; generated)
   Active: inactive (dead)
     Docs: man:systemd-sysv-generator(8)

[root@mysql_m local]# mysql -uroot -p
mysql: error while loading shared libraries: libtinfo.so.5: cannot open shared object file: No such file or directory

[root@mysql_m local]#  ln -s /usr/lib64/libtinfo.so.6.1 /usr/lib64/libtinfo.so.5

#3.6、集群实例添加和调试

Server version: 8.0.32 MySQL Community Server - GPL

Copyright (c) 2000, 2023, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> alter user root@localhost identified by 'mysql';
Query OK, 0 rows affected (0.00 sec)

mysql>  flush privileges;
Query OK, 0 rows affected (0.00 sec)

#3.7、通过mysql-shell测试数据库的链接状态正常

[root@mysql_m bin]# ./mysqlsh 
 MySQL  JS >  shell.connect('[email protected]:3306');
 MySQL  127.0.0.1:3306 ssl  JS > 
 MySQL  127.0.0.1:3306 ssl  SQL > 

MySQL  127.0.0.1:3306 ssl  SQL > show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
4 rows in set (0.0011 sec)


mysql>  create user 'mgr'@'localhost' identified by 'mysql';
Query OK, 0 rows affected (0.01 sec)

mysql>  create user 'mgr'@'%' identified by 'mysql';
Query OK, 0 rows affected (0.01 sec)

mysql>  grant all on *.* to 'mgr'@'%' with grant option;
Query OK, 0 rows affected (0.00 sec)

mysql>  grant all on *.* to 'mgr'@'localhost' with grant option;
Query OK, 0 rows affected (0.00 sec)

mysql> select uuid();
+--------------------------------------+
| uuid()                               |
+--------------------------------------+
| 0a8540e7-ab66-11ed-8fb0-0242ac110004 |
+--------------------------------------+
1 row in set (0.00 sec)
========================================
三台机器：
 MySQL  127.0.0.1:3306 ssl  JS > dba.configureLocalInstance()
3、连接3台机器中的primary或master节点进行集群创建
bin/mysqlsh 
# 连接01
 MySQL  JS > shell.connect('[email protected]:3306');
# 创建一个 cluster，命名为 'myCluster'
 MySQL  127.0.0.1:3306 ssl  JS > var cluster=dba.createCluster('mycluster');
# 创建成功后，查看cluster状态
 MySQL  127.0.0.1:3306 ssl  JS > cluster.status();

3.8 配置2个secondary备节点.

1、local_address地址信息(可以是主机名，也可以是IP地址)
[root@slave1 mysql]# grep local_address /etc/my.cnf
loose-group_replication_local_address="master:33061"
[root@slave1 mysql]# vi /etc/my.cnf
[root@slave1 mysql]# grep local_address /etc/my.cnf
loose-group_replication_local_address="slave1:33061"
[root@slave1 mysql]# 
2、设置server-id ，三台主机的server-id必须使用不同的数值代替
[root@master mysql]# grep server_id /etc/my.cnf
server_id=1
[root@master mysql]# 
[root@slave1 mysql]# grep server_id /etc/my.cnf
server_id=2
[root@slave2 mysql]# grep server_id /etc/my.cnf
server_id=3
=======================================================
3、两个备节点添加一行加入集群的参数
[root@slave1 mysql]# vi /etc/my.cnf
[root@slave1 mysql]# grep disjoin /etc/my.cnf
loose-group_replication_allow_local_disjoint_gtids_join=ON
=======================================================
MySQL  127.0.0.1:3306 ssl  JS > dba.configureLocalInstance()
然后在master节点进行slave1、slave2的添加操作
 MySQL  master:3306 ssl  JS > cluster.addInstance('root@slave1:3306');
 MySQL  master:3306 ssl  JS > cluster.addInstance('root@slave2:3306');
==============================================================
 MySQL  127.0.0.1:3306 ssl  JS > cluster.status();
{
    
    
    "clusterName": "mycluster", 
    "defaultReplicaSet": {
    
    
        "name": "default", 
        "primary": "mysql_m:3306", 
        "ssl": "REQUIRED", 
        "status": "OK", 
        "statusText": "Cluster is ONLINE and can tolerate up to ONE failure.", 
        "topology": {
    
    
            "mysql_m:3306": {
    
    
                "address": "mysql_m:3306", 
                "memberRole": "PRIMARY", 
                "mode": "R/W", 
                "readReplicas": {
    
    }, 
                "replicationLag": "applier_queue_applied", 
                "role": "HA", 
                "status": "ONLINE", 
                "version": "8.0.32"
            }, 
            "mysql_s1:3306": {
    
    
                "address": "mysql_s1:3306", 
                "memberRole": "SECONDARY", 
                "mode": "R/O", 
                "readReplicas": {
    
    }, 
                "replicationLag": "applier_queue_applied", 
                "role": "HA", 
                "status": "ONLINE", 
                "version": "8.0.32"
            }, 
            "mysql_s2:3306": {
    
    
                "address": "mysql_s2:3306", 
                "memberRole": "SECONDARY", 
                "mode": "R/O", 
                "readReplicas": {
    
    }, 
                "replicationLag": "applier_queue_applied", 
                "role": "HA", 
                "status": "ONLINE", 
                "version": "8.0.32"
            }
        }, 
        "topologyMode": "Single-Primary"
    }, 
    "groupInformationSourceMember": "mysql_m:3306"

1.添加一个实例

dba.dropMetadataSchema();

[root@mysql_m bin]# ./mysqlsh --mysqlx -umgr -h 127.0.0.1 -p33060 -p
MySQL Shell 8.0.32
所有节点
 shell.connect('[email protected]:3306');
 MySQL  127.0.0.1:33060+ ssl  JS > dba.configureLocalInstance();

2. 连接3台机器中的primary或master节点进行集群创建
   单主模式，第一个加入写节点

 MySQL  localhost:33060+ ssl  JS >  var cluster=dba.createCluster('mycluster');
A new InnoDB Cluster will be created on instance 'mysql_m:3306'.

Disabling super_read_only mode on instance 'mysql_m:3306'.
Validating instance configuration at localhost:3306...

This instance reports its own address as mysql_m:3306

Instance configuration is suitable.
NOTE: Group Replication will communicate with other members using 'mysql_m:3306'. Use the localAddress option to override.

Creating InnoDB Cluster 'mycluster' on 'mysql_m:3306'...

Adding Seed Instance...
Dba.createCluster: This option cannot be set while START or STOP GROUP_REPLICATION is ongoing. (MYSQLSH 3724)
================================
 MySQL  MYSQL03:33060+ ssl  JS > cluster=dba.getCluster();	
 MySQL  127.0.0.1:3306 ssl  JS > cluster.status();
{
    
    
    "clusterName": "mycluster", 
    "defaultReplicaSet": {
    
    
        "name": "default", 
        "primary": "mysql_m:3306", 
        "ssl": "REQUIRED", 
        "status": "OK_NO_TOLERANCE", 
        "statusText": "Cluster is NOT tolerant to any failures.", 
        "topology": {
    
    
            "mysql_m:3306": {
    
    
                "address": "mysql_m:3306", 
                "memberRole": "PRIMARY", 
                "mode": "R/W", 
                "readReplicas": {
    
    }, 
                "replicationLag": "applier_queue_applied", 
                "role": "HA", 
                "status": "ONLINE", 
                "version": "8.0.32"
            }
        }, 
        "topologyMode": "Single-Primary"
    }, 
    "groupInformationSourceMember": "mysql_m:3306"
}

错误1

[ERROR] [MY-011735] [Repl] Plugin group_replication reported: ‘[GCS] The member has failed to gracefully leave the group.’

解决的方法是memberC也执行stop group_replication停掉这个组，再重新组成一个新的组。

mysql> set global group_replication_bootstrap_group=ON;

Query OK, 0 rows affected (0.00 sec)

mysql> start group_replication;

Query OK, 0 rows affected (3.06 sec)

mysql> set global group_replication_bootstrap_group=OFF;

Query OK, 0 rows affected (0.00 sec)

此时memberA再重新加入就成功了：

mysql> start group_replication;

Query OK, 0 rows affected (4.30 sec)
错误2
ERROR: RuntimeError: Cannot add an instance with the same server UUID
vi /var/lib/mysql/auto.cnf // 修改server_uuid

Dba.getCluster: This function is not available through a session to a standalone instance (metadata exists, instance does not belong to that metadata, and GR is not active) (RuntimeError)
此时可删除从节点元数据，重新加入集群中：
错误3

dba.dropMetadataSchema();

执行过程中会报一个ERROR，提示是否禁用super_reade_only，输入y即可。

对于非当前节点报的此错误，首先要\connect连接到错误节点。

 MySQL  localhost:33060+ ssl  JS > var cluster=dba.getCluster()
Dba.getCluster: This function is not available through a session to a standalone instance(MYSQLSH 51300)