1. 结合pv/pvc Mysql 主从架构
1.1 StatefulSet 概念和特点
Pod调度运行时,如果应用不需要任何稳定的标示,有序部署,删除和扩展,则应该使用一组无状态副本的控制器来部署应用,例如Deployment和ReplicaSet更适合无状态服务需求.而StatefulSet适合管理所有有状态的服务,比如Mysql,MongoDB集群等.
StatefulSet本质上就是Deployment的一个变体,它为了解决有状态服务器的问题,让管理的POD拥有固定的POD名称,启停顺序,在StatefulSet中,Pod名字称为网路表示(hostname),还必须要用到共享存储.
在Deployment中,与之对应的服务是service,而在StatefulSet中与之对应的是headless service,即无头服务,与service的区别就是他没有cluster ip,解析它的名称则返回headless Service对应的全部Pod的Endpoint列表
StatefulSet 特点:
- 给每个pod分配固定切唯一的网络标识符
- 给每个pod分配固定切持久化的外部存储
- 对pod进行有序的部署和扩展
- 对pod进行有序的删除和终止
- 对pod进行有序的自动滚动更新
1.2 StatefulSet组成部分
Headless Service: 用来定义Pod网络标识(DNS domain/Hostname),指的是短的service(丢了domainname)
StatefulSet: 定义具体有多少个pod副本,并为每个Pod定义了一个域名
volumeClaimTemplates: 存储卷申请模板,创建pvc指定pvc名称,大小,将自动创建pvc,切pvc必须由存储类供应.
2. 镜像准备
下载xtrabackup:1.0和mysql5.7镜像,并上传到harbor
docker pull registry.cn-shanghai.aliyuncs.com/qiuqin/xtrabackup:1.0
docker pull mysql:5.7
docker tag registry.cn-shanghai.aliyuncs.com/qiuqin/xtrabackup:1.0 harbor.intra.com/baseimages/xtrabackup:1.0
docker push harbor.intra.com/baseimages/xtrabackup:1.0
docker tag mysql:5.7 harbor.intra.com/baseimages/mysql:5.7.36
docker push harbor.intra.com/baseimages/mysql:5.7.36
3. PV准备
## NFS服务器上创建目录
root@haproxy-1:~# mkdir -p /data/k8s/wework/mysql-datadir-{1..6}
mysql-persistentvolume.yaml 配置文件
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-1
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-1
server: 192.168.31.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-2
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-2
server: 192.168.31.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-3
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-3
server: 192.168.31.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-4
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-4
server: 192.168.31.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-5
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-5
server: 192.168.31.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-datadir-6
namespace: wework
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8s/wework/mysql-datadir-6
server: 192.168.31.109
创建PV
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql/pv# kubectl apply -f mysql-persistentvolume.yaml
persistentvolume/mysql-datadir-1 created
persistentvolume/mysql-datadir-2 created
persistentvolume/mysql-datadir-3 created
persistentvolume/mysql-datadir-4 created
persistentvolume/mysql-datadir-5 created
persistentvolume/mysql-datadir-6 created
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql/pv# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
mysql-datadir-1 50Gi RWO Retain Available 7s
mysql-datadir-2 50Gi RWO Retain Available 7s
mysql-datadir-3 50Gi RWO Retain Available 7s
mysql-datadir-4 50Gi RWO Retain Available 7s
mysql-datadir-5 50Gi RWO Retain Available 7s
mysql-datadir-6 50Gi RWO Retain Available 7s
redis-cluster-pv0 5Gi RWO Retain Bound wework/data-redis-0 24h
redis-cluster-pv1 5Gi RWO Retain Bound wework/data-redis-1 24h
redis-cluster-pv2 5Gi RWO Retain Bound wework/data-redis-4 24h
redis-cluster-pv3 5Gi RWO Retain Bound wework/data-redis-5 24h
redis-cluster-pv4 5Gi RWO Retain Bound wework/data-redis-2 24h
redis-cluster-pv5 5Gi RWO Retain Bound wework/data-redis-3 24h
redis-datadir-pv-1 10Gi RWO Retain Bound wework/redis-datadir-pvc-1 25h
test 1Gi RWX Retain Available nfs 58d
zookeeper-datadir-pv-1 20Gi RWO Retain Bound wework/zookeeper-datadir-pvc-1 40h
zookeeper-datadir-pv-2 20Gi RWO Retain Bound wework/zookeeper-datadir-pvc-2 40h
zookeeper-datadir-pv-3 20Gi RWO Retain Bound wework/zookeeper-datadir-pvc-3 40h
4. 配置Mysql
4.1 ConfigMap 配置
mysql-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: mysql
namespace: wework
labels:
app: mysql
data:
master.cnf: |
# Apply this config only on the master.
[mysqld]
log-bin
log_bin_trust_function_creators=1
lower_case_table_names=1
slave.cnf: |
# Apply this config only on slaves.
[mysqld]
super-read-only
log_bin_trust_function_creators=1
生成configmap
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl apply -f mysql-configmap.yaml
configmap/mysql created
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl get configmap -n wework
NAME DATA AGE
kube-root-ca.crt 1 3d2h
mysql 2 55s
redis-conf 1 25h
4.2 Service配置
mysql-services.yaml
# Headless service for stable DNS entries of StatefulSet members.
apiVersion: v1
kind: Service
metadata:
namespace: wework
name: mysql
labels:
app: mysql
spec:
ports:
- name: mysql
port: 3306
clusterIP: None
selector:
app: mysql
---
# Client service for connecting to any MySQL instance for reads.
# For writes, you must instead connect to the master: mysql-0.mysql.
apiVersion: v1
kind: Service
metadata:
name: mysql-read
namespace: wework
labels:
app: mysql
spec:
ports:
- name: mysql
port: 3306
selector:
app: mysql
部署service
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl apply -f mysql-services.yaml
service/mysql created
service/mysql-read created
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl get svc -n wework
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mysql ClusterIP None <none> 3306/TCP 15s
mysql-read ClusterIP 10.200.195.5 <none> 3306/TCP 15s
redis ClusterIP None <none> 6379/TCP 25h
redis-access ClusterIP 10.200.88.25 <none> 6379/TCP 25h
srv-devops-redis NodePort 10.200.67.224 <none> 6379:36379/TCP 26h
wework-nginx-service NodePort 10.200.89.252 <none> 80:30090/TCP,443:30091/TCP 3d1h
wework-tomcat-app1-service ClusterIP 10.200.21.158 <none> 80/TCP 2d6h
zookeeper ClusterIP 10.200.117.19 <none> 2181/TCP 28h
zookeeper1 NodePort 10.200.167.230 <none> 2181:32181/TCP,2888:31774/TCP,3888:56670/TCP 28h
zookeeper2 NodePort 10.200.36.129 <none> 2181:32182/TCP,2888:46321/TCP,3888:30984/TCP 28h
zookeeper3 NodePort 10.200.190.129 <none> 2181:32183/TCP,2888:61447/TCP,3888:51393/TCP 28h
4.3 mysql-StatefulSet配置
mysql-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mysql
namespace: wework
spec:
selector:
matchLabels:
app: mysql
serviceName: mysql
replicas: 3
template:
metadata:
labels:
app: mysql
spec:
initContainers:
- name: init-mysql
image: harbor.intra.com/baseimages/mysql:5.7.36
command:
- bash
- "-c"
- |
set -ex
# Generate mysql server-id from pod ordinal index.
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
echo [mysqld] > /mnt/conf.d/server-id.cnf
# Add an offset to avoid reserved server-id=0 value.
echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
# Copy appropriate conf.d files from config-map to emptyDir.
if [[ $ordinal -eq 0 ]]; then
cp /mnt/config-map/master.cnf /mnt/conf.d/
else
cp /mnt/config-map/slave.cnf /mnt/conf.d/
fi
volumeMounts:
- name: conf
mountPath: /mnt/conf.d
- name: config-map
mountPath: /mnt/config-map
- name: clone-mysql
image: harbor.intra.com/baseimages/xtrabackup:1.0
command:
- bash
- "-c"
- |
set -ex
# Skip the clone if data already exists.
[[ -d /var/lib/mysql/mysql ]] && exit 0
# Skip the clone on master (ordinal index 0).
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
[[ $ordinal -eq 0 ]] && exit 0
# Clone data from previous peer.
ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
# Prepare the backup.
xtrabackup --prepare --target-dir=/var/lib/mysql
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
containers:
- name: mysql
image: harbor.intra.com/baseimages/mysql:5.7.36
env:
- name: MYSQL_ALLOW_EMPTY_PASSWORD
value: "1"
ports:
- name: mysql
containerPort: 3306
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: 500m
memory: 1Gi
livenessProbe:
exec:
command: ["mysqladmin", "ping"]
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
exec:
# Check we can execute queries over TCP (skip-networking is off).
command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
initialDelaySeconds: 5
periodSeconds: 2
timeoutSeconds: 1
- name: xtrabackup
image: harbor.intra.com/baseimages/xtrabackup:1.0
ports:
- name: xtrabackup
containerPort: 3307
command:
- bash
- "-c"
- |
set -ex
cd /var/lib/mysql
# Determine binlog position of cloned data, if any.
if [[ -f xtrabackup_slave_info ]]; then
# XtraBackup already generated a partial "CHANGE MASTER TO" query
# because we're cloning from an existing slave.
mv xtrabackup_slave_info change_master_to.sql.in
# Ignore xtrabackup_binlog_info in this case (it's useless).
rm -f xtrabackup_binlog_info
elif [[ -f xtrabackup_binlog_info ]]; then
# We're cloning directly from master. Parse binlog position.
[[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
rm xtrabackup_binlog_info
echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
fi
# Check if we need to complete a clone by starting replication.
if [[ -f change_master_to.sql.in ]]; then
echo "Waiting for mysqld to be ready (accepting connections)"
until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
echo "Initializing replication from clone position"
# In case of container restart, attempt this at-most-once.
mv change_master_to.sql.in change_master_to.sql.orig
mysql -h 127.0.0.1 <<EOF
$(<change_master_to.sql.orig),
MASTER_HOST='mysql-0.mysql',
MASTER_USER='root',
MASTER_PASSWORD='',
MASTER_CONNECT_RETRY=10;
START SLAVE;
EOF
fi
# Start a server to send backups when requested by peers.
exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
"xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: 100m
memory: 100Mi
volumes:
- name: conf
emptyDir: {
}
- name: config-map
configMap:
name: mysql
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 10Gi
部署mysql-StatefulSet
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl apply -f mysql-statefulset.yaml
statefulset.apps/mysql created
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl get pods -n wework
NAME READY STATUS RESTARTS AGE
mysql-0 2/2 Running 0 3m47s
mysql-1 2/2 Running 1 3m33s
mysql-2 2/2 Running 1 3m4s
zookeeper1-699d46468c-wd6jc 1/1 Running 0 7h13m
zookeeper2-7cc484778-b5fb7 1/1 Running 0 7h13m
zookeeper3-cdf484f7c-hx5cf 1/1 Running 0 7h13m
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl get sts -n wework
NAME READY AGE
mysql 3/3 4m16s
4.4 测试数据库同步
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl exec -it mysql-2 -c mysql -n wework bash
root@mysql-2:/# mysql
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 1228
Server version: 5.7.36 MySQL Community Server (GPL)
Copyright (c) 2000, 2021, Oracle and/or its affiliates.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show slave status \G;
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: mysql-0.mysql
Master_User: root
Master_Port: 3306
Connect_Retry: 10
Master_Log_File: mysql-0-bin.000003
Read_Master_Log_Pos: 154
Relay_Log_File: mysql-2-relay-bin.000002
Relay_Log_Pos: 322
Relay_Master_Log_File: mysql-0-bin.000003
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 154
Relay_Log_Space: 531
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 100
Master_UUID: 45c28d34-1949-11ed-9aaa-f64e9e84e712
Master_Info_File: /var/lib/mysql/master.info
SQL_Delay: 0
SQL_Remaining_Delay: NULL
Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
Master_Retry_Count: 86400
Master_Bind:
Last_IO_Error_Timestamp:
Last_SQL_Error_Timestamp:
Master_SSL_Crl:
Master_SSL_Crlpath:
Retrieved_Gtid_Set:
Executed_Gtid_Set:
Auto_Position: 0
Replicate_Rewrite_DB:
Channel_Name:
Master_TLS_Version:
1 row in set (0.00 sec)
## 连1看也是
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl exec -it mysql-1 -c mysql -n wework bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@mysql-1:/# mysql -e "show slave status\G;"
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: mysql-0.mysql
Master_User: root
Master_Port: 3306
Connect_Retry: 10
Master_Log_File: mysql-0-bin.000003
Read_Master_Log_Pos: 154
Relay_Log_File: mysql-1-relay-bin.000002
Relay_Log_Pos: 322
Relay_Master_Log_File: mysql-0-bin.000003
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
## 连0主节点
root@k8s-master-01:/opt/k8s-data/yaml/wework/mysql# kubectl exec -it mysql-0 -c mysql -n wework bash
mysql> show master status \G;
*************************** 1. row ***************************
File: mysql-0-bin.000003
Position: 154
Binlog_Do_DB:
Binlog_Ignore_DB:
Executed_Gtid_Set:
1 row in set (0.00 sec)
# 创建数据库
mysql> create database devops;
Query OK, 1 row affected (0.01 sec)
## 再到2上查看
mysql> show databases;
+------------------------+
| Database |
+------------------------+
| information_schema |
| devops |
| mysql |
| performance_schema |
| sys |
| xtrabackup_backupfiles |
+------------------------+
6 rows in set (0.00 sec)
## 副本数调整到4,在mysql-3上也是可以正常读取数据库
root@mysql-3:/# mysql
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 27
Server version: 5.7.36 MySQL Community Server (GPL)
Copyright (c) 2000, 2021, Oracle and/or its affiliates.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show databases;
+------------------------+
| Database |
+------------------------+
| information_schema |
| devops |
| mysql |
| performance_schema |
| sys |
| xtrabackup_backupfiles |
+------------------------+
6 rows in set (0.01 sec)