- 增加node节点的前提是,先要有一套kubernetes集群
- 关于kubernetes集群的部署,可以查看我另外几篇博客
0、前景提要
- master 节点需要的操作:
- 更新
kube-apiserver
证书,将新增节点ip加入到证书内
- node 节点需要的操作:
- 部署
flannel
、 docker
、 kubelet
、 kube-proxy
1、准备node节点环境
1.0、修改配置脚本参数
- 如果集群不是根据我的博客部署的,不需要进行这一步操作
- 后面的操作,只需要在k8s-01节点上操作即可
k8s-01:~
k8s-01:/opt/k8s/bin
export NODE_IPS=( 192.168.72.44 192.168.72.45 )
export NODE_NAMES=( k8s-06 k8s-07 )
1.1、配置免密
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@${host}
expect {
\"*yes/no*\" {send \"yes\r\"; exp_continue}
\"*Password*\" {send \"123.com\r\"; exp_continue}
\"*Password*\" {send \"123.com\r\";}
}"
done
1.2、添加hosts解析
k8s-01:~
> 192.168.72.44 k8s-06
> 192.168.72.45 k8s-07
> EOF
for host in k8s-02 k8s-03 k8s-04 k8s-05 k8s-06 k8s-07
do
printf "\e[1;34m${host}\e[0m\n"
scp /etc/hosts ${host}:/etc/hosts
done
1.3、修改主机名
for host in 6 7
do
printf "\e[1;34mk8s-0${host}\e[0m\n"
ssh root@k8s-0${host} "hostnamectl set-hostname --static k8s-0${host}"
done
1.4、更新PATH变量
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "echo 'PATH=$PATH:/opt/k8s/bin' >> /etc/profile"
done
1.5、安装依赖包
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "zypper in -y ntp ipset iptables curl sysstat wget"
done
1.6、关闭防火墙以及swap分区
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "systemctl disable SuSEfirewall2.service --now"
ssh root@${host} "iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat"
ssh root@${host} "iptables -P FORWARD ACCEPT"
ssh root@${host} "swapoff -a"
ssh root@${host} "sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab"
done
1.7、开启内核模块
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "modprobe ip_vs_rr"
ssh root@${host} "modprobe br_netfilter"
ssh root@${host} "echo 'modprobe ip_vs_rr' >> /etc/rc.local"
ssh root@${host} "echo 'modprobe br_netfilter' >> /etc/rc.local"
ssh root@${host} "chmod +x /etc/rc.local"
done
1.8、内核优化
- k8s-01节点上已经独立配置过k8s的内核优化文件,因此,直接scp过去,使配置生效即可
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
scp /etc/sysctl.d/kubernetes.conf ${host}:/etc/sysctl.d/kubernetes.conf
ssh root@${host} "sysctl -p /etc/sysctl.d/kubernetes.conf"
done
1.9、创建部署所需目录
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "mkdir -p /opt/k8s/bin /etc/kubernetes/cert"
done
2、更新 kube-apiserver 证书
2.0、创建新的kubernetes证书和私钥
k8s-01:~
k8s-01:/opt/k8s/ssl
k8s-01:/opt/k8s/ssl # cat > kubernetes-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.72.39",
"192.168.72.40",
"192.168.72.41",
"192.168.72.42",
"192.168.72.43",
"192.168.72.44",
"192.168.72.45",
"${CLUSTER_KUBERNETES_SVC_IP}",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "ShangHai",
"L": "ShangHai",
"O": "k8s",
"OU": "bandian"
}
]
}
EOF
2.1、生成新的kubernetes证书和私钥
k8s-01:/opt/k8s/ssl
-ca-key=/opt/k8s/ssl/ca-key.pem \
-config=/opt/k8s/ssl/ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
2.2、分发新的kubernetes证书和私钥
source /opt/k8s/bin/k8s-env.sh
for host in ${MASTER_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
scp /opt/k8s/ssl/kubernetes*.pem ${host}:/etc/kubernetes/cert/
done
2.3、重启kube-apiserver
source /opt/k8s/bin/k8s-env.sh
for host in ${MASTER_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "systemctl daemon-reload && \
systemctl restart kube-apiserver && \
systemctl status kube-apiserver | grep Active"
done
3、部署flannel网络
- flannel需要配置的, 在一开始都已经就绪了,只需要分发文件,启动新节点的flannel服务即可
3.0、分发证书文件到新的节点
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "mkdir -p /etc/flanneld/cert"
scp /opt/k8s/ssl/ca.pem ${host}:/etc/kubernetes/cert/
scp /opt/k8s/ssl/flanneld*.pem ${host}:/etc/flanneld/cert/
scp /opt/k8s/packages/flannel/{
flanneld,mk-docker-opts.sh} ${host}:/opt/k8s/bin/
scp /opt/k8s/conf/flanneld.service ${host}:/etc/systemd/system/
done
3.1、启动flanneld服务
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "systemctl daemon-reload && \
systemctl enable flanneld --now && \
systemctl status flanneld | grep Active"
done
3.2、查看新增node节点是否存在flannel网卡
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "ip a | grep flannel | grep -w inet"
done
4、部署docker
4.0、分发文件到新的节点
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "mkdir /etc/docker"
scp /opt/k8s/packages/docker/* ${host}:/usr/bin/
scp /opt/k8s/conf/daemon.json ${host}:/etc/docker/
scp /opt/k8s/conf/docker.service ${host}:/etc/systemd/system/
done
4.1、启动docker服务
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "systemctl daemon-reload && \
systemctl enable docker --now && \
systemctl status docker | grep Active"
done
4.2、查看新节点的docker和flannel网卡是否为同一网段
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} 'ifconfig | egrep "docker*|flannel*" -A 1'
done
5、部署kubelet组件
5.0、创建kubelet bootstrap kubeconfig文件
source /opt/k8s/bin/k8s-env.sh
for node_name in ${NODE_NAMES[@]}
do
printf "\e[1;34m${node_name}\e[0m\n"
export BOOTSTRAP_TOKEN=$(kubeadm token create \
--description kubelet-bootstrap-token \
--groups system:bootstrappers:${
node_name} \
--kubeconfig ~/.kube/config)
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/cert/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=/opt/k8s/ssl/kubelet-bootstrap-${node_name}.kubeconfig
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=/opt/k8s/ssl/kubelet-bootstrap-${node_name}.kubeconfig
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=/opt/k8s/ssl/kubelet-bootstrap-${node_name}.kubeconfig
kubectl config use-context default --kubeconfig=/opt/k8s/ssl/kubelet-bootstrap-${node_name}.kubeconfig
done
"查看kubeadm为新节点创建的token"
k8s-01:/opt/k8s/ssl
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
6sp12t.btr31aj1hc403tar 23h 2021-02-16T01:34:59+08:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-06
bajiy9.b4fhfy8serfmyve0 23h 2021-02-16T01:35:00+08:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-07
5.1、分发文件到新的节点
source /opt/k8s/bin/k8s-env.sh
for (( i=0; i < 2; i++ ))
do
sed -e "s/##NODE_IP##/${NODE_IPS[i]}/" /opt/k8s/conf/kubelet.service.template > \
/opt/k8s/conf/kubelet-${NODE_IPS[i]}.service
sed -e "s/##NODE_IP##/${NODE_IPS[i]}/" /opt/k8s/conf/kubelet-config.yaml.template > \
/opt/k8s/conf/kubelet-config-${NODE_IPS[i]}.yaml.template
done
for node_name in ${NODE_NAMES[@]}
do
printf "\e[1;34m${node_name}\e[0m\n"
scp /opt/k8s/ssl/kubelet-bootstrap-${node_name}.kubeconfig \
${node_name}:/etc/kubernetes/cert/kubelet-bootstrap.kubeconfig
done
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
scp /opt/k8s/bin/kubelet ${host}:/opt/k8s/bin/kubelet
scp /opt/k8s/conf/kubelet-${host}.service ${host}:/etc/systemd/system/kubelet.service
scp /opt/k8s/conf/kubelet-config-${host}.yaml.template ${host}:/etc/kubernetes/kubelet-config.yaml
scp /opt/k8s/packages/pause.tar ${host}:/opt/k8s/
ssh root@${host} "docker load -i /opt/k8s/pause.tar"
done
5.2、启动kubelet服务
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/"
ssh root@${host} "systemctl daemon-reload && \
systemctl enable kubelet --now && \
systemctl status kubelet | grep Active"
done
5.3、查看新增节点是否ready了
k8s-01:~
NAME STATUS ROLES AGE VERSION
192.168.72.39 Ready <none> 2d v1.19.7
192.168.72.40 Ready <none> 2d v1.19.7
192.168.72.41 Ready <none> 2d v1.19.7
192.168.72.42 Ready <none> 2d v1.19.7
192.168.72.43 Ready <none> 2d v1.19.7
192.168.72.44 Ready <none> 81s v1.19.7
192.168.72.45 Ready <none> 79s v1.19.7
5.4、手动approve server cert csr
k8s-01:~
6、部署kube-proxy
- 同样,只需要分发文件后,启动kube-proxy即可
6.0、分发文件到新的节点
source /opt/k8s/bin/k8s-env.sh
for (( i=0; i < 2; i++ ))
do
sed -e "s/##NODE_IP##/${NODE_IPS[i]}/" /opt/k8s/conf/kube-proxy.service.template > \
/opt/k8s/conf/kube-proxy-${NODE_IPS[i]}.service
sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" \
/opt/k8s/conf/kube-proxy-config.yaml.template > /opt/k8s/conf/kube-proxy-config-${NODE_IPS[i]}.yaml.template
done
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
scp /opt/k8s/ssl/kube-proxy.kubeconfig ${host}:/etc/kubernetes/cert
scp /opt/k8s/conf/kube-proxy-${host}.service ${host}:/etc/systemd/system/kube-proxy.service
scp /opt/k8s/conf/kube-proxy-config-${host}.yaml.template \
${host}:/etc/kubernetes/kube-proxy-config.yaml
scp /opt/k8s/packages/conntrack ${host}:/opt/k8s/bin/
scp /opt/k8s/packages/kubernetes/server/bin/kube-proxy ${host}:/opt/k8s/bin/
ssh root@${host} "chmod +x /opt/k8s/bin/*"
done
6.1、启动kube-proxy服务
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "mkdir -p ${K8S_DIR}/kube-proxy"
ssh root@${host} "modprobe ip_vs_rr"
ssh root@${host} "systemctl daemon-reload && \
systemctl enable kube-proxy --now && \
systemctl status kube-proxy | grep Active"
done
6.2、查看kube-proxy端口
source /opt/k8s/bin/k8s-env.sh
for host in ${NODE_IPS[@]}
do
printf "\e[1;34m${host}\e[0m\n"
ssh root@${host} "ss -nltp | grep kube-proxy"
done