shell install 3 nodes k8s

# change time zone
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
timedatectl set-timezone Asia/Shanghai
rm /etc/yum.repos.d/CentOS-Base.repo
cp /vagrant/yum/*.* /etc/yum.repos.d/
mv /etc/yum.repos.d/CentOS7-Base-163.repo /etc/yum.repos.d/CentOS-Base.repo

echo 'disable selinux'
setenforce 0
sed -i 's/=enforcing/=disabled/g' /etc/selinux/config

systemctl stop firewalld
systemctl disable firewalld


echo "关闭 swap 分区"
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

echo "关闭 SELinux"

echo "install sshd"
rm -rf /etc/ssh/sshd_config 
yum -y update openssh-server
rpm -e --nodeps -f openssh-server
yum -y update openssh-server
yum -y install openssh-server
echo "start sshd"
systemctl restart sshd
echo "安装依赖包"
yum install -y epel-release
yum install -y conntrack ipvsadm ipset jq sysstat curl iptables libseccomp

echo "set iptables"
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT


echo "install wget"
yum -y install wget
echo 'sync time'
yum -y install ntp
systemctl start ntpd
systemctl enable ntpd

echo "add user k8s"
useradd -m k8s
echo "set password"
sh -c 'echo 123456 | passwd k8s --stdin' # 为 k8s 账户设置密码

echo "add k8s to group wheel"
gpasswd -a k8s wheel
echo "add user docker	"  
useradd -m docker
echo "add k8s to group docker"
gpasswd -a k8s docker
echo  "new dir"
mkdir -p  /etc/docker/


echo "创建目录"
mkdir -p /opt/k8s/bin
chown -R k8s /opt/k8s
mkdir -p /etc/kubernetes/cert
chown -R k8s /etc/kubernetes
mkdir -p /etc/etcd/cert
chown -R k8s /etc/etcd/cert
mkdir -p /var/lib/etcd && chown -R k8s /etc/etcd/cert

#######################single-mode env-config-file#################

echo "store environment.sh "
cp /vagrant/cluster-environment.sh /opt/k8s/bin/
source /opt/k8s/bin/cluster-environment.sh

# define upload file to file server

function uploadFiles(){
  arr=$1
  for fileName in ${arr[*]}
  do
     _file=$2/$fileName
     curl -F "subDirName=$3" -F "file=@$_file" ${FILE_UPLOAD_URL}
  done
}

# define download file from file server

function downloadFiles(){
 arr=$1
   for str in ${arr[*]}
    do echo $str
    wget -O ${str} ${FILE_DOWNLOAD_URL}${str}?subDirName=$3
    mv ${str} $2
  done
}

function downloadFile(){
    wget -O $1 ${FILE_DOWNLOAD_URL}$1?subDirName=$2
}

cat >> /etc/hosts <<EOF
172.27.130.105 cluster-node1	
172.27.130.111 cluster-node2	
172.27.130.112 cluster-node3	
EOF
echo "将可执行文件路径 /opt/k8s/bin 添加到 PATH 变量中"
sh -c "echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/root/.bashrc"
echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>~/.bashrc
echo "k8s user add evn path"
sh -c "echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/home/k8s/.bashrc"
echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/home/k8s/.bashrc



echo "加载内核模块"
modprobe br_netfilter
modprobe ip_vs

echo "设置系统参数"
downloadFile kubernetes.conf basic-config

cp kubernetes.conf  /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
mount -t cgroup -o cpu,cpuacct none /sys/fs/cgroup/cpu,cpuacct

#########################02.创建 CA 证书和秘钥##############################
echo "#########################02.创建 CA 证书和秘钥##############################"
if [ $1 -eq 1 ];then
echo "安装 cfssl 工具集"

mkdir -p /opt/k8s/cert && chown -R k8s /opt/k8s && cd /opt/k8s
mv /vagrant/cfssl/cfssl_linux-amd64 /opt/k8s/bin/cfssl

mv /vagrant/cfssl/cfssljson_linux-amd64 /opt/k8s/bin/cfssljson

mv /vagrant/cfssl/cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo

chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH
echo "创建根证书 (CA)"
downloadFile ca-config.json basic-config

echo "创建证书签名请求文件"
downloadFile ca-csr.json basic-config

echo "生成 CA 证书和私钥"
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*
mkdir -p /etc/kubernetes/cert && chown -R k8s /etc/kubernetes
cp ca*.pem ca-config.json /etc/kubernetes/cert
uploadFiles "${CA_FILES[*]}" /etc/kubernetes/cert ${CA}
else
  echo "give out to node2 and node3"
	mkdir -p /etc/kubernetes/cert && chown -R k8s /etc/kubernetes
	downloadFiles "${CA_FILES[*]}" /etc/kubernetes/cert ${CA}
fi
################03.部署 kubectl 命令行工具 ALL install #############
echo "################03.部署 kubectl 命令行工具 ALL install #############"
#!important use absolute path

cd /vagrant/kubernetes
echo "cp /vagrant/kubernetes/client/bin/kubectl /opt/k8s/bin/"
cp /vagrant/kubernetes/client/bin/kubectl /opt/k8s/bin/
echo "chmod +x /opt/k8s/bin/*"
chmod +x /opt/k8s/bin/*
echo "创建证书签名请求============"
if [ $1 -eq 1 ];then
	echo "创建证书签名请求"
	downloadFile admin-csr.json basic-config
	echo "生成证书和私钥"
	cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
		-ca-key=/etc/kubernetes/cert/ca-key.pem \
		-config=/etc/kubernetes/cert/ca-config.json \
		-profile=kubernetes admin-csr.json | cfssljson -bare admin
	ls admin*
	echo "设置集群参数"
	kubectl config set-cluster kubernetes \
		--certificate-authority=/etc/kubernetes/cert/ca.pem \
		--embed-certs=true \
		--server=${KUBE_APISERVER} \
		--kubeconfig=kubectl.kubeconfig

	echo "设置客户端认证参数"
	kubectl config set-credentials admin \
		--client-certificate=admin.pem \
		--client-key=admin-key.pem \
		--embed-certs=true \
		--kubeconfig=kubectl.kubeconfig

	echo "设置上下文参数"
	kubectl config set-context kubernetes \
		--cluster=kubernetes \
		--user=admin \
		--kubeconfig=kubectl.kubeconfig
		
	echo "设置默认上下文"
	kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig
	echo "make dir /.kube"
	#ll -a
	#!problem
	mkdir -p /home/k8s/.kube
	cp kubectl.kubeconfig /home/k8s/.kube/config
	mkdir -p /root/.kube
	cp kubectl.kubeconfig /root/.kube/config
  echo "put kubectl.kubeconfig to dir /vagrant/kubectl-config for node2,node3"
  uploadFiles "${CTL_CONF_FILES[*]}" /root/.kube/ ${CTL_CONF}
  
else
	mkdir -p /home/k8s/.kube
	mkdir -p /root/.kube
	downloadFiles "${CTL_CONF_FILES[*]}" /home/k8s/.kube/ ${CTL_CONF}
	downloadFiles "${CTL_CONF_FILES[*]}" /root/.kube/ ${CTL_CONF}
fi
#################04.部署 etcd 集群########################
echo "#################04.部署 etcd 集群########################"
#etcd-v3.3.7-linux-amd64
cp /vagrant/etcd-v3.3.7-linux-amd64/etcd* /opt/k8s/bin
chmod +x /opt/k8s/bin/*
if [ $1 -eq 1 ];then
	downloadFile etcd-csr.json basic-config
	echo "生成证书和私钥"
	cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
		  -ca-key=/etc/kubernetes/cert/ca-key.pem \
		  -config=/etc/kubernetes/cert/ca-config.json \
		  -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
	ls etcd*
	echo "生成的证书和私钥"
	mkdir -p /etc/etcd/cert && chown -R k8s /etc/etcd/cert
	cp etcd*.pem /etc/etcd/cert/
	uploadFiles "${ETCD_CA_FILES[*]}" /etc/etcd/cert/ ${ETCD_CA}
else
	mkdir -p /etc/etcd/cert && chown -R k8s /etc/etcd/cert
  downloadFiles "${ETCD_CA_FILES[*]}" /etc/etcd/cert/ ${ETCD_CA}
fi
echo "ips = ${ETCD_NODES}"
downloadFile etcd.service.template basic-config
idx=`expr $1 - 1`
echo "创建 systemd unit 文件"
sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##ETCD_NODES##|${ETCD_NODES}|" etcd.service.template > etcd-${NODE_IPS[$idx]}.service 
ls *.service
mkdir -p /var/lib/etcd && chown -R k8s /var/lib/etcd
cp etcd-${NODE_IPS[$idx]}.service /etc/systemd/system/etcd.service

echo "!important add execute permission"
chown -R k8s /etc/etcd/cert/
chmod +x -R /etc/etcd/cert/
#!important
chown -R k8s /etc/kubernetes/cert/
chmod -R +x /etc/kubernetes/cert

cat /etc/systemd/system/etcd.service
echo "start etcd"
systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd &
echo "etcd status"
systemctl status etcd|grep Active
echo "look ectd log"
journalctl -u etcd

#################05.部署 flannel 网络#########################
echo "#################05.部署 flannel 网络#########################"
mkdir /vagrant/flannel
echo "tar flannel.."
echo "copy to path"
cp  /vagrant/flannel/{flanneld,mk-docker-opts.sh} /opt/k8s/bin/
echo "add execute permission.."
chmod +x /opt/k8s/bin/*
if [ $1 -eq 1 ];then
	echo "创建证书签名请求"
	downloadFile flanneld-csr.json basic-config
	echo "生成证书和私钥"
	cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
		-ca-key=/etc/kubernetes/cert/ca-key.pem \
		-config=/etc/kubernetes/cert/ca-config.json \
		-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
	ls flanneld*pem
	echo "将生成的证书和私钥分发到所有节点"
	mkdir -p /etc/flanneld/cert && chown -R k8s /etc/flanneld
	cp flanneld*.pem /etc/flanneld/cert
  uploadFiles "${FLAN_CA_FILES[*]}" /etc/flanneld/cert ${FLAN_CA}
	echo "向 etcd 写入集群 Pod 网段信息---@again"
	etcdctl \
		--endpoints=${ETCD_ENDPOINTS} \
		--ca-file=/etc/kubernetes/cert/ca.pem \
		--cert-file=/etc/flanneld/cert/flanneld.pem \
		--key-file=/etc/flanneld/cert/flanneld-key.pem \
		set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'

else
  mkdir -p /etc/flanneld/cert && chown -R k8s /etc/flanneld
  downloadFiles "${FLAN_CA_FILES[*]}" /etc/flanneld/cert ${FLAN_CA}
fi

echo "创建 flanneld 的 systemd unit 文件"
#eth0
export IFACE=enp0s8
downloadFile flanneld.service.template basic-config

sed -e "s|##ETCD_ENDPOINTS##|${ETCD_ENDPOINTS}|" -e "s|##FLANNEL_ETCD_PREFIX##|${FLANNEL_ETCD_PREFIX}|" -e "s/##IFACE##/${IFACE}/" flanneld.service.template > flanneld.service

echo "分发 flanneld systemd unit 文件到所有节点"
cp flanneld.service /etc/systemd/system/

echo "启动 flanneld 服务"
systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld
echo "检查启动结果"
systemctl status flanneld|grep Active
echo "查看日志"
journalctl -u flanneld

################06-0.部署 master 节点#################
echo "################06-0.部署 master 节点#################"
echo "install k8s "
cd /vagrant/kubernetes
echo "拷贝到所有 master 节点"
cp /vagrant/kubernetes/server/bin/* /opt/k8s/bin/
chmod +x /opt/k8s/bin/*



################06-1.部署高可用组件####################
echo "################06-1.部署高可用组件####################"
echo "安装软件包 keepalived haproxy"
yum install -y keepalived haproxy

if [ $1 -eq 1 ];then
	echo "haproxy 配置文件"
	downloadFile haproxy.cfg basic-config

	echo "下发 haproxy.cfg 到所有 master 节点"
	cp haproxy.cfg /etc/haproxy
else
  downloadFile haproxy.cfg basic-config
  mv haproxy.cfg /etc/haproxy
fi
echo "起 haproxy 服务"
systemctl restart haproxy
echo "检查 haproxy 服务状态"
systemctl status haproxy|grep Active
echo "查看日志"
journalctl -u haproxy

if [ $1 -eq 1 ];then
	echo "keepalived conf file"
	downloadFile keepalived-master.conf.template basic-config

	sed -e "s/##VIP_IF##/${VIP_IF}/" -e "s/##MASTER_VIP##/${MASTER_VIP}/" keepalived-master.conf.template > keepalived-master.conf
	echo "下发 keepalived 配置文件"
	cp keepalived-master.conf /etc/keepalived/keepalived.conf
else
	echo "backup 配置文件"
	downloadFile keepalived-backup.conf.template basic-config
  sed -e "s/##VIP_IF##/${VIP_IF}/" -e "s/##MASTER_VIP##/${MASTER_VIP}/" keepalived-backup.conf.template > keepalived-backup.conf
	cp keepalived-backup.conf /etc/keepalived/keepalived.conf
fi
echo "起 keepalived 服务"
systemctl restart keepalived
echo "检查 keepalived 服务"
systemctl status keepalived|grep Active
echo "查看日志"
journalctl -u keepalived
echo "ping 通 VIP"
/usr/sbin/ip addr show ${VIP_IF}
ping -c 1 ${MASTER_VIP}
##################06-2.部署 kube-apiserver 组件##############################
if [ $1 -eq 1 ];then
	echo "##################06-2.部署 kube-apiserver 组件##############################"
	echo "创建证书签名请求"
	downloadFile kubernetes-csr.json.template basic-config
  sed -e "s/##MASTER_VIP##/${MASTER_VIP}/" -e "s/##CLUSTER_KUBERNETES_SVC_IP##/${CLUSTER_KUBERNETES_SVC_IP}/" kubernetes-csr.json.template > kubernetes-csr.json

	echo "生成证书和私钥"

	cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
		-ca-key=/etc/kubernetes/cert/ca-key.pem \
		-config=/etc/kubernetes/cert/ca-config.json \
		-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
	ls kubernetes*pem
	echo "将生成的证书和私钥文件拷贝到 master 节点"
	cp kubernetes*.pem /etc/kubernetes/cert/
	uploadFiles "${API_CA_FILES[*]}" /etc/kubernetes/cert/ ${API_CA}
else
	downloadFiles "${API_CA_FILES[*]}" /etc/kubernetes/cert/ ${API_CA}
fi
echo "创建加密配置文件"
downloadFile encryption-config.yaml.template basic-config
sed -e "s/##ENCRYPTION_KEY##/${ENCRYPTION_KEY}/"  encryption-config.yaml.template > encryption-config.yaml
echo "将加密配置文件拷贝到 master 节点的 /etc/kubernetes 目录下"
cp encryption-config.yaml /etc/kubernetes/


echo "创建 kube-apiserver systemd unit 模板文件"
downloadFile kube-apiserver.service.template basic-config

echo "替换模板文件中的变量，为各节点创建 systemd unit 文件"
idx=`expr $1 - 1`
sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##SERVICE_CIDR##|${SERVICE_CIDR}|" -e "s|##NODE_PORT_RANGE##|${NODE_PORT_RANGE}|" -e "s|##ETCD_ENDPOINTS##|${ETCD_ENDPOINTS}|" kube-apiserver.service.template > kube-apiserver-${NODE_IPS[$idx]}.service 
ls kube-apiserver*.service

echo "分发生成的 systemd unit 文件"
mkdir -p /var/log/kubernetes && chown -R k8s /var/log/kubernetes
cp kube-apiserver-${NODE_IPS[$idx]}.service /etc/systemd/system/kube-apiserver.service

echo "give permission to exeucte"
chown -R k8s /etc/kubernetes/cert/
chmod -R +x /etc/kubernetes/cert

echo "启动 kube-apiserver 服务"
systemctl daemon-reload && systemctl enable kube-apiserver && systemctl restart kube-apiserver
echo "检查 kube-apiserver 运行状态"
systemctl status kube-apiserver |grep Active
echo " master 节点查看日志"
journalctl -u kube-apiserver
ETCDCTL_API=3 etcdctl \
    --endpoints=${ETCD_ENDPOINTS} \
    --cacert=/etc/kubernetes/cert/ca.pem \
    --cert=/etc/etcd/cert/etcd.pem \
    --key=/etc/etcd/cert/etcd-key.pem \
    get /registry/ --prefix --keys-only

echo "检查集群信息"
kubectl cluster-info
kubectl get all --all-namespaces
kubectl get componentstatuses

#echo "检查 kube-apiserver 监听的端口"
#netstat -lnpt|grep kube
echo "授予 kubernetes 证书访问 kubelet API 的权限"
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

###################06-3.部署高可用 kube-controller-manager 集群################
echo "###################06-3.部署高可用 kube-controller-manager 集群################"
if [ $1 -eq 1 ];then
	echo "创建 kube-controller-manager 证书和私钥"
	echo "single node mode"
	downloadFile kube-controller-manager-csr.json basic-config

	echo "生成证书和私钥"
	cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
		-ca-key=/etc/kubernetes/cert/ca-key.pem \
		-config=/etc/kubernetes/cert/ca-config.json \
		-profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

	echo "copy pem files to cert dir"
	cp kube-controller-manager*.pem /etc/kubernetes/cert/
	chmod +x /etc/kubernetes/cert/*.pem
	uploadFiles "${CM_CA_FILES[*]}" /etc/kubernetes/cert/ ${CM_CA}
else
  downloadFiles "${CM_CA_FILES[*]}" /etc/kubernetes/cert/ ${CM_CA}
	chmod +x /etc/kubernetes/cert/*.pem
fi
if [ $1 -eq 1 ];then
	echo "创建和分发 kubeconfig 文件"
	kubectl config set-cluster kubernetes \
		--certificate-authority=/etc/kubernetes/cert/ca.pem \
		--embed-certs=true \
		--server=${KUBE_APISERVER} \
		--kubeconfig=kube-controller-manager.kubeconfig

	kubectl config set-credentials system:kube-controller-manager \
		--client-certificate=/etc/kubernetes/cert/kube-controller-manager.pem \
		--client-key=/etc/kubernetes/cert/kube-controller-manager-key.pem \
		--embed-certs=true \
		--kubeconfig=kube-controller-manager.kubeconfig

	kubectl config set-context system:kube-controller-manager \
		--cluster=kubernetes \
		--user=system:kube-controller-manager \
		--kubeconfig=kube-controller-manager.kubeconfig

	kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig

	echo "分发 kubeconfig 到所有 master 节点"
	cp kube-controller-manager.kubeconfig /etc/kubernetes/
	uploadFiles "${CM_CONF_FILES[*]}" /etc/kubernetes/ ${CM_CONF}
else
	downloadFiles "${CM_CONF_FILES[*]}" /etc/kubernetes/ ${CM_CONF}
fi
echo "创建和分发 kube-controller-manager systemd unit 文件"
downloadFile kube-controller-manager.service.template basic-config
sed -e "s|##SERVICE_CIDR##|${SERVICE_CIDR}|"  kube-controller-manager.service.template > kube-controller-manager.service 
echo "分发 systemd unit 文件到所有 master 节点"
cp kube-controller-manager.service /etc/systemd/system/
#!important
chown -R k8s /etc/kubernetes/cert/
chmod -R +x /etc/kubernetes/cert

chown -R k8s /etc/kubernetes
chmod -R +x /etc/kubernetes
echo "启动 kube-controller-manager 服务"
systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl restart kube-controller-manager
echo "检查服务运行状态"
systemctl status kube-controller-manager|grep Active
echo "查看日志"
journalctl -u kube-controller-manager
echo "查看输出的 metric"
curl -s --cacert /etc/kubernetes/cert/ca.pem https://127.0.0.1:10252/metrics |head
echo "测试 kube-controller-manager 集群的高可用===查看当前的 leader"
kubectl get endpoints kube-controller-manager --namespace=kube-system  -o yaml

#######################06-4.部署高可用 kube-scheduler 集群######################
echo "#######################06-4.部署高可用 kube-scheduler 集群######################"
if [ $1 -eq 1 ];then
	echo "创建 kube-scheduler 证书和私钥"

	downloadFile kube-scheduler-csr.json basic-config
	echo "generate scheduler ca"
	cfssl gencert \
		-ca=/etc/kubernetes/cert/ca.pem \
		-ca-key=/etc/kubernetes/cert/ca-key.pem \
		-config=/etc/kubernetes/cert/ca-config.json \
		-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

	echo "copy pem files to cert dir"
	cp kube-scheduler*.pem /etc/kubernetes/cert/
	chmod +x /etc/kubernetes/cert/*.pem

	echo "创建和分发 kubeconfig 文件"
	kubectl config set-cluster kubernetes \
		--certificate-authority=/etc/kubernetes/cert/ca.pem \
		--embed-certs=true \
		--server=${KUBE_APISERVER} \
		--kubeconfig=kube-scheduler.kubeconfig

	kubectl config set-credentials system:kube-scheduler \
		--client-certificate=kube-scheduler.pem \
		--client-key=kube-scheduler-key.pem \
		--embed-certs=true \
		--kubeconfig=kube-scheduler.kubeconfig

	kubectl config set-context system:kube-scheduler \
		--cluster=kubernetes \
		--user=system:kube-scheduler \
		--kubeconfig=kube-scheduler.kubeconfig

	kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
	echo "分发 kubeconfig 到所有 master 节点"
	cp kube-scheduler.kubeconfig /etc/kubernetes/
  uploadFiles "${SCH_CONF_FILES[*]}" /etc/kubernetes ${SCH_CONF}
else
	downloadFiles "${SCH_CONF_FILES[*]}" /etc/kubernetes ${SCH_CONF}
fi
echo "创建和分发 kube-scheduler systemd unit 文件"
downloadFile kube-scheduler.service basic-config
echo "分发 systemd unit 文件到所有 master 节点"
cp kube-scheduler.service /etc/systemd/system/
#!important
chown -R k8s /etc/kubernetes/cert/
chmod -R +x /etc/kubernetes/cert

chown -R k8s /etc/kubernetes
chmod -R +x /etc/kubernetes
echo "启动 kube-scheduler 服务"
systemctl daemon-reload && systemctl enable kube-scheduler && systemctl restart kube-scheduler

echo "检查服务运行状态"
systemctl status kube-scheduler|grep Active

journalctl -u kube-scheduler

echo "查看输出的 metric"
curl -s http://127.0.0.1:10251/metrics |head
echo "测试 kube-scheduler 集群的高可用====查看当前的 leader"
kubectl get endpoints kube-scheduler --namespace=kube-system  -o yaml


######################install client####################################

echo "######################install client####################################"
echo "#################### 部署 docker ###########################"
#tar -xzvf /vagrant/docker-18.03.1-ce.tgz -C /vagrant
#cp /vagrant/docker/docker*  /opt/k8s/bin/
echo "install the latest version of docker"
yum install -y yum-utils \
  device-mapper-persistent-data \
  lvm2
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

yum -y install docker-ce
rm -f /opt/k8s/bin/docker*
cp /usr/bin/docker* /opt/k8s/bin
chmod +x /opt/k8s/bin/*
echo "创建和分发docker systemd unit 文件"
downloadFile docker.service basic-config

cp docker.service /etc/systemd/system/

echo "append mirror to daemon"

downloadFile docker-daemon.json basic-config
mv docker-daemon.json /etc/docker/daemon.json

echo "start docker service"
systemctl daemon-reload && systemctl enable docker && systemctl restart docker
sysctl -p /etc/sysctl.d/kubernetes.conf
echo "check docker status"
systemctl status docker|grep Active
echo "check docker log"
journalctl -u docker
echo "检查 docker docker0 网桥"
ip addr show flannel.1 && /usr/sbin/ip addr show docker0
######################07-2.部署 kubelet 组件######################
echo "######################07-2.部署 kubelet 组件######################"
echo "创建 kubelet bootstrap kubeconfig 文件"
echo "# 创建 token"
idx=`expr $1 - 1`
export BOOTSTRAP_TOKEN=$(kubeadm token create \
--description kubelet-bootstrap-token \
--groups system:bootstrappers:${NODE_NAMES[$idx]} \
--kubeconfig ~/.kube/config)

echo "# 设置集群参数"
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/cert/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig

echo "# 设置客户端认证参数"
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig

echo "# 设置上下文参数"
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig

echo "# 设置默认上下文"
kubectl config use-context default --kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig

echo "查看 kubeadm 为各节点创建的 token"
kubeadm token list --kubeconfig ~/.kube/config

echo "各 token 关联的 Secret"
kubectl get secrets  -n kube-system

echo "分发 bootstrap kubeconfig 文件到所有 worker 节点"
cp kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig /etc/kubernetes/kubelet-bootstrap.kubeconfig

echo "!important k8s.V1.8 not use this file"
echo "创建和分发 kubelet 参数配置文件"
downloadFile kubelet.config.json.template basic-config

echo "为各节点创建和分发 kubelet 配置文件"
idx=`expr $1 - 1`
sed -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s/##CLUSTER_DNS_DOMAIN##/${CLUSTER_DNS_DOMAIN}/" -e "s/##CLUSTER_DNS_SVC_IP##/${CLUSTER_DNS_SVC_IP}/" kubelet.config.json.template > kubelet.config-${NODE_IPS[$idx]}.json
cp kubelet.config-${NODE_IPS[$idx]}.json /etc/kubernetes/kubelet.config.json


echo "创建和分发 kubelet systemd unit 文件"
downloadFile kubelet.service.template basic-config


echo "为各节点创建和分发 kubelet systemd unit 文件"
idx=`expr $1 - 1`
sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" kubelet.service.template > kubelet-${NODE_NAMES[$idx]}.service
cp kubelet-${NODE_NAMES[$idx]}.service /etc/systemd/system/kubelet.service
cp /vagrant/kubelet.kubeconfig /etc/kubernetes/

echo "Bootstrap Token Auth 和授予权限"
journalctl -u kubelet -a |grep -A 2 'certificatesigningrequests'
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers

echo "启动 kubelet 服务"

chown -R k8s /etc/kubernetes/
chmod -R +x /etc/kubernetes/

mkdir -p /var/lib/kubelet
systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet

kubectl get csr
kubectl get nodes

echo "approve kubelet CSR 请求"
echo "查看 CSR 列表"
kubectl get csr
kubectl describe  csr node-csr-QzuuQiuUfcSdp3j5W4B2UOuvQ_n9aTNHAlrLzVFiqrk
echo "自动 approve CSR 请求"
downloadFile csr-crb.yaml basic-config

kubectl apply -f csr-crb.yaml

kubectl get csr

kubectl get nodes


ls -l /etc/kubernetes/kubelet.kubeconfig

ls -l /etc/kubernetes/cert/|grep kubelet

######################07-3.部署 kube-proxy 组件###################################

echo "创建 kube-proxy 证书"
downloadFile kube-proxy-csr.json basic-config

echo "生成证书和私钥"
cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
  -ca-key=/etc/kubernetes/cert/ca-key.pem \
  -config=/etc/kubernetes/cert/ca-config.json \
  -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

echo "创建和分发 kubeconfig 文件"

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/cert/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \
  --client-certificate=kube-proxy.pem \
  --client-key=kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
echo "分发 kubeconfig 文件!important chown -R "
cp kube-proxy.kubeconfig /etc/kubernetes/

echo "创建 kube-proxy 配置文件"
downloadFile kube-proxy.config.yaml.template basic-config
idx=`expr $1 - 1`
sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##CLUSTER_CIDR##|${CLUSTER_CIDR}|" kube-proxy.config.yaml.template > kube-proxy-${NODE_NAMES[$idx]}.config.yaml
cp kube-proxy-${NODE_NAMES[$idx]}.config.yaml /etc/kubernetes/kube-proxy.config.yaml

echo "创建和分发 kube-proxy systemd unit 文件"
downloadFile kube-proxy.service basic-config
echo "分发 kube-proxy systemd unit 文件"
cp kube-proxy.service /etc/systemd/system/


chown -R k8s /etc/kubernetes/
chmod -R +x /etc/kubernetes/

echo "启动 kube-proxy 服务"
mkdir -p /var/lib/kube-proxy
mkdir -p /var/log/kubernetes && chown -R k8s /var/log/kubernetes
systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy
echo "检查启动结果"
systemctl status kube-proxy|grep Active
journalctl -u kube-proxy