Kubernetes---使用单节点二进制方式部署k8s集群

一、单master节点用二进制部署K8s集群

1.1 拓扑图

1.2 环境部署

主机名	IP地址	所需部署组件
master	192.168.140.20	kube-apiserver、kube-controller-manager、kube-scheduler、etcd
node01	192.168.140.21	kubelet、kube-proxy、docker、flannel、etcd
node02	192.168.140.22	kubelet、kube-proxy、docker、flannel、etcd

1.3 自签SSL证书

• 部署K8S集群中会用到

组件	使用的证书
etcd	ca.pem，server.pem，server-key.pem
flannel	ca.pem，server.pem，server-key.pem
kube-apiserver	ca.pem，server.pem，server-key.pem
kubelet	ca.pem，ca-key.pem
kube-proxy	ca.pem，kube-proxy.pem，kube-proxy-key.pem
kubectl	ca.pem，admin-pem，admin-key.pem

1.4 开局优化

• 修改主机名

hostnamectl set-hostname master

hostnamectl set-hostname node01

hostnamectl set-hostname node02

• 关闭防火墙与核心防护（三个节点都做）

systemctl stop firewalld && systemctl disable firewalld

setenforce 0 && sed -i "s/SELINUX=enforcing/SELNIUX=disabled/g" /etc/selinux/config

1.5 etcd 集群部署

• master主机创建k8s文件夹并上传etcd脚本，下载cffssl官方证书生成工具

#创建工作目录上传脚本
[root@master ~]# mkdir k8s
[root@master ~]# cd k8s/
[root@master k8s]# rz -E	'//上传etcd脚本'
rz waiting to receive.
[root@master k8s]# ls
etcd-cert.sh  etcd.sh
[root@master k8s]# mkdir etcd-cert
[root@master k8s]# mv etcd-cert.sh etcd-cert

#下载证书制作工具
[root@master k8s]# vi cfssl.sh
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

#下载cfssl官方包
[root@master k8s]# bash cfssl.sh
[root@master k8s]# ls /usr/local/bin/
 cfssl  cfssl-certinfo  cfssljson
//cfssl 生成证书工具   cfssljson通过传入json文件生成证书  cfssl-certinfo查看证书信息

[root@master k8s]# chmod +x /usr/local/bin/cfssl
[root@master k8s]# chmod +x /usr/local/bin/cfssl-certinfo
[root@master k8s]# chmod +x /usr/local/bin/cfssljson

#如果你已经下载好了这三个工具，可以直接导入到 /usr/local/bin/下，注意需要赋予可执行权限

• 创建证书

[root@master ~]# cd /root/k8s/etcd-cert/
[root@master etcd-cert]# cat > ca-config.json <<EOF
{
    
    
   "signing": {
    
    
    "default": {
    
    
       "expiry": "87600h"
     },
    "profiles": {
    
    
       "www": {
    
    
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"     
        ]  
       } 
     }         
   }
}
EOF

'//定义ca证书配置文件'

[root@master etcd-cert]# ls
ca-config.json etcd-cert.sh
[root@master etcd-cert]# cat > ca-csr.json <<EOF 
{
    
       
    "CN": "etcd CA",
    "key": {
    
    
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
    
    
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

'//实现证书签名'

[root@master etcd-cert]# ls
ca-config.json  ca-csr.json  etcd-cert.sh
[root@master etcd-cert]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca - 
2021/03/18 11:38:03 [INFO] generating a new CA key and certificate from CSR
2021/03/18 11:38:03 [INFO] generate received request
2021/03/18 11:38:03 [INFO] received CSR
2021/03/18 11:38:03 [INFO] generating key: rsa-2048
2021/03/18 11:38:03 [INFO] encoded CSR
2021/03/18 11:38:03 [INFO] signed certificate with serial number 646726809365034111674974961625583230830644619161

'//生成证书：ca-key.pem、ca.pem'

[root@master etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd-cert.sh

• 指定 etcd 三个节点之间的通信验证

'#配置服务器端的签名文件'
[root@master etcd-cert]# cat > server-csr.json <<EOF
{
    
    
    "CN": "etcd",
    "hosts": [
    "192.168.140.20",
    "192.168.140.21",
    "192.168.140.22"
    ],
    "key": {
    
    
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
    
    
            "C": "CN",
            "L": "Chengdu",
            "ST": "Chengdu"
        }
    ]
}
EOF

[root@master etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd-cert.sh  server-csr.json

'#生成ETCD证书 server-key.pem   server.pem'
[root@master etcd-cert]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2021/03/18 11:44:52 [INFO] generate received request
2021/03/18 11:44:52 [INFO] received CSR
2021/03/18 11:44:52 [INFO] generating key: rsa-2048
2021/03/18 11:44:53 [INFO] encoded CSR
2021/03/18 11:44:53 [INFO] signed certificate with serial number 474867500554792348214975026719573693837760757686
2021/03/18 11:44:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@master etcd-cert]# ls
ca-config.json  ca-csr.json  ca.pem        server.csr       server-key.pem
ca.csr          ca-key.pem   etcd-cert.sh  server-csr.json  server.pem

• 下载并解压ETCD二进制包
  下载地址：https://github.com/etcd-io/etcd/releases

[root@master ~]# cd k8s/
[root@master k8s]# rz -E		'//我已经下载好了，直接上传'
rz waiting to receive.
[root@master k8s]# ls
cfssl.sh   etcd.sh                          flannel-v0.10.0-linux-amd64.tar.gz
etcd-cert  etcd-v3.3.10-linux-amd64.tar.gz  kubernetes-server-linux-amd64.tar.gz

[root@master k8s]# tar zxvf etcd-v3.3.10-linux-amd64.tar.gz		'//解压软件'

• 配置文件，命令文件，证书

[root@master k8s]# mkdir -p /opt/etcd/{cfg,bin,ssl}		//递归创建目录
[root@master k8s]# ls /opt/etcd/
bin  cfg  ssl
[root@master k8s]# ls etcd-v3.3.10-linux-amd64
Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md

[root@master k8s]# mv etcd-v3.3.10-linux-amd64/etcd* /opt/etcd/bin		'//移动命令到创建的bin目录'
[root@master k8s]# ls /opt/etcd/bin/
etcd  etcdctl

[root@master k8s]# cp etcd-cert/*.pem /opt/etcd/ssl	'//将证书文件拷贝到创建的ssl目录'
[root@master k8s]# ls /opt/etcd/ssl
ca-key.pem  ca.pem  server-key.pem  server.pem

[root@master k8s]# vim etcd.sh	'//查看配置文件'

• 主节点进入卡住状态等待其他节点加入

[root@master k8s]# bash etcd.sh etcd01 192.168.140.20 etcd02=https://192.168.140.21:2380,etcd03=https://192.168.140.22:2380

#使用另外一个会话打开，会发现etcd进程已经开启
[root@master ~]# ps -ef | grep etcd

• 此时还需要拷贝证书和启动脚本到两个工作节点

[root@master k8s]# scp -r /opt/etcd/ root@192.168.140.21:/opt
The authenticity of host '192.168.140.21 (192.168.140.21)' can't be established.
ECDSA key fingerprint is SHA256:5MAR/SpyvPKdJkN2J2yoeWttwiO2/2LgyHt6hNvawmQ.
ECDSA key fingerprint is MD5:f1:ff:6f:fd:14:28:f6:90:4d:f9:51:66:33:21:df:bf.
Are you sure you want to continue connecting (yes/no)? yes			'//输入yes'
Warning: Permanently added '192.168.140.21' (ECDSA) to the list of known hosts.
root@192.168.140.21's password: 		'//输入拷贝的节点root密码'
etcd                                                                                    100%  516   708.3KB/s   00:00    
etcd                                                                                    100%   18MB  81.0MB/s   00:00    
etcdctl                                                                                 100%   15MB 127.8MB/s   00:00    
ca-key.pem                                                                              100% 1679     2.2MB/s   00:00    
ca.pem                                                                                  100% 1265   382.5KB/s   00:00    
server-key.pem                                                                          100% 1675     3.5MB/s   00:00    
server.pem                                                                              100% 1338   495.1KB/s   00:00    
[root@master k8s]# scp -r /opt/etcd/ root@192.168.140.22:/opt

[root@master k8s]# scp /usr/lib/systemd/system/etcd.service root@192.168.140.21:/usr/lib/systemd/system
[root@master k8s]# scp /usr/lib/systemd/system/etcd.service root@192.168.140.22:/usr/lib/systemd/system
root@192.168.140.22's password: 
etcd.service                                                                            100%  923     1.0MB/s   00:00

• 修改node01和node02两个工作节点上的etcd配置文件

#修改相应的名称和IP地址
[root@node01 ~]# vim /opt/etcd/cfg/etcd

[root@node02 ~]# vim /opt/etcd/cfg/etcd

node01

node02

• 启动etcd服务
  先开启主节点的集群脚本，然后两个节点启动etcd

[root@master k8s]# bash etcd.sh etcd01 192.168.140.20 etcd02=https://192.168.140.21:2380,etcd03=https://192.168.140.22:2380

[root@node01 ~]# systemctl start etcd

[root@node02 ~]# systemctl start etcd

• 最后检查群集状态

[root@master k8s]# cd etcd-cert/
[root@master etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd-cert.sh  server.csr  server-csr.json  server-key.pem  server.pem
[root@master etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.140.20:2379,https://192.168.140.21:2379,https://192.168.140.22:2379" cluster-health
member 7e1b55f14a638ac2 is healthy: got healthy result from https://192.168.140.20:2379
member bef76abee0df27ac is healthy: got healthy result from https://192.168.140.22:2379
member f5d4e61e4d6e5f2e is healthy: got healthy result from https://192.168.140.21:2379
cluster is healthy

出现 cluster is healthy 表示ETCD集群已经搭建成功

1.6 Docker相关部署

• 在node节点上安装docker-ce，可以参考之前的安装，这里不再赘述
  docker安装详见

1.7 部署 flannel 容器集群网络

1.7.1 flannel网络理论介绍

• Overlay Network：覆盖网络，在基础网络上叠加的一种虚拟化网络技术模式，该网络中的主机通过虚拟链路连接起来；
• VXLAN：将源数据包封装到UDP中，并使用基础网络的IP/MAC作为外层报文头进行封装，然后在以太网上进行传输，到达目的地后由隧道端点解封装并将数据发送给目标地址；
• Flannel：是Overlay网络的一种，也是将源数据包封装在另一种网络包里面进行路由转发和通信，目前已经支持UDP、VXLAN、AWS VPC和GCE路由等数据转发方式；

1.7.2 flannel工作流程与功能

• Flannel功能

  • Flannel是CoreOS团队针对 Kubernetes设计的一个网络规划服务，
  • 简单来说，它的功能是让集群中的不同节点主机创建的 Docker容器都具有全集群唯一的虚拟IP地址。而且它还能在这些IP地址之间建立一个覆盖网络（overlay Network），通过这个覆盖网络，将数据包原封不动地传递到目标容器内

• ETCD在 flannel 里起的作用：为Flannel提供说明

  • 存储管理 Flannel可分配的IP地址段资源
  • 监控ETCD中每个Pod的实际地址，并在内存中建立维护Pod节点路由表

1.7.3 部署flannel

• master节点上写入分配的子网段到ETCD中，供 flannel 使用

[root@master ]# cd /root/k8s/etcd-cert
[root@master etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.140.20:2379,https://192.168.140.21:2379,https://192.168.140.22:2379" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'

{
    
     "Network": "172.17.0.0/16", "Backend": {
    
    "Type": "vxlan"}} 	'//写入分配的网段'

[root@master etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.140.20:2379,https://192.168.140.21:2379,https://192.168.140.22:2379" get /coreos.com/network/config

{
    
     "Network": "172.17.0.0/16", "Backend": {
    
    "Type": "vxlan"}}		'//查看写入的信息'

• 在node节点上部署 flannel

#将msater节点上的flannel拷贝到所有node节点
[root@master k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz root@192.168.140.21:/opt
root@192.168.140.21's password: 
flannel-v0.10.0-linux-amd64.tar.gz                                                              100% 9479KB  60.3MB/s   00:00    
[root@master k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz root@192.168.140.22:/opt
root@192.168.140.22's password: 
flannel-v0.10.0-linux-amd64.tar.gz

#然后在node01和node02节点上解压，这里以node01节点为例
[root@node01 ~]# cd /opt/
[root@node01 opt]# ls
containerd  etcd  flannel-v0.10.0-linux-amd64.tar.gz  rh
[root@node01 opt]# tar zxvf flannel-v0.10.0-linux-amd64.tar.gz 
flanneld
mk-docker-opts.sh
README.md

• node节点上的其他操作

#创建k8s工作目录，并移动脚本到对应工作目录
[root@node01 opt]# mkdir -p /opt/k8s/{
    
    cfg,bin,ssl}
[root@node01 opt]# mv mk-docker-opts.sh flanneld ./k8s/bin/

#编辑flannel.sh脚本：创建配置文件与启动脚本，定义的端口是2379，节点对外提供的端口
[root@node01 opt]# vi flannel.sh		

#!/bin/bash

ETCD_ENDPOINTS=${
    
    1:-"http://127.0.0.1:2379"}

cat <<EOF >/opt/k8s/cfg/flanneld

FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/opt/etcd/ssl/ca.pem \
-etcd-certfile=/opt/etcd/ssl/server.pem \
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"

EOF

cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/opt/k8s/cfg/flanneld
ExecStart=/opt/k8s/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld

#执行脚本，开启flannel网络功能
[root@node01 opt]# bash flannel.sh https://192.168.140.20:2379,https://192.168.140.21:2379,https://192.168.140.22:2379
Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service.

[root@node01 opt]# systemctl status flanneld		'//查看状态'

#配置docker连接flannel
[root@node01 ~]# vim /usr/lib/systemd/system/docker.service
...
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/subnet.env		'//添加的配置'
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS  -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID		'//修改配置'
...//省略部分信息

[root@node01 ~]# cat /run/flannel/subnet.env
DOCKER_OPT_BIP="--bip=172.17.80.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_NETWORK_OPTIONS=" --bip=172.17.80.1/24 --ip-masq=false --mtu=1450"
//bip：指定启动时的子网

#重启Docker服务，再次查看flannel网络是否有变化
[root@node01 ]# systemctl daemon-reload
[root@node01 ]# systemctl restart docker

注意node2上也要进行相同的操作

1.7.4 测试

• 测试ping通对方docker0网卡 证明flannel起到路由作用
• 先创建容器

[root@node01 ~]# docker run -it centos:7 /bin/bash		'//创建并运容器'
Unable to find image 'centos:7' locally
7: Pulling from library/centos
2d473b07cdd5: Pull complete 
Digest: sha256:0f4ec88e21daf75124b8a9e5ca03c37a5e937e0e108a255d890492430789b60e
Status: Downloaded newer image for centos:7
[root@b6c0c30808c5 /]# yum install net-tools -y			'//安装网络工具'
[root@b6c0c30808c5 /]# ifconfig

[root@node02 ~]# docker run -it centos:7 /bin/bash		'//创建并运容器'
[root@de7c49ccf719 /]# yum install net-tools -y			'//安装网络工具'
[root@de7c49ccf719 /]# ifconfig

• 再测试两个node节点是否可以互联互通

[root@de7c49ccf719 /]# ping 172.17.80.2		'//在node02容器里测试ping node01容器'
PING 172.17.80.2 (172.17.80.2) 56(84) bytes of data.
64 bytes from 172.17.80.2: icmp_seq=1 ttl=62 time=0.499 ms
64 bytes from 172.17.80.2: icmp_seq=2 ttl=62 time=0.741 ms
64 bytes from 172.17.80.2: icmp_seq=3 ttl=62 time=0.502 ms

证明flannel网络部署成功