（三）二进制安装k8s-1.11版本之flanneld部署

在跳板机上生成flanneld证书

#生成flanneld证书请求
#cat >/server/ssl/flanneld-csr.json <<EOF
{
    "CN": "flanneld",
    "hosts": [],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Hangzhou",
            "L": "Hangzhou",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
#cd /server/ssl 
#生成flanneld私钥和证书
#cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json  -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
#cp flanneld*.pem /opt/kubernetes/ssl 


生成的证书发送到master和node节点
#scp flanneld*pem master01:/opt/kubernetes/ssl
#scp flanneld*pem master02:/opt/kubernetes/ssl
#scp flanneld*pem node01:/opt/kubernetes/ssl
#scp flanneld*pem node02:/opt/kubernetes/ssl

选择一台etcd集群节点机器, 向etcd注册flannel相关信息并验证

配置环境变量

export CLUSTER_CIDR="172.30.0.0/16"
export ETCD_ENDPOINTS="https://192.168.213.131:2379,https://192.168.213.132:2379,https://192.168.213.133:2379"
export FLANNEL_ETCD_PREFIX="/kubernetes/network"

向etcd注册flannel相关信息

etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/flanneld.pem \
--key-file=/opt/kubernetes/ssl/flanneld-key.pem \
set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'

验证

etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/flanneld.pem \
--key-file=/opt/kubernetes/ssl/flanneld-key.pem \
get ${FLANNEL_ETCD_PREFIX}/config

下载和配置flanneld

在跳板机上下载flanneld

cd /tools
wget  https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz
tar xf flannel-v0.10.0-linux-amd64.tar.gz

在跳板机上准备flanneld配置文件

#cat  >/server/ssl/flanneld<<EOF
FLANNEL_OPTIONS="-etcd-cafile=/opt/kubernetes/ssl/ca.pem   -etcd-certfile=/opt/kubernetes/ssl/flanneld.pem   -etcd-keyfile=/opt/kubernetes/ssl/flanneld-key.pem   -etcd-endpoints=https://192.168.213.131:2379,https://192.168.213.132:2379,https://192.168.213.133:2379  -etcd-prefix=/kubernetes/network"
EOF

在跳板机上准备flanneld启动脚本

#cat >/server/ssl/flanneld.service<<EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

在跳板机上准备docker启动脚本文件

#cat > /server/ssl/docker.service <<EOF

[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target
EOF

在跳板机上把flanneld证书和私钥文件, flanneld配置文件, flanneld启动脚本文件, docker的启动配置文件发送给master和node节点

scp /server/ssl/flanneld*pem     master01:/opt/kubernetes/ssl/
scp /server/ssl/flanneld master01:/opt/kubernetes/cfg/
scp  /server/ssl/flanneld.service /server/ssl/docker.service master01:/usr/lib/systemd/system/
scp /tools/mk-docker-opts.sh  /tools/flanneld  master01:/opt/kubernetes/bin/

scp /server/ssl/flanneld*pem     master02:/opt/kubernetes/ssl/
scp /server/ssl/flanneld master02:/opt/kubernetes/cfg/
scp  /server/ssl/flanneld.service /server/ssl/docker.service master02:/usr/lib/systemd/system/
scp /tools/mk-docker-opts.sh  /tools/flanneld  master02:/opt/kubernetes/bin/

scp /server/ssl/flanneld*pem     node01:/opt/kubernetes/ssl/
scp /server/ssl/flanneld node01:/opt/kubernetes/cfg/
scp  /server/ssl/flanneld.service /server/ssl/docker.service node01:/usr/lib/systemd/system/
scp /tools/mk-docker-opts.sh  /tools/flanneld  node01:/opt/kubernetes/bin/

scp /server/ssl/flanneld*pem     node02:/opt/kubernetes/ssl/
scp /server/ssl/flanneld node02:/opt/kubernetes/cfg/
scp  /server/ssl/flanneld.service /server/ssl/docker.service node02:/usr/lib/systemd/system/
scp /tools/mk-docker-opts.sh  /tools/flanneld  node02:/opt/kubernetes/bin/

#跳板机也使用flannel
\cp flanneld*pem /opt/kubernetes/ssl/
cp flanneld /opt/kubernetes/cfg/
cp *service /usr/lib/systemd/system
cp /tools/flanneld /opt/kubernetes/bin/
cp /tools/mk-docker-opts.sh /opt/kubernetes/bin/

在master和node节点重启flanend和docker服务

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
systemctl restart docker
systemctl status docker 
systemctl status flanneld

验证：获取flannel在每个节点的网段信息

export CLUSTER_CIDR="172.30.0.0/16"
export ETCD_ENDPOINTS="https://192.168.213.131:2379,https://192.168.213.132:2379,https://192.168.213.133:2379"
export FLANNEL_ETCD_PREFIX="/kubernetes/network"
etcdctl    --endpoints=${ETCD_ENDPOINTS}    --ca-file=/opt/kubernetes/ssl/ca.pem    --cert-file=/opt/kubernetes/ssl/flanneld.pem    --key-file=/opt/kubernetes/ssl/flanneld-key.pem  ls ${FLANNEL_ETCD_PREFIX}/subnets

在master和node节点上查看路由表: 在每个节点上都能看到其他节点的路由

master节点和node节点, docker0的ip地址使用的是flannel网段地址范围, 也是pod容器的网关

master节点和node节点之间的flannel网段之间都能ping通, 说明flannel网络部署完成