4、 k8s集群手动部署笔记之Flannel配置

配置Flannel之前,先用etcd给Flannel分配网段

分配网段直接在master节点配置
[root@master ssl]# /opt/kubernetes/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'

配置Flannel

下载Flannel二进制文件:
https://github.com/coreos/flannel/releases

解压文件
[root@master ~]# tar zxvf flannel-v0.9.1-linux-amd64.tar.gz

把flannel文件复制给node1和node2节点
[root@master ~]# scp flanneld mk-docker-opts.sh [email protected]:/opt/kubernetes/bin/
[root@master ~]# scp flanneld mk-docker-opts.sh [email protected]:/opt/kubernetes/bin/

直接用Flannel脚本:flannel.sh

#!/bin/bash

ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}

cat <<EOF >/opt/kubernetes/cfg/flanneld

FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/opt/kubernetes/ssl/ca.pem \
-etcd-certfile=/opt/kubernetes/ssl/server.pem \
-etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"

EOF

cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target

EOF

cat <<EOF >/usr/lib/systemd/system/docker.service

[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
systemctl restart docker

node1节点配置flannel运行上面的脚本

[root@node1 ~]# chmod +x flannel.sh 

运行把其他节点ip都加上去
[root@node1 ~]# ./flannel.sh https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379

查看是否flannel是否启动
[root@node1 ~]# ps -ef|grep kube                                                      
root     21512     1  0 Jul09 ?        00:23:32 /opt/kubernetes/bin/etcd --name=etcd02 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.10.61:2380 --listen-client-urls=https://192.168.10.61:2379,http://127.0.0.1:2379 --advertise-client-urls=https://192.168.10.61:2379 --initial-advertise-peer-urls=https://192.168.10.61:2380 --initial-cluster=etcd01=https://192.168.10.60:2380,etcd02=https://192.168.10.61:2380,etcd03=https://192.168.10.62:2380 --initial-cluster-token=etcd01=https://192.168.10.60:2380,etcd02=https://192.168.10.61:2380,etcd03=https://192.168.10.62:2380 --initial-cluster-state=new --cert-file=/opt/kubernetes/ssl/server.pem --key-file=/opt/kubernetes/ssl/server-key.pem --peer-cert-file=/opt/kubernetes/ssl/server.pem --peer-key-file=/opt/kubernetes/ssl/server-key.pem --trusted-ca-file=/opt/kubernetes/ssl/ca.pem --peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem                                         
root     22461     1  0 10:46 ?        00:00:00 /opt/kubernetes/bin/flanneld --ip-masq --etcd-endpoints=https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379 -etcd-cafile=/opt/kubernetes/ssl/ca.pem -etcd-certfile=/opt/kubernetes/ssl/server.pem -etcd-keyfile=/opt/kubernetes/ssl/server-key.pem                         
root     22887 22377  0 10:49 pts/0    00:00:00 grep --color=auto kube

检查flannel和docker是否在同网络中
[root@node1 ~]# ip addr
......
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:91:11:69:9b brd ff:ff:ff:ff:ff:ff
    inet 172.17.5.1/24 brd 172.17.5.255 scope global docker0
       valid_lft forever preferred_lft forever
5: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default 
    link/ether f2:e0:13:e7:1a:b5 brd ff:ff:ff:ff:ff:ff
    inet 172.17.5.0/32 scope global flannel.1
       valid_lft forever preferred_lft forever
    inet6 fe80::f0e0:13ff:fee7:1ab5/64 scope link 
       valid_lft forever preferred_lft forever

查看flannel分配ip的配置文件    
[root@node1 ~]# cat /run/flannel/subnet.env
'''显示如下'''
DOCKER_OPT_BIP="--bip=172.17.5.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_NETWORK_OPTIONS=" --bip=172.17.5.1/24 --ip-masq=false --mtu=1450"   

node2节点配置flannel跟上面一样

[root@node1 ~]# chmod +x flannel.sh 

运行把其他节点ip都加上去
[root@node1 ~]# ./flannel.sh https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379

查看节点网段分配情况，在master节点查看需要证书查看就到ssl目录

查看flannel分配网段
[root@master ssl]# /opt/kubernetes/bin/etcdctl \
--ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem \
--endpoints="https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379" ls /coreos.com/network/subnets

'''显示如下'''
/coreos.com/network/subnets/172.17.5.0-24
/coreos.com/network/subnets/172.17.23.0-24

查看flannel分配网段在哪台节点上
[root@master ssl]# /opt/kubernetes/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.10.60:2379,https://192.168.10.61:2379,https://192.168.10.62:2379" get /coreos.com/network/subnets/172.17.5.0-24
'''显示如下'''
{"PublicIP":"192.168.10.61","BackendType":"vxlan","BackendData":{"VtepMAC":"f2:e0:13:e7:1a:b5"}}