seven. Deploy the flannel network
Kubernetes supports flannel and weave networks based on vxlan, and Calico networks based on BGP routing. This section uses flannel networks.
Flannel network uses etcd and other kv storage for centralized control, generates 1 subnet on each host, and the subnet on each host is opened up through vxlan.
1. Create flannel TLS certificate and private key
The etcd cluster has mutual TLS authentication enabled, and the CA and key for communicating with the etcd cluster need to be specified for the flannel network.
1) Create a flannel certificate signing request
[root@kubenode1 ~]# mkdir -p /etc/kubernetes/flannel [root@kubenode1 ~]# cd /etc/kubernetes/flannel/ [root@kubenode1 flannel]# touch flanneld-csr.json # hosts字段留空 [root@kubenode1 flannel]# vim flanneld-csr.json { "CN": "flanneld", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ChengDu", "L": "ChengDu", "O": "k8s", "OU": "cloudteam" } ] }
2) Generate flannel certificate and private key
[root@kubenode1 flannel]# cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \ -ca-key=/etc/kubernetes/ssl/ca-key.pem \ -config=/etc/kubernetes/ssl/ca-config.json \ -profile=kubernetes flanneld-csr.json | cfssljson -bare flannel
# 分发flanneld.pem，flanneld-key.pem [root@kubenode1 flannel]# scp flanneld.pem flanneld-key.pem [email protected]:/etc/kubernetes/flannel/ [root@kubenode1 flannel]# scp flanneld.pem flanneld-key.pem [email protected]:/etc/kubernetes/flannel/
2. Write the cluster Pod network segment information in etcd
#Writing Pod network segment information to etcd cluster only needs to operate on one node ; # etcdctl uses etcd v2 api; #Pay attention to the environment variables used, ${CLUSTER_CIDR} needs to be with the parameters of kube-controller-manager- -cluster-cidr consistent; [root@kubenode1 flannel] # etcdctl --endpoints=${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/ssl/ ca.pem \ --cert-file=/etc/kubernetes /flannel/ flanneld.pem \ --key-file=/etc/kubernetes/flannel/flanneld- key.pem \ set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
3. Download and deploy flannel
[root@kubenode1 ~]# cd /usr/local/src/ [root@kubenode1 src]# wget https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz [root@kubenode1 src]# mkdir -p /usr/local/flannel [root@kubenode1 src]# tar -zxvf flannel-v0.10.0-linux-amd64.tar.gz [root@kubenode1 src]# cd /usr/local/flannel/
4. Configure the systemd unit file of flanneld
[root@kubenode1 src]# cd /usr/local/flannel/ [root@kubenode1 flannel]# touch /usr/lib/systemd/system/flanneld.service # EnvironmentFile: Here, the startup parameters of flanneld are placed outside the unit file, and there is no need to reload after modification; # ExecStart: The location of the startup file, and the variable with the specified parameter; # ExecStartPost: The operation that needs to be performed after startup, here is the use of mk The -docker-opts.sh script writes the Pod subnet information assigned to flanneld to the /run/flannel/docker file. When the docker service starts, the parameters in the /run/flannel/docker file are called to set the docker0 bridge; # RequiredBy: docker.service depends on the flanneld service; # flanneld uses the interface where the system default route is located to encapsulate vxlan to communicate with other nodes. When there are multiple network ports, the --iface option can be used to specify the communication interface [root@kubenode1 flannel] # vim /usr/lib/systemd /system/flanneld.service [Unit] Description=Flanneld overlay address etcd agent Documentation=https://github.com/coreos After=network.target After=network-online.target Wants=network-online.target After=etcd.service Before=docker.service [Service] Type=notify EnvironmentFile=/usr/local/flannel/flanneld.conf ExecStart=/usr/local/flannel/flanneld $FLANNELD_ARGS ExecStartPost=/usr/local/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker Restart=on-failure [Install] WantedBy=multi-user.target RequiredBy=docker.service #Startup parameter file [root@kubenode1 flannel] # touch /usr/local/flannel/flanneld.conf [root@kubenode1 flannel] # vim /usr/local/flannel/flanneld.conf FLANNELD_ARGS= " -etcd-cafile=/etc /kubernetes/ssl/ca.pem \ -etcd-certfile = / etc / kubernetes / flannel / flanneld.pem \ -etcd-keyfile=/etc/kubernetes/flannel/flanneld-key.pem \ -etcd-endpoints=https://172.30.200.21:2379,https://172.30.200.22:2379,https://172.30.200.23:2379 \ -etcd-prefix=/kubernetes/network"
5. Modify the systemd unit file of docker
#Add the EnvironmentFile item: specify the location of the startup parameters, and pay attention to the file corresponding to the file set by flanneld; #Modify the ExecStart item: unix calls the dockerd interface for the open local client, and the ip address calls the dockerd interface for the open remote client. The parameter name set by flanneld corresponds to [root@kubenode1 flannel] # vim /usr/lib/systemd/system/docker.service [Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com After=network.target [Service] Type=notify EnvironmentFile=-/run/flannel/docker ExecStart=/usr/bin/dockerd -H unix:///var/run/docker.sock -H 0.0.0.0:2375 $DOCKER_NETWORK_OPTIONS ExecReload=/bin/kill -s HUP $MAINPID LimitNOFILE = infinity LimitNPROC = infinity LimitCORE=infinity TimeoutStartSec=0 Delegate=yes KillMode=process [Install] WantedBy=multi-user.target # docker startup parameter file, just create the corresponding directory and file without editing; # mk-docker-opts.sh will use the information allocated from etcd to edit this file [root@kubenode1 flannel] # mkdir -p /run/ flannel [root@kubenode1 flannel] # touch /run/flannel/docker
6. Start flanneld
# docker服务应该在flanneld服务之后启动 [root@kubenode1 flannel]# systemctl daemon-reload [root@kubenode1 flannel]# systemctl enable flanneld [root@kubenode1 flannel]# systemctl stop docker [root@kubenode1 flannel]# systemctl start flanneld [root@kubenode1 flannel]# systemctl start docker
7. Verify
1) Network port
# docker0 has obtained the address of a 24-bit subnet in the Pod network segment from etcd, replacing the default address of 172.17.0.0/16; # Externally (except cross-host container communication), use the default bridge network to communicate with the outside in the form of nat; # At the same time, a flannel.1 network port is generated, and the cross-host communication is sent through the vxlan encapsulation of flannel.1; # flannel assigns an independent 24-bit subnet to each host, and flannel.1 connects these subnets and can be routed to each other; # flannel does not provide isolation [root@kubenode1 ~] # ip address show
2) Routing
#After the other nodes start the flanneld service, the relevant routes will be pushed to each host; #The following is the route from kubenode1 to kubenode2&kubenode3, encapsulated by flannel.1 [root@kubenode1 ~] # ip route
3) View the Pod network segment information allocated to each flanneld by etcd
#Cluster Pod network segment, that is, the Pod network segment information written to etcd [root@kubenode1 ~] # etcdctl \ --endpoints= ${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/ssl/ ca.pem \ --cert-file=/etc/kubernetes/flannel/ flanneld.pem \ --key-file=/etc/kubernetes/flannel/flanneld- key.pem \ get ${FLANNEL_ETCD_PREFIX}/config
# 已分配的Pod subnet列表 [root@kubenode1 ~]# etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/ssl/ca.pem \ --cert-file=/etc/kubernetes/flannel/flanneld.pem \ --key-file=/etc/kubernetes/flannel/flanneld-key.pem \ ls ${FLANNEL_ETCD_PREFIX}/subnets
#The ip and network parameters monitored by the flanneld process corresponding to a Pod subnet; #Select a Pod subnet list, such as 10.254.3.0/24, you can query the host node and vtep mac address where it is located [root@kubenode1 ~] # etcdctl \ --endpoints= ${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/ssl/ ca.pem \ --cert-file=/etc/kubernetes/flannel/ flanneld.pem \ --key-file= /etc/kubernetes/flannel/flanneld-key.pem \ get ${FLANNEL_ETCD_PREFIX}/subnets/10.254.3.0-24
4) ping test
# The last rule of the iptables input chain is "-A INPUT -j REJECT --reject-with icmp-host-prohibited", that is, it does not meet the rules defined by the input, and access is denied at the end of the chain; between flannel.1 It is vxlan communication encapsulated by udp, the input chain does not allow udp by default, resulting in the ping packets between docker0 will be discarded; # Cannot restart iptables, the docker service starts, or the Pod subnet changes when the rules are added to iptables, restarting will cause the relevant rules to be lost; # At the same time, you can comment out the following two rules in /etc/sysconfig/iptables [root@kubenode1 ~ ] # iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited [root@kubenode1 ~] # iptables -D FORWARD -j REJECT --reject-with icmp-host-prohibited # ping测试 [root@kubenode1 ~]# ping 10.254.3.1 [root@kubenode1 ~]# ping 10.254.71.1
