Kubernetes安装 | k8s组件安装

                      --昨夜西风凋碧树，独上高楼，望尽天涯路

请先看上篇博客：如何搭建基础环境以及etcd集群

• 准备kubernetes的证书

在master节点操作

（1）创建相关目录

mkdir  $HOME/ssl && cd $HOME/ssl

（2）配置 &生成 root ca

#配置root ca
cat > ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
     "expiry": "87600h"
  }
}
EOF

#生成root ca
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*.pem

（3）配置kube-apiserver证书

#配置apiserver
#10.96.0.1 是 kube-apiserver 指定的 service-cluster-ip-range 网段的第一个IP
cat > kube-apiserver-csr.json << EOF
{
    "CN": "kube-apiserver",
    "hosts": [
      "127.0.0.1",
      "192.168.2.207",
      "192.168.2.181",
      "192.168.2.182",
      "10.96.0.1",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

#生成apiserver
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
ls kube-apiserver*.pem

（4）配置&生成 kube-controller-manager证书

#配置kube-controller-manager
cat > kube-controller-manager-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "hosts": [
      "127.0.0.1",
      "192.168.2.207",
      "192.168.2.181",
      "192.168.2.182"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-controller-manager",
            "OU": "System"
        }
    ]
}
EOF

#生成kube-controller-manager
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
ls kube-controller-manager*.pem

(5)配置&生成kube-scheduler证书

#配置kube-scheduler
cat > kube-scheduler-csr.json << EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "192.168.2.207",
      "192.168.2.181",
      "192.168.2.182"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-scheduler",
            "OU": "System"
        }
    ]
}
EOF

#生成kube-scheduler
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
ls kube-scheduler*.pem

（6）配置&生成 kube-proxy 证书

#配置kube-proxy
cat > kube-proxy-csr.json << EOF
{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-proxy",
            "OU": "System"
        }
    ]
}
EOF

#生成kube-proxy
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
ls kube-proxy*.pem

（7）配置&生成 admin 证书

#配置admin
cat > admin-csr.json << EOF
{
    "CN": "admin",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:masters",
            "OU": "System"
        }
    ]
}
EOF

#生成admin
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
ls admin*.pem

（8）复制生成的证书文件，并分发至其他节点

mkdir -pv /etc/kubernetes/pki
cp ca*.pem admin*.pem kube-proxy*.pem kube-scheduler*.pem kube-controller-manager*.pem kube-apiserver*.pem /etc/kubernetes/pki
scp -r /etc/kubernetes 192.168.2.181:/etc/
scp -r /etc/kubernetes 192.168.2.182:/etc/

• 安装master

下载解压server包并配置环境变量

对于无法访问google的童鞋，请去该地址下载server&node包：

https://pan.baidu.com/s/1DvZ8FenR_zAjtMOGf-sV8A

cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-server-linux-amd64.tar.gz
tar -xf kubernetes-server-linux-amd64.tar.gz -C /usr/local
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes
	
cat > /etc/profile.d/kubernetes.sh << EOF
k8s_home=/usr/local/kubernetes
export PATH=\$k8s_home/server/bin:\$PATH
source <(kubectl completion bash)
EOF
	
source /etc/profile.d/kubernetes.sh
kubectl version

生成kubeconfig：

#使用 TLS Bootstrapping
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')

cat > /etc/kubernetes/token.csv << EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF


#创建 kubelet bootstrapping kubeconfig
cd /etc/kubernetes

export KUBE_APISERVER="https://192.168.2.207:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap.conf
  
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap.conf
  
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap.conf

kubectl config use-context default --kubeconfig=kubelet-bootstrap.conf


#创建 kube-controller-manager kubeconfig
export KUBE_APISERVER="https://192.168.2.207:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.conf
  
 kubectl config set-credentials kube-controller-manager \
--client-certificate=/etc/kubernetes/pki/kube-controller-manager.pem \
--client-key=/etc/kubernetes/pki/kube-controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-controller-manager.conf
  
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-controller-manager \
--kubeconfig=kube-controller-manager.conf
  
kubectl config use-context default --kubeconfig=kube-controller-manager.conf

#创建 kube-scheduler kubeconfig
export KUBE_APISERVER="https://192.168.2.207:6443"

kubectl config set-cluster kubernetes \
 --certificate-authority=/etc/kubernetes/pki/ca.pem \
 --embed-certs=true \
 --server=${KUBE_APISERVER} \
 --kubeconfig=kube-scheduler.conf
  
kubectl config set-credentials kube-scheduler \
--client-certificate=/etc/kubernetes/pki/kube-scheduler.pem \
--client-key=/etc/kubernetes/pki/kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.conf
  
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-scheduler \
--kubeconfig=kube-scheduler.conf
  
kubectl config use-context default --kubeconfig=kube-scheduler.conf

#创建 kube-proxy kubeconfig
export KUBE_APISERVER="https://192.168.2.207:6443"
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/pki/ca.pem \
  --embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.conf
  
kubectl config set-credentials kube-proxy \
  --client-certificate=/etc/kubernetes/pki/kube-proxy.pem \
  --client-key=/etc/kubernetes/pki/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.conf
  
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.conf
  
kubectl config use-context default --kubeconfig=kube-proxy.conf

#创建 admin kubeconfig
export KUBE_APISERVER="https://192.168.2.207:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/pki/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=admin.conf
  
kubectl config set-credentials admin \
  --client-certificate=/etc/kubernetes/pki/admin.pem \
  --client-key=/etc/kubernetes/pki/admin-key.pem \
  --embed-certs=true \
  --kubeconfig=admin.conf
  
kubectl config set-context default \
  --cluster=kubernetes \
  --user=admin \
  --kubeconfig=admin.conf
  
kubectl config use-context default --kubeconfig=admin.conf

配置启动kube-apiserver

#复制 etcd ca
mkdir -pv /etc/kubernetes/pki/etcd
cd /etc/etcd/ssl
cp etcd-ca.pem etcd-key.pem etcd.pem /etc/kubernetes/pki/etcd

#生成 service account key
openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub
ls /etc/kubernetes/pki/sa.*
cd $HOME

#启动文件
cat > /etc/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Service
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
ExecStart=/usr/local/kubernetes/server/bin/kube-apiserver \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBE_ETCD_ARGS \\
      \$KUBE_API_ADDRESS \\
      \$KUBE_SERVICE_ADDRESSES \\
      \$KUBE_ADMISSION_CONTROL \\
      \$KUBE_APISERVER_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#该配置文件同时被 kube-apiserver, kube-controller-manager, kube-scheduler, kubelet, kube-proxy 使用

cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
EOF

cat > /etc/kubernetes/apiserver << EOF
KUBE_API_ADDRESS="--advertise-address=192.168.2.207"
KUBE_ETCD_ARGS="--etcd-servers=https://192.168.2.207:2379,https://192.168.2.181:2379,https://192.168.2.182:2379 --etcd-cafile=/etc/kubernetes/pki/etcd/etcd-ca.pem --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.96.0.0/12"
KUBE_ADMISSION_CONTROL="--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
KUBE_APISERVER_ARGS="--allow-privileged=true --authorization-mode=Node,RBAC --enable-bootstrap-token-auth=true --token-auth-file=/etc/kubernetes/token.csv --service-node-port-range=30000-32767 --tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem --tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem --client-ca-file=/etc/kubernetes/pki/ca.pem --service-account-key-file=/etc/kubernetes/pki/sa.pub --enable-swagger-ui=true --secure-port=6443 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --anonymous-auth=false --kubelet-client-certificate=/etc/kubernetes/pki/admin.pem --kubelet-client-key=/etc/kubernetes/pki/admin-key.pem"
EOF

#启动
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

#访问测试
curl -k https://192.168.2.207:6443/

出现一下内容说明搭建成功：
{
  "kind": "Status",
"apiVersion": "v1",
"metadata": {

},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}

配置启动kube-controller-manager

#启动文件
cat > /etc/systemd/system/kube-controller-manager.service << EOF
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/controller-manager
ExecStart=/usr/local/kubernetes/server/bin/kube-controller-manager \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#配置文件
cat >/etc/kubernetes/controller-manager<<EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-controller-manager.conf"
KUBE_CONTROLLER_MANAGER_ARGS="--address=127.0.0.1 --cluster-cidr=10.0.0.0/8 --cluster-name=kubernetes --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem --service-account-private-key-file=/etc/kubernetes/pki/sa.key --root-ca-file=/etc/kubernetes/pki/ca.pem --leader-elect=true --use-service-account-credentials=true --node-monitor-grace-period=10s --pod-eviction-timeout=10s --allocate-node-cidrs=true --controllers=*,bootstrapsigner,tokencleaner"
EOF

#启动
systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager

配置启动kube-scheduler

#systemctl启动文件
cat > /etc/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler Plugin
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/scheduler
ExecStart=/usr/local/kubernetes/server/bin/kube-scheduler \\
            \$KUBE_LOGTOSTDERR \\
            \$KUBE_LOG_LEVEL \\
            \$KUBECONFIG \\
            \$KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#配置文件
cat > /etc/kubernetes/scheduler << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-scheduler.conf"
KUBE_SCHEDULER_ARGS="--leader-elect=true --address=127.0.0.1"
EOF

#启动
systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler

配置kubectl

rm -rf $HOME/.kube
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get node

查看各个组件的状态

kubectl get componentstatuses

配置kubelet使用bootstrap

kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

• 配置cni和kubelet

(1)master端操作

#下载cni包
cd /root
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin


#配置启动kubelet
#配置启动文件
cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/server/bin/kubelet \\
            \$KUBE_LOGTOSTDERR \\
            \$KUBE_LOG_LEVEL \\
            \$KUBELET_CONFIG \\
            \$KUBELET_HOSTNAME \\
            \$KUBELET_POD_INFRA_CONTAINER \\
            \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF


cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
EOF


cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.2.207"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF


cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.2.207
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
  - 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
  anonymous:
    enbaled: false
  webhook:
    enbaled: false
EOF

#启动kubelet
systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

(2)在node1上操作

#下载cni包
cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-node-linux-amd64.tar.gz
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz

tar -xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

#配置启动kubelet
#配置systemctl启动文件

cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/node/bin/kubelet \\
            \$KUBE_LOGTOSTDERR \\
            \$KUBE_LOG_LEVEL \\
            \$KUBELET_CONFIG \\
            \$KUBELET_HOSTNAME \\
            \$KUBELET_POD_INFRA_CONTAINER \\
            \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF


cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
EOF

cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.2.181"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF


cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.2.181
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
  - 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
  anonymous:
    enbaled: false
  webhook:
    enbaled: false
EOF

#启动kubelet
systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet	

(3)在node2上操作

#下载cni包
cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-node-linux-amd64.tar.gz
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz

tar -xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

#配置启动kubelet
#配置systemctl启动文件

cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/node/bin/kubelet \\
            \$KUBE_LOGTOSTDERR \\
            \$KUBE_LOG_LEVEL \\
            \$KUBELET_CONFIG \\
            \$KUBELET_HOSTNAME \\
            \$KUBELET_POD_INFRA_CONTAINER \\
            \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF


cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
EOF

cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.2.182"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF


cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.2.182
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
  - 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
  anonymous:
    enbaled: false
  webhook:
    enbaled: false
EOF

#启动kubelet
systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

(4)通过证书验证添加各个节点

#在master节点操作

kubectl get csr

	
#通过验证并添加进集群

kubectl get csr | awk '/node/{print $1}' | xargs kubectl certificate approve

###单独执行命令例子：
    kubectl certificate approve node-csr-Yiiv675wUCvQl3HH11jDr0cC9p3kbrXWrxvG3EjWGoE


#查看节点
#此时节点状态为 NotReady，因为还没有配置网络


kubectl get nodes

	
# 在node节点查看生成的文件

ls -l /etc/kubernetes/kubelet.conf
ls -l /etc/kubernetes/pki/kubelet*

• 配置kube-proxy

在master节点操作

#安装conntrack-tools
yum install -y conntrack-tools

#启动文件
cat > /etc/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/local/kubernetes/server/bin/kube-proxy \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#启用ipvs主要就是把kube-proxy的--proxy-mode配置选项修改为ipvs
#并且要启用--masquerade-all，使用iptables辅助ipvs运行

cat > /etc/kubernetes/proxy << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-proxy.conf"
KUBE_PROXY_ARGS="--proxy-mode=ipvs  --masquerade-all=true --cluster-cidr=10.0.0.0/8"
EOF

#启动
systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
systemctl status kube-proxy

在所有的node上操作

#安装
yum install -y conntrack-tools

#启动文件
cat > /etc/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/local/kubernetes/node/bin/kube-proxy \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF


#启用ipvs主要就是把kube-proxy的--proxy-mode配置选项修改为ipvs
#并且要启用--masquerade-all，使用iptables辅助ipvs运行

cat > /etc/kubernetes/proxy << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-proxy.conf"
KUBE_PROXY_ARGS="--proxy-mode=ipvs --masquerade-all=true --cluster-cidr=10.0.0.0/8"
EOF

#启动
systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
systemctl status kube-proxy

• 设置集群角色

在master节点操作

#设置 192.168.2.207 为 master
kubectl label nodes 192.168.2.207 node-role.kubernetes.io/master=

#设置 192.168.2.181 - 182 为 node
kubectl label nodes 192.168.2.181 node-role.kubernetes.io/node=
kubectl label nodes 192.168.2.182 node-role.kubernetes.io/node=

#设置 master 一般情况下不接受负载
kubectl taint nodes 192.168.2.207 node-role.kubernetes.io/master=true:NoSchedule

查看节点

• 配置网络

以下网络二选一：

使用flannel网络

cd /root/
mkdir flannel
cd flannel
wget https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml

sed -ri 's#("Network": ")10.244.0.0/16#\110.0.0.0/8#' kube-flannel.yml
#修改kube-flannel文件中的网段为我们需要的网段

kubectl apply -f .

使用canal网络

cd /root/
mkdir canal
cd canal

wget https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/rbac.yaml
wget https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/canal.yaml

sed -ri 's#("Network": ")10.244.0.0/16#\110.0.0.0/8#' canal.yaml
#修改cannl文件中的网段为我们需要的网段

kubectl apply -f .

#查看网络容器是否为running状态
kubectl get -n kube-system pod -o wide	
	
[root@master ~]# kubectl get -n kube-system pod -o wide		
NAME          READY     STATUS    RESTARTS   AGE       IP            NODE
canal-45fde   3/3       Running   0          7m        192.168.2.207   192.168.2.207
canal-fsdgs   3/3       Running   0          7m        192.168.2.181   192.168.2.181
canal-fasfa   3/3       Running   0          7m        192.168.2.182   192.168.2.182

#查看各个节点是否为Ready状态
kubectl get node 

[root@master ~]# 
NAME          STATUS    ROLES     AGE       VERSION
192.168.2.207   Ready     master    5h        v1.11.1
192.168.2.181   Ready     node      5h        v1.11.1
192.168.2.182   Ready     node      5h        v1.11.1

 

• 配置使用coredns

#10.96.0.10 是kubelet中配置的dns
#安装coredns

cd /root && mkdir coredns && cd coredns
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/deploy.sh
chmod +x deploy.sh
./deploy.sh -i 10.96.0.10 > coredns.yml
kubectl apply -f coredns.yml


#查看

kubectl get svc,pods -n kube-system
	

[root@master coredns]# kubectl get svc,pods -n kube-system
NAME               TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE
service/kube-dns   ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP   2m

NAME                           READY     STATUS    RESTARTS   AGE
pod/canal-4fafa                3/3       Running   0          17h
pod/canal-3ewhz                3/3       Running   0          17h
pod/canal-3rewf                3/3       Running   2          17h
pod/coredns-6975654877-jfdfc   1/1       Running   0          2m
pod/coredns-6975654877-3dsaf   1/1       Running   0          2m