首先在主控节点200上签发证书
1. 编辑kubelet-csr.json文件
{
"CN": "k8s-kubelet",
"hosts": [
"127.0.0.1",
"192.168.1.10",
"192.168.1.21",
"192.168.1.22",
"192.168.1.23",
"192.168.1.24"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}
2. 签发证书
[root@hdss-1-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json |cfssl-json -bare kubelet
[root@hdss-1-200 certs]# ls -l|grep kubelet
-rw-r--r-- 1 root root 1115 Feb 10 15:28 kubelet.csr
-rw-r--r-- 1 root root 479 Feb 10 15:25 kubelet-csr.json
-rw------- 1 root root 1679 Feb 10 15:28 kubelet-key.pem
-rw-r--r-- 1 root root 1468 Feb 10 15:28 kubelet.pem
3.拷贝证书,私钥,注意文件属性600(以21节点为例)
[root@hdss-1-21 cert]# scp [email protected]:/opt/certs/kubelet-key.pem .
[root@hdss-1-21 cert]# scp [email protected]:/opt/certs/kubelet.pem .
[root@hdss-1-21 cert]# ll
total 32
-rw------- 1 root root 1675 Feb 10 09:49 apiserver-key.pem
-rw-r--r-- 1 root root 1598 Feb 10 09:49 apiserver.pem
-rw------- 1 root root 1675 Feb 10 09:49 ca-key.pem
-rw-r--r-- 1 root root 1346 Feb 10 09:48 ca.pem
-rw------- 1 root root 1675 Feb 10 09:49 client-key.pem
-rw-r--r-- 1 root root 1363 Feb 10 09:49 client.pem
-rw------- 1 root root 1679 Feb 10 15:35 kubelet-key.pem
-rw-r--r-- 1 root root 1468 Feb 10 15:35 kubelet.pem
[root@hdss-1-21 cert]#
4.分发证书
分发证书有四步:
(1)set cluster
[root@hdss-1-21 conf]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
--embed-certs=true \
--server=https://192.168.1.10:7443 \
--kubeconfig=kubelet.Rubeconfig
查看升成的文件内容
[root@hdss-1-21 conf]# ll
total 8
-rw-r--r-- 1 root root 2224 Feb 10 10:48 audit.yaml
-rw------- 1 root root 1989 Feb 10 15:53 kubelet.Rubeconfig
[root@hdss-1-21 conf]# cat kubelet.Rubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR0RENDQXB5Z0F3SUJBZ0lVRGkzQ0lwbGZsWGZxMzNWZnNXMVFXYWNGZDJnd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1lERUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjJKbGFXcHBibWN4RURBT0JnTlZCQWNUQjJKbAphV3BwYm1jeEN6QUpCZ05WQkFvVEFtOWtNUXd3Q2dZRFZRUUxFd052Y0hNeEVqQVFCZ05WQkFNVENVOXNaR0p2CmVVVmtkVEFlRncweU1UQXlNRGt3TnpJMU1EQmFGdzAwTVRBeU1EUXdOekkxTURCYU1HQXhDekFKQmdOVkJBWVQKQWtOT01SQXdEZ1lEVlFRSUV3ZGlaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZGlaV2xxYVc1bk1Rc3dDUVlEVlFRSwpFd0p2WkRFTU1Bb0dBMVVFQ3hNRGIzQnpNUkl3RUFZRFZRUURFd2xQYkdSaWIzbEZaSFV3Z2dFaU1BMEdDU3FHClNJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURPd2lKZ0FMb3Y0akVoQjEzNzNhdlgvK05EK0phMjU2RUgKMmk0RHV5eFJZTGZtNE5MSUpkZ3FpODM4SW1KNG5mdDV5L1Fhc0pHUUNNQTlRei8xbkoxeEpHU2Y4OXB3QW9MdwpkcFJockJiaDJhSk9rSUhnTDlCaVVUeXc4MllhbjlzN25iQVQ5VUhyeUUzeXg3Njl5YXNtTlNmc3JCRTNDNExsCnVGdHc1SXBmQmI4SU1yYzFZeHBNNkFtZHRFaHJ2eE9XUTRtWWlUbEJlcUdOejZmcTdHQXpEazhNb05uYkxaL0cKUFlnYjRKYlk0SHBkTXlpUEhLUVpBMjZNMXVqeXVrNFhTVnBpK0tvMktkZjBFQ0VJeFpPU0JqNUY4WDlOeWU3UwpmK1VQajBRVVZEYUpYMlU3cE1VdS9obFVYNHU0dlJKL1dhTWtHUEREeUJxV3BCcTVHdFhMQWdNQkFBR2paakJrCk1BNEdBMVVkRHdFQi93UUVBd0lCQmpBU0JnTlZIUk1CQWY4RUNEQUdBUUgvQWdFQ01CMEdBMVVkRGdRV0JCVGkKOHkvdGlmNUlWRlREdno2UjdGakx5TFhWbHpBZkJnTlZIU01FR0RBV2dCVGk4eS90aWY1SVZGVER2ejZSN0ZqTAp5TFhWbHpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQUdlOWNTN0VuVEROR0FqWDlqanZXejQ1OW9ZTlk3cndYCmJLQkc4MGpBcDVSMDRNbUdvQW9TMm5LbUx0N1FXaEd6TisyS2VlN0VFd0tzOURYUFZPbzMwT1FiNVloc0tWeUYKOWt5WjMzWHNObHpRODFiNE1IS0xQWDY5MnVQdVUrRTJ3bzdQR1ZzbFNQUTZ5cmFpY2NVZkdxNEphWVE0aURPZwovRXVacnkrWENsMVF6ZExSVFRpakFEVW9za0gwSURWa2luaHRtbE1DS2MvTXN2Q0RGTEk2WEx6Y0RaeXI1dGFICjFzc1h0eXVEQ25uMlpXWHljYnMzNUMzVTVKR0dROUF4NUM5ZXFBNXA2VE9sNWt6OTRyVTMweUpjWXh3dm9Oa3UKcUQ4NTgxczFSTWFTakp6SUZUb1loRzV0OUxGYUY3Tk9ESmM0TlNCQ2dLcVBGVndDNWMyZUlRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
server: https://192.168.1.10:7443
name: myk8s
contexts: []
current-context: ""
kind: Config
preferences: {}
users: []
[root@hdss-1-21 conf]#
(2)set-credentials
[root@hdss-1-21 conf]# kubectl config set-credentials k8s-node \
--client-certificate=/opt/kubernetes/server/bin/cert/client.pem \
--client-key=/opt/kubernetes/server/bin/cert/client-key.pem \
--embed-certs=true \
--kubeconfig=kubelet.kubeconfig
[root@hdss-1-21 conf]# ll
total 16
-rw-r--r-- 1 root root 2224 Feb 10 10:48 audit.yaml
-rw------- 1 root root 4230 Feb 10 15:59 kubelet.kubeconfig
-rw------- 1 root root 1989 Feb 10 15:53 kubelet.Rubeconfig
(3) set-context
[root@hdss-1-21 conf]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=k8s-node \
--kubeconfig=kubelet.kubeconfig
(4) use-context
[root@hdss-1-21 conf]# kubectl config use-context myk8s-context --kubeconfig=kubelet.kubeconfig
查看kubelet.kubeconfig 完整信息
[root@hdss-1-21 conf]# cat kubelet.kubeconfig
apiVersion: v1
clusters: []
contexts:
- context:
cluster: myk8s
user: k8s-node
name: myk8s-context
current-context: myk8s-context
kind: Config
preferences: {}
users:
- name: k8s-node
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR3akNDQXFxZ0F3SUJBZ0lVSjRIdDhFdlI2ZjNCMm5FZHF1c3FtcEJ5YTNjd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1lERUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjJKbGFXcHBibWN4RURBT0JnTlZCQWNUQjJKbAphV3BwYm1jeEN6QUpCZ05WQkFvVEFtOWtNUXd3Q2dZRFZRUUxFd052Y0hNeEVqQVFCZ05WQkFNVENVOXNaR0p2CmVVVmtkVEFlRncweU1UQXlNVEF3TVRJNU1EQmFGdzAwTVRBeU1EVXdNVEk1TURCYU1GOHhDekFKQmdOVkJBWVQKQWtOT01SQXdEZ1lEVlFRSUV3ZGlaV2xxYVc1bk1SQXdEZ1lEVlFRSEV3ZGlaV2xxYVc1bk1Rc3dDUVlEVlFRSwpFd0p2WkRFTU1Bb0dBMVVFQ3hNRGIzQnpNUkV3RHdZRFZRUURFd2hyT0hNdGJtOWtaVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSzk5b1orNU1ZeXl6eWFxRUNsaTYyME4vVDJadFc0QkV3UVoKYmU5dlR4OVE1bE1uS2EyVEZ3em13a2pLSVZZM29BbWdlT01zRGlVc09ETERNMnFORnVpVVNhcjVCRVd5cVJpYgp0WU9PeTJtY0lCd2IvSEwram10UGp1cWJBV21OdSs3TjRiRDJ0b0pOa2JXYkR0NnBodUlrd0FTaHd1QVlXZXd0CkVkTDdCN3JYNm1EVXlISVd5ZXJPb3B0ZTBkNnhRa2ZWUC9MNkp4YTEyUlFnd3BKU3I2L0loL2VmaHpVTkNlUlAKOUIwU1VOTFZuc1QwUk54SC9KTUZTWUg5N0FsMTNKTVNTR0ZoaDFzcXMwb2ZKNzNnTzRIUjkwVVdTSjBRMkoyRgpZV0ZvYjBJZTZMTUNEdjRMTHMzS2sxU3lrQ2V3SFRZdzIxRCtCREl0aFJyOGtFOUlQejhDQXdFQUFhTjFNSE13CkRnWURWUjBQQVFIL0JBUURBZ1dnTUJNR0ExVWRKUVFNTUFvR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUMKTUFBd0hRWURWUjBPQkJZRUZFaFYrL3hmWkd5MnBQNGpnUldYVUpUQkNzNGVNQjhHQTFVZEl3UVlNQmFBRk9MegpMKzJKL2toVVZNTy9QcEhzV012SXRkV1hNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUJNRUdia0RqQWhGVUxLCnV3MHl0RGt5UXQwNlo0dVhoNzJSYnJuSnNzTUplMllFenFzUVpTaEV3MGpHNC8wNStOdEpUYzhMazBPK0hMMjQKL3lKLzIrektCS24wQ3BMOER0VnZ2OGlHWThRUi8rMWo1MlVVSUxsSm9YYkR3b1pXajJNdCtNWUZvdHlIQTF3agplZG82S2NjSkJJSUdmcVFrZ3hqb2ZXcWlLYXJ1cUxUempSdkZrWldwWHFGOVhIcHd0ck8yNllubGNWdDNOTDIzCkk2cERSS0NEZy8zOXJkUFBFczI4U3pGTnhtcUx2VFhiQ2FFSTk4UlI4b1lOaXY1T0ViQ0YvWVpqeGprZ1NjRGwKZmo2Z3lLbzIyc0dpaXFzSVp0dTAxZ2dVaWs4V3hHV3hVbFhLeXhKR2FOWUdGUmR1bmd3UDFRUXVJMDNjb1lFWQphcWhZV2FqcQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBcjMyaG43a3hqTExQSnFvUUtXTHJiUTM5UFptMWJnRVRCQmx0NzI5UEgxRG1VeWNwCnJaTVhET2JDU01vaFZqZWdDYUI0NHl3T0pTdzRNc016YW8wVzZKUkpxdmtFUmJLcEdKdTFnNDdMYVp3Z0hCdjgKY3Y2T2EwK082cHNCYVkyNzdzM2hzUGEyZ2syUnRac08zcW1HNGlUQUJLSEM0QmhaN0MwUjB2c0h1dGZxWU5USQpjaGJKNnM2aW0xN1IzckZDUjlVLzh2b25GclhaRkNEQ2tsS3ZyOGlIOTUrSE5RMEo1RS8wSFJKUTB0V2V4UFJFCjNFZjhrd1ZKZ2Yzc0NYWGNreEpJWVdHSFd5cXpTaDhudmVBN2dkSDNSUlpJblJEWW5ZVmhZV2h2UWg3b3N3SU8KL2dzdXpjcVRWTEtRSjdBZE5qRGJVUDRFTWkyRkd2eVFUMGcvUHdJREFRQUJBb0lCQUdyRTg4MmF2Z1BOVW0xQwpzTHlOVzVFcEhyY1l5cUtYOHJyZXdkMFlDcnFSM09BZ2hxSDlTTnZubGJMMWo1ekdFUTlxVzhKVExuUDlXRi9DCkZZa3hPQXZmRkliNkpGaEhYOHdDQ0VoM1cxand6VFJzN29ZRVRDQUw4bmJZUWQ2M2hleS94QjJIclBqUU91T1QKNkZNZUwxdHRZb291NU1LMDdoamNQdWp2SS8vUGxYa1ZyelBVYjMweEwxSjh4Q2taem5naGRHSXc4WVA2aTUzbwpveFZPNm0xbDZvakFrbi8yaGhnQ0pIeE1taUJNcHJzK0l4YXZIekNVRnhEWmFuLzVwb2RmUklEcG8vRGYrMEtwCjRtSkVQY2gxZVQ0YU9FS2tRdXZyeW45Yk9rVmdPZ291NllYVlkxQjVKS3FZR0IvTDRTTlJIUEpLOFZBNDZqVmkKMDM3U1YwRUNnWUVBNFc4cGJ4Mmt6NWRlZmViU3dyQ3V5bzdsNGoyblJHQ281YmhocEVLNWo2ZFdYYVQycE9YOQpHRDRrWlMxSzArOFpkVnEzbTZNM3ZaR3JHNndJVzVaeGR3SFRvNEJTaW9WMVRRelFPM0VBd3oxdEx3K1JlZmNHCnQzeGIrRVVBZ2JIam9DWDRXZEZnbEI4MDRCa3pDTlM1WGlvWEU3anZDc0gxMUFKdWdGVytmWjhDZ1lFQXgwanQKZ2xpdHQ2d0F3TGtEMzFLeUtHZkdnb1RGVGg1RnRaeFNOMFJ1MzlCMVAxNzhkT0JaV1F2RmJycXBzQ2d3eWJHegovcDE3dkt4anR6R2lBQzVFU0dMMklnNWtWMWJ6a1N6RXVWV1UxUjdsb05YblI5OXRrOXF1UFQ3MlZhSkxtNWJzCjlZaTNUeWp0dmZGOFNjSjhaTGUxcFVtSnM5cG94RFVjMGdiWG1tRUNnWUF2dVVxNlBUcFF3Q2krMm9pdnNwdXUKUWdZa21ReTZDd1IyYlV3dWEvbkFrWnBWQk1FNk0zNGJEYnpDdG8zcXcrbE4wTUNoOERHeXJnM2RNOVFwSDNZeAp6Vzh5WTg0VmlSOUdDbnBpdXcrQVJaYkJxYmdRYTkwK0p0T0pnWUlUOHNiUGRucFcrVjVLdlRpQ1lKTlBRVWxrCngrQk95bzNBWjBOVmVKWlFFTmJjM1FLQmdGUWU1S0x3NitOc0o2VXlkQUFBMkZOVit1VjlSZ3dEbTBsZUF4WngKUEp3ZFVtQzlwN051UUVqbVB5RE9pbkpoaExhZTNHdk43ajFtUjNFODhqVkJVbkY3MUlucnFVOTAxTHVqT2lkQgppT0d0ZVNDN0tqV2NpcFN1Nm4zUDVndWZ4bXFXK3M5cU5uYWp3RGM5aUVsYVgwZmtoRGlSSWg5WlFvQVFLMm84Cm1TWUJBb0dBTzh0TTZ0SlZNN0hyai9uT21YdU9yVGRtQXBHdEZaWUhEYnd1NkZja2RQUEZMNXIza1NQMDBvK1cKNUJZd3hueCtRVHNEek5KSGhDUTNnV3VmbXRpN3liY0kxVkxKMXZuUGdrVGlubFhObGxsYWFacHlDeVRPdS8rSgo4UlNhQUNIWHp2NUxZMlY2ZnRMR21JVFQ3ZHhlQStjbyt3QU8yWmZBR3k0cFJCUVJvdlE9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
角色绑定
创建 k8s-node.yaml
[root@hdss-1-21 conf]# vim k8s-node.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: k8s-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: k8s-node
[root@hdss-1-21 conf]# kubectl create -f k8s-node.yaml
clusterrolebinding.rbac.authorization.k8s.io/k8s-node created
创建一个clusterrolebinding的用户资源
[root@hdss-1-21 conf]# kubectl get clusterrolebinding k8s-node -o yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: "2021-02-10T08:28:50Z"
name: k8s-node
resourceVersion: "11170"
selfLink: /apis/rbac.authorization.k8s.io/v1/clusterrolebindings/k8s-node
uid: e28af683-eeb4-47ed-b5ee-c13ebc0b4b0a
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: k8s-node
22节点上将分发的证书拷过来即可
[root@hdss-1-22 conf]# scp hdss-1-21:/opt/kubernetes/server/bin/conf/kubelet.kubeconfig .
[root@hdss-1-22 conf]# ll
total 12
-rw-r--r-- 1 root root 2223 Feb 10 10:14 audit.yaml
-rw------- 1 root root 4309 Feb 10 16:37 kubelet.kubeconfig
准备pause镜像在运维主机200上,kubelet再启动的时候需要有一个基础镜像去启动pob
[root@hdss-1-200 ~]# docker pull kubernetes/pause
Using default tag: latest
latest: Pulling from kubernetes/pause
4f4fb700ef54: Pull complete
b9c8ec465f6b: Pull complete
Digest: sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105
Status: Downloaded newer image for kubernetes/pause:latest
docker.io/kubernetes/pause:latest
[root@hdss-1-200 ~]#
登录到harbo私有仓库进行打包上传操作
[root@hdss-1-200 ~]# docker login harbor.od.com
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@hdss-1-200 ~]# docker images|grep pause
kubernetes/pause latest f9d5de079539 6 years ago 240kB
[root@hdss-1-200 ~]# docker tag f9d5de079539 harbor.od.com/public/pause:latest
[root@hdss-1-200 ~]# docker push harbor.od.com/public/pause:latest
The push refers to repository [harbor.od.com/public/pause]
5f70bf18a086: Mounted from public/nginx
e16a89738269: Pushed
latest: digest: sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105 size: 938
[root@hdss-1-200 ~]#
启动kubernetes服务
添加启动脚本并赋执行权限
[root@hdss-1-22 bin]# vim kubelet.sh
#!/bin/sh
./kubelet \
--anonymous-auth=false \
--cgroup-driver systemd \
--cluster-dns 192.168.0.2 \
--cluster-domain cluster.local \
--runtime-cgroups=/systemd/system.slice \
--kubelet-cgroups=/systemd/system.slice \
--fail-swap-on="false" \
--client-ca-file ./cert/ca.pem \
--tls-cert-file ./cert/kubelet.pem \
--tls-private-key-file ./cert/kubelet-key.pem \
--hostname-override hdss-1-22.host.com \
--image-gc-high-threshold 20 \
--image-gc-low-threshold 10 \
--kubeconfig ./conf/kubelet.kubeconfig \
--log-dir /data/logs/kubernetes/kube-kubelet \
--pod-infra-container-image harbor.od.com/public/pause:latest \
--root-dir /data/kubelet
[root@hdss-1-22 bin]# chmod u+x kubelet.sh
创建脚本中的目录
[root@hdss-1-22 bin]# mkdir -p /data/logs/kubernetes/kube-kubelet
[root@hdss-1-22 bin]# mkdir /data/kubelet
编辑后台启动脚本
[root@hdss-1-22 bin]# vi /etc/supervisord.d/kube-kubelet.ini
[program:kube-kubelet-1-22]
command=/opt/kubernetes/server/bin/kubelet.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
启动
[root@hdss-1-22 bin]# supervisorctl update
kube-scheduler-1-22: added process group
[root@hdss-1-22 bin]# supervisorctl status
给角色打标签
[root@hdss-1-21 bin]# kubectl label node hdss-1-21.host.com node-role.kubernetes.io/master=
node/hdss-1-21.host.com labeled
[root@hdss-1-21 bin]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
hdss-1-21.host.com Ready master 3m52s v1.15.2
hdss-1-22.host.com Ready <none> 4m19s v1.15.2
[root@hdss-1-21 bin]#