1, o ambiente básico
1. cfssl Instalação (simplesmente K8S-master01 nó pode)
$ wget -O /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
$ wget -O /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
$ wget -O /bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
$ for cfssl in `ls /bin/cfssl*`;do chmod +x $cfssl;done;
arquivo 2. Configurar anfitriões
cat >>/etc/hosts<< EOF
k8s-master01 10.0.0.31
k8s-master02 10.0.0.32
k8s-master03 10.0.0.39
EOF
2, geração de certificados ETCD
1. Instale ETCD
$ yum install -y etcd
2. Configure Certificate
vim etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.0.0.31",
"10.0.0.32",
"10.0.0.39"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "shenzh",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
vim ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
vim ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "shenzh",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
3. Criar um certificado CA e uma chave privada
$ cfssl gencert -initca ca-csr.json | cfssljson -bare ca
4. certificado e ETCD geradora de chave privada
$ cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
// 查看生成的证书
$ ls *.pem
ca-key.pem ca.pem etcd-key.pem etcd.pem
$ mkdir -pv /etc/etcd/ssl
$ cp -r ./{ca-key,ca,etcd-key,etcd}.pem /etc/etcd/ssl/
5. Copie o certificado para outro nó
scp -r ./ [email protected]:/etc/etcd/ssl
scp -r ./ [email protected]:/etc/etcd/ssl
6. Modificação dos respectivos nós ETCD etcd.conf perfil
k8s-master01
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.31:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.31:2379"
ETCD_NAME="k8s-master01"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.31:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.31:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
k8s-master02
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.32:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.32:2379"
ETCD_NAME="k8s-master02"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.32:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.32:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
k8s-master03
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.39:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.39:2379"
ETCD_NAME="k8s-master03"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.39:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.39:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
Configuração Parâmetro Significado
· ETCD_NAME 节点名称
· ETCD_DATA_DIR 数据目录
· ETCD_LISTEN_PEER_URLS 集群通信监听地址
· ETCD_LISTEN_CLIENT_URLS 客户端访问监听地址
· ETCD_INITIAL_ADVERTISE_PEER_URLS 集群通告地址
· ETCD_ADVERTISE_CLIENT_URLS 客户端通告地址
· ETCD_INITIAL_CLUSTER 集群节点地址
· ETCD_INITIAL_CLUSTER_TOKEN 集群Token
· ETCD_INITIAL_CLUSTER_STATE 加入集群的当前状态,new是新集群,existing表示加入已有集群
Iniciar ETCD e start-up em cada nó, execute o seguinte comando
$ systemctl start etcd
$ systemctl enable etcd
Ver ETCD está funcionando corretamente
$ etcdctl --endpoints "https://10.0.0.31:2379,https://10.0.0.32:2379,https://10.0.0.39:2379" --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
// 输出如下
member 61105fb5ea81da2 is healthy: got healthy result from https://10.0.0.39:2379
member 1f46bee47a4f04aa is healthy: got healthy result from https://10.0.0.31:2379
member 6443b97f5544707b is healthy: got healthy result from https://10.0.0.32:2379
cluster is healthy
Então aglomerado muito ETCD configuração é concluída
3, de alta disponibilidade mestre
1. Duas LB (carga de máquina de equilíbrio) montado haproxy
$ yum install haproxy
$ cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 10000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode tcp
log global
retries 3
timeout connect 10s
timeout client 1m
timeout server 1m
frontend kube-apiserver
bind *:6443 # 指定前端端口
mode tcp
default_backend master
backend master # 指定后端机器及端口,负载方式为轮询
balance roundrobin
server k8s-master01 10.0.0.31:6443 check maxconn 2000
server k8s-master02 10.0.0.32:6443 check maxconn 2000
server k8s-master03 10.0.0.39:6443 check maxconn 2000
Iniciar haproxy
$ systemctl enable haproxy
$ systemctl start haproxy
implantação 2. Dois lb
$ yum install keepalived
主lb
$ vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email_from [email protected]
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.101 #漂移ip
}
}
备lb
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.101 #漂移ip
}
}
Iniciar keepalived
systemctl start keepalived
systemctl enable keepalived
3. Certificado de movimento ETCD/etc/kubernetes/pki/
$ mkdir -p /etc/kubernetes/pki/
$ cp -r {ca,etcd,etcd-key}.pem /etc/kubernetes/pki/
4. O arquivo de configuração kubeadm-config.yaml
$ cat /root/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.31
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "10.0.0.101:6443" # 该地址和vip要一致
controllerManager: {}
dns:
type: CoreDNS
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.15.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
etcd:
external:
endpoints:
- https://10.0.0.31:2379
- https://10.0.0.32:2379
- https://10.0.0.39:2379
caFile: /etc/kubernetes/pki/ca.pem
certFile: /etc/kubernetes/pki/etcd.pem
keyFile: /etc/kubernetes/pki/etcd-key.pem
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
mestre de inicialização
$ kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs
Após as instruções de inicialização executado
$ mkdir -p $HOME/.kube
$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config
5. Adicionar outro nó mestre
Kubeadm versão original, a junção de comando só é usado para unir o nó de trabalho, e a nova versão adiciona os parâmetros --control-plano, o plano de controlo (mestre) nó a ser adicionado ao cluster kubeadm juntar comando.
$ kubeadm join 10.0.0.101:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:699cdd59cfa20509cc25794c5b153678a8ff354c9401e215cb1e41d750cbeb54 \
--control-plane --certificate-key 87d0f654fd4d2d563969ce24fa226321a3fd098477e0528479476fce3bf404c3
6. Adicione nó nó
$ kubeadm join 10.0.0.101:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:699cdd59cfa20509cc25794c5b153678a8ff354c9401e215cb1e41d750cbeb54