15.Calico组件的安装
15.1 安装calico
[root@ansible-server ansible]
[root@ansible-server ansible]
[root@ansible-server calico]
tasks templates vars
[root@ansible-server calico]
HARBOR_DOMAIN: harbor.raymonds.cc
[root@ansible-server calico]
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
---
kind: ConfigMap
apiVersion: v1
metadata:
name: calico-config
namespace: kube-system
data:
etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"
etcd_ca: ""
etcd_cert: ""
etcd_key: ""
typha_service_name: "none"
calico_backend: "bird"
veth_mtu: "1440"
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "calico",
"log_level": "info",
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "calico-ipam"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {
"portMappings": true}
},
{
"type": "bandwidth",
"capabilities": {
"bandwidth": true}
}
]
}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-kube-controllers
rules:
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
- serviceaccounts
verbs:
- watch
- list
- get
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs:
- watch
- list
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-kube-controllers
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-kube-controllers
subjects:
- kind: ServiceAccount
name: calico-kube-controllers
namespace: kube-system
---
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-node
rules:
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs:
- get
- apiGroups: [""]
resources:
- endpoints
- services
verbs:
- watch
- list
- apiGroups: [""]
resources:
- configmaps
verbs:
- get
- apiGroups: [""]
resources:
- nodes/status
verbs:
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: calico-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-node
subjects:
- kind: ServiceAccount
name: calico-node
namespace: kube-system
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: calico-node
namespace: kube-system
labels:
k8s-app: calico-node
spec:
selector:
matchLabels:
k8s-app: calico-node
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
k8s-app: calico-node
spec:
nodeSelector:
kubernetes.io/os: linux
hostNetwork: true
tolerations:
- effect: NoSchedule
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
operator: Exists
serviceAccountName: calico-node
terminationGracePeriodSeconds: 0
priorityClassName: system-node-critical
initContainers:
- name: install-cni
image: registry.cn-beijing.aliyuncs.com/dotbalo/cni:v3.15.3
command: ["/install-cni.sh"]
env:
- name: CNI_CONF_NAME
value: "10-calico.conflist"
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: calico-config
key: cni_network_config
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: CNI_MTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
- name: SLEEP
value: "false"
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /calico-secrets
name: etcd-certs
securityContext:
privileged: true
- name: flexvol-driver
image: registry.cn-beijing.aliyuncs.com/dotbalo/pod2daemon-flexvol:v3.15.3
volumeMounts:
- name: flexvol-driver-host
mountPath: /host/driver
securityContext:
privileged: true
containers:
- name: calico-node
image: registry.cn-beijing.aliyuncs.com/dotbalo/node:v3.15.3
env:
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
- name: CALICO_K8S_NODE_REF
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CALICO_NETWORKING_BACKEND
valueFrom:
configMapKeyRef:
name: calico-config
key: calico_backend
- name: CLUSTER_TYPE
value: "k8s,bgp"
- name: IP
value: "autodetect"
- name: CALICO_IPV4POOL_IPIP
value: "Always"
- name: CALICO_IPV4POOL_VXLAN
value: "Never"
- name: FELIX_IPINIPMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
- name: FELIX_VXLANMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
- name: FELIX_WIREGUARDMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
value: "ACCEPT"
- name: FELIX_IPV6SUPPORT
value: "false"
- name: FELIX_LOGSEVERITYSCREEN
value: "info"
- name: FELIX_HEALTHENABLED
value: "true"
securityContext:
privileged: true
resources:
requests:
cpu: 250m
livenessProbe:
exec:
command:
- /bin/calico-node
- -felix-live
- -bird-live
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
readinessProbe:
exec:
command:
- /bin/calico-node
- -felix-ready
- -bird-ready
periodSeconds: 10
volumeMounts:
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
- name: policysync
mountPath: /var/run/nodeagent
volumes:
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: var-lib-calico
hostPath:
path: /var/lib/calico
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0400
- name: policysync
hostPath:
type: DirectoryOrCreate
path: /var/run/nodeagent
- name: flexvol-driver-host
hostPath:
type: DirectoryOrCreate
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-node
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
labels:
k8s-app: calico-kube-controllers
spec:
replicas: 1
selector:
matchLabels:
k8s-app: calico-kube-controllers
strategy:
type: Recreate
template:
metadata:
name: calico-kube-controllers
namespace: kube-system
labels:
k8s-app: calico-kube-controllers
spec:
nodeSelector:
kubernetes.io/os: linux
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: calico-kube-controllers
priorityClassName: system-cluster-critical
hostNetwork: true
containers:
- name: calico-kube-controllers
image: registry.cn-beijing.aliyuncs.com/dotbalo/kube-controllers:v3.15.3
env:
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
- name: ENABLED_CONTROLLERS
value: policy,namespace,serviceaccount,workloadendpoint,node
volumeMounts:
- mountPath: /calico-secrets
name: etcd-certs
readinessProbe:
exec:
command:
- /usr/bin/check-status
- -r
volumes:
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0400
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-kube-controllers
namespace: kube-system
---
---
---
[root@ansible-server calico]
etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"
[root@ansible-server calico]
[root@ansible-server calico]
etcd_endpoints: {
% for i in groups.master %}https://{
{
hostvars[i].ansible_default_ipv4.address }}{
% if not loop.last %},{
% endif %}{
% endfor %}
[root@ansible-server calico]
- name: copy calico-etcd.yaml file
template:
src: calico-etcd.yaml.j2
dest: /root/calico-etcd.yaml
when:
- ansible_hostname=="k8s-master01"
[root@ansible-server calico]
- name: get ETCD_KEY key
shell:
cmd: cat /etc/kubernetes/pki/etcd/server.key | base64 | tr -d '\n'
register: ETCD_KEY
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*etcd-key:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '# (etcd-key:) null'
replace: '\1 {
{ ETCD_KEY.stdout }}'
when:
- ansible_hostname=="k8s-master01"
- name: get ETCD_CERT key
shell:
cmd: cat /etc/kubernetes/pki/etcd/server.crt | base64 | tr -d '\n'
register: ETCD_CERT
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*etcd-cert:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '# (etcd-cert:) null'
replace: '\1 {
{ ETCD_CERT.stdout }}'
when:
- ansible_hostname=="k8s-master01"
- name: get ETCD_CA key
shell:
cmd: cat /etc/kubernetes/pki/etcd/ca.crt | base64 | tr -d '\n'
when:
- ansible_hostname=="k8s-master01"
register: ETCD_CA
- name: Modify the ".*etcd-ca:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '# (etcd-ca:) null'
replace: '\1 {
{ ETCD_CA.stdout }}'
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*etcd_ca:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '(etcd_ca:) ""'
replace: '\1 "/calico-secrets/etcd-ca"'
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*etcd_cert:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '(etcd_cert:) ""'
replace: '\1 "/calico-secrets/etcd-cert"'
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*etcd_key:.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '(etcd_key:) ""'
replace: '\1 "/calico-secrets/etcd-key"'
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*CALICO_IPV4POOL_CIDR.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '# (- name: CALICO_IPV4POOL_CIDR)'
replace: '\1'
when:
- ansible_hostname=="k8s-master01"
- name: get POD_SUBNET
shell:
cmd: cat /etc/kubernetes/manifests/kube-controller-manager.yaml | grep cluster-cidr= | awk -F= '{print $NF}'
register: POD_SUBNET
when:
- ansible_hostname=="k8s-master01"
- name: Modify the ".*192.168.0.0.*" line
replace:
path: /root/calico-etcd.yaml
regexp: '# (value:) "192.168.0.0/16"'
replace: ' \1 "{
{ POD_SUBNET.stdout }}"'
when:
- ansible_hostname=="k8s-master01"
- name: Modify the "image:" line
replace:
path: /root/calico-etcd.yaml
regexp: '(.*image:) docker.io/calico(/.*)'
replace: '\1 {
{ HARBOR_DOMAIN }}/google_containers\2'
when:
- ansible_hostname=="k8s-master01"
[root@ansible-server calico]
- name: get calico version
shell:
chdir: /root
cmd: awk -F "/" '/image:/{print $NF}' calico-etcd.yaml
register: CALICO_VERSION
when:
- ansible_hostname=="k8s-master01"
- name: download calico image
shell: |
{
% for i in CALICO_VERSION.stdout_lines %}
docker pull registry.cn-beijing.aliyuncs.com/raymond9/{
{
i }}
docker tag registry.cn-beijing.aliyuncs.com/raymond9/{
{
i }} {
{
HARBOR_DOMAIN }}/google_containers/{
{
i }}
docker rmi registry.cn-beijing.aliyuncs.com/raymond9/{
{
i }}
docker push {
{
HARBOR_DOMAIN }}/google_containers/{
{
i }}
{
% endfor %}
when:
- ansible_hostname=="k8s-master01"
[root@ansible-server calico]
- name: install calico
shell:
chdir: /root
cmd: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f calico-etcd.yaml"
when:
- ansible_hostname=="k8s-master01"
[root@ansible-server calico]
- include: calico_file.yml
- include: config.yml
- include: download_images.yml
- include: install_calico.yml
[root@ansible-server calico]
[root@ansible-server ansible]
roles/calico
├── tasks
│ ├── calico_file.yml
│ ├── config.yml
│ ├── download_images.yml
│ ├── install_calico.yml
│ └── main.yml
├── templates
│ └── calico-etcd.yaml.j2
└── vars
└── main.yml
3 directories, 7 files
[root@ansible-server ansible]
---
- hosts: master
roles:
- role: calico
[root@ansible-server ansible]
15.2 验证calico
[root@k8s-master01 ~]
kube-system calico-kube-controllers-55bfb655fc-9gmdn 1/1 Running 0 2m42s
kube-system calico-node-2gwjm 1/1 Running 0 2m42s
kube-system calico-node-k8vtb 1/1 Running 0 2m42s
kube-system calico-node-l5q6m 1/1 Running 0 2m42s
kube-system calico-node-mrzrl 1/1 Running 0 2m42s
kube-system calico-node-vk4n4 1/1 Running 0 2m42s
kube-system calico-node-zwn7s 1/1 Running 0 2m42s
[root@k8s-master01 ~]
NAME STATUS ROLES AGE VERSION
k8s-master01.example.local Ready control-plane,master 70m v1.20.14
k8s-master02.example.local Ready control-plane,master 64m v1.20.14
k8s-master03.example.local Ready control-plane,master 63m v1.20.14
k8s-node01.example.local Ready <none> 62m v1.20.14
k8s-node02.example.local Ready <none> 61m v1.20.14
k8s-node03.example.local Ready <none> 61m v1.20.14