K8S use Ceph RBD as back-end storage

First, the preparatory work

Ceph version: v13.2.5 mimic Stable

1, Ceph storage pool on preparation

[root@ceph-node1 ceph]# ceph osd pool create k8s 128 128
pool 'k8s' created
[root@ceph-node1 ceph]# ceph osd pool ls
k8s

2, ready K8S client's account on the Ceph

In this environment, the direct use of the Ceph admin account, of course, the production environment or to assign different functions depending on client accounts:
ceph auth get-or-create client.k8s mon 'allow r' osd 'allow rwx pool=k8s' -o ceph.client.k8s.keyring
acquiring key accounts:

[root@ceph-node1 ceph]# ceph auth get-key client.admin | base64
QVFDMmIrWmNEL3JTS2hBQWwwdmR3eGJGMmVYNUM3SjdDUGZZbkE9PQ==

3, to provide command controller-manager rbd

When PV dynamically created using StorageClass, controller-manager automatically creates image on Ceph, so we want to be ready rbd command.
(1) If the cluster is kubeadm deployment, because the controller-manager rbd no official mirror command, so we have to import a foreign configuration:

kind: ClusterRole 
apiVersion: rbac.authorization.k8s.io/v1 
metadata: 
  name: rbd-provisioner 
rules: 
  - apiGroups: [""] 
    resources: ["persistentvolumes"] 
    verbs: ["get", "list", "watch", "create", "delete"] 
  - apiGroups: [""] 
    resources: ["persistentvolumeclaims"] 
    verbs: ["get", "list", "watch", "update"] 
  - apiGroups: ["storage.k8s.io"] 
    resources: ["storageclasses"] 
    verbs: ["get", "list", "watch"] 
  - apiGroups: [""] 
    resources: ["events"] 
    verbs: ["create", "update", "patch"] 
  - apiGroups: [""] 
    resources: ["services"] 
    resourceNames: ["kube-dns","coredns"] 
    verbs: ["list", "get"] 
--- 
kind: ClusterRoleBinding 
apiVersion: rbac.authorization.k8s.io/v1 
metadata: 
  name: rbd-provisioner 
subjects: 
  - kind: ServiceAccount 
    name: rbd-provisioner 
    namespace: default 
roleRef: 
  kind: ClusterRole 
  name: rbd-provisioner 
  apiGroup: rbac.authorization.k8s.io 
--- 
apiVersion: rbac.authorization.k8s.io/v1 
kind: Role 
metadata: 
  name: rbd-provisioner 
rules: 
- apiGroups: [""] 
  resources: ["secrets"] 
  verbs: ["get"] 
- apiGroups: [""] 
  resources: ["endpoints"] 
  verbs: ["get", "list", "watch", "create", "update", "patch"] 
--- 
apiVersion: rbac.authorization.k8s.io/v1 
kind: RoleBinding 
metadata: 
  name: rbd-provisioner 
roleRef: 
  apiGroup: rbac.authorization.k8s.io 
  kind: Role 
  name: rbd-provisioner 
subjects: 
  - kind: ServiceAccount 
    name: rbd-provisioner 
    namespace: default 
--- 
apiVersion: extensions/v1beta1 
kind: Deployment 
metadata: 
  name: rbd-provisioner 
spec: 
  replicas: 1 
  strategy: 
    type: Recreate 
  template: 
    metadata: 
      labels: 
        app: rbd-provisioner 
    spec: 
      containers: 
      - name: rbd-provisioner 
        image: quay.io/external_storage/rbd-provisioner:latest 
        env: 
        - name: PROVISIONER_NAME 
          value: ceph.com/rbd 
      serviceAccount: rbd-provisioner 
--- 
apiVersion: v1 
kind: ServiceAccount 
metadata: 
  name: rbd-provisioner

kubectl apply -f rbd-provisioner.yaml
Note: rbd-provisioner and ceph to be mirrored version of the adapter, using the latest image here, according to official support has prompted ceph mimic version.

(2) If the cluster is deployed in binary mode, the master node can be directly mounted ceph-common.
YUM Source:

[Ceph]
name=Ceph packages for $basearch
baseurl=http://download.ceph.com/rpm-mimic/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
priority=1
[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://download.ceph.com/rpm-mimic/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=http://download.ceph.com/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
priority=1

# Install the client
yum -y install ceph-common-13.2.5
# keyring file copy
will ceph of ceph.client.admin.keyring files are copied to the master / etc / ceph directory.

4, to provide rbd command kubelet

When creating a pod, kubelet need to detect and mount command rbd pv corresponding ceph image, so to install the client ceph ceph-common-13.2.5 all the worker nodes.

Second, the trial Ceph RBD stored on K8S

1. Create a storage class

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: ceph-sc
  namespace: default
  annotations: 
    storageclass.kubernetes.io/is-default-class: "false"
provisioner: ceph.com/rbd
reclaimPolicy: Retain
parameters:
  monitors: 172.16.1.31:6789,172.16.1.32:6789,172.16.1.33:6789
  adminId: admin
  adminSecretName: storage-secret
  adminSecretNamespace: default
  pool: k8s
  fsType: xfs
  userId: admin
  userSecretName: storage-secret
  imageFormat: "2"
  imageFeatures: "layering"

kubectl apply -f storage_class.yaml

2, to provide for the secret storage class

apiVersion: v1
kind: Secret
metadata:
  name: storage-secret
  namespace: default
data:
  key: QVFDMmIrWmNEL3JTS2hBQWwwdmR3eGJGMmVYNUM3SjdDUGZZbkE9PQ==
type:
  kubernetes.io/rbd

kubectl apply -f storage_secret.yaml
Note: provisioner values and values to be set as rbd-provisioner

3. Create PVC

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: ceph-pvc
  namespace: default
spec:
  storageClassName: ceph-sc
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi

kubectl apply -f storage_pvc.yaml
After # Create a PVC, PV is automatically created:

[root@k8s-master03 ceph]# kubectl get pv       
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                  STORAGECLASS   REASON   AGE
pvc-315991e9-7d4b-11e9-b6cc-0050569ba238   1Gi        RWO            Retain           Bound    default/ceph-sc-test   prom-sc                 13h

# Normally PVC is also in Bound state

[root@k8s-master03 ceph]# kubectl get pvc
NAME           STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
ceph-sc-test   Bound    pvc-315991e9-7d4b-11e9-b6cc-0050569ba238   1Gi        RWO            prom-sc        17s

4, create a test application

apiVersion: v1
kind: Pod
metadata:
  name: ceph-pod1
spec:
  nodeName: k8s-node02
  containers:
  - name: nginx
    image: nginx:1.14
    volumeMounts:
    - name: ceph-rdb-vol1
      mountPath: /usr/share/nginx/html
      readOnly: false
  volumes:
  - name: ceph-rdb-vol1
    persistentVolumeClaim:
      claimName: ceph-pvc

-f storage_pod.yaml the Apply kubectl
# View pod status

[root@k8s-master03 ceph]# kubectl get pods -o wide
NAME                             READY   STATUS    RESTARTS   AGE     IP            NODE         NOMINATED NODE   READINESS GATES
ceph-pod1                        1/1     Running   0          3d23h   10.244.4.75   k8s-node02   <none>           <none>

# Mount into the container to view the situation, you can see rbd mounted to / usr / share / nginx / html directory.

[root@k8s-master03 ceph]# kubectl exec -it ceph-pod1 -- /bin/bash
root@ceph-pod1:/# df –hT
/dev/rbd0            xfs     1014M   33M  982M   4% /usr/share/nginx/html
#在挂载目录下添加一个测试文件
root@ceph-pod1:/# cat /usr/share/nginx/html/index.html
hello ceph!

Check image corresponding to the node # mounted on Ceph, i.e. the current in k8s-node02 172.16.1.22.

[root@ceph-node1 ~]# rbd status k8s/kubernetes-dynamic-pvc-2410765c-7dec-11e9-aa80-26a98c3bc9e4
Watchers:
        watcher=172.16.1.22:0/264870305 client.24553 cookie=18446462598732840961

# And then we delete this one pod

[root@k8s-master03 ceph]# kubectl delete -f  storage_pod.yaml   
pod "ceph-pod1" deleted

# Modify the manifest file storage_pod.yaml, the pod scheduled on k8s-node01, and applications.
# Later, view the status of the pod, change the pod has been deployed on the k8s-node01.

[root@k8s-master01 ~]# kubectl get pods -o wide
NAME                             READY   STATUS    RESTARTS   AGE    IP            NODE         NOMINATED NODE   READINESS GATES
ceph-pod1                        1/1     Running   0          34s    10.244.3.28   k8s-node01   <none>           <none>

Check the image node # mount again on Ceph, the current that is k8s-node01 at 172.16.1.21

[root@ceph-node1 ~]# rbd status k8s/kubernetes-dynamic-pvc-2410765c-7dec-11e9-aa80-26a98c3bc9e4
Watchers:
        watcher=172.16.1.21:0/1812501701 client.114340 cookie=18446462598732840963

# Into the container, check the file exists and is not lost, indicating that the use of the original image after pod switching node.

[root@k8s-master03 ceph]# kubectl exec -it ceph-pod1 -- /bin/bash
root@ceph-pod1:/# cat /usr/share/nginx/html/index.html
hello ceph!