本文章案例可用于参考Jenkins for kubernetes部署。因每个公司的架构和环境不一样，需要改变一些部署的方式。
Jenkins for kubernetes的好处：
- Jenkins-master的高可用。k8s 的rc或deployment可以监控副本的存活状态（通过探针）和副本数量，如果master出现无法提供服务的情况，就会重启或者迁移到其他节点。
- Jenkins-slave的动态伸缩。 每次构建都会启动一个pod用于部署slave，构建完成后就会释放掉。那么pod在创建的时候，k8s就会选择集群内资源剩余较多的节点创建slave的pod，构建完成后pod会自动删除。
- 扩展性好。 因为可以同时拥有很多个slave，可以配置Jenkins同时执行很多构建操作，减少排队等待构建的时间。
部署思路：
首先在k8s中部署Jenkins-master然后使用Kubernetes plugin插件进行slave的动态伸缩。并且使用NFS作为后端存储的PersistentVolume来挂载Jenkins-master的jenkins_home目录、构建时slave的maven缓存m2目录（可以利用缓存加快每次构建的速度）、保留slave每次构建产生的数据（workspace目录中的每个job）。
使用PersistentVolume的原因是k8s任何节点都可以访问到挂载的目录，不会因为master迁移节点导致数据丢失。NFS方便部署而且性能也满足Jenkins的使用需求所以选择了NFS，也可以使用其他的后端存储。
部署
部署方式可以自定义也可以使用Kubernetes plugin官网提供的部署yml。自定义使用deployment也是可以的，但是官网的部署方式使用了StatefulSet。Jenkins是一个有状态的应用，我感觉使用StatefulSet部署更加严谨一点。我这里使用了官网提供的文档进行部署的，但是也根据实际情况修改了一些东西。
首先需要在k8s所有节点部署NFS客户端：
yum -y install nfs-utils
systemctl start nfs-utils
systemctl enable nfs-utils
rpcinfo -p
NFS服务端配置文件增加配置：
/data/dev_jenkins 10.0.0.0/24(rw,sync,no_root_squash,no_subtree_check)
#dev环境Jenkins slave节点挂载workspace
/data/dev_jenkins/workspace 0.0.0.0/0(rw,sync,no_root_squash,no_subtree_check)
#dev环境Jenkins slave节点挂载m2 maven缓存目录
/data/dev_jenkins/m2 0.0.0.0/0(rw,sync,no_root_squash,no_subtree_check)
共享目录一定要给777权限。不然容器内部会报错没有写入权限。
service-account.yml 此文件用于创建k8s的rbac，授权给后面的Jenkins应用可以创建和删除slave的pod。
# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: jenkins #与jenkins.yml中的serviceAccountName: jenkins相对应
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
jenkins-pv.yml和jenkins-pvc.yml用于创建挂载jenkins_home目录
[root@dev-master1 kubernetes]# cat jenkins-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-home
spec:
capacity: #指定容量
storage: 20Gi
accessModes:
- ReadWriteOnce #访问模式，还有ReadOnlyMany ##ReadOnlymany
# persistenVolumeReclaimPolicy: Recycle
# storageClassName: nfs ##指定存储的类型
nfs:
path: /data/dev_jenkins #指明nfs的路径
server: 10.0.0.250 #指明nfs的ip
[root@dev-master1 kubernetes]# cat jenkins-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: jenkins-home
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
创建Jenkins的master，可以根据实际情况限制Jenkins的资源使用
[root@dev-master1 kubernetes]# cat jenkins.yml
# jenkins
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
selector:
matchLabels:
name: jenkins
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: 10.0.0.59/jenkins/jenkins:lts-alpine #官方镜像为jenkins/jenkins:lts-alpine，为了节省下载时间已经push到自己到harbor仓库
imagePullPolicy: Always
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts: #挂载pvc存储到Jenkins容器的/var/jenkins_home
- name: jenkinshome
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 600 #存活探针时间改为600s，如果服务器配置低，Jenkins还没有启动成功就被重启了。
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
securityContext:
fsGroup: 1000
volumes: #此处声明Jenkins的pvc存储
- name: jenkinshome
persistentVolumeClaim:
claimName: jenkins-home
# imagePullSecrets: 如果使用私有仓库，并且仓库对镜像设置了访问权限，需要在k8s master创建一个secret
# - name: registry-secret
jenkins-sv.yml 用于创建Jenkins的Service
[root@dev-master1 kubernetes]# cat jenkins-sv.yml
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
sessionAffinity: "ClientIP"
type: NodePort
selector:
name: jenkins
ports:
-
name: http
port: 80
nodePort: 31006
protocol: TCP
-
name: agent
port: 50000
nodePort: 31007
protocol: TCP
挂载maven缓存目录
[root@dev-master1 kubernetes]# cat m2-pv.yml
#m2是maven的缓存，挂载以提高build速度
apiVersion: v1
kind: PersistentVolume
metadata:
name: maven-m2
spec:
capacity: #指定容量
storage: 200Gi
accessModes:
- ReadWriteOnce #访问模式，还有ReadOnlyMany ##ReadOnlymany
# persistenVolumeReclaimPolicy: Recycle
# storageClassName: nfs ##指定存储的类型
nfs:
path: /data/dev_jenkins/m2 #指明nfs的路径
server: 10.0.0.250 #指明nfs的ip
[root@dev-master1 kubernetes]# cat m2-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: maven-m2
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200Gi
挂载slave节点保存构建结果的目录
[root@dev-master1 kubernetes]# cat workspace-pv.yml
#m2是maven的缓存，挂载以提高build速度
apiVersion: v1
kind: PersistentVolume
metadata:
name: workspace
spec:
capacity: #指定容量
storage: 200Gi
accessModes:
- ReadWriteOnce #访问模式，还有ReadOnlyMany ##ReadOnlymany
# persistenVolumeReclaimPolicy: Recycle
# storageClassName: nfs ##指定存储的类型
nfs:
path: /data/dev_jenkins/workspace #指明nfs的路径
server: 10.0.0.250 #指明nfs的ip
[root@dev-master1 kubernetes]# cat workspace-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: kubernetes-plugin
name: workspace
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200Gi
创建Jenkins的Ingress。因为我的k8s集群里面使用的是traefik，所以我把traefik的配置文件和kubernetes-plugin官网给出的Ingress一起贴出来。
[root@dev-master1 kubernetes]# cat jenkins-traefik.yml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
namespace: kubernetes-plugin
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: jenkins-dev.doudou.com
http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 80
[root@dev-master1 kubernetes]# cat jenkins-Ingress.yml
#因为集群使用traefik所以此Ingress配置文件不创建,此文件为官方原版
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: jenkins
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
kubernetes.io/tls-acme: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
nginx.ingress.kubernetes.io/proxy-body-size: 50m
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
# For nginx-ingress controller < 0.9.0.beta-18
ingress.kubernetes.io/ssl-redirect: "true"
# "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
ingress.kubernetes.io/proxy-body-size: 50m
ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
rules:
- http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 80
host: jenkins.example.com
tls:
- hosts:
- jenkins.example.com
secretName: tls-jenkins
创建以上的配置文件：
kubectl create namespace kubernetes-plugin #创建kubernetes-plugin namespace，下面创建的所有东西都归属到这个namespace
kubectl config set-context $(kubectl config current-context) --namespace=kubernetes-plugin #修改k8s默认的namespace为kubernetes-plugin，这样下面创建的都默认为kubernetes-plugin命名空间
kubectl create -f service-account.yml
#kubectl create -f jenkins-Ingress.yml
kubectl create -f jenkins-pv.yml
kubectl create -f jenkins-pvc.yml
kubectl create -f jenkins-sv.yml
kubectl create -f jenkins.yml
kubectl create -f m2-pvc.yml
kubectl create -f m2-pv.yml
kubectl create -f workspace-pvc.yml
kubectl create -f workspace-pv.yml
查看创建状态
[root@dev-master1 ~]# kubectl get service,pod,StatefulSet -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/jenkins NodePort 10.105.123.193 <none> 80:31006/TCP,50000:31007/TCP 9d name=jenkins
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/jenkins-0 1/1 Running 0 6d5h 100.78.0.141 dev-node4 <none> <none>
NAME READY AGE CONTAINERS IMAGES
statefulset.apps/jenkins 1/1 7d jenkins 10.0.0.59/jenkins/jenkins:lts-alpine
[root@dev-master1 ~]# kubectl get pv,pvc
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/jenkins-home 20Gi RWO Retain Bound kubernetes-plugin/jenkins-home 13d
persistentvolume/maven-m2 200Gi RWO Retain Bound kubernetes-plugin/maven-m2 7d5h
persistentvolume/workspace 200Gi RWO Retain Bound kubernetes-plugin/workspace 7d5h
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/jenkins-home Bound jenkins-home 20Gi RWO 13d
persistentvolumeclaim/maven-m2 Bound maven-m2 200Gi RWO 7d5h
persistentvolumeclaim/workspace Bound workspace 200Gi RWO 7d5h
pv的状态为Bound状态表示已经绑定到对应的PVC上。Jenkins的pod状态为1/1就说明启动成功了，可以通过绑定Ingress的域名访问了。或者使用Service配置中的nodePort端口访问 k8s任意节点IP：nodePort
查看jenkins密码：
kubectl exec -it jenkins-0 -n kubernetes-plugin -- cat /var/jenkins_home/secrets/initialAdminPassword