Kubernetes—部署自动化平台（Jenkins）

1.项目发布方案概述

（1）蓝绿发布
项目逻辑上分为AB组，在项目升级时，首先把A组从负载均衡中摘除，进行新版本的部署。B组仍然继续提供服务。A组升级完成上线，B组从负载均衡中摘除。
特点：
• 策略简单
• 升级/回滚速度快
• 用户无感知，平滑过渡
缺点：
• 需要两倍以上服务器资源
• 短时间内浪费一定资源成本


（2）灰度发布
灰度发布：只升级部分服务，即让一部分用户继续用老版本，一部分用户开始用新版本，如果用户对新版本没有什么意见，那么逐步扩大范围，把所有用户都迁移到新版本上面来。
特点：
• 保证整体系统稳定性
• 用户无感知，平滑过渡
缺点：
• 自动化要求高


（3）滚动发布

滚动发布：每次只升级一个或多个服务，升级完成后加入生产环境，不断执行这个过程，直到集群中的全部旧版升级新版本。
特点：
• 用户无感知，平滑过渡
缺点：
• 部署周期长
• 发布策略较复杂
• 不易回滚

2.发布流程设计

192.168.74.230  k8s-master
192.168.74.246  k8s-node1
192.168.74.247  k8s-node2
192.168.74.248  harbor  git   nfs

3.准备代码版本仓库Git和容器镜像仓库Harbor

在192.168.74.248搭建harbor仓库和git仓库

#配置密钥对互信
 ssh-keygen  
ssh-copy-id [email protected]

#配置git仓库
mkdir java-demo.git
cd java-demo.git/
git --bare init

chmod +x docker-compose
mv docker-compose /usr/local/bin/
（1）给这个节点安装docker 准备harbor的环境
yum install -y yum-utils device-mapper-persistent-data lvm2
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
mv docker-ce.repo /etc/yum.repos.d/
systemctl start docker && systemctl enable docker
vim /etc/docker/daemon.json
{ “registry-mirrors”: [“https://h5d0mk6d.mirror.aliyuncs.com”] }
systemctl restart docker
（2）制作证书
mkdir /application/ssl -p
cd /application/ssl

openssl req -newkey rsa:4096 -nodes -sha512 -subj “/C=CN/ST=/L=/O=/OU=/CN=qushuaibo.com” -keyout ca.key -x509 -days 3650 -out ca.crt

openssl req -newkey rsa:4096 -nodes -sha512 -subj “/C=CN/ST=/L=/O=e/OU=/CN=qushuaibo.com” -keyout qushuaibo.com.key -out qushuaibo.com.csr

openssl x509 -req -days 3650 -in qushuaibo.com.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out qushuaibo.com.crt
（3）配置harbor（这里采用的https方式域名访问）
tar xf harbor-offline-installer-v1.7.5.tgz
cd harbor/
sed -i ‘s#hostname = reg.mydomain.com#hostname = qushuaibo.com#g’ /root/harbor/harbor.cfg
sed -i ‘s#hostname = reg.mydomain.com#hostname = qushuaibo.com#g’ /root/harbor/harbor.cfg
sed -i ‘s#ui_url_protocol = http#ui_url_protocol = https#g’ /root/harbor/harbor.cfg
sed -i ‘s#ssl_cert = /data/cert/server.crt#ssl_cert = /application/ssl/qushuaibo.com.crt#g’ /root/harbor/harbor.cfg
sed -i ‘s#ssl_cert_key = /data/cert/server.key#ssl_cert_key = /application/ssl/qushuaibo.com.key#g’ /root/harbor/harbor.cfg
sed -i ‘s#harbor_admin_password = Harbor12345#harbor_admin_password = 123456#g’ /root/harbor/harbor.cfg
（4）将证书分发至个个node和master节点并更新证书，信任此证书
mkdir -r /etc/docker/certs.d/qushuaibo.com/ #k8s集群内部都创建放证书的地方
scp qushuaibo.com.crt 192.168.74.230:/etc/docker/certs.d/qushuaibo.com/ #master
scp qushuaibo.com.crt 192.168.74.246:/etc/docker/certs.d/qushuaibo.com/ #node1
scp qushuaibo.com.crt 192.168.74.247:/etc/docker/certs.d/qushuaibo.com/ #node2
cd /etc/docker/certs.d/qushuaibo.com/
update-ca-trust
（5）启动harbor
./prepare
./install.sh

4.Jenkins在K8S中动态创建代理

https://github.com/jenkinsci/kubernetes-plugin/tree/fc40c869edfd9e3904a9a56b0f80c5a25e988fa1/src/main/kubernetes
在k8s中部署jenkins
kubectl apply -f .
Ingress.yaml

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: jenkins
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    kubernetes.io/tls-acme: "true"
    # 如果上传插件超出默认会报"413 Request Entity Too Large", 增加 client_max_body_size
    nginx.ingress.kubernetes.io/proxy-body-size: 50m
    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
    # nginx-ingress controller版本小于 0.9.0.beta-18 的配置
    ingress.kubernetes.io/ssl-redirect: "true"
    ingress.kubernetes.io/proxy-body-size: 50m
    ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
  rules:
  - host: jenkins.example.com
    http:
      paths:
      - path: /
        backend:
          serviceName: jenkins
          servicePort: 80

rbac.yml

**# 创建名为jenkins的ServiceAccount**
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins

---
# 创建名为jenkins的Role，授予允许管理API组的资源Pod
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---
# 将名为jenkins的Role绑定到名为jenkins的ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins

service-account.yml

# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins

service.yml  
apiVersion: v1
kind: Service
metadata:
  name: jenkins
spec:
  selector:
    name: jenkins
  type: NodePort
  ports:
    -
      name: http
      port: 80
      targetPort: 8080
      protocol: TCP
      nodePort: 30006
    -
      name: agent
      port: 50000
      protocol: TCP

statefulset.yml
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  serviceName: jenkins
  replicas: 1
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccountName: jenkins
      containers:
        - name: jenkins
          image: jenkins/jenkins:lts-alpine
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
            - containerPort: 50000
          resources:
            limits:
              cpu: 1
              memory: 1Gi
            requests:
              cpu: 0.5
              memory: 500Mi
          env:
            - name: LIMITS_MEMORY
              valueFrom:
                resourceFieldRef:
                  resource: limits.memory
                  divisor: 1Mi
            - name: JAVA_OPTS
              value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          volumeMounts:
            - name: jenkins-home
              mountPath: /var/jenkins_home
          livenessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 200
            timeoutSeconds: 5
            failureThreshold: 12
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12
      securityContext:
        fsGroup: 1000
  volumeClaimTemplates:
  - metadata:
      name: jenkins-home
    spec:
      storageClassName: "managed-nfs-storage"
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi

修改jenkins代理以及安装插件git pip kubernetes Kubernetes Continuous Deploy

在配置系统中拉到最下面

5.构建Jenkins-Slave镜像

制作jenkins-slave镜像并上传至haror
docker build -t qushuaibo.com/jenkins/jenkins_slave:v1 .
Dockerfile为

FROM centos:7
LABEL maintainer lizhenliang

RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \ 
    yum clean all && \
    rm -rf /var/cache/yum/* && \
    mkdir -p /usr/share/jenkins

COPY slave.jar /usr/share/jenkins/slave.jar  
COPY jenkins-slave /usr/bin/jenkins-slave        #这个是个脚本为了就是访问slave.jar
COPY settings.xml /etc/maven/settings.xml       #这个是为了maven的源
RUN chmod +x /usr/bin/jenkins-slave          

ENTRYPOINT ["jenkins-slave"]

6.Jenkins Pipeline构建流水线发布

• Jenkins Pipeline是一套插件，支持在Jenkins中实现集成和持续交付管道；
• Pipeline通过特定语法对简单到复杂的传输管道进行建模；
• 声明式：遵循与Groovy相同语法。pipeline { }
• 脚本式：支持Groovy大部分功能，也是非常表达和灵活的工具。node { }
• Jenkins Pipeline的定义被写入一个文本文件，称为Jenkinsfile。


先测试jenkins是否可以拉起slave

podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
    containerTemplate(
        name: 'jnlp', 
        image: "qushuaibo.com/jenkins/jenkins_slave:v1"
    ),
  ],
  volumes: [
    hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    hostPathVolume(mountPath: '/usr/bin/docker', hostPath: '/usr/bin/docker')
  ],
) 
{
  node("jenkins-slave"){
     stage('拉取代码'){
         
     }
  }
}

标签上下对应才可以把下面的步骤在上面这个slave中

设置保存用户名密码（登陆harbor用的）

7.Jenkins在Kubernetes中持续部署

❖ 使用Jenkins三个插件
• Kubernetes
• Pipeline
• Kubernetes Continuous Deploy
• git
• 也可以装一个新皮肤Blue Ocean
❖ CI/CD环境特点
• Slave弹性伸缩
• 基于镜像隔离构建环境
• 流水线发布，易维护
❖Jenkins参数化构建可帮助你完成更复杂环境CI/CD
持续部署是需要Kubernetes Continuous Deploy 这个插件来做
那么我们就需要把认证信息给jenkins，才可以确保jenkins能连接k8s
cat .kube/config

上图为jenkins连接k8s的凭证

创建拉取镜像凭证
kubectl create secret docker-registry registry-pull-secret --docker-username=admin --docker-server=https://qushuaibo.com --docker-password=123456

Pipeline脚本如下

// 公共
def registry = "qushuaibo.com"                 //仓库坐标
// 项目 
def project = "welcome"                     //项目仓库名字
def app_name = "demo"                      //项目名称
def image_name = "${registry}/${project}/${app_name}:${Branch}-${BUILD_NUMBER}"
//k8s部署用的镜像名称通过值用sed来给yaml传递   
//Branch是分支我们需要设置的参数化构建
//BUILD_NUMBER 这个是jenkins的内置变量
def git_address = "[email protected]:/home/git/java-demo.git"
//git仓库位置
// 认证
def secret_name = "registry-pull-secret"          //k8s部署应用拉镜像的凭证
def docker_registry_auth = "c87990b8-0802-4ab7-9d6c-dfe48e581ff2"
//harbor的username和password两个变量的传入，以及这样可以不将私有仓库账号密码暴露在外面
def git_auth = "dcfb69a8-8a74-47c6-ad09-1780d91802c9"
//git仓库的凭据，等于是把下面的拿上来可以不同的仓库多次利用
def k8s_auth = "da3ad51c-69aa-4cc3-b938-744b9f622cda"
//jenkins连接k8s所需要的认证
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
    containerTemplate(
        name: 'jnlp', 
        image: "${registry}/jenkins/jenkins_slave:v1"
    ),
  ],
  volumes: [
    hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    hostPathVolume(mountPath: '/usr/bin/docker', hostPath: '/usr/bin/docker'),
hostPathVolume(mountPath: '/root/.m2', hostPath: '/tmp/m2')
//将宿主机的docker环境挂载在jenkins-slave中就不用在jenkins-slave中安装docker
//挂载.m2是为了将maven拉取的依赖包同步在宿主机的/tmp/m2中这样子可以多次利用，也可以用共享存储让多节点一起使用
  ],    
    ImagePullSecrets: [‘registry-pull-secret’],    //私有镜像需要加这个，共有仓库不需要
) 
{
  node("jenkins-slave"){         
//这里的jeknins-slave对应上面的label这样才可以白傲冥是用上面的从来做这个任务，根据不同的项目，做不同的从jenkins环境，这里是java所以从配置的是maven和java环境
      // 第一步
      stage('拉取代码'){
        checkout([$class: 'GitSCM', branches: [[name: "${Branch}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
      }
      // 第二步
      stage('代码编译'){
          sh "mvn clean package -Dmaven.test.skip=true"
      }
      // 第三步
     stage('构建镜像'){
          withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
           sh """
              echo '
                FROM lizhenliang/tomcat 
                RUN rm -rf /usr/local/tomcat/webapps/*
                ADD target/*.war /usr/local/tomcat/webapps/ROOT.war 
              ' > Dockerfile
              docker build -t ${image_name} .
              docker login -u ${username} -p '${password}' https://${registry}
              docker push ${image_name}
            """
            }
      }
      // 第四步
      stage('部署到K8S平台'){
          sh """
          sed -i 's#\$IMAGE_NAME#${image_name}#' deploy.yml
          sed -i 's#\$SECRET_NAME#${secret_name}#' deploy.yml
          """
          kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"
      }
   }
}