基于k8s、docker、jenkins构建springboot服务

Jenkins + github + docker + k8s + springboot

本文介绍基于k8s、docker、jenkins、springboot构建docker服务。

环境准备

server-1 k8s-master Centos7 ip地址10.12.5.110
server-2    k8s-node Centos7 ip地址10.12.5.115

两台服务执行如下命令

$ setenforce 0
$ systemctl stop firewalld
$ systemctl disable firewalld

server-1 k8s-master 安装

k8s-master节点安装kubernets、docker、etcd、git、maven等软件。

安装docker

$ yum install docker
// 修改docker配置文件 /etc/sysconfig/docker

OPTIONS='--registry-mirror=https://docker.mirrors.ustc.edu.cn --selinux-enabled --log-driver=journald --signature-verification=false'
ADD_REGISTRY='--add-registry 10.12.5.110:5000'
INSECURE_REGISTRY='--insecure-registry 10.12.5.110:5000'

$ systemctl enable docker
$ service docker start

安装registry

$ docker pull registry
$ mkdir -p /data/docker/registry
$ docker run -d -p 5000:5000 -v /data/docker/registry:/var/lib/registry registry

安装etcd

$ yum install etcd
$ vi /etc/etcd/etcd.conf
// 修改如下配置
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"

$ systemctl enable etcd
$ systemctl start etcd

安装kubernetes

$ yum install kubernetes
// 修改配置文件apiserver
$ vi /etc/kubernetes/apiserver
// 修改如下配置
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
KUBE_API_PORT="--port=8080"
KUBELET_PORT="--kubelet-port=10250"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota"

// 启动服务
$ systemctl enable kube-apiserver kube-controller-manager kube-scheduler
$ systemctl start kube-apiserver kube-controller-manager kube-scheduler

安装git和mvn

$ yum install git
$ yum install maven

server-2 k8s-node 安装

k8s-master节点安装kubernets、docker等软件。

安装docker

$ yum install docker
// 修改docker配置文件 /etc/sysconfig/docker

ADD_REGISTRY='--add-registry 10.12.5.110:5000'
INSECURE_REGISTRY='--insecure-registry 10.12.5.110:5000'

$ systemctl enable docker
$ systemctl start docker

安装kubernetes

$ yum install kubernetes
// 修改配置文件kubelet
$ vi /etc/kubernetes/kubelet
// 修改如下配置
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_PORT="--port=10250"
KUBELET_HOSTNAME="--hostname-override=10.12.5.115"
KUBELET_API_SERVER="--api-servers=http://10.12.5.110:8080"
// 修改配置文件config
$ vi /etc/kubernetes/config
// 修改如下配置
KUBE_MASTER="--master=http://10.12.5.110:8080"
// 启动服务
$ systemctl enable kube-proxy kubelet
$ systemctl start kube-proxy kubelet

检查k8s集群运行情况

// 在k8s-master节点执行
$ kubectl get nodes
NAME          STATUS    AGE
10.12.5.115   Ready     3m 

在k8s-master节点builder springcloudenv镜像

* 下载jdk8
$ mdkir -p /home/docker/docker-jdk8
$ wget http://javadl.oracle.com/webapps/download/AutoDL?BundleId=233162_512cd62ec5174c3487ac17c61aaa89e8 -O jre-8u161-linux-x64.tar.gz
$ tar zxvf jre-8u161-linux-x64.tar.gz
* 删除jdk中用不到的文件，尽量减少docker镜像文件大小
$ cd jre1.8.0_171
$ rm -rf lib/plugin.jar lib/ext/jfxrt.jar bin/javaws lib/javaws.jar lib/desktop plugin lib/deploy* lib/*javafx* lib/*jfx* lib/amd64/libdecora_sse.so lib/amd64/libprism_*.so lib/amd64/libfxplugins.so lib/amd64/libglass.so lib/amd64/libgstreamer-lite.so lib/amd64/libjavafx*.so lib/amd64/libjfx*.so
* 编写Dockerfile文件
$ cd /home/docker/docker-jdk8
$ vi Dockerfile

FROM centos
MAINTAINER by wangtw
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo 'Asia/Shanghai' >/etc/timezone && \
    yum -y install kde-l10n-Chinese && \
    localedef -c -f UTF-8 -i zh_CN zh_CN.utf8 && \
    mkdir -p /usr/java
COPY jre1.8.0_171 /usr/java/jre1.8.0_171
ENV LANG zh_CN.UTF-8
ENV JAVA_HOME /usr/java/jre1.8.0_171
ENV PATH $JAVA_HOME/bin:$PATH

* 创建镜像springcloudenv，并上传到registry，确保docker registry已经启动

$ docker build -t springcloudenv .
$ docker tag springcloudenv springcloudenv:v1
$ docker push springcloudenv:v1
// 查看registry中的镜像
$ curl http://10.12.5.110:5000/v2/_catalog
{"repositories":["springcloudenv"]}

安装nfs

在k8s-master节点安装nfs server

$ yum install -y nfs-utils rpcbind
$ mkdir -p /data/mysql-pv
$ chmod 777 /data/mysql-pv/
$ mkdir -p /data/nfs
$ chmod 666 /data/nfs/
$ vi /etc/exports
// 增加如下行
/data/nfs 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)
/data/mysql-pv 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)
$ systemctl enable rpcbind nfs
$ systemctl start rpcbind nfs
$ mkdir -p /data/mysql
$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock

在k8s-node节点安装nfs

$ yum install -y nfs-utils
$ mkdir -p /data/mysql
$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock

安装mysql docker

以持久化方式运行myql

在k8s-master节点创建相关文件

$ mkdir -p /home/k8s/yml/services/mysql
$ cd /home/k8s/yml/services/mysql
$ vi mysql-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
    name: mysql-pv
spec:
    accessModes:
    - ReadWriteOnce
    capacity:
    storage: 1Gi
    persistentVolumeReclaimPolicy: Retain
#storageClassName: nfs
    nfs:
    path: /data/mysql-pv
    server: 10.12.5.110

$ vi mysql-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
    name: mysql-pvc
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  #storageClassName: nfs

$ vi mysql.yaml

apiVersion: v1
kind: Service
metadata:
  name: mysql
spec:
  ports:
  - port: 3306
  selector:
    app: mysql
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: mysql
spec:
  replicas: 1
  selector:
    matchLabels:
      app: mysql
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - name: mysql
        image: mysql:5.6
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: password
        ports:
        - containerPort: 3306
          name: mysql
        volumeMounts:
         - name: mysql-persistent-storage
          mountPath: /var/lib/mysql
      volumes:
      - name: mysql-persistent-storage
        persistentVolumeClaim:
          claimName: mysql-pvc

创建mysql service和pod

$ cd /home/k8s/yml/services/mysql
$ kubectl create -f mysq-pv.yaml
$ kubectl get pv

NAME       CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS    CLAIM               REASON    AGE
mysql-pv   1Gi        RWO           Retain          Bound     default/mysql-pvc             2d

$ kubectl create -f mysq-pvc.yaml
$ kubectl get pvc

NAME        STATUS    VOLUME     CAPACITY   ACCESSMODES   AGE
mysql-pvc   Bound     mysql-pv   1Gi        RWO           2d

$ kubectl create -f mysq.yaml
$ kubectl get service

NAME         CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
mysql        10.254.151.113   <none>        3306/TCP   2d

$ kubectl get pod

NAME                           READY     STATUS             RESTARTS   AGE
mysql-3827607452-hd5ct         1/1       Running            80         19h

k8s-master节点安装和配置jenkins

$ mkdir /home/jenkins
$ cd
$ vi .bash_profile
export JENKINS_HOME=/home/jenkins
$ cd /home/jenkins
$ wget http://mirrors.jenkins.io/war-stable/latest/jenkins.war
$ java -jar jenkins.war --httpPort=9090&
浏览器打开地址：http://10.12.5.110:9090
把文件/home/jenkins/secrets/initialAdminPassword的内容复制到安装页面
解决插件安装离线问题

打开地址：http://10.12.5.110:9090/pluginManager/advanced
把https://updates.jenkins.io/update-center.json修改为http://updates.jenkins.io/update-center.json

安装插件：Publish Over SSH

创建jenkins任务,源代码在github上

* 任务名称：springboottest
* 任务类型：自由风格
* 创建github访问Credential

Credential类型：username with password

* 源码管理 选择git
Repository URL 值为https://github.com/wangtiewu/springboottest.git
Credentials 值为上一步创建的 Credential
* 创建构建脚本

mvn clean package
tag=$( date +'%Y%m%d%H%M' )
app=springcloudtest
echo $tag > version_$app.txt
docker_name=$app
docker build -t $docker_name:$tag .
docker push $docker_name:$tag

* 创建ssh server

系统管理-〉系统设置菜单-〉publish over ssh

* 增加构建后操作，Send build artifacts over SSH

SSH Server:name 值为k8s master
Transfer Set:Source files 值为version_springcloudtest.txt
Transfer Set:Remote directory   值为springcloudtest
Transfer Set:Exec command 值为
cd /home/k8s/yml/services/springcloudtest
app=springcloudtest
deploy_file=$app.yaml
version_file=version_$app.txt
patch_file=$app.patch
new_version=$( cat $version_file )
old_version=$( cat $deploy_file | grep image | awk -F ":" '{print $3}' )
sed -i "s/$old_version/$new_version/" $deploy_file
deployment_num=$( /usr/bin/kubectl get deployment | grep springcloudtest | wc -l )
if [ $deployment_num -eq 1 ];then
    p_old_version=$( cat $patch_file | jq '.spec.template.spec.containers[0].image' )
    p_new_version=\"$app:$new_version\"
    sed -i "s/$p_old_version/$p_new_version/" $patch_file
    patch=$( cat $patch_file )
    /usr/bin/kubectl patch deployment $app -p $patch
else
    /usr/bin/kubectl create -f $deploy_file
fi

创建springclouttest service和pod

$ mkdir -p /home/k8s/yml/services/springcloudtest
$ cd /home/k8s/yml/services/springcloudtest
$ vi springcloudtest-svr.yaml

apiVersion: v1
kind: Service
metadata:
  name: springcloudtest
spec:
  ports:
  - name: springcloudtest-svr
    port: 9091
    nodePort: 30000
    targetPort: 9091
  selector:
    app: springcloudtest
  type: NodePort

$ vi springcloudtest.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: springcloudtest
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: springcloudtest
    spec:
      containers:
      - name: springclouttest
        image: springcloudtest:201805272108
        ports: 
        - containerPort: 9091
          protocol: TCP

$ vi springcloudtest.patch

{"spec":{"template":{"spec":{"containers":[{"name":"springclouttest","image":"springcloudtest:201805052039"}]}}}}

构建jenkins任务：springclouttest

构建完成后，kubectl get pod 查看

springcloudtest-1053936621-w19sh   1/1       Running   1          1d

创建springclouttest service

$ cd /home/k8s/yml/services/springcloudtest
$ kubectl create -f springcloudtest-svr.yaml
查看服务状态
$ kubectl get service
springcloudtest   10.254.126.13   <nodes>       9091:30000/TCP   45m
在k8s-node查看容器情况
$ docker ps

CONTAINER ID        IMAGE                                                        COMMAND                  CREATED             STATUS              PORTS               NAMES
269366fa1b49        mysql:5.6                                                    "docker-entrypoint..."   About an hour ago   Up About an hour                        k8s_mysql.affc4af4_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_404d6f6a
2ff21fb61984        springcloudtest:201805272108                                 "/bin/sh -c 'java ..."   About an hour ago   Up About an hour                        k8s_springclouttest.c0ee0b5d_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_a99e1b37
88e9ba47faf1        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           About an hour ago   Up About an hour                        k8s_POD.389b0ddb_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_7387fdc6
130a60acd94d        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           About an hour ago   Up About an hour                        k8s_POD.2aaa0bac_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_fa8aba88

解决 NodePort方式下,Node所在节点可以访问服务,节点外客户端请求服务都会失败问题
$ iptables -P FORWARD ACCEPT

创建数据库和表

$ docker exec -it 269366fa1b49 /bin/bash
$ mysql -uroot -p
输入密码

mysql>create database test1 default charset utf8 COLLATE utf8_general_ci;
mysql>use test1
mysql>create table HELLO(ID bigint, NAME varchar(128), primary key (ID)); 

测试服务

浏览器打开 http://10.12.5.115:30000/
浏览器显示 Hello World!

$ curl --data '{"id":1,"name":"Hello World"}' -H "Content-type: Application/json" http://10.12.5.115:30000/create/hello