Linux-k8s, Jenkins, Gitlab, Harbor implement CI/CD

CI / CD

1. Environment construction:

Host	IP	service
master	192.168.1.40	K8s、Harbor
node1	192.168.1.41	K8s
Jenkins	192.168.1.42	jenkins
Gitlab	192.168.1.43	gitlab

1. Build K8s

1.1 Install Docker

The rest need Docker environment to execute according to this step

[root@master ~]# wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
[root@master ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@master ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo 
[root@master ~]# yum repolist
[root@master ~]# yum makecache
[root@master ~]# yum list docker-ce.x86_64 --showduplicates | sort -r
[root@master ~]# yum -y install docker-ce-18.09.0-3.el7 docker-ce-cli-18.09.0-3.el7 containerd.io-1.2.10-el7
[root@master ~]# docker version
Client:
 Version:           18.09.0
 API version:       1.39
 Go version:        go1.10.4
 Git commit:        4d60db4
 Built:             Wed Nov  7 00:48:22 2018
 OS/Arch:           linux/amd64
 Experimental:      false

Server: Docker Engine - Community
 Engine:
  Version:          18.09.0
  API version:      1.39 (minimum version 1.12)
  Go version:       go1.10.4
  Git commit:       4d60db4
  Built:            Wed Nov  7 00:19:08 2018
  OS/Arch:          linux/amd64
  Experimental:     false
[root@master ~]# vim /etc/docker/daemon.json  
{
"registry-mirrors": ["https://z1pa8k3e.mirror.aliyuncs.com"]
}
[root@master ~]# systemctl  start docker.service 
[root@master ~]# systemctl  enable  docker.service 
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.

1.2 Install K8s master node

1.2.1 Preparing for installing K8s

[root@master ~]# systemctl  stop firewalld
[root@master ~]# systemctl  disable firewalld
[root@master ~]# setenforce 0
setenforce: SELinux is disabled
[root@master ~]# iptables -F   //清空
[root@master ~]# iptables-save	//保存
[root@master ~]# swapoff -a
[root@master ~]# vim /etc/fstab
......
#/dev/mapper/centos-swap swap                    swap    defaults        0 0
......
[root@master ~]# free -h    //验证swap确实关闭
              total        used        free      shared  buff/cache   available
Mem:           2.7G        534M        1.7G         13M        562M        2.0G
Swap:            0B          0B          0B
[root@master ~]# vim  /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.40 master
192.168.1.41 node01
[root@master ~]# ssh-keygen -t rsa
[root@master ~]# ssh-copy-id root@node01
[root@master ~]# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
[root@master ~]# sysctl -p
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1

1.2.2 Install master node

[root@master ~]# cat > /etc/yum.repos.d/kubernetes.repo  <<EOF
> [kubernetes]
> name=Kubernetes
> baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
> enabled=1
> gpgcheck=1
> repo_gpgcheck=1
> gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
> EOF
[root@master ~]# scp -rp /etc/yum.repos.d/kubernetes.repo node01:/etc/yum.repos.d/
kubernetes.repo                                                   100%  276   135.6KB/s   00:00   
[root@master ~]# yum -y install kubelet-1.15.0 kubeadm-1.15.0 kubectl-1.15.0
[root@master ~]# systemctl enable kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.

#下载K8s环境基础镜像
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.15.0
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.15.0
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.15.0
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/etcd:3.3.10
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/coredns:1.3.1
[root@master ~]# docker pull registry.aliyuncs.com/google_containers/pause:3.1

#因为是从阿里云下载的镜像所以需要改名
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/kube-apiserver:v1.15.0 k8s.gcr.io/kube-apiserver:v1.15.0
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 k8s.gcr.io/kube-proxy:v1.15.0
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/kube-controller-manager:v1.15.0 k8s.gcr.io/kube-controller-manager:v1.15.0
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/kube-scheduler:v1.15.0 k8s.gcr.io/kube-scheduler:v1.15.0
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/etcd:3.3.10 k8s.gcr.io/etcd:3.3.10
[root@master ~]# docker tag  registry.aliyuncs.com/google_containers/coredns:1.3.1 k8s.gcr.io/coredns:1.3.1
[root@master ~]# docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1

#初始化K8s
[root@master ~]# kubeadm init --kubernetes-version=v1.15.0 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/15 --ignore-preflight-errors=Swap
......
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.1.40:6443 --token agclkg.2oy26svx0xvw5ixv \
    --discovery-token-ca-cert-hash sha256:584021231c5f1da1dd92dcbb8fc2694c1b67f44d3b34dba07fe650842453abf2 
[root@master ~]#  mkdir -p $HOME/.kube
[root@master ~]#   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master ~]#   sudo chown $(id -u):$(id -g) $HOME/.kube/config

#添加网络组件（flannel）

[root@master ~]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
[root@master ~]# kubectl  get  nodes
NAME     STATUS   ROLES    AGE   VERSION
master   Ready    master   83s   v1.15.0
[root@master ~]# kubectl get pod --all-namespaces
NAMESPACE     NAME                             READY   STATUS    RESTARTS   AGE
kube-system   coredns-5c98db65d4-c9rzw         1/1     Running   0          94s
kube-system   coredns-5c98db65d4-v6gvm         1/1     Running   0          94s
kube-system   etcd-master                      1/1     Running   0          43s
kube-system   kube-apiserver-master            1/1     Running   0          35s
kube-system   kube-controller-manager-master   1/1     Running   0          36s
kube-system   kube-flannel-ds-gqlvz            1/1     Running   0          62s
kube-system   kube-proxy-t29np                 1/1     Running   0          94s
kube-system   kube-scheduler-master            1/1     Running   0          40s

#设置kubectl命令行工具自动补全功能
[root@master ~]# yum install -y bash-completion
[root@master ~]# source /usr/share/bash-completion/bash_completion
[root@master ~]# source <(kubectl completion bash)
[root@master ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc
[root@master ~]# cat  > .vimrc  <<EOF
> set tabstop=2
> EOF

1.3 Install K8s node

Only one node is configured for the test environment here! ! !

[root@node01 ~]# yum install -y kubelet-1.15.0 kubeadm-1.15.0
[root@node01 ~]# systemctl  enable  kubelet.service 
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
[root@node01 ~]# docker pull registry.aliyuncs.com/google_containers/pause:3.1 
[root@node01 ~]# docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0
[root@node01 ~]# docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
[root@node01 ~]# docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 k8s.gcr.io/kube-proxy:v1.15.0
[root@node01 ~]# kubeadm join 192.168.1.40:6443 --token agclkg.2oy26svx0xvw5ixv \
>     --discovery-token-ca-cert-hash sha256:584021231c5f1da1dd92dcbb8fc2694c1b67f44d3b34dba07fe650842453abf2 

2. Build Jenkins

Install the docker environment according to 1.1 steps

2.1 Install jdk (java)

jdk URL

[root@jenkins ~]# mkdir software
[root@jenkins ~]# cd software/
[root@jenkins software]# ls
jdk-8u231-linux-x64.tar.gz  jenkins-2.190.3-1.1.noarch_2.rpm
[root@jenkins software]# tar -zxf jdk-8u231-linux-x64.tar.gz 
[root@jenkins software]# mv jdk1.8.0_231   /usr/java
[root@jenkins software]# vim /etc/profile
[root@jenkins software]# tail -4 /etc/profile
export JAVA_HOME=/usr/java
export JRE_HOME=/usr/java/jre
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar
[root@jenkins software]# source  /etc/profile
[root@jenkins software]# java -version
java version "1.8.0_231"
Java(TM) SE Runtime Environment (build 1.8.0_231-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.231-b11, mixed mode)

2.2 Install jenkins

[root@jenkins software]# ls
jdk-8u231-linux-x64.tar.gz  jenkins-2.190.3-1.1.noarch_2.rpm
[root@jenkins software]# rpm -ivh jenkins-2.190.3-1.1.noarch_2.rpm 
警告：jenkins-2.190.3-1.1.noarch_2.rpm: 头V4 DSA/SHA1 Signature, 密钥 ID d50582e6: NOKEY
准备中...                          ################################# [100%]
正在升级/安装...
   1:jenkins-2.190.3-1.1              ################################# [100%]

2.3 Add the JDK path that jenkins starts to depend on.

[root@jenkins software]# vim /etc/init.d/jenkins
......
 83 /usr/bin/java
 84 /usr/java/bin/java
......
[root@jenkins software]# systemctl  start  jenkins
[root@jenkins software]# ss -lnt | grep 8080
LISTEN     0      50          :::8080                    :::*             
[root@jenkins software]# cat /var/lib/jenkins/secrets/initialAdminPassword 
ddbbf205ccfb433c893b5a1b6aa7545f

Download needs to support Gitlab and Chinese package

For network reasons, choose the downloaded one here! ! !

[root@jenkins software]# cd /var/lib/jenkins/
[root@jenkins jenkins]# ls
jenkins-plugins.tar.gz                         
[root@jenkins jenkins]# mv plugins plugins.bak
[root@jenkins jenkins]# tar -zxf jenkins-plugins.tar.gz 
[root@jenkins jenkins]# ls plugins
ace-editor                              matrix-auth.jpi
ace-editor.jpi                          matrix-project
ant                                     matrix-project.jpi
antisamy-markup-formatter               momentjs
antisamy-markup-formatter.jpi           momentjs.jpi
ant.jpi                                 pam-auth.jpi.tmp
......
[root@jenkins jenkins]# service  jenkins  restart
Restarting jenkins (via systemctl):                        [  确定  ]

3. Build Gitlab

[root@gitlab ~]# yum -y install curl policycoreutils openssh-server openssh-clients postfix  
[root@gitlab ~]# wget https://mirrors.tuna.tsinghua.edu.cn/gitlab-ce/yum/el7/gitlab-ce-10.2.3-ce.0.el7.x86_64.rpm
[root@gitlab ~]# yum -y install  gitlab-ce-10.2.3-ce.0.el7.x86_64.rpm
[root@gitlab ~]# vim /etc/gitlab/gitlab.rb
external_url 'http://192.168.1.43'
[root@gitlab ~]# gitlab-ctl reconfigure

4. Build Harbor

This is a test environment, so the warehouse is installed on the master node of K8s.

4.1 Download Docker-compse tool

[root@master ~]# curl -L  https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
[root@master ~]# tar -zxPf docker-compose.1.25.0.tar.gz -C /usr/local/bin/
[root@master ~]# chmod  +x /usr/local/bin/docker-compose 
[root@master ~]# docker-compose -v  
docker-compose version 1.25.0, build 0a186604
[root@master ~]# yum -y install yum-utlis device-mapper-persistent-data lvm2

4.2 Download Harbor

[root@master ~]# wget https://storage.googleapis.com/harbor-releases/release-1.7.0/harbor-offline-installer-v1.7.4.tgz  
[root@master ~]# ls
harbor-offline-installer-v1.7.4.tgz  docker-compose.1.25.0.tar.gz  
[root@master ~]# tar -zxf harbor-offline-installer-v1.7.4.tgz  -C /usr/local/
[root@master ~]# cd /usr/local/harbor/
[root@master harbor]# vim harbor.cfg 
......
hostname = 192.168.1.40
......
[root@master harbor]# ./install.sh 

Two, simulate web

1. Associate Harbor

1.2 k8s connect to Harbor

Create a certificate

PS: In the node server and jenkins server also need to add the warehouse address! ! ! !

[root@master ~]# vim /usr/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd --insecure-registry 192.168.1.40
[root@master ~]# systemctl  daemon-reload 
[root@master ~]# systemctl  restart docker
[root@master ~]# docker login -u admin -p Harbor12345 192.168.1.40
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded
[root@master ~]#  cat ~/.docker/config.json
{
	"auths": {
		"192.168.1.40": {
			"auth": "YWRtaW46SGFyYm9yMTIzNDU="
		}
	},
	"HttpHeaders": {
		"User-Agent": "Docker-Client/18.09.0 (linux)"
	}
}

Base encryption

[root@master ~]# cat .docker/config.json | base64
ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjEuNDAiOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2U0dG
eVltOXlNVEl6TkRVPSIKCQl9Cgl9LAoJIkh0dHBIZWFkZXJzIjogewoJCSJVc2VyLUFnZW50Ijog
IkRvY2tlci1DbGllbnQvMTguMDkuMCAobGludXgpIgoJfQp9

Create Secret resource

[root@master ~]#  vim secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: registry-secret
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjEuNDAiOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2U0dGeVltOXlNVEl6TkRVPSIKCQl9Cgl9LAoJIkh0dHBIZWFkZXJzIjogewoJCSJVc2VyLUFnZW50IjogIkRvY2tlci1DbGllbnQvMTguMDkuMCAobGludXgpIgoJfQp9
type: kubernetes.io/dockerconfigjson
[root@master ~]# kubectl apply -f secret.yaml
secret/registry-secret created

1.3 Upload private image

[root@master ~]# docker pull nginx
[root@master ~]# docker run  -itd  --name  web   nginx:latest 
c5233f10cc7bff9bca61e2348b71e6ea41d3dc56dff1f5e3147f946552cd062d
[root@master ~]# docker exec  -it web bash
root@c5233f10cc7b:/# echo 1111 > /usr/share/nginx/html/index.html 
root@c5233f10cc7b:/# cat  /usr/share/nginx/html/index.html 
1111
root@c5233f10cc7b:/# exit
exit
[root@master ~]# docker commit  web  web:v1
sha256:5718b129e875fb3032210a618b14bc4173698588aba545958856512f5c24ebd8
[root@master ~]# docker tag  web:v1 192.168.1.40/web/v1

2. Simulate web services

[root@master ~]#  mkdir web
[root@master ~]# cd web/
[root@master web]# vim myapp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: myapp
spec:
  replicas: 2
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: myapp
        image: 192.168.1.40/web/v1
        imagePullPolicy: Always
      imagePullSecrets:
      - name: registry-secret
---
kind: Service
apiVersion: v1
metadata:
  name: mysvc
spec:
  selector:
    app: nginx
  type: NodePort
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
    nodePort: 30039

[root@master web]# kubectl  apply  -f  myapp.yaml 
deployment.extensions/myapp created
service/mysvc created
[root@master ~]# kubectl  get pod 
NAME                     READY   STATUS    RESTARTS   AGE
myapp-667db6c49d-wtgrp   1/1     Running   0          3m17s
myapp-667db6c49d-wzbpx   1/1     Running   0          3m17s
[root@master ~]# kubectl  get svc
NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1      <none>        443/TCP        36d
mysvc        NodePort    10.96.78.132   <none>        80:30039/TCP   3m19s
[root@master ~]# curl  127.0.0.1:30039
1111

3. Linkage between Jenkins and Gitlab platform

The linkage of these two services requires SSH verification. We need to bind the ssh public key of the jenkins server on gitlab. The public and private keys of the root user are used here. Remember that the production environment does not allow you to use root casually

[root@jenkins ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:Ls8H6FHL3i0c7lnE2OR6nemmbjYL+YALTtEL7MToD9E root@jenkins
The key's randomart image is:
+---[RSA 2048]----+
|                 |
|                 |
|            .    |
|     = ..  *     |
|    o E+S.. =    |
|   . +o++o.+ . o |
|    o.=o+==oo +  |
|     =.=..*B=..  |
|      o +oo===.  |
+----[SHA256]-----+
[root@jenkins ~]# cat ~/.ssh/id_rsa.pub 
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTfcKQI34QV0W2g5xFDXvjodV1bjhXMCDfdPdhfMLA592PjTjraHYJpOUOTfExHNW5Xj6kbxKaxapVk/NFy/QIsOXzZexi5pp5smhbcVYKISh5zcbT/tjw6LSJ8jG3FDMrVXBQbHLFBaW+/6/d6gecrxu3pJd5xDALKrmeypJncJ7TkOUiYcG3Q4cO/THbumEGnZXX0lFZRMSOyuM3WERq8slU7l51Yb4Av/hm9sj+1+wlDKWEXdV71N90Ufcb8C4Ltf6NIm26r++u5k29jUpRAA/AU3SWvVMoFzZwcrThdp+6VLtBAjfJQvgpDy8+M1WP9GcBZbsUmOIsFaNVvT7l root@jenkins

4. In gitlab, create a project

Add project file

You can use the git command to clone the project locally to verify and view

[root@jenkins ~]# mkdir web
[root@jenkins ~]# cd web/
[root@jenkins web]# git clone [email protected]:root/test.git
正克隆到 'test'...
The authenticity of host '192.168.1.43 (192.168.1.43)' can't be established.
ECDSA key fingerprint is SHA256:4YkYpIBzWiwJtzfIG1R+GdAFUVtGIu2lxKDjsytMW88.
ECDSA key fingerprint is MD5:14:a7:e4:91:f9:32:76:41:b0:0c:8b:0e:2c:0a:8b:fd.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.43' (ECDSA) to the list of known hosts.
remote: Counting objects: 3, done.
remote: Total 3 (delta 0), reused 0 (delta 0)
接收对象中: 100% (3/3), done.
[root@jenkins web]# ls
test
[root@jenkins web]# cat test/index.html 
<h1> Hello Word!!!</h1>

5. Create a project in jenkins

#!/bin/bash
backupcode="/data/backcode/$JOB_NAME/$BUILD_NUMBER"
mkdir -p $backupcode
chmod 644 "$JENKINS_HOME"/workspace/"$JOB_NAME"/*
rsync -acP   "$JENKINS_HOME"/workspace/"$JOB_NAME"/*  $backupcode
echo From  192.168.1.40/web/v1 > "$JENKINS_HOME"/workspace/Dockerfile
echo COPY ./"$JOB_NAME"/* /usr/share/nginx/html/ >> "$JENKINS_HOME"/workspace/Dockerfile
docker rmi 192.168.1.40/web/v1
docker build -t 192.168.1.40/web/v1 /"$JENKINS_HOME"/workspace/.
docker push 192.168.1.40/web/v1
ssh [email protected] kubectl delete deployment myapp
ssh [email protected] kubectl apply -f /root/web/myapp.yaml

6. Modify the settings of jenkins

7. Gitlab add hook

8. Test visit

Script running result

9.Jenkins permission settings

9.1. Do password-free login with master node

[root@jenkins ~]# ssh-copy-id [email protected]

9.2 Change user of jenkins

[root@jenkins ~]# vim /etc/sysconfig/jenkins
...
29 JENKINS_USER="root"
...
[root@jenkins ~]# service  jenkins  restart
Restarting jenkins (via systemctl):                        [  确定  ]

Three, upload code for automated deployment