K8S+ jenkins+gitlub+Harbor实现CI/CD

---

一、环境部署

主机名	IP地址	服务
master	192.168.1.10	docker、k8s、Harbor
node01	192.168.1.20	docker、k8s
node02	192.168.1.30	docker、k8s
jenkins	192.168.1.40	gitlub
gitlub	192.168.1.50	docker、jenkins

1、部署docker

1、安装docker软件

~]# yum install -y yum-utils device-mapper-persistent-data lvm2
~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
~]# yum repolist
~]# yum makecache
~]# yum -y install docker-ce
~]# systemctl start docker
~]# systemctl enable docker

2、开启路由转发

~]# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
~]# sysctl -p

3、配置镜像加速器

sudo mkdir -p /etc/docker
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://12azv802.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

2、部署k8s集群

1、关闭防火墙，清空iptables，禁用selinux

~]# systemctl stop firewalld
~]# systemctl disable firewalld
~]# iptables -F
~]# iptables-save
~]# vim /etc/selinux/config
SELINUX=disabled

2、禁用swap

~]# swapoff -a
~]# vim /etc/fstab
#/dev/mapper/centos-swap swap                    swap    defaults        0 0

3、编辑对应域名解析

[root@master ~]# vim /etc/hosts
192.168.1.10 master
192.168.1.20 node01
192.168.1.30 node02
[root@master ~]# scp /etc/hosts node01:/etc/hosts
[root@master ~]# scp /etc/hosts node02:/etc/hosts

4、开启无密码传送

[root@master ~]# ssh-keygen -t rsa
[root@master ~]# ssh-copy-id root@node01
[root@master ~]# ssh-copy-id root@node02

5、添加kubernetes的yum源

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

6、安装master节点

[root@master ~]# yum -y install kubelet-1.15.0 kubeadm-1.15.0 kubectl-1.15.0 
[root@master ~]# systemctl enable kubelet
docker pull registry.aliyuncs.com/google_containers/coredns:1.3.1
docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0
docker pull registry.aliyuncs.com/google_containers/etcd:3.3.10
docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.15.0
docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.15.0
docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.15.0
docker pull registry.aliyuncs.com/google_containers/pause:3.1
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.15.0 k8s.gcr.io/kube-apiserver:v1.15.0
docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 k8s.gcr.io/kube-proxy:v1.15.0
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.15.0 k8s.gcr.io/kube-controller-manager:v1.15.0
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.15.0 k8s.gcr.io/kube-scheduler:v1.15.0
docker tag registry.aliyuncs.com/google_containers/etcd:3.3.10 k8s.gcr.io/etcd:3.3.10
docker tag registry.aliyuncs.com/google_containers/coredns:1.3.1 k8s.gcr.io/coredns:1.3.1
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
[root@master ~]# kubeadm init --kubernetes-version=v1.15.0 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --ignore-preflight-errors=Swap
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@master ~]# yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc
[root@master ~]# vim .vimrc
set tabstop=2
[root@master ~]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
[root@master ~]# wget https://github.com/coreos/flannel/releases/download/v0.13.0/flanneld-v0.13.0-amd64.docker
[root@master ~]# docker load < flanneld-v0.13.0-amd64.docker
[root@master ~]# docker save quay.io/coreos/flannel > flannel.tar
[root@master ~]# scp flannel.tar node01:
[root@master ~]# scp flannel.tar node02:

7、安装node节点

[root@node01 ~]# yum -y install kubelet-1.15.0 kubeadm-1.15.0
[root@node01 ~]# systemctl enable kubelet
docker pull registry.aliyuncs.com/google_containers/pause:3.1
docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.15.0 k8s.gcr.io/kube-proxy:v1.15.0
[root@node01 ~]# docker load < flannel.tar
[root@node01 ~]# kubeadm join 192.168.1.10:6443 --token 24bj0y.67cd6dsp5bao7ypu \
>     --discovery-token-ca-cert-hash sha256:668f9ee00d17a77b81d47e792f71aa32dc9750a604875793a4eea97b55b0f50e

3、部署Harbor私有仓库

1、下载docker-compse工具

[root@master ~]# wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-Linux-x86_64
[root@master ~]# mv docker-compose-Linux-x86_64 docker-compose
[root@master ~]# mv docker-compose /usr/local/sbin/
[root@master ~]# chmod +x /usr/local/sbin/docker-compose 
[root@master ~]# docker-compose -v
docker-compose version 1.26.2, build eefe0d31
[root@master ~]# yum -y install yum-utils device-mapper-persistent-data lvm2

2、下载harbor

[root@master ~]# wget https://github.com/goharbor/harbor/releases/download/v2.0.2/harbor-offline-installer-v2.0.2.tgz
[root@master ~]# tar -zxf harbor-offline-installer-v2.0.2.tgz -C /usr/local/
[root@master ~]# cd /usr/local/harbor/
[root@master harbor]# cp harbor.yml.tmpl harbor.yml
[root@master harbor]# vim harbor.yml
hostname: 192.168.1.10        #harbor服务器主机IP或域名
harbor_admin_password: 123456       #harbor管理员登录密码
[root@localhost ~]# vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd --insecure-registry 192.168.1.10
[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart docker
[root@master certs]# cd /usr/local/harbor/
[root@master harbor]# ./install.sh 

3、准备私有仓库镜像

[root@master ~]# docker pull httpd
[root@master ~]# docker run -itd --name httpd httpd
[root@master ~]# docker exec -it httpd /bin/bash
root@b92195cd3071:/usr/local/apache2# cd /usr/local/apache2/htdocs/
root@b92195cd3071:/usr/local/apache2/htdocs# echo 111 > index.html 
root@b92195cd3071:/usr/local/apache2/htdocs# exit
exit
[root@master ~]# docker commit httpd httpd:v1
[root@master ~]# docker tag httpd:v1 192.168.1.10/httpd/v1
[root@master ~]# docker push 192.168.1.10/httpd/v1

4、部署gitlub

1、安装程序

[root@gitlab ~]# yum -y install curl policycoreutils openssh-server openssh-clients postfix           
[root@gitlab ~]# wget https://mirrors.tuna.tsinghua.edu.cn/gitlab-ce/yum/el7/gitlab-ce-10.2.3-ce.0.el7.x86_64.rpm
[root@gitlab ~]# yum -y install gitlab-ce-10.2.3-ce.0.el7.x86_64.rpm
[root@gitlab ~]# vim /etc/gitlab/gitlab.rb
external_url 'http://192.168.1.40'
[root@gitlab ~]# gitlab-ctl reconfigure

2、gitlab汉化

[root@gitlab ~]# gitlab-ctl stop
[root@gitlab ~]# tar zxf gitlab-patch-zh.tat.gz 
[root@gitlab ~]# cd gitlab/
[root@gitlab gitlab]# git diff v10.2.3 v10.2.3-zh > ../10.2.3-zh.diff
[root@gitlab gitlab]# patch -d /opt/gitlab/embedded/service/gitlab-rails -p1 < /root/10.2.3-zh.diff
[root@git gitlab]# gitlab-ctl restart

5、部署jenkins

1、安装Jenkins

[root@jenkins ~]# cd /etc/yum.repos.d/
[root@jenkins yum.repos.d]# wget http://pkg.jenkins.io/redhat/jenkins.repo
[root@jenkins yum.repos.d]# rpm --import http://pkg.jenkins.io/redhat/jenkins.io.key
[root@jenkins yum.repos.d]# yum -y install jenkins
[root@jenkins ~]# vim /etc/sysconfig/jenkins
JENKINS_USER="root"
[root@jenkins ~]# systemctl start jenkins
[root@jenkins ~]# systemctl enable jenkins
[root@jenkins ~]# cat /var/lib/jenkins/secrets/initialAdminPassword
6cc1036577174dad86be36179dabc7b3

2、复制插件包到服务器

[root@jenkins ~]# mv plugins.zip /var/lib/jenkins/
[root@jenkins ~]# cd /var/lib/jenkins/
[root@jenkins ~]# unzip plugins.zip

二、模拟web服务

1、配置k8s连接Harbor

1、创建证书

[root@master ~]# docker login -u admin -p 123456 192.168.1.10
[root@master ~]# cat ~/.docker/config.json 
{
    
    
	"auths": {
    
    
		"192.168.1.10": {
    
    
			"auth": "YWRtaW46MTIzNDU2"
		}
	},
	"HttpHeaders": {
    
    
		"User-Agent": "Docker-Client/19.03.13 (linux)"
	}
}

2、进行base加密

[root@master ~]# cat .docker/config.json | base64
ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjEuMTAiOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2TVRJ
ek5EVTIiCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXIt
Q2xpZW50LzE5LjAzLjEzIChsaW51eCkiCgl9Cn0=

3、创建Secret资源

[root@master ~]# vim secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: registry-secret
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjEuMTAiOiB7CgkJCSJhdXRoIjogIllXUnRhVzQ2TVRJek5EVTIiCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXItQ2xpZW50LzE4LjA5LjAgKGxpbnV4KSIKCX0KfQo=
type: kubernetes.io/dockerconfigjson
[root@master ~]# kubectl apply -f secret.yaml

2、部署registry服务

[root@master ~]# mkdir /opt/autoweb
[root@master ~]# cd /opt/autoweb/
[root@master autoweb]# vim myapp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: myapp
spec:
  replicas: 2
  template:
    metadata:
      labels:
        app: httpd
    spec:
      containers:
      - name: myapp
        image: 192.168.1.10/httpd/v1
        imagePullPolicy: Always
      imagePullSecrets:
      - name: registry-secret
[root@master autoweb]# vim mysvc.yaml
kind: Service
apiVersion: v1
metadata:
  name: mysvc
spec:
  selector:
    app: httpd
  type: NodePort
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
    nodePort: 30039
[root@master autoweb]# kubectl apply -f myapp.yaml 
[root@master autoweb]# kubectl apply -f mysvc.yaml 
[root@master autoweb]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        138m
mysvc        NodePort    10.103.212.48   <none>        80:30039/TCP   30s
[root@master autoweb]# curl 10.103.212.48
111

三、配置jenkins关联gitlab

1、在gitlab上绑定服务器的ssh公钥

[root@jenkins ~]# ssh-keygen -t rsa
[root@jenkins ~]# cat ~/.ssh/id_rsa.pub

2、创建项目

3、创建新版本库

[root@gitlab ~]# git clone [email protected]:root/test.git
[root@gitlab ~]# cd test
[root@gitlab ~]# touch README.md
[root@gitlab ~]# git add README.md
[root@gitlab ~]# git commit -m "add README"
[root@gitlab ~]# git push -u origin master

4、jenkins创建工程

#!/bin/bash
backupcode="/data/backcode/$JOB_NAME/$BUILD_NUMBER"
mkdir -p $backupcode
chmod 644 "$JENKINS_HOME"/workspace/"$JOB_NAME"/*
rsync -acP   "$JENKINS_HOME"/workspace/"$JOB_NAME"/*  $backupcode
echo From  192.168.1.10/httpd/v1 > "$JENKINS_HOME"/workspace/Dockerfile
echo COPY ./"$JOB_NAME"/* /usr/local/apache2/htdocs/ >> "$JENKINS_HOME"/workspace/Dockerfile
docker rmi 192.168.1.10/httpd/v1
docker build -t 192.168.1.10/httpd/v1 /"$JENKINS_HOME"/workspace/.
docker push 192.168.1.10/httpd/v1
ssh [email protected] kubectl delete deployment myapp
ssh [email protected] kubectl apply -f /opt/autoweb/myapp.yaml

5、修改jenkins的设置

6、gitlab添加钩子

7、测试访问

四、jenkins权限设置

1、与master节点做免密登录

[root@jenkins ~]# ssh-copy-id [email protected]

2、登录私有仓库

[root@jenkins ~]# docker login -u admin -p 123456 192.168.1.10

五、上传代码进行自动化部署

[root@gitlab ~]# git clone https://gitee.com/kangjie1209/monitor.git
[root@gitlab ~]# cp -rp /root/monitor/* test/
[root@gitlab ~]# cd test/
[root@gitlab test]# git init
重新初始化现存的 Git 版本库于 /root/test/.git/
[root@gitlab test]# git add .
[root@gitlab test]# git commit -m "Initial commit"
[root@gitlab test]# git push -u origin master