1. 使用Helm安装Mysql:
上一篇博文谈到了如何使用Helm安装Redis和RabbitMQ，下来我们来聊聊如何用Helm安装mysql.
本人对于Mysql数据库不是非常熟悉，因为我们公司的分工比较明确，数据库这块的工作主要由DBA负责，运维同学只负责应用的维护。
按照我们前面博文的描述，首先是在官方文档查看helm安装mysql的书名: https://github.com/helm/charts/tree/master/stable/mysql
我根据官方文档的描述配置的value.yaml文件如下：
## mysql image version
## ref: https://hub.docker.com/r/library/mysql/tags/
##
image: "k8s.harbor.maimaiti.site/system/mysql"
imageTag: "5.7.14"
busybox:
image: "k8s.harbor.maimaiti.site/system/busybox"
tag: "1.29.3"
testFramework:
image: "k8s.harbor.maimaiti.site/system/bats"
tag: "0.4.0"
## Specify password for root user
##
## Default: random 10 character string
mysqlRootPassword: admin123
## Create a database user
##
mysqlUser: test
## Default: random 10 character string
mysqlPassword: test123
## Allow unauthenticated access, uncomment to enable
##
# mysqlAllowEmptyPassword: true
## Create a database
##
mysqlDatabase: test
## Specify an imagePullPolicy (Required)
## It's recommended to change this to 'Always' if the image tag is 'latest'
## ref: http://kubernetes.io/docs/user-guide/images/#updating-images
##
imagePullPolicy: IfNotPresent
extraVolumes: |
# - name: extras
# emptyDir: {}
extraVolumeMounts: |
# - name: extras
# mountPath: /usr/share/extras
# readOnly: true
extraInitContainers: |
# - name: do-something
# image: busybox
# command: ['do', 'something']
# Optionally specify an array of imagePullSecrets.
# Secrets must be manually created in the namespace.
# ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
# imagePullSecrets:
# - name: myRegistryKeySecretName
## Node selector
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
nodeSelector: {}
## Tolerations for pod assignment
## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
##
tolerations: []
livenessProbe:
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
readinessProbe:
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
## Persist data to a persistent volume
persistence:
enabled: true
## database data Persistent Volume Storage Class
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
storageClass: "dynamic"
accessMode: ReadWriteOnce
size: 8Gi
annotations: {}
## Configure resource requests and limits
## ref: http://kubernetes.io/docs/user-guide/compute-resources/
##
resources:
requests:
memory: 256Mi
cpu: 100m
# Custom mysql configuration files used to override default mysql settings
configurationFiles: {}
# mysql.cnf: |-
# [mysqld]
# skip-name-resolve
# ssl-ca=/ssl/ca.pem
# ssl-cert=/ssl/server-cert.pem
# ssl-key=/ssl/server-key.pem
# Custom mysql init SQL files used to initialize the database
initializationFiles: {}
# first-db.sql: |-
# CREATE DATABASE IF NOT EXISTS first DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
# second-db.sql: |-
# CREATE DATABASE IF NOT EXISTS second DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
metrics:
enabled: true
image: k8s.harbor.maimaiti.site/system/mysqld-exporter
imageTag: v0.10.0
imagePullPolicy: IfNotPresent
resources: {}
annotations: {}
# prometheus.io/scrape: "true"
# prometheus.io/port: "9104"
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
readinessProbe:
initialDelaySeconds: 5
timeoutSeconds: 1
## Configure the service
## ref: http://kubernetes.io/docs/user-guide/services/
service:
annotations: {}
## Specify a service type
## ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types
type: ClusterIP
port: 3306
# nodePort: 32000
ssl:
enabled: false
secret: mysql-ssl-certs
certificates:
# - name: mysql-ssl-certs
# ca: |-
# -----BEGIN CERTIFICATE-----
# ...
# -----END CERTIFICATE-----
# cert: |-
# -----BEGIN CERTIFICATE-----
# ...
# -----END CERTIFICATE-----
# key: |-
# -----BEGIN RSA PRIVATE KEY-----
# ...
# -----END RSA PRIVATE KEY-----
## Populates the 'TZ' system timezone environment variable
## ref: https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html
##
## Default: nil (mysql will use image's default timezone, normally UTC)
## Example: 'Australia/Sydney'
# timezone:
# To be added to the database server pod(s)
podAnnotations: {}
podLabels: {}
## Set pod priorityClassName
# priorityClassName: {}
主要修改了如下几点的配置：
- 将所有的镜像都改为了私服镜像地址；
- 配置了mysql的初始化root密码，普通用户账户和密码，创建一个测试数据库；
- 配置了持久化存储；
使用helm install的时候也可以自定义参数，具体参数如何使用要看官方文档；比如:
helm install --values=mysql.yaml --set mysqlRootPassword=abc123 --name r1 stable/mysql查看安装好之后的mysql如何连接，有一个my-mysql的服务直接调用即可；
[root@master-01 mysql]# helm status my-mysql
LAST DEPLOYED: Thu Apr 25 15:08:27 2019
NAMESPACE: kube-system
STATUS: DEPLOYED
RESOURCES:
==> v1/Pod(related)
NAME READY STATUS RESTARTS AGE
my-mysql-5fd54bd9cb-948td 2/2 Running 3 6d22h
==> v1/Secret
NAME TYPE DATA AGE
my-mysql Opaque 2 6d22h
==> v1/ConfigMap
NAME DATA AGE
my-mysql-test 1 6d22h
==> v1/PersistentVolumeClaim
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
my-mysql Bound pvc-ed8a9252-6728-11e9-8b25-480fcf659569 8Gi RWO dynamic 6d22h
==> v1/Service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
my-mysql ClusterIP 10.200.200.169 <none> 3306/TCP,9104/TCP 6d22h
==> v1beta1/Deployment
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
my-mysql 1 1 1 1 6d22h
NOTES:
MySQL can be accessed via port 3306 on the following DNS name from within your cluster:
my-mysql.kube-system.svc.cluster.local
To get your root password run:
MYSQL_ROOT_PASSWORD=$(kubectl get secret --namespace kube-system my-mysql -o jsonpath="{.data.mysql-root-password}" | base64 --decode; echo)
To connect to your database:
1. Run an Ubuntu pod that you can use as a client:
kubectl run -i --tty ubuntu --image=ubuntu:16.04 --restart=Never -- bash -il
2. Install the mysql client:
$ apt-get update && apt-get install mysql-client -y
3. Connect using the mysql cli, then provide your password:
$ mysql -h my-mysql -p
To connect to your database directly from outside the K8s cluster:
MYSQL_HOST=127.0.0.1
MYSQL_PORT=3306
# Execute the following command to route the connection:
kubectl port-forward svc/my-mysql 3306
mysql -h ${MYSQL_HOST} -P${MYSQL_PORT} -u root -p${MYSQL_ROOT_PASSWORD}
[root@master-01 mysql]#2. 使用prometheus监控公共组件:
2.1 在prometheus的configmap里面增加配置:
- job_name: "mysql"
static_configs:
- targets: ['my-mysql:9104']
- job_name: "redis"
static_configs:
- targets: ['my-redis-redis-ha:9121']
然后重新删除和建立configmap,热更新prometheus的配置；
kubectl replace -f configmap.yaml --force
curl -X POST "http://10.109.108.37:9090/-/reload"3. 使用helm安装jenkins
首先还是搜索一下jenkins的chart
[root@master-01 k8sdemo2]# helm search jenkins
NAME CHART VERSION APP VERSION DESCRIPTION
stable/jenkins 1.1.10 lts Open source continuous integration server. It supports mu...
[root@master-01 k8sdemo2]#然后我们把它下载下来并解压
扫描二维码关注公众号，回复：
6098990 查看本文章
helm fetch stable/jenkins --untar --untardir ./在kubernetes集群里面使用jenkins的原理主要包括:
-
在k8s集群里面运行jenkins master的pod，使用持久化存储，保证jenkins pod重启或者迁移之后，jenkins的插件和页面的job配置不会丢失;
- 在K8S集群里面运行jenkins slave Pod,当有一个job运行的时候，就会启动一个jenkins slave的Pod，当job运行完成之后这个slave pod会自动销毁，可以节省资源；
-
因为jenkins主要是CI/CD工具，所以jenkins完成的任务包括调用gitlab下载代码--->使用maven编译打包--->使用sonar代码检查（可选）--->使用docker build构建镜像--->使用docker push上传镜像到私有仓库--->
使用kubectl命令发布应用到k8s集群 - 我们发布一个k8s应用一般需要用到Dockerfile、k8s YAML清单配置文件、jenkins pipeline流水线配置文件；
-
Dockerfile文件的主要目的是构建镜像，将jenkins maven打包之后的war包或者jar包ADD到镜像内，然后定义镜像的启动命令，环境变量（比如JAVA_HOME)等；
-
YAML文件主要定义的K8S部署应用的规则，比如部署几个副本，应用的资源限制是多少，应用启动之后的健康检查是curl还是tcp端口检查；应用是否需要挂载存储等；除了配置应用的deployment文件之外，
一般还要配置service清单文件，用于其他应用调用服务名来访问本应用，如果应用还需要对外提供访问，还需要配置Ingress文件，甚至还包括配置文件configmap需要创建，应用依赖的数据库，MQ账户信息等需要
使用secrets配置清单文件等等，所以建议熟悉Helm的同学在jenkins里面还是调用helm命令部署应用是最好的； -
pipeline文件就是jenkins的配置内容了，现在使用jenkins都是推荐使用流水线模式，因为流水线模式非常利于jenkins的迁移等；pipeline文件主要定义了step,包括上面描述的打包、构建、镜像制作、
k8s应用发布等动作，这些动作的实现都是依靠jenkins slave这个POD。所以这个jenkins slave镜像的制作就是非常重要的了。 - jenkins slave镜像主要包含JAVA命令、maven命令、docker命令、kubectl命令等；还要挂载docker.sock文件和kubectl config文件等；
紧接着就是参考官方文档的说明修改value.yaml文件: https://github.com/helm/charts/tree/master/stable/jenkins
# Default values for jenkins.
# This is a YAML-formatted file.
# Declare name/value pairs to be passed into your templates.
# name: value
## Overrides for generated resource names
# See templates/_helpers.tpl
# nameOverride:
# fullnameOverride:
master:
# Used for label app.kubernetes.io/component
componentName: "jenkins-master"
image: "k8s.harbor.maimaiti.site/system/jenkins"
imageTag: "lts"
imagePullPolicy: "Always"
imagePullSecretName:
# Optionally configure lifetime for master-container
lifecycle:
# postStart:
# exec:
# command:
# - "uname"
# - "-a"
numExecutors: 0
# configAutoReload requires UseSecurity is set to true:
useSecurity: true
# Allows to configure different SecurityRealm using Jenkins XML
securityRealm: |-
<securityRealm class="hudson.security.LegacySecurityRealm"/>
# Allows to configure different AuthorizationStrategy using Jenkins XML
authorizationStrategy: |-
<authorizationStrategy class="hudson.security.FullControlOnceLoggedInAuthorizationStrategy">
<denyAnonymousReadAccess>true</denyAnonymousReadAccess>
</authorizationStrategy>
hostNetworking: false
# When enabling LDAP or another non-Jenkins identity source, the built-in admin account will no longer exist.
# Since the AdminUser is used by configAutoReload, in order to use configAutoReload you must change the
# .master.adminUser to a valid username on your LDAP (or other) server. This user does not need
# to have administrator rights in Jenkins (the default Overall:Read is sufficient) nor will it be granted any
# additional rights. Failure to do this will cause the sidecar container to fail to authenticate via SSH and enter
# a restart loop. Likewise if you disable the non-Jenkins identity store and instead use the Jenkins internal one,
# you should revert master.adminUser to your preferred admin user:
adminUser: "admin"
adminPassword: maimaiti.cn@4321
# adminSshKey: <defaults to auto-generated>
# If CasC auto-reload is enabled, an SSH (RSA) keypair is needed. Can either provide your own, or leave unconfigured to allow a random key to be auto-generated.
# If you supply your own, it is recommended that the values file that contains your key not be committed to source control in an unencrypted format
rollingUpdate: {}
# Ignored if Persistence is enabled
# maxSurge: 1
# maxUnavailable: 25%
resources:
requests:
cpu: "2000m"
memory: "2048Mi"
limits:
cpu: "2000m"
memory: "4096Mi"
# Environment variables that get added to the init container (useful for e.g. http_proxy)
# initContainerEnv:
# - name: http_proxy
# value: "http://192.168.64.1:3128"
# containerEnv:
# - name: http_proxy
# value: "http://192.168.64.1:3128"
# Set min/max heap here if needed with:
# javaOpts: "-Xms512m -Xmx512m"
# jenkinsOpts: ""
# jenkinsUrl: ""
# If you set this prefix and use ingress controller then you might want to set the ingress path below
# jenkinsUriPrefix: "/jenkins"
# Enable pod security context (must be `true` if runAsUser or fsGroup are set)
usePodSecurityContext: true
# Set runAsUser to 1000 to let Jenkins run as non-root user 'jenkins' which exists in 'jenkins/jenkins' docker image.
# When setting runAsUser to a different value than 0 also set fsGroup to the same value:
# runAsUser: <defaults to 0>
# fsGroup: <will be omitted in deployment if runAsUser is 0>
servicePort: 8080
# For minikube, set this to NodePort, elsewhere use LoadBalancer
# Use ClusterIP if your setup includes ingress controller
serviceType: LoadBalancer
# Jenkins master service annotations
serviceAnnotations: {}
# Jenkins master custom labels
deploymentLabels: {}
# foo: bar
# bar: foo
# Jenkins master service labels
serviceLabels: {}
# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: https
# Put labels on Jenkins master pod
podLabels: {}
# Used to create Ingress record (should used with ServiceType: ClusterIP)
# hostName: jenkins.cluster.local
# nodePort: <to set explicitly, choose port between 30000-32767
# Enable Kubernetes Liveness and Readiness Probes
# ~ 2 minutes to allow Jenkins to restart when upgrading plugins. Set ReadinessTimeout to be shorter than LivenessTimeout.
healthProbes: true
healthProbesLivenessTimeout: 90
healthProbesReadinessTimeout: 60
healthProbeReadinessPeriodSeconds: 10
healthProbeLivenessFailureThreshold: 12
slaveListenerPort: 50000
slaveHostPort:
disabledAgentProtocols:
- JNLP-connect
- JNLP2-connect
csrf:
defaultCrumbIssuer:
enabled: true
proxyCompatability: true
cli: false
# Kubernetes service type for the JNLP slave service
# slaveListenerServiceType is the Kubernetes Service type for the JNLP slave service,
# either 'LoadBalancer', 'NodePort', or 'ClusterIP'
# Note if you set this to 'LoadBalancer', you *must* define annotations to secure it. By default
# this will be an external load balancer and allowing inbound 0.0.0.0/0, a HUGE
# security risk: https://github.com/kubernetes/charts/issues/1341
slaveListenerServiceType: "ClusterIP"
slaveListenerServiceAnnotations: {}
slaveKubernetesNamespace:
# Example of 'LoadBalancer' type of slave listener with annotations securing it
# slaveListenerServiceType: LoadBalancer
# slaveListenerServiceAnnotations:
# service.beta.kubernetes.io/aws-load-balancer-internal: "True"
# service.beta.kubernetes.io/load-balancer-source-ranges: "172.0.0.0/8, 10.0.0.0/8"
# LoadBalancerSourcesRange is a list of allowed CIDR values, which are combined with ServicePort to
# set allowed inbound rules on the security group assigned to the master load balancer
loadBalancerSourceRanges:
- 0.0.0.0/0
# Optionally assign a known public LB IP
# loadBalancerIP: 1.2.3.4
# Optionally configure a JMX port
# requires additional javaOpts, ie
# javaOpts: >
# -Dcom.sun.management.jmxremote.port=4000
# -Dcom.sun.management.jmxremote.authenticate=false
# -Dcom.sun.management.jmxremote.ssl=false
# jmxPort: 4000
# Optionally configure other ports to expose in the master container
extraPorts:
# - name: BuildInfoProxy
# port: 9000
# List of plugins to be install during Jenkins master start
installPlugins:
- kubernetes:1.14.0
- workflow-job:2.31
- workflow-aggregator:2.6
- credentials-binding:1.17
- git:3.9.1
# Enable to always override the installed plugins with the values of 'master.installPlugins' on upgrade or redeployment.
# overwritePlugins: true
# Enable HTML parsing using OWASP Markup Formatter Plugin (antisamy-markup-formatter), useful with ghprb plugin.
# The plugin is not installed by default, please update master.installPlugins.
enableRawHtmlMarkupFormatter: false
# Used to approve a list of groovy functions in pipelines used the script-security plugin. Can be viewed under /scriptApproval
scriptApproval:
# - "method groovy.json.JsonSlurperClassic parseText java.lang.String"
# - "new groovy.json.JsonSlurperClassic"
# List of groovy init scripts to be executed during Jenkins master start
initScripts:
# - |
# print 'adding global pipeline libraries, register properties, bootstrap jobs...'
# Kubernetes secret that contains a 'credentials.xml' for Jenkins
# credentialsXmlSecret: jenkins-credentials
# Kubernetes secret that contains files to be put in the Jenkins 'secrets' directory,
# useful to manage encryption keys used for credentials.xml for instance (such as
# master.key and hudson.util.Secret)
# secretsFilesSecret: jenkins-secrets
# Jenkins XML job configs to provision
jobs:
# test: |-
# <<xml here>>
# Below is the implementation of Jenkins Configuration as Code. Add a key under configScripts for each configuration area,
# where each corresponds to a plugin or section of the UI. Each key (prior to | character) is just a label, and can be any value.
# Keys are only used to give the section a meaningful name. The only restriction is they may only contain RFC 1123 \ DNS label
# characters: lowercase letters, numbers, and hyphens. The keys become the name of a configuration yaml file on the master in
# /var/jenkins_home/casc_configs (by default) and will be processed by the Configuration as Code Plugin. The lines after each |
# become the content of the configuration yaml file. The first line after this is a JCasC root element, eg jenkins, credentials,
# etc. Best reference is https://<jenkins_url>/configuration-as-code/reference. The example below creates a welcome message:
JCasC:
enabled: false
pluginVersion: 1.5
supportPluginVersion: 1.5
configScripts:
welcome-message: |
jenkins:
systemMessage: Welcome to our CI\CD server. This Jenkins is configured and managed 'as code'.
# Optionally specify additional init-containers
customInitContainers: []
# - name: CustomInit
# image: "alpine:3.7"
# imagePullPolicy: Always
# command: [ "uname", "-a" ]
sidecars:
configAutoReload:
# If enabled: true, Jenkins Configuration as Code will be reloaded on-the-fly without a reboot. If false or not-specified,
# jcasc changes will cause a reboot and will only be applied at the subsequent start-up. Auto-reload uses the Jenkins CLI
# over SSH to reapply config when changes to the configScripts are detected. The admin user (or account you specify in
# master.adminUser) will have a random SSH private key (RSA 4096) assigned unless you specify adminSshKey. This will be saved to a k8s secret.
enabled: false
image: shadwell/k8s-sidecar:0.0.2
imagePullPolicy: IfNotPresent
resources:
# limits:
# cpu: 100m
# memory: 100Mi
# requests:
# cpu: 50m
# memory: 50Mi
# SSH port value can be set to any unused TCP port. The default, 1044, is a non-standard SSH port that has been chosen at random.
# Is only used to reload jcasc config from the sidecar container running in the Jenkins master pod.
# This TCP port will not be open in the pod (unless you specifically configure this), so Jenkins will not be
# accessible via SSH from outside of the pod. Note if you use non-root pod privileges (runAsUser & fsGroup),
# this must be > 1024:
sshTcpPort: 1044
# folder in the pod that should hold the collected dashboards:
folder: "/var/jenkins_home/casc_configs"
# If specified, the sidecar will search for JCasC config-maps inside this namespace.
# Otherwise the namespace in which the sidecar is running will be used.
# It's also possible to specify ALL to search in all namespaces:
# searchNamespace:
# Allows you to inject additional/other sidecars
other:
## The example below runs the client for https://smee.io as sidecar container next to Jenkins,
## that allows to trigger build behind a secure firewall.
## https://jenkins.io/blog/2019/01/07/webhook-firewalls/#triggering-builds-with-webhooks-behind-a-secure-firewall
##
## Note: To use it you should go to https://smee.io/new and update the url to the generete one.
# - name: smee
# image: docker.io/twalter/smee-client:1.0.2
# args: ["--port", "{{ .Values.master.servicePort }}", "--path", "/github-webhook/", "--url", "https://smee.io/new"]
# resources:
# limits:
# cpu: 50m
# memory: 128Mi
# requests:
# cpu: 10m
# memory: 32Mi
# Node labels and tolerations for pod assignment
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature
nodeSelector: {}
tolerations: []
# Leverage a priorityClass to ensure your pods survive resource shortages
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
# priorityClass: system-cluster-critical
podAnnotations: {}
# The below two configuration-related values are deprecated and replaced by Jenkins Configuration as Code (see above
# JCasC key). They will be deleted in an upcoming version.
customConfigMap: false
# By default, the configMap is only used to set the initial config the first time
# that the chart is installed. Setting `overwriteConfig` to `true` will overwrite
# the jenkins config with the contents of the configMap every time the pod starts.
# This will also overwrite all init scripts
overwriteConfig: false
# By default, the Jobs Map is only used to set the initial jobs the first time
# that the chart is installed. Setting `overwriteJobs` to `true` will overwrite
# the jenkins jobs configuration with the contents of Jobs every time the pod starts.
overwriteJobs: false
ingress:
enabled: true
# For Kubernetes v1.14+, use 'networking.k8s.io/v1beta1'
apiVersion: "extensions/v1beta1"
labels: {}
annotations:
kubernetes.io/ingress.class: traefik
# kubernetes.io/tls-acme: "true"
# Set this path to jenkinsUriPrefix above or use annotations to rewrite path
# path: "/jenkins"
hostName: k8s.jenkins.maimaiti.site
tls:
# - secretName: jenkins.cluster.local
# hosts:
# - jenkins.cluster.local
# Openshift route
route:
enabled: false
labels: {}
annotations: {}
# path: "/jenkins"
additionalConfig: {}
# master.hostAliases allows for adding entries to Pod /etc/hosts:
# https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
hostAliases: []
# - ip: 192.168.50.50
# hostnames:
# - something.local
# - ip: 10.0.50.50
# hostnames:
# - other.local
agent:
enabled: true
image: "10.83.74.102/jenkins/jnlp"
imageTag: "v11"
customJenkinsLabels: []
# name of the secret to be used for image pulling
imagePullSecretName:
componentName: "jenkins-slave"
privileged: false
resources:
requests:
cpu: "2000m"
memory: "4096Mi"
limits:
cpu: "2000m"
memory: "4096Mi"
# You may want to change this to true while testing a new image
alwaysPullImage: false
# Controls how slave pods are retained after the Jenkins build completes
# Possible values: Always, Never, OnFailure
podRetention: "Never"
# You can define the volumes that you want to mount for this container
# Allowed types are: ConfigMap, EmptyDir, HostPath, Nfs, Pod, Secret
# Configure the attributes as they appear in the corresponding Java class for that type
# https://github.com/jenkinsci/kubernetes-plugin/tree/master/src/main/java/org/csanchez/jenkins/plugins/kubernetes/volumes
# Pod-wide ennvironment, these vars are visible to any container in the slave pod
envVars:
# - name: PATH
# value: /usr/local/bin
volumes:
- type: HostPath
hostPath: /var/run/docker.sock
mountPath: /var/run/docker.sock
- type: HostPath
hostPath: /root/.kube
mountPath: /root/.kube
- type: Nfs
mountPath: /root/.m2
serverAddress: 10.83.32.224
serverPath: /data/m2
# - type: Secret
# secretName: mysecret
# mountPath: /var/myapp/mysecret
nodeSelector: {}
# Key Value selectors. Ex:
# jenkins-agent: v1
# Executed command when side container gets started
command:
args:
# Side container name
sideContainerName: "jnlp"
# Doesn't allocate pseudo TTY by default
TTYEnabled: false
# Max number of spawned agent
containerCap: 10
# Pod name
podName: "jenkins-slave"
persistence:
enabled: true
## A manually managed Persistent Volume and Claim
## Requires persistence.enabled: true
## If defined, PVC must be created manually before volume will be bound
existingClaim:
## jenkins data Persistent Volume Storage Class
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
storageClass: "dynamic"
annotations: {}
accessMode: "ReadWriteOnce"
size: "8Gi"
volumes:
# - name: nothing
# emptyDir: {}
mounts:
# - mountPath: /var/nothing
# name: nothing
# readOnly: true
networkPolicy:
# Enable creation of NetworkPolicy resources.
enabled: false
# For Kubernetes v1.4, v1.5 and v1.6, use 'extensions/v1beta1'
# For Kubernetes v1.7, use 'networking.k8s.io/v1'
apiVersion: networking.k8s.io/v1
## Install Default RBAC roles and bindings
rbac:
create: true
serviceAccount:
create: true
# The name of the service account is autogenerated by default
name:
annotations: {}
## Backup cronjob configuration
## Ref: https://github.com/nuvo/kube-tasks
backup:
# Backup must use RBAC
# So by enabling backup you are enabling RBAC specific for backup
enabled: false
# Used for label app.kubernetes.io/component
componentName: "backup"
# Schedule to run jobs. Must be in cron time format
# Ref: https://crontab.guru/
schedule: "0 2 * * *"
annotations:
# Example for authorization to AWS S3 using kube2iam
# Can also be done using environment variables
iam.amazonaws.com/role: "jenkins"
image:
repository: "nuvo/kube-tasks"
tag: "0.1.2"
# Additional arguments for kube-tasks
# Ref: https://github.com/nuvo/kube-tasks#simple-backup
extraArgs: []
# Add additional environment variables
env:
# Example environment variable required for AWS credentials chain
- name: "AWS_REGION"
value: "us-east-1"
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 1Gi
cpu: 1
# Destination to store the backup artifacts
# Supported cloud storage services: AWS S3, Minio S3, Azure Blob Storage
# Additional support can added. Visit this repository for details
# Ref: https://github.com/nuvo/skbn
destination: "s3://nuvo-jenkins-data/backup"
checkDeprecation: true
我主要修改了value.yaml以下几个配置参数，主要包括：
-
修改了镜像的参数为私服仓库的镜像。这里有一个特别注意的点就是jenkins slave镜像，这个镜像如果只是使用
官方的镜像还是不行的，需要自己制作镜像。镜像里面要包含kubectl命令、docker命令、mvn打包命令等； -
配置了jenkins的登录密码；
-
配置了资源限制情况，这里要特别注意一点，jenkins slave镜像的默认资源限制太小，经常会因为这个资源不足导致jinkins slave Pod终端；所以需要将jenkins slave镜像的资源限制调高一点；
-
再就是配置ingress,因为我们需要通过k8s集群外访问jenkins应用；
-
最主要的是agent这一块的配置，这里的镜像要自己制作，并且要把资源限额调大，并且要挂载docker.sock和kubectl的配置文件，将jenkins slaves
的.m2目录配置成nfs存储。这样的话，重新启动的jenkins slave构建的时候就不用每次都在apache官网下载依赖的jar包了 - 配置持久化存储，因为持久化存储需要存储插件和jenkins的页面配置；
在jenkins这个应用部署到k8s集群过程中，我踩过了几个坑，现在把这些坑都总结一下，给大家参考:
-
我在测试jenkins的时候，删除一个chart。由于一直习惯于使用命令
helm delete jenkins --purge来删除应用，结果我使用了这个命令之后，再次使用
helm install --name jenkins --namespace=kube-system ./jenkins当我登录jenkins的时候，发现所有的配置都没有了。包括我已经配置的job、安装的插件等等；所以大家在使用helm delete的时候一定要记得--purge慎用，加了--purge就代表pvc也会一并删除；
- 我在制作jenkins slave镜像的时候，运行起来的slave pod，当执行docker命令的时候，总是提示如下报错:
+ docker version
Client:
Version: 18.06.0-ce
API version: 1.38
Go version: go1.10.3
Git commit: 0ffa825
Built: Wed Jul 18 19:04:39 2018
OS/Arch: linux/amd64
Experimental: false
Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.38/version: dial unix /var/run/docker.sock: connect: permission denied
Build step 'Execute shell' marked build as failure
Finished: FAILURE其实这个报错的原因是因为官方的jenkins slave镜像是已jenkins(id为10010)的用户启动的，但是我们映射到
slave镜像的/var/run/docker.sock文件，必须要root用户才有权限读取。所以就产生了这个问题；解决方案有两种:
- 把所有的k8s宿主机的/var/run/docker.sock文件权限修改为chmod -R 777 /var/run/docker.sock 再mount进去，因为不安全不推荐这样;
- 在jenkins slave镜像中设置jenkins用户可以使用sudo命令执行root权限的命令
jenkins ALL=(ALL) NOPASSWD: ALL - 直接定义jenkins slave镜像由root用户启动 USER root即可；
我这里选择了第三种方案；其实第二种方案是最安全的，只是因为我的pipeline文件里面调用了多个docker插件，来实现docker build和docker login等，不知道插件里面怎么实现sudo命令；自定义jenkins slave镜像的Dockerfile可以这样写:
FROM 10.83.74.102/jenkins/jnlp:v2
MAINTAINER Yang Gao "[email protected]"
USER root
ADD jdk /root/
ADD maven /root/maven
ENV JAVA_HOME /root/jdk/
ENV MAVEN_HOME /root/maven/
ENV PATH $PATH:$JAVA_HOME/bin:$MAVEN_HOME/bin
RUN echo "deb http://apt.dockerproject.org/repo debian-jessie main" \
> /etc/apt/sources.list.d/docker.list \
&& apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 \
--recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
&& apt-get update \
&& apt-get install -y apt-transport-https \
&& apt-get install -y sudo \
&& apt-get install -y docker-engine \
&& rm -rf /var/lib/apt/lists/*
RUN echo "jenkins ALL=NOPASSWD: ALL" >> /etc/sudoers
RUN curl -L https://github.com/docker/compose/releases/download/1.8.0/\
docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose; \
chmod +x /usr/local/bin/docker-compose然后使用docker build命令构建镜像
docker build -t harbor.k8s.maimaiti.site/system/jenkins-jnlp:v11
docker push harbor.k8s.maimaiti.site/system/jenkins-jnlp:v11- jenkins slave pod模板可以在两个地方配置，一个就是helm chart value.yaml文件里面定义，包括slave
镜像的版本，挂载等；还有一种就是在jenkins页面安装了kubernetes插件之后有个云的配置，里面也可以配置jenkins slave pod模板。
但是有个问题，如果你使用了helm upgrade 命令修改了helm里面的参数，包括slave的版本等，实际上是不生效的。因为jenkins还是
会使用管理页面的那个配置。当使用helm install jenkins的时候，第一次登陆jenkins页面，这个jenkins的helm默认就安装了
kubernetes插件，所以可以直接到系统管理---系统配置里面找云的配置，默认就有了slave pod的配置，这个配置和helm value.yaml里面是一样的。
但是后续再helm upgrade jenkins的时候，实际上是不能更新jenkins页面的这个地方的slave 配置，所以会发现一直更改jenkins slave不生效。
讲完了我探究jenkins in kubernetes遇到的坑，再看看一个k8s应用发布，到底都需要配置哪些jenkins的设置。这里我列举了一个我们公司的内部项目
其中一些地方做了脱敏处理，大家只需要关注流程和架构即可，具体的配置已各公司的应用不同会有所不通；
-
登录jenkins,配置插件；系统管理---插件管理---available插件---安装docker、kubernetes插件等
- 配置凭据：配置包括gitlab和harbor的账户和密码，后面再pipeline里面会使用到
- 创建一个pipeline的job，配置包括:
- 配置发布应用的模块变量
- 配置发布应用的版本分支名称；
- 配置pipeline的内容；
k8s应用清单文件：
node {
try {
stage('代码拉取') {
git branch: "${BranchName}", credentialsId: 'k8sgitlab', url: 'http://k8s.gitlab.test.site/root/test.git'
}
stage('项目构建') {
if ("${MODULE}".contains('test-ui')){
dir('test-ui'){
sh "npm i"
sh "npm run sit"
}
}else {
dir('test-parent'){
sh "mvn clean install -Psit"
}
}
}
def regPrefix = 'k8s.harbor.test.site/test/'
stage('构建镜像'){
docker.withRegistry('http://k8s.harbor.test.site/','k8sharbor'){
if ("${MODULE}".contains('test-admin')){
dir('test-parent/test-admin/target') {
sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
sh "jar -uvf admin.jar BOOT-INF/classes/application.yml"
def imageName = docker.build("${regPrefix}admin:V1.0-${env.BUILD_ID}")
imageName.push("V1.0-${env.BUILD_ID}")
//imageName.push("latest")
sh "/usr/bin/docker rmi ${regPrefix}admin:V1.0-${env.BUILD_ID}"
}
}
if ("${MODULE}".contains('test-eureka')){
dir('test-parent/test-eureka/target') {
sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
sh "jar -uvf testEurekaServer.jar BOOT-INF/classes/application.yml"
def imageName = docker.build("${regPrefix}eureka:V1.0-${env.BUILD_ID}")
imageName.push("V1.0-${env.BUILD_ID}")
//imageName.push("latest")
sh "/usr/bin/docker rmi ${regPrefix}eureka:V1.0-${env.BUILD_ID}"
}
}
if ("${MODULE}".contains('test-quality')){
dir('test-parent/test-quality/target') {
sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
sh "jar -uvf quality.jar BOOT-INF/classes/application.yml"
def imageName = docker.build("${regPrefix}quality:V1.0-${env.BUILD_ID}")
imageName.push("V1.0-${env.BUILD_ID}")
//imageName.push("latest")
sh "/usr/bin/docker rmi ${regPrefix}quality:V1.0-${env.BUILD_ID}"
}
}
if ("${MODULE}".contains('test-schedule')){
dir('test-parent/test-schedule/target') {
sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
sh "jar -uvf schedule.jar BOOT-INF/classes/application.yml "
def imageName = docker.build("${regPrefix}schedule:V1.0-${env.BUILD_ID}")
imageName.push("V1.0-${env.BUILD_ID}")
//imageName.push("latest")
sh "/usr/bin/docker rmi ${regPrefix}schedule:V1.0-${env.BUILD_ID}"
}
}
if ("${MODULE}".contains('test-zuul')){
dir('test-parent/test-zuul/target') {
sh "cp ../Dockerfile . && cp -rf ../BOOT-INF ./ &&cp -rf ../../pinpoint-agent ./"
sh "jar -uvf test-api.jar BOOT-INF/classes/application.yml "
def imageName = docker.build("${regPrefix}zuul:V1.0-${env.BUILD_ID}")
imageName.push("V1.0-${env.BUILD_ID}")
//imageName.push("latest")
sh "/usr/bin/docker rmi ${regPrefix}zuul:V1.0-${env.BUILD_ID}"
}
}
}
}
stage('重启应用'){
if ("${MODULE}".contains('test-admin')){
sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-admin.yml "
sh "/usr/local/bin/kubectl --kubeconfig=test-parent/DockerCompose/config apply -f test-parent/DockerCompose/test-admin.yml --record "
}
if ("${MODULE}".contains('test-eureka')){
sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-eureka.yml "
sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-eureka.yml --record "
}
if ("${MODULE}".contains('test-quality')){
sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-quality.yml "
sh "/usr/local/bin/kubectl apply -ftest-parent/DockerCompose/test-quality.yml --record "
}
if ("${MODULE}".contains('test-schedule')){
sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-schedule.yml "
sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-schedule.yml --record "
}
if ("${MODULE}".contains('test-zuul')){
sh "sed -i \'s/latest/V1.0-${env.BUILD_ID}/g\' test-parent/DockerCompose/test-zuul.yml "
sh "/usr/local/bin/kubectl apply -f test-parent/DockerCompose/test-zuul.yml --record "
}
}
}catch (any) {
currentBuild.result = 'FAILURE'
throw any}
}
Dockerfile的内容主要如下：
FROM 10.10.10.10/library/java:8-jdk-alpine
ENV TZ=Asia/Shanghai
VOLUME /tmp
ADD admin.jar /app/test/admin.jar
ADD skywalking-agent/ /app/icm/skywalking-agent
ENTRYPOINT ["java","-javaagent:/app/test/skywalking-agent/skywalking-agent.jar","-Djava.security.egd=file:/dev/./urandom","-XX:+UnlockExperimentalVMOptions","-XX:+UseCGroupMemoryLimitForHeap","-jar","/app/test/admin.jar"]
Yaml清单文件的内容主要如下:
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: test-admin
namespace: sit
labels:
k8s-app: test-admin
spec:
replicas: 3
revisionHistoryLimit: 3
#滚动升级时70s后认为该pod就绪
minReadySeconds: 70
strategy:
##由于replicas为3,则整个升级,pod个数在2-4个之间
rollingUpdate:
#滚动升级时会先启动1个pod
maxSurge: 1
#滚动升级时允许的最大Unavailable的pod个数
maxUnavailable: 1
selector:
matchLabels:
k8s-app: test-admin
template:
metadata:
labels:
k8s-app: test-admin
spec:
containers:
- name: test-admin
image: k8s.harbor.test.site/test/admin:latest
resources:
# need more cpu upon initialization, therefore burstable class
#limits:
# memory: 1024Mi
# cpu: 200m
#requests:
# cpu: 100m
# memory: 256Mi
ports:
#容器的端口
- containerPort: 8281
#name: ui
protocol: TCP
livenessProbe:
httpGet:
path: /admin/health
port: 8281
scheme: HTTP
initialDelaySeconds: 180
timeoutSeconds: 5
periodSeconds: 15
successThreshold: 1
failureThreshold: 2
#volumeMounts:
#- mountPath: "/download"
# name: data
#volumes:
#- name: data
# persistentVolumeClaim:
# claimName: download-pvc
---
apiVersion: v1
kind: Service
metadata:
name: test-admin
namespace: sit
labels:
k8s-app: test-admin
spec:
type: NodePort
ports:
#集群IP的端口
- port: 8281
protocol: TCP
#容器的端口
targetPort: 8281
#nodePort: 28281
selector:
k8s-app: test-admin