kubekey—创建kubernetes集群、安装kubesphere3.3

参考官网
https://v3-0.docs.kubesphere.io/zh/docs/quick-start/all-in-one-on-linux/#%E6%AD%A5%E9%AA%A4-2%E4%B8%8B%E8%BD%BD-kubekey

一、基础配置

1.修改主机名

hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-master02
hostnamectl set-hostname k8s-master03
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02

2.添加 主机名与IP地址解析

cat > /etc/hosts <<EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6


17.2.1.220 k8s-master01
17.2.1.221 k8s-master02
17.2.1.222 k8s-master03
17.2.1.223 k8s-node01
17.2.1.224 k8s-node02
EOF

3.升级服务器，时间同步，关闭防火墙，重启服务器

#添加访问互联路由
cat > /etc/resolv.conf <<EOF
nameserver 114.114.114.114
nameserver 8.8.8.8
EOF

cat /etc/resolv.conf

# ssh连接Linux比较慢
#sed -i "s|#UseDNS yes|UseDNS no|" /etc/ssh/sshd_config
#sed -i "s|GSSAPIAuthentication yes|GSSAPIAuthentication no|" /etc/ssh/sshd_config


#设置为阿里云yum源

rm -rf /etc/yum.repos.d/bak && mkdir -p /etc/yum.repos.d/bak && mv /etc/yum.repos.d/* /etc/yum.repos.d/bak

curl -o /etc/yum.repos.d/CentOS-7.repo http://mirrors.aliyun.com/repo/Centos-7.repo


yum clean all && yum makecache

cd /etc/yum.repos.d


#CentOS7使用/etc/rc.d/rc.local设置开机自动启动
chmod +x /etc/rc.d/rc.local

#安装依赖包

yum -y install vim net-tools lrzsz unzip gcc telnet wget sshpass ntpdate ntp curl

yum -y install conntrack ipvsadm ipset  iptables  sysstat libseccomp git  

yum install socat -y
yum install conntrack -y

#时间同步
echo '*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com >/dev/null 2>&1'>/var/spool/cron/root && crontab -l



#设置防火墙为 Iptables 并设置空规则
systemctl  stop firewalld  &&  systemctl  disable firewalld

yum -y install iptables-services  &&  systemctl  start iptables  &&  systemctl  enable iptables  &&  iptables -F  &&  service iptables save


#关闭 SELINUX
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

#调整内核参数，对于 K8S
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
#net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM  
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

modprobe ip_vs_rr && modprobe br_netfilter && sysctl -p /etc/sysctl.d/kubernetes.conf


#关闭系统不需要服务
systemctl stop postfix && systemctl disable postfix



#  k8s持久化保存日志的目录
mkdir /var/log/journal 
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent
 
# 压缩历史日志
Compress=yes
 
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
 
# 最大占用空间 10G
SystemMaxUse=10G
 
# 单日志文件最大 200M
SystemMaxFileSize=200M
 
# 日志保存时间 36 周
MaxRetentionSec=36week
 
# 不将日志转发到 syslog
ForwardToSyslog=no
EOF

systemctl restart systemd-journald

二、安装kubekey

1.下载kubekey

export KKZONE=cn

curl -sfL https://get-kk.kubesphere.io | VERSION=v2.2.1 sh -

chmod +x kk

2.创建配置

#./kk create config [--with-kubernetes version] [--with-kubesphere version]
#如果您在这一步的命令中不添加标志 --with-kubesphere，则不会部署 KubeSphere，KubeKey 将只安装 Kubernetes。如果您添加标志 --with-kubesphere 时不指定 KubeSphere 版本，则会安装最新版本的 KubeSphere。

#初始化安装的kubesphere，存储为本地的pv和pvc，所有后面在安装kubesphere3.3
#./kk create config --with-kubernetes v1.19.8 --with-kubesphere v3.3.0



./kk create config --with-kubernetes v1.19.8

2.1参考-修改配置

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {
    
    name: k8s-master01, address: 17.2.1.220, internalAddress: 17.2.1.220, user: root, password: 1qaz@WSX}
  - {
    
    name: k8s-master02, address: 17.2.1.221, internalAddress: 17.2.1.221, user: root, password: 1qaz@WSX}
  - {
    
    name: k8s-master03, address: 17.2.1.222, internalAddress: 17.2.1.222, user: root, password: 1qaz@WSX}
  - {
    
    name: k8s-node01, address: 17.2.1.223, internalAddress: 17.2.1.223, user: root, password: 1qaz@WSX}
  - {
    
    name: k8s-node02, address: 17.2.1.224, internalAddress: 17.2.1.224, user: root, password: 1qaz@WSX}
  roleGroups:
    etcd:
    - k8s-master01
    - k8s-master02
    - k8s-master03
    master: 
    - k8s-master01
    - k8s-master02
    - k8s-master03
    worker:
    - k8s-node01
    - k8s-node02
  controlPlaneEndpoint:
    ##启用内置haproxy模式实现高可用
    internalLoadbalancer: haproxy

    domain: lb.kubesphere.local
    address: ""
    port: 6443

3.安装

./kk create cluster -f config-sample.yaml

4.验证集群

kubectl get nodes

kubectl get componentstatuses

kubectl get pods -A

5.kubernetes v1.15以上更新证书的方法

更新/etc/kubernetes/pki目录下的所有证书（不包含ca证书）
注意：需要在每一个节点上都更新

# 查看现有证书到期时间
$ kubeadm alpha certs check-expiration
# 使用二进制更新证书
$ kubeadm alpha certs renew all
# 每月的最后1天
crontab  -e

* * 1  * *  /usr/bin/kubeadm alpha certs renew all

三、安装kubesphere3.3

1.部署kubesphere时需要默认 StorageClass

kubectl edit sc nfs-boge

  metadata:
    annotations:
      storageclass.beta.kubernetes.io/is-default-class: "true"

2.下载yaml

最小化安装

您的 Kubernetes 版本必须为：v1.19.x、v1.20.x、v1.21.x、* v1.22.x、* v1.23.x 和 * v1.24.x。带星号的版本可能出现边缘节点部分功能不可用的情况。因此，如需使用边缘节点，推荐安装 v1.21.x。
wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/kubesphere-installer.yaml
wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/cluster-configuration.yaml



#修改cluster-configuration.yaml
#将ectd下的 endpointIps改为你的master节点的私有IP地址。
#endpointIps: XX.X.X.X
  etcd:
    monitoring: true       # Enable or disable etcd monitoring dashboard installation. You have to create a Secret for etcd before you enable it.
    endpointIps: '17.2.1.220,17.2.1.221,17.2.1.222'

参考修改

apiVersion: installer.kubesphere.io/v1alpha1
kind: ClusterConfiguration
metadata:
  name: ks-installer
  namespace: kubesphere-system
  labels:
    version: v3.3.0
spec:
  persistence:
    storageClass: ""        #这里保持默认即可，因为我们有了默认的存储类
  authentication:
    jwtSecret: ""           # Keep the jwtSecret consistent with the Host Cluster. Retrieve the jwtSecret by executing "kubectl -n kubesphere-system get cm kubesphere-config -o yaml | grep -v "apiVersion" | grep jwtSecret" on the Host Cluster.
  local_registry: ""        # Add your private registry address if it is needed.
  etcd:
    monitoring: true       # 改为"true"，表示开启etcd的监控功能
    endpointIps: '17.2.1.220,17.2.1.221,17.2.1.222'  # 改为自己的master节点IP地址
    port: 2379              # etcd port.
    tlsEnable: true
  common:
    redis:
      enabled: true         #改为"true"，开启redis功能
    openldap:
      enabled: true         #改为"true"，开启轻量级目录协议
    minioVolumeSize: 20Gi # Minio PVC size.
    openldapVolumeSize: 2Gi   # openldap PVC size.
    redisVolumSize: 2Gi # Redis PVC size.
    monitoring:
      # type: external   # Whether to specify the external prometheus stack, and need to modify the endpoint at the next line.
      endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 # Prometheus endpoint to get metrics data.
    es:   # Storage backend for logging, events and auditing.
      # elasticsearchMasterReplicas: 1   # The total number of master nodes. Even numbers are not allowed.
      # elasticsearchDataReplicas: 1     # The total number of data nodes.
      elasticsearchMasterVolumeSize: 4Gi   # The volume size of Elasticsearch master nodes.
      elasticsearchDataVolumeSize: 20Gi    # The volume size of Elasticsearch data nodes.
      logMaxAge: 7                     # Log retention time in built-in Elasticsearch. It is 7 days by default.
      elkPrefix: logstash              # The string making up index names. The index name will be formatted as ks-<elk_prefix>-log.
      basicAuth:
        enabled: false
        username: ""
        password: ""
      externalElasticsearchUrl: ""
      externalElasticsearchPort: ""
  console:
    enableMultiLogin: true  # Enable or disable simultaneous logins. It allows different users to log in with the same account at the same time.
    port: 30880
  alerting:                # (CPU: 0.1 Core, Memory: 100 MiB) It enables users to customize alerting policies to send messages to receivers in time with different time intervals and alerting levels to choose from.
    enabled: true         # 改为"true"，开启告警功能
    # thanosruler:
    #   replicas: 1
    #   resources: {}
  auditing:                # Provide a security-relevant chronological set of records，recording the sequence of activities happening on the platform, initiated by different tenants.
    enabled: true         #  改为"true"，开启审计功能
  devops:                  # (CPU: 0.47 Core, Memory: 8.6 G) Provide an out-of-the-box CI/CD system based on Jenkins, and automated workflow tools including Source-to-Image & Binary-to-Image.
    enabled: true             # 改为"true"，开启DevOps功能
    jenkinsMemoryLim: 2Gi      # Jenkins memory limit.
    jenkinsMemoryReq: 1500Mi   # Jenkins memory request.
    jenkinsVolumeSize: 8Gi     # Jenkins volume size.
    jenkinsJavaOpts_Xms: 512m  # The following three fields are JVM parameters.
    jenkinsJavaOpts_Xmx: 512m
    jenkinsJavaOpts_MaxRAM: 2g
  events:                  # Provide a graphical web console for Kubernetes Events exporting, filtering and alerting in multi-tenant Kubernetes clusters.
    enabled: true         # 改为"true"，开启集群的事件功能
    ruler:
      enabled: true
      replicas: 2
  logging:                 # (CPU: 57 m, Memory: 2.76 G) Flexible logging functions are provided for log query, collection and management in a unified console. Additional log collectors can be added, such as Elasticsearch, Kafka and Fluentd.
    enabled: true        # 改为"true"，开启日志功能
    logsidecar:
      enabled: true
      replicas: 2
  metrics_server:                    # (CPU: 56 m, Memory: 44.35 MiB) It enables HPA (Horizontal Pod Autoscaler).
    enabled: false                   # 这个不用修改，因为在上面我们已经安装过了，如果这里开启，镜像是官方的，会拉取镜像失败
  monitoring:
    storageClass: ""                 # If there is an independent StorageClass you need for Prometheus, you can specify it here. The default StorageClass is used by default.
    # prometheusReplicas: 1          # Prometheus replicas are responsible for monitoring different segments of data source and providing high availability.
    prometheusMemoryRequest: 400Mi   # Prometheus request memory.
    prometheusVolumeSize: 20Gi       # Prometheus PVC size.
    # alertmanagerReplicas: 1          # AlertManager Replicas.
  multicluster:
    clusterRole: none  # host | member | none  # You can install a solo cluster, or specify it as the Host or Member Cluster.
  network:
    networkpolicy: # Network policies allow network isolation within the same cluster, which means firewalls can be set up between certain instances (Pods).
      # Make sure that the CNI network plugin used by the cluster supports NetworkPolicy. There are a number of CNI network plugins that support NetworkPolicy, including Calico, Cilium, Kube-router, Romana and Weave Net.
      enabled: true # 改为"true"，开启网络策略
    ippool: # Use Pod IP Pools to manage the Pod network address space. Pods to be created can be assigned IP addresses from a Pod IP Pool.
      type: none #如果你的网络插件是calico，需要修改为"calico"，这里我是Flannel，保持默认。
    topology: # Use Service Topology to view Service-to-Service communication based on Weave Scope.
      type: none # Specify "weave-scope" for this field to enable Service Topology. "none" means that Service Topology is disabled.
  openpitrix: # An App Store that is accessible to all platform tenants. You can use it to manage apps across their entire lifecycle.
    store:
      enabled: true # 改为"true"，开启应用商店
  servicemesh:         # (0.3 Core, 300 MiB) Provide fine-grained traffic management, observability and tracing, and visualized traffic topology.
    enabled: true     # 改为"true"，开启微服务治理
  kubeedge:          # Add edge nodes to your cluster and deploy workloads on edge nodes.
    enabled: false   # 这个就不修改了，这个是边缘服务，我们也没有边缘的设备。
    cloudCore:
      nodeSelector: {
    
    "node-role.kubernetes.io/worker": ""}
      tolerations: []
      cloudhubPort: "10000"
      cloudhubQuicPort: "10001"
      cloudhubHttpsPort: "10002"
      cloudstreamPort: "10003"
      tunnelPort: "10004"
      cloudHub:
        advertiseAddress: # At least a public IP address or an IP address which can be accessed by edge nodes must be provided.
          - ""            # Note that once KubeEdge is enabled, CloudCore will malfunction if the address is not provided.
        nodeLimit: "100"
      service:
        cloudhubNodePort: "30000"
        cloudhubQuicNodePort: "30001"
        cloudhubHttpsNodePort: "30002"
        cloudstreamNodePort: "30003"
        tunnelNodePort: "30004"
    edgeWatcher:
      nodeSelector: {
    
    "node-role.kubernetes.io/worker": ""}
      tolerations: []
      edgeWatcherAgent:
        nodeSelector: {
    
    "node-role.kubernetes.io/worker": ""}
        tolerations: []
        

3.运行yaml

kubectl apply -f kubesphere-installer.yaml

kubectl apply -f cluster-configuration.yaml

4. 查看日志

kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f


#访问任意机器的 30880端口
#账号 ： admin
#密码 ： P@88w0rd

5.解决etcd监控证书找不到问题

kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs  \
--from-file=etcd-client-ca.crt=/etc/ssl/etcd/ssl/ca.pem  \
--from-file=etcd-client.crt=/etc/ssl/etcd/ssl/member-k8s-master01.pem  \
--from-file=etcd-client.key=/etc/ssl/etcd/ssl/member-k8s-master01-key.pem

6.查看jenkins密码

https://v2-1.docs.kubesphere.io/docs/zh-CN/devops/jenkins-setting/

#编辑yaml，随便添加一个labels,然后确定

登陆 Jenkins 重新加载
1、Installer 安装将会同时部署 Jenkins Dashboard，Jenkins 已对接了 KubeSphere 的 LDAP，因此可使用用户名 admin 和 KubeSphere 集群管理员的密码登录 Jenkins Dashboard，访问公网 IP (EIP) + Nodeport (30180) 并登陆 Jenkins Dashboard。登陆后，在左侧导航栏点击 系统管理。

#查看端口
kubectl -n kubesphere-devops-system get svc


#登入Jenkins
http://17.2.1.220:30180/