k8s v1.14.1，1.20.8版本kubeadm编译，解决k8s证书一年有效期问题

编译kubeadm，初始化k8s集群，解决证书一年有效期问题

一、安装go环境

1.1、安装编译工具

yum install -y gcc make rsync git

1.2、下载go

wget https://dl.google.com/go/go1.15.15.linux-amd64.tar.gz

1.3、解压go安装包

tar zxvf go1.15.15.linux-amd64.tar.gz -C /usr/local

1.4、配置环境变量

vi /etc/profile
export GOROOT=/usr/local/go
export GOPATH=/usr/local/gopath
export PATH=$PATH:$GOROOT/bin

1.5、使环境变量生效

source /etc/profile

1.6、查看go版本

go version

二、下载kubernetes源码

2.1、源码500多M

go get -d k8s.io/kubernetes

2.2、进入kubernetes源码目录

cd /usr/local/gopath/src/k8s.io/kubernetes

2.3、查看当前分支

git branch

2.4、查看所有分支

git tag -l

2.5、切换到指定分支

#切换至1.14.1分支
git checkout v1.14.1
#切换到1.20.8分支
git checkout v1.20.8

2.6、查看当前分支

git branch

三、修改源码

3.1、修改源代码-cert.go

文件：staging/src/k8s.io/client-go/util/cert/cert.go

NewSelfSignedCACert 方法，签发以下证书，且默认为10年有效期：

front-proxy-ca.crt
front-proxy-client.crt
ca.crt
etcd/ca.crt
etcd/peer.crt

# 1.14.0版为例

vim vendor/k8s.io/client-go/util/cert/cert.go
 
const duration365d = time.Hour * 24 * 365
 
// Config contains the basic fields required for creating a certificate
type Config struct {
CommonName string
Organization []string
AltNames AltNames
Usages []x509.ExtKeyUsage
}
 
// AltNames contains the domain names and IP addresses that will be added
// to the API Server's x509 certificate SubAltNames field. The values will
// be passed directly to the x509.Certificate object.
type AltNames struct {
DNSNames []string
IPs []net.IP
}
 
// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
now := time.Now()
tmpl := x509.Certificate{
SerialNumber: new(big.Int).SetInt64(0),
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
NotBefore: now.UTC(),
# 默认已调整有效期为10年，将其修改为100年；
# 但只影响部分证书：
//NotAfter: now.Add(duration365d * 10).UTC(),
NotAfter: now.Add(duration365d * 100).UTC(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
 
certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
if err != nil {
return nil, err
}
return x509.ParseCertificate(certDERBytes)
}

# 1.20.8版为例

vim staging/src/k8s.io/client-go/util/cert/cert.go
 
// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
now := time.Now()
tmpl := x509.Certificate{
SerialNumber: new(big.Int).SetInt64(0),
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
NotBefore: now.UTC(),
# 默认已调整有效期为10年，将其修改为100年；
# 但只影响部分证书：
// NotAfter: now.Add(duration365d * 10).UTC(),
NotAfter: now.Add(duration365d * 100).UTC(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
 
certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
if err != nil {
return nil, err
}
return x509.ParseCertificate(certDERBytes)
}

3.2 修改源代码-pki_helpers.go

以下证书由 NewSignedCert 方法签发，但签发的证书默认只有一年有效期：

apiserver.crt
apiserver-etcd-client.crt
etcd/server.crt
etcd/healthcheck-client.crt
apiserver-kubelet-client.crt

# `NewSignedCert` 方法：

# 部分证书是通过NewSignedCert这个方法签发，而这个方法签发的证书默认只有一年有效期，查看代码逻辑

# 1.14.0版为例

vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go
 
// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
if err != nil {
return nil, err
}
if len(cfg.CommonName) == 0 {
return nil, errors.New("must specify a CommonName")
}
if len(cfg.Usages) == 0 {
return nil, errors.New("must specify at least one ExtKeyUsage")
}
 
certTmpl := x509.Certificate{
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
DNSNames: cfg.AltNames.DNSNames,
IPAddresses: cfg.AltNames.IPs,
SerialNumber: serial,
NotBefore: caCert.NotBefore,
# 修改签发相关证书的默认有效期为100年
// NotAfter: time.Now().Add(duration365d).UTC(),
NotAfter: time.Now().Add(duration365d * 100).UTC(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: cfg.Usages,
}
certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
if err != nil {
return nil, err
}
return x509.ParseCertificate(certDERBytes)
}

# 1.20.8版为例

vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go
 
// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
const effectyear = time.Hour * 24 * 365 * 100  #添加此行信息
serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64))
if err != nil {
return nil, err
}
if len(cfg.CommonName) == 0 {
return nil, errors.New("must specify a CommonName")
}
if len(cfg.Usages) == 0 {
return nil, errors.New("must specify at least one ExtKeyUsage")
}
 
RemoveDuplicateAltNames(&cfg.AltNames)
 
certTmpl := x509.Certificate{
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
DNSNames: cfg.AltNames.DNSNames,
IPAddresses: cfg.AltNames.IPs,
SerialNumber: serial,
NotBefore: caCert.NotBefore,
// NotAfter: time.Now().Add(kubeadmconstants.CertificateValidity).UTC(), #注释掉此行
NotAfter: time.Now().Add(effectyear).UTC(), #添加此行
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: cfg.Usages,
}

四、编译

go环境已经准备好

4.1、编译kubeadm

make WHAT=cmd/kubeadm GOFLAGS=-v

4.2、编译kubelet

make all WHAT=cmd/kubelet GOFLAGS=-v

4.3、编译kubectl

make all WHAT=cmd/kubectl GOFLAGS=-v

4.4、编译生成的二进制文件在 _output/bin/ 目录下

五、更新集群证书

以下操作需要在k8s服务运行的情况下执行，不要关闭k8s服务。

5.1、查看所有证书有效期

1.14.1版本查看证书时间

for i in $(ls /etc/kubernetes/pki/*.crt); do echo "===== $i ====="; openssl x509 -in $i -text -noout | grep -A 3 'Validity' ; done

1.20.8版本查看证书时间

kubeadm alpha certs check-expiration

5.2、更新kubeadm

# 将kubeadm 文件拷贝替换系统中原有kubeadm

cp /usr/bin/kubeadm /usr/bin/kubeadm.origin
cp _output/bin/kubeadm /usr/bin/kubeadm

5.3、更新 kube-master节点证书

5.3.1、备份

cp -r /etc/kubernetes/pki /etc/kubernetes/pki.origin
mkdir /root/kubeadm/

5.3.2、生成集群的配置文件

kubeadm config view > /root/kubeadm/kubeadm-config.yaml

5.4、更新证书；

# "--config" 指定 "kubeadm" 的配置文件，建议使用部署集群时使用的配置文件；

# 其他参数可参考官方文档

cd /etc/kubernetes/pki
kubeadm alpha certs renew all --config=/root/kubeadm/kubeadm-config.yaml

5.5、验证新的证书有效期，以"apiserver.crt" 为例

openssl x509 -in apiserver.crt -text -noout | grep Not

5.6、查看所有证书有效期

1.14.1版本查看证书时间

for i in $(ls /etc/kubernetes/pki/*.crt); do echo "===== $i ====="; openssl x509 -in $i -text -noout | grep -A 3 'Validity' ; done

1.20.8版本查看证书时间

kubeadm alpha certs check-expiration

六、更新配置文件，验证服务

6.1、备份 /etc/kubernetes/*.conf 文件；

# 必须备份(mv)，否则无法更新"*.conf" 文件

ll /etc/kubernetes/*.conf | awk '{print $9}' | xargs -i mv {} {}.`date "+%Y%m%d"`

6.2、更新/etc/kubernetes/*.conf文件；

# 如果没有 "mv" , 输出为"[kubeconfig] Using existing kubeconfig file"，
kubeadm init phase kubeconfig all --config=/root/kubeadm/kubeadm-config.yaml
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file

6.3、在(所有)kube-master 节点重启 "kube-apiserver","kube-controller-manager", "kube-scheduler","etcd" 4个容器

docker ps | egrep "etcd|kube-apiserver|kube-controller-manager|kube-scheduler" | grep -v pause | awk '{print $1}' | xargs -i docker restart {}

6.4、覆盖"$HOME/.kube/config" 文件，验证服务

cp -i /etc/kubernetes/admin.conf $HOME/.kube/config