Kubernetes production environment installation installation ready to deploy based on the Kubernetes v1.14.0

Based on CentOS 7.6.1810 version is installed

1, system upgrades and settings

Modify the server name

hostnamectl set-hostname changes name

CentOS and kernel upgrade package

yum -y update
yum -y install yum-plugin-fastestmirror
yum install -y epel-release
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
yum -y --enablerepo=elrepo-kernel install kernel-ml

Set the default boot kernel to install the latest version

grub2-set-default 0
grub2-mkconfig -o /boot/grub2/grub.cfg

Installation tools

yum -y install yum-utils ipvsadm telnet wget net-tools

Set system.conf

cat >> /etc/systemd/system.conf << EOF
DefaultLimitMEMLOCK=infinity
DefaultLimitCORE=infinity
DefaultCPUAccounting=yes
DefaultMemoryAccounting=yes
DefaultLimitNOFILE=1024000
DefaultLimitNPROC=1024000
EOF

Set off the firewall and SELINUX

sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
setenforce 0

Close Swap

swapoff -a && sysctl -w vm.swappiness=0
vi /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0

set ulimit disable large memory pages to optimize disk io

Description: sd modify the server name corresponding to the device

echo "ulimit -SHn 1024000" >> /etc/rc.local
cat >> /etc/rc.local << EOF
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
block=`(find /sys/block/ -name "sd")`
for sblock in \$block
do
echo 16384 >\$sblock/queue/read_ahead_kb
echo 512 >\$sblock/queue/nr_requests
done
block=`(find /dev/ -name "sd")`
for sblock in \$block
do
/sbin/blockdev --setra 16384 \$sblock
done
EOF
chmod +x /etc/rc.local

Sysctl.conf kernel configuration settings open bbr

true > /etc/sysctl.conf
cat >> /etc/sysctl.conf << EOF
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
fs.file-max = 2048000
fs.nr_open = 2048000
fs.quota.free_dquots = 0
vm.min_free_kbytes = 512000
vm.swappiness = 0
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5
vm.dirty_writeback_centisecs=200
vm.dirty_expire_centisecs = 500
vm.vfs_cache_pressure=200
vm.max_map_count = 2048000
vm.overcommit_memory = 1
vm.zone_reclaim_mode = 0
vm.panic_on_oom =0
vm.oom_kill_allocating_task = 1
kernel.randomize_va_space = 1
kernel.sem =5010 641280 5010 128
kernel.pid_max = 4194303
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_max_tw_buckets = 2621440
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 1024000 8738000 1677721600
net.ipv4.tcp_wmem = 1024000 8738000 1677721600
net.ipv4.udp_mem = 1024000 8738000 1677721600
net.ipv4.tcp_mem = 1024000 8738000 1677721600
net.ipv4.route.flush=1
net.core.wmem_default = 167772160
net.core.rmem_default = 167772160
net.core.rmem_max = 167772160
net.core.wmem_max = 167772160
net.ipv4.udp_rmem_min = 167772160
net.ipv4.udp_wmem_min = 167772160
net.core.optmem_max = 2048000
net.core.netdev_max_backlog = 2048000
net.core.somaxconn = 65535
net.core.dev_weight = 64
net.core.message_cost = 5
net.core.message_burst = 10
net.core.busy_poll = 0
net.core.busy_read = 0
net.core.netdev_budget = 300
net.core.default_qdisc = fq
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 2048000
net.unix.max_dgram_qlen = 2048000
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.neigh.default.gc_stale_time=120
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.lo.arp_announce=2
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.ip_forward=1
net.bridge.bridge-nf-call-arptables=1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.all.rp_filter=0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.ip_default_ttl = 64
net.ipv4.ip_dynaddr = 0
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.inet_peer_threshold = 65664
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_maxttl = 600
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_frto = 2
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_mtu_probing = 1
net.ipv4.tcp_base_mss = 512
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.tcp_challenge_ack_limit = 100
net.ipv4.tcp_limit_output_bytes = 131072
net.ipv4.tcp_slow_start_after_idle = 1
net.netfilter.nf_conntrack_tcp_timeout_last_ack = 10
net.ipv4.tcp_available_congestion_control = bbr reno cubic
net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_max_ssthresh = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_thin_dupack = 0
net.ipv4.tcp_min_tso_segs = 2
net.ipv4.conf.all.forwarding = 0
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.src_valid_mark = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.neigh.default.gc_thresh1 = 80000
net.ipv4.neigh.default.gc_thresh2 = 90000
net.ipv4.neigh.default.gc_thresh3 = 100000
net.ipv4.conf.all.tag = 0
EOF
/sbin/sysctl -p
lsmod | grep bbr ## 查看bbr是否加载

Set limits.conf

cat >> /etc/security/limits.conf << EOF
 *           soft   nproc       1024000
 *           hard   nproc       1024000
 *           soft   nofile      1024000
 *           hard   nofile      1024000
 *           soft   core        1024000
 *           hard   core        1024000
 ######big mem ########
 #*           hard    memlock    unlimited  
 #*           soft    memlock    unlimited
EOF

Set 20-nproc.conf

But -i 's / 4096/1024000 / /etc/security/limits.d/20-nproc.conf

Set journal log size and storage path

echo SystemMaxUse=600M >>/etc/systemd/journald.conf
mkdir -p /var/log/journal
chown root:systemd-journal /var/log/journal
chmod 2755 /var/log/journal
systemctl restart systemd-journald

Close NetworkManager

systemctl disable NetworkManager.service
systemctl stop NetworkManager.service
service network restart
chkconfig network on

profile modification

cat >> /etc/profile << EOF
ulimit -d unlimited
ulimit -m unlimited
ulimit -s unlimited
ulimit -v unlimited
ulimit -t unlimited
ulimit -c unlimited
ulimit -l unlimited
EOF
. /etc/profile
reboot

2 k8s-operation environment ready

2.1, the compiler LXCSF distributed to the node after node

Download Compilation Tools

yum install -y git automake libtool fuse-devel ansible

Compile LXCSF

git clone git://github.com/lxc/lxcfs
cd lxcfs/
./bootstrap.sh
./configure
make
mkdir ../binlxfs/lib/lxcfs
cp -pdr lxcfs ../binlxfs
cp -pdr .libs/iblxcfs.so ../binlxfs/lib/lxcfs
cp -pdr liblxcfs.la ../binlxfs/lib/lxcfs

Creating LXCFS start the service

cd ../binlxfs
cat << EOF | tee lxcfs.service
[Unit]
Description=FUSE filesystem for LXC
ConditionVirtualization=!container
Before=lxc.service
Documentation=man:lxcfs(1)

[Service]
ExecStart=/usr/local/bin/lxcfs /var/lib/lxcfs/
KillMode=process
Restart=on-failure
ExecStopPost=-/bin/fusermount -u /var/lib/lxcfs
Delegate=yes

[Install]
WantedBy=multi-user.target
EOF

Description: lxcfs.service Please distributed to all node node /usr/lib/systemd/system/lxcfs.service

lxcfs binlxfs folder distributed to / usr / local / bin / lxcfs

binlxfs / lib distributed to / usr / local / lib

Creating / var / lib / lxcfs / directory mkdir -p / var / lib / lxcfs /

All node node configuration boot lxcfs

systemctl daemon-reload && systemctl start lxcfs && systemctl enable lxcfs

2.2 Installation and configuration CFSSL

yum install go
vi ~/.bash_profile
GOBIN=/root/go/bin/
PATH=$PATH:$GOBIN:$HOME/bin
export PATH
go get -u github.com/cloudflare/cfssl/cmd/cfssl
go get -u github.com/cloudflare/cfssl/cmd/cfssljson

When viewing the cluster state after use 2.3 client installed kubectl

wget https://dl.k8s.io/v1.14.0/kubernetes-client-linux-amd64.tar.gz
tar -xzvf kubernetes-client-linux-amd64.tar.gz
mv kubernetes / client / bin / kubectl / usr / local / bin /
rm -rf kubernetes *

Baidu network disk Download

Links: https://pan.baidu.com/s/1BvV1zVMG-q9Bx1nZZI_B2Q extraction code: kr6q

Next: Kubernetes production installation to deploy based on the deployment of etcd cluster Kubernetes v1.14.0