postgresql 高可用 etcd + patroni 之 etcd

os: centos 7.4
etcd:3.2.18

etcd + patroni 是在一个postgrsql 开源大会上亚信的一个哥们讲解的高可用方案。
依然是基于 postgreql stream replication。

这篇blog 主要是讲 etcd 的配置，下一篇blog讲解 patroni 。

ip规划
192.168.56.101 node1 master
192.168.56.102 node2 slave
192.168.56.102 node3 slave

下载、安装

# yum install etcd 

# yum list installed |grep -i etcd
etcd.x86_64                        3.2.18-1.el7                        @extras

etcd 配置

# cp /etc/etcd/etcd.conf /etc/etcd/etcd.conf.bak
# vi /etc/etcd/etcd.conf 

# cat etcd.conf
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/node1.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="http://192.168.56.101:2380,http://127.0.0.1:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.56.101:2379,http://127.0.0.1:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="node1"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.56.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.56.101:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="node1=http://192.168.56.101:2380,node2=http://192.168.56.102:2380,node3=http://192.168.56.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
#ETCD_CERT_FILE=""
#ETCD_KEY_FILE=""
#ETCD_CLIENT_CERT_AUTH="false"
#ETCD_TRUSTED_CA_FILE=""
#ETCD_AUTO_TLS="false"
#ETCD_PEER_CERT_FILE=""
#ETCD_PEER_KEY_FILE=""
#ETCD_PEER_CLIENT_CERT_AUTH="false"
#ETCD_PEER_TRUSTED_CA_FILE=""
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"

# egrep ^[A-Z] ./etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/node1.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.56.101:2380,http://127.0.0.1:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.56.101:2379,http://127.0.0.1:2379"
ETCD_NAME="node1"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.56.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.56.101:2379"
ETCD_INITIAL_CLUSTER="node1=http://192.168.56.101:2380,node2=http://192.168.56.102:2380,node3=http://192.168.56.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

修改 etcd.service

# systemctl status etcd.service 
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)
   Active: inactive (dead)

# vi /usr/lib/systemd/system/etcd.service
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd \
--name=\"${ETCD_NAME}\" \
--data-dir=\"${ETCD_DATA_DIR}\" \
--listen-peer-urls=\"${ETCD_LISTEN_PEER_URLS}\" \
--listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" \
--initial-advertise-peer-urls=\"${ETCD_INITIAL_ADVERTISE_PEER_URLS}\" \
--advertise-client-urls=\"${ETCD_ADVERTISE_CLIENT_URLS}\" \
--initial-cluster=\"${ETCD_INITIAL_CLUSTER}\"  \
--initial-cluster-token=\"${ETCD_INITIAL_CLUSTER_TOKEN}\" \
--initial-cluster-state=\"${ETCD_INITIAL_CLUSTER_STATE}\""
Restart=on-failure
LimitNOFILE=65536

以上操作在 node1、node2、node3 节点上都需要操作,需要修改对应的ip。

启动 etcd

依次启动 node1、node2、node3 节点的 etcd

# systemctl start etcd.service
# systemctl status etcd.service
# systemctl enable etcd.service

验证 etcd

# etcdctl ls /
# etcdctl cluster-health
member ca933ab8cfffe553 is healthy: got healthy result from http://192.168.56.101:2379
member f63afbe816fb463d is healthy: got healthy result from http://192.168.56.102:2379
member d44832212a08c43f is healthy: got healthy result from http://192.168.56.103:2379
cluster is healthy

# etcdctl member list
ca933ab8cfffe553: name=node1 peerURLs=http://192.168.56.101:2380 clientURLs=http://192.168.56.101:2379 isLeader=false
d44832212a08c43f: name=node3 peerURLs=http://192.168.56.103:2380 clientURLs=http://192.168.56.103:2379 isLeader=true
f63afbe816fb463d: name=node2 peerURLs=http://192.168.56.102:2380 clientURLs=http://192.168.56.102:2379 isLeader=false

ok，etcd 运行正常。

node1 上关掉 etcd 服务

# systemctl stop etcd
# etcdctl cluster-health
cluster may be unhealthy: failed to list members
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
; error #1: dial tcp 127.0.0.1:4001: getsockopt: connection refused

error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
error #1: dial tcp 127.0.0.1:4001: getsockopt: connection refused

node2 上检查etcd集群信息

# etcdctl cluster-health
failed to check the health of member ca933ab8cfffe553 on http://192.168.56.101:2379: Get http://192.168.56.101:2379/health: dial tcp 192.168.56.101:2379: getsockopt: connection refused
member ca933ab8cfffe553 is unreachable: [http://192.168.56.101:2379] are all unreachable
member d44832212a08c43f is healthy: got healthy result from http://192.168.56.103:2379
member f63afbe816fb463d is healthy: got healthy result from http://192.168.56.102:2379
cluster is healthy

会看到有如下告警信息，是因为各节点的时间有差异。

Jul 11 03:03:40 node1 etcd[1657]: the clock difference against peer f63afbe816fb463d is too high [3.132011837s > 1s]
Jul 11 03:04:05 node1 etcd[1657]: the clock difference against peer d44832212a08c43f is too high [4.491729602s > 1s]
Jul 11 03:04:10 node1 etcd[1657]: the clock difference against peer f63afbe816fb463d is too high [3.132795377s > 1s]
Jul 11 03:04:35 node1 etcd[1657]: the clock difference against peer d44832212a08c43f is too high [4.492031513s > 1s]
Jul 11 03:04:40 node1 etcd[1657]: the clock difference against peer f63afbe816fb463d is too high [3.132249351s > 1s]

crontab 定时同步时间

# crontab -e
0 1 * * * /usr/sbin/ntpdate ntp.sjtu.edu.cn >> /var/log/ntpdate.log &

参考：
https://github.com/coreos/etcd
https://github.com/coreos/etcd/blob/master/Documentation/docs.md

https://zhidao.baidu.com/question/492858132927230132.html