分布式Redis解决方案:Codis初体验

1. 安装 Go 运行环境

[root@localhost appInstall]# 
[root@localhost appInstall]# pwd
/appInstall
[root@localhost appInstall]# mkdir gohome
[root@localhost appInstall]# 
>> 解压go安装包
[root@localhost appInstall]# tar -zxvf go1.9.7.linux-amd64.tar.gz -C ./gohome/
[root@localhost appInstall]# cd gohome/
[root@localhost ~]# ls
go

>> 创建工作空间目录
[root@localhost ~]# mkdir work
[root@localhost ~]# ls
go  work

>> 配置环境变量：新增如下
[root@localhost appInstall]# vim /etc/profile
# go conf path
export GOHOME=/appInstall/gohome
# 指定工作目录
export GOPATH=$GOHOME/work
# 指定go目录
export GOROOT=$GOHOME/go
export PATH=$PATH:$GOROOT/bin:$GOPATH/bin

>> 生效配置
[root@localhost ~]# vim /etc/profile
[root@localhost ~]# source /etc/profile
[root@localhost ~]# 
[root@localhost ~]# go version
go version go1.9.7 linux/amd64
[root@localhost ~]# 
[root@localhost ~]# go env GOPATH
/appInstall/gohome/work
[root@localhost ~]#

2.下载 Codis 源代码

codis源代码需要下载到 $GOPATH/src/github.com/CodisLabs/codis

[root@localhost codis] mkdir -p $GOPATH/src/github.com/CodisLabs
[root@localhost codis] cd $_ && git clone https://github.com/CodisLabs/codis.git -b release3.2
[root@localhost CodisLabs]# pwd
/appInstall/gohome/work/src/github.com/CodisLabs
[root@localhost CodisLabs]# 
[root@localhost CodisLabs]# 
[root@localhost CodisLabs]# ls
codis
[root@localhost CodisLabs]#

3.编译 Codis 源代码

[root@localhost codis] cd $GOPATH/src/github.com/CodisLabs/codis
[root@localhost codis]# pwd
/appInstall/gohome/work/src/github.com/CodisLabs/codis
[root@localhost codis]#
>> 执行该命令报如下异常
[root@localhost codis] make
......
Hint: It's a good idea to run 'make test' ;)
make[2]: 离开目录“/appInstall/gohome/work/src/github.com/CodisLabs/codis/extern/redis-3.2.11/src”
make[1]: 离开目录“/appInstall/gohome/work/src/github.com/CodisLabs/codis/extern/redis-3.2.11”
autoconf
./autogen.sh:行5: autoconf: 未找到命令
Error 0 in autoconf
make[2]: *** [config] 错误 1
make[1]: *** [build] 错误 2
make: *** [codis-deps] 错误 2
[root@localhost codis]# make test
make: *** 没有规则可以创建目标“test”。停止。
[root@localhost codis]#
>> 安装依赖即可解决

[root@localhost codis]# yum install autoconf automake libtool -y

[root@localhost codis]# make
...
===============================================================================
go build -i -o bin/codis-dashboard ./cmd/dashboard
go build -i -tags "cgo_jemalloc" -o bin/codis-proxy ./cmd/proxy
go build -i -o bin/codis-admin ./cmd/admin
go build -i -o bin/codis-ha ./cmd/ha
go build -i -o bin/codis-fe ./cmd/fe
[root@localhost codis]#

>> 编译后可执行文件如下
[root@localhost codis]# ls -lrht bin/
总用量 92M
-rwxr-xr-x. 1 root root 5.2M 7月 17 12:51 codis-server
-rwxr-xr-x. 1 root root 2.4M 7月 17 12:51 redis-benchmark
-rwxr-xr-x. 1 root root 2.5M 7月 17 12:51 redis-cli
-rwxr-xr-x. 1 root root 5.2M 7月 17 12:51 redis-sentinel
-rw-r--r--. 1 root root 166 7月 17 12:51 version
-rwxr-xr-x. 1 root root 16M 7月 17 12:52 codis-dashboard
-rwxr-xr-x. 1 root root 18M 7月 17 12:52 codis-proxy
-rwxr-xr-x. 1 root root 15M 7月 17 12:52 codis-admin
-rwxr-xr-x. 1 root root 14M 7月 17 12:52 codis-ha
-rwxr-xr-x. 1 root root 15M 7月 17 12:52 codis-fe
drwxr-xr-x. 4 root root 117 7月 17 12:52 assets
[root@localhost codis]#

>> 拷贝可执行文件到新目录，便于管理
[root@localhost codis]# mkdir /appInstall/codis-app
[root@localhost codis]# mkdir /appInstall/codis-app/{conf,logs,pids,dbs,runlog}
[root@localhost codis]#
[root@localhost codis]# ls /appInstall/codis-app/
bin conf dbs logs pids runlog
[root@localhost codis]#
[root@localhost codis]# cp -r .bin/* /appInstall/codis-app/bin/
[root@localhost codis]#
[root@localhost codis]# ls /appInstall/codis-app/bin/
assets codis-dashboard codis-ha codis-server redis-cli version
codis-admin codis-fe codis-proxy redis-benchmark redis-sentinel
[root@localhost codis]#

4. 启动zookeeper服务

目前以zookeeper 为数据存储，本实例以单zk演示，后续改为zookeeper集群

[root@localhost zookeeper-3.4.8]# 
[root@localhost zookeeper-3.4.8]# ./bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /appInstall/zookeeper-3.4.8/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@localhost zookeeper-3.4.8]#

5. 启动codis-dashboard服务

[root@localhost conf]# 
[root@localhost conf]# pwd
/appInstall/codis-app/conf
>> 从源码中复制dashboard配置文件
[root@localhost conf]# cp /appInstall/gohome/work/src/github.com/CodisLabs/codis/config/dashboard.toml ./dashboard.toml
>> 或者通过查看默认配置，来创建文件
[root@localhost conf]# ./bin/codis-dashboard --default-config | tee dashboard.toml
[root@localhost conf]...
[root@localhost conf]# vim dashboard.toml 
[root@localhost conf]# 
##################################################
# #
# Codis-Dashboard #
# #
##################################################

# Set Coordinator, only accept "zookeeper" & "etcd" & "filesystem".
# for zookeeper/etcd, coorinator_auth accept "user:password" 
# Quick Start
#coordinator_name = "filesystem"
#coordinator_addr = "/tmp/codis"

# 外部存储类型，接受 zookeeper/etcd
coordinator_name = "zookeeper"
coordinator_addr = "127.0.0.1:2181"
#coordinator_auth = ""

# Set Codis Product Name/Auth.
# 集群名称，满足正则
product_name = "liu-codis"
# 集群密码，默认为空
product_auth = ""

# Set bind address for admin(rpc), tcp only.
# RESTful API 端口
admin_addr = "0.0.0.0:18080"

# Set arguments for data migration (only accept 'sync' & 'semi-async').
migration_method = "semi-async"
migration_parallel_slots = 100
migration_async_maxbulks = 200
migration_async_maxbytes = "32mb"
migration_async_numkeys = 500
migration_timeout = "30s"

# Set configs for redis sentinel.
sentinel_client_timeout = "10s"
sentinel_quorum = 2
sentinel_parallel_syncs = 1
sentinel_down_after = "30s"
sentinel_failover_timeout = "5m"
sentinel_notification_script = ""
sentinel_client_reconfig_script = ""
[root@localhost conf]#
>> 启动codis-dashboard服务
[root@localhost conf]# cd ..
[root@localhost codis-app]# pwd
/appInstall/codis-app
[root@localhost codis-app]# 
[root@localhost codis-app]# nohup ./bin/codis-dashboard --ncpu=4 --config=./conf/dashboard.toml --log=./logs/dashboard.log --log-level=WARN  >>./runlog/dashboard.log &
[1] 2933
nohup: 忽略输入重定向错误到标准输出端
[root@localhost codis-app]# ps -ef|grep codis
root      2933  2582  1 20:36 pts/0    00:00:00 ./bin/codis-dashboard --ncpu=4 --config=./conf/dashboard.toml --log=./logs/dashboard.log --log-level=WARN
root      2950  2582  0 20:36 pts/0    00:00:00 grep --color=auto codis
[root@localhost codis-app]#

6. 启动 codis-fe服务

[root@localhost codis-app]# pwd
/appInstall/codis-app
[root@localhost codis-app]# nohup ./bin/codis-fe --ncpu=4 --log=./logs/fe.log --log-level=WARN --zookeeper=127.0.0.1:2181 --listen=192.168.1.8:5080 >>./runlog/codis_fe.log &
[2] 3039
nohup: 忽略输入重定向错误到标准输出端
[root@localhost codis-app]# 
[root@localhost codis-app]# ps -ef|grep codis
root      2933  2582  0 20:36 pts/0    00:00:00 ./bin/codis-dashboard --ncpu=4 --config=./conf/dashboard.toml --log=./logs/dashboard.log --log-level=WARN
root      3039  2582  0 20:40 pts/0    00:00:00 ./bin/codis-fe --ncpu=4 --log=./logs/fe.log --log-level=WARN --zookeeper=127.0.0.1:2181 --listen=192.168.1.8:5080
root      3057  2582  0 20:40 pts/0    00:00:00 grep --color=auto codis
[root@localhost codis-app]# 
[root@localhost codis-app]#

7. 启动codis-proxy服务

按需求启动多台实例

>> 从源码中复制proxy配置文件
[root@localhost codis-app]# cp /appInstall/gohome/work/src/github.com/CodisLabs/codis/config/proxy.toml ./conf/proxy.toml
>> 或者通过查看默认配置，来创建文件
[root@localhost codis-app]# ./bin/codis-proxy --default-config | tee proxy.toml
[root@localhost codis-app]# vim conf/proxy.toml 
##################################################
#                                                #
#                  Codis-Proxy                   #
#                                                #
##################################################

# Set Codis Product Name/Auth.
# 集群名称
product_name = "liu-codis"
product_auth = ""

# Set auth for client session
#   1. product_auth is used for auth validation among codis-dashboard,
#      codis-proxy and codis-server.
#   2. session_auth is different from product_auth, it requires clients
#      to issue AUTH <PASSWORD> before processing any other commands.
session_auth = ""

# Set bind address for admin(rpc), tcp only.
# RESTful API 端口
admin_addr = "0.0.0.0:11010"

# Set bind address for proxy, proto_type can be "tcp", "tcp4", "tcp6", "unix" or "unixpacket".
proto_type = "tcp4"
# Redis 端口地址或者路径
proxy_addr = "0.0.0.0:19010"
[root@localhost codis-app]# mv conf/proxy.toml conf/proxy_19010.toml
启动codis-proxy服务
[root@localhost codis-app]# nohup ./bin/codis-proxy --ncpu=4 --config=./conf/proxy_19010.toml --log=./logs/proxy_19010.log --log-level=WARN >>./runlog/proxy_19010.log &
[3] 3326
nohup: 忽略输入重定向错误到标准输出端
[root@localhost codis-app]# ps -ef|grep codis
root      2933  2582  0 20:36 pts/0    00:00:00 ./bin/codis-dashboard --ncpu=4 --config=./conf/dashboard.toml --log=./logs/dashboard.log --log-level=WARN
root      3039  2582  0 20:40 pts/0    00:00:00 ./bin/codis-fe --ncpu=4 --log=./logs/fe.log --log-level=WARN --zookeeper=127.0.0.1:2181 --listen=192.168.1.8:5080
root      3326  2582  0 20:48 pts/0    00:00:00 ./bin/codis-proxy --ncpu=4 --config=./conf/proxy_19010.toml --log=./logs/proxy_19010.log --log-level=WARN
root      3342  2582  0 20:48 pts/0    00:00:00 grep --color=auto codis
[root@localhost codis-app]#

8. 启动codis-server服务

启动 ./bin/codis-server，与启动普通 redis 的方法一致。按需求启动多台实例
启动完成后，可以通过 codis-fe 提供的界面或者 codis-admin 命令行工具添加到集群中。

>> 从源码中复制proxy配置文件
[root@localhost codis-app]# cp /appInstall/gohome/work/src/github.com/CodisLabs/codis/config/redis.conf ./conf/redis.conf
[root@localhost codis-app]# 
>> 具体调整,可按照实际增减
bind 127.0.0.1
protected-mode no
# pid文件
pidfile /appInstall/codis-app/pids/redis_6381.pid
# 日志
logfile "/appInstall/codis-app/logs/redis_6381.log"
# 数据文件名称
dbfilename dump_6381.rdb
# 持久化数据存储目录
dir /appInstall/codis-app/dbs/
[root@localhost codis-app]# mv ./conf/redis.conf ./conf/redis_6381.conf
>> 启动codis-server
[root@localhost codis-app]# nohup ./bin/codis-server ./conf/redis_6381.conf >> ./runlog/redis_6381.log &
[root@localhost codis-app]#

9.添加codis-proxy到集群

通过codis-admin 命令行工具添加如： ./bin/codis-admin --dashboard=192.168.1.8:18080 --create-proxy -x 192.168.1.8:11010
通过codis-fe 提供的界面如：http://192.168.1.8:5080/#liu-codis

10.添加codis-Server到集群

通过codis-admin 命令行工具添加具体见： ./bin/codis-admin help
通过codis-fe 提供的界面如：http://192.168.1.8:5080/#liu-codis

先添加一个group, 然后添加redis-server到group

通过fe初始化slot

命令集中：

# 启动codis-dashboard实例
nohup ./bin/codis-dashboard --ncpu=4 --config=./conf/dashboard.toml --log=./logs/dashboard.log --log-level=WARN  >>./runlog/dashboard.log &

# 启动codis-fe实例
nohup ./bin/codis-fe --ncpu=4 --log=./logs/fe.log --log-level=WARN --zookeeper=127.0.0.1:2181 --listen=192.168.1.8:5080 >>./runlog/codis_fe.log &

# 启动codis-proxy实例
nohup ./bin/codis-proxy --ncpu=4 --config=./conf/proxy_19010.toml --log=./logs/proxy_19010.log --log-level=WARN >>./runlog/proxy_19010.log &
nohup ./bin/codis-proxy --ncpu=4 --config=./conf/proxy_19020.toml --log=./logs/proxy_19020.log --log-level=WARN >>./runlog/proxy_19020.log &

# 工具codis-admin添加proxy
./bin/codis-admin --dashboard=192.168.1.8:18080 --create-proxy -x 192.168.1.8:11010
./bin/codis-admin --dashboard=192.168.1.8:18080 --create-proxy -x 172.19.32.73:11020

>> redis参数调整
bind 127.0.0.1
protected-mode no
# pid文件
pidfile /appInstall/codis-app/pids/redis_6381.pid
# 日志
logfile "/appInstall/codis-app/logs/redis_6381.log"
# 数据文件名称
dbfilename dump_6381.rdb
# 持久化数据存储目录
dir /appInstall/codis-app/dbs/

# 启动codis-server实例
nohup ./bin/codis-server ./conf/redis_6381.conf >> ./runlog/redis_6381.log &
nohup ./bin/codis-server ./conf/redis_6382.conf >> ./runlog/redis_6382.log &