Currently tcp long connection used by the application program is nginx (ngx_stream_core_module), static configuration; built-in health check can only be based on port, but the application program errors, or sometimes suspended animation, the port is OK, cause the client a lot of error, so the new dynamically configurable routing scheme may be required, by the monitoring node checker dynamic configuration management backend.
http dynamic routing (also known as dynamic upstream) program more, there are ready-made plug-ins, or lua support for this piece was also good; but relatively few tcp programs under the full study, summarized under the following two fly program .
1. Nginx, lua library under openresty, write your own script lua
realize the function:
- When the configuration server Nginx startup to initialize redis, hash structure: {ip1: 0, ip2: 0, ip3: 0, ......} (0: OK, 1: not available);
- Each request is acquired from the server nginx, the manner in rotation, Redis supports connection pooling;
- Redis access connection fails, or no available access server key (hash is empty, or all value 1), with the local static configuration;
- Can not access a server (ip port or unreasonable, too busy or inaccessible), the other nodes will try, try number can be equipped;
- Unrealized and static ports configured as health check mechanism automatically kicked out (for two reasons: 1, it will conflict with the external independent monitoring inspection procedures; 2, from the timing of tasks required in nginx init stage, but does not support socket module init stage access redis, the right solution is not found)
Script Description:
High concurrency scenarios need to focus on the use of variables (as far as possible not to use global variables) and processing logic, especially in rotation, failure retry two functions.
stream {
lua_package_path "/usr/local/lib/lua/?.lua;;";
lua_shared_dict dict_ups 4m;
lua_shared_dict dict_try 10m;
lua_shared_dict rotate_lock 100k;
lua_add_variable $dhq_proxypass;
lua_add_variable $backend_server;
lua_add_variable $try_cnt;
lua_add_variable $dhq_conn_cnt;
log_format main '$remote_addr [$time_local] $server_addr:$server_port $dhq_proxypass $dhq_conn_cnt $try_cnt $backend_server $status';
access_log logs/access.log main;
upstream mytcp_static {
server 10.40.20.201:22 max_fails=3 fail_timeout=5s;
server 10.40.20.202:22 max_fails=3 fail_timeout=5s;
server 10.40.20.203:22 max_fails=3 fail_timeout=5s;
server 10.40.20.204:22 max_fails=3 fail_timeout=5s;
}
upstream mytcp_lua {
# just a place holder,not work
server 1.1.1.1:1111;
balancer_by_lua_block {
local backend_port = 22
local try_cnt = ngx.shared.dict_try:get("conn" .. ngx.var.dhq_conn_cnt)
if try_cnt > 16 then
return
end
local balancer = require "ngx.balancer"
balancer.set_timeouts(3, 3, 3)
balancer.set_more_tries(4)
if g_ups_cur_dhq then
local state_name, status_code = balancer.get_last_failure()
if state_name == nil then
balancer.set_current_peer(g_ups_cur_dhq, backend_port)
ngx.var.backend_server = g_ups_cur_dhq .. ":" .. backend_port
else
local table_len = table.getn(g_ups_dhq_active_table)
local ups_cur_dhq = g_ups_dhq_active_table[(try_cnt - 1) % table_len + 1]
balancer.set_current_peer(ups_cur_dhq, backend_port)
ngx.var.backend_server = ups_cur_dhq .. ":" .. backend_port
try_cnt = try_cnt + 1
ngx.shared.dict_try:set("conn" .. ngx.var.dhq_conn_cnt, try_cnt)
end
ngx.var.try_cnt = try_cnt
else
ngx.log(ngx.ERR, "[error]: no server in upstream. ")
return
end
}
}
server {
listen 12345;
proxy_connect_timeout 3s;
proxy_timeout 120s;
proxy_next_upstream_tries 5;
preread_by_lua_block {
ups_dhq_table = {}
ups_dhq_table["10.40.20.201"] = 0
ups_dhq_table["10.40.20.202"] = 0
ups_dhq_table["10.40.20.203"] = 0
ups_dhq_table["10.40.30.204"] = 0
local ups_name = "mytcp"
local redis_ups_key = "upstream_denghaoqi"
ngx.var.dhq_proxypass = ups_name .. "_lua"
function func_get_redis()
local Redis = require "resty.redis"
local redis = Redis:new()
local pool_options = { pool_size = 300, blck_log = 20000 }
redis:set_timeout(3000)
local ok, err = redis:connect("10.40.16.45", 36379, pool_options)
if not ok then
ngx.var.dhq_proxypass = ups_name .. "_static"
ngx.log(ngx.ERR, "connect to redis failed, ", err)
return
end
return redis
end
local dhq_conn_cnt, err = ngx.shared.dict_ups:incr("dhq_conn_cnt",1,0,0)
ngx.var.dhq_conn_cnt = dhq_conn_cnt
-- sync to redis when nginx start
if (dhq_conn_cnt == 1) then
local redis = func_get_redis()
if redis == nil then
return
end
local ok, err = redis:del(redis_ups_key)
local ok, err = redis:hmset(redis_ups_key, ups_dhq_table)
-- local ok, err = redis:close()
redis:set_keepalive(30000, 300)
end
-- get a server in rotation
local redis = func_get_redis()
if redis == nil then
return
end
local ok, err = redis:array_to_hash(redis:hgetall(redis_ups_key))
if not ok then
ngx.var.dhq_proxypass = ups_name .. "_static"
ngx.log(ngx.ERR, "get redis key failed. ")
return
end
redis:set_keepalive(30000, 300)
if type(ok) == "table" then
if ok[1] == false then
ngx.log(ngx.ERR, "error: ", ok[2])
else
local ups_dhq_active_table = {}
for key, value in pairs(ok) do
if value == "0" then
table.insert(ups_dhq_active_table,key)
end
end
if table.getn(ups_dhq_active_table) == 0 then
ngx.var.dhq_proxypass = ups_name .. "_static"
ngx.log(ngx.ERR, "redis key has no valid server. ")
return
end
table.sort(ups_dhq_active_table)
local ind = (dhq_conn_cnt - 1) % table.getn(ups_dhq_active_table) + 1
g_ups_cur_dhq = ups_dhq_active_table[ind]
g_ups_dhq_active_table = ups_dhq_active_table
ngx.shared.dict_try:set("conn" .. dhq_conn_cnt, 1)
end
end
}
proxy_pass $dhq_proxypass;
}
}2. Haproxy, dataplaneapi haproxy ecosystem
Description:
Dataplaneapi achieve a restful api, through a friendly interface to delete server, increased server, dataplaneapi and haproxy deployed on a single server, is one relationship, it is necessary for each node haproxy api of operation;
After haproxy manually edit the configuration file, execute the following command strong brush dataplaneapi cache: kill -SIGUSR2 dataplaneapi process, or restart dataplaneapi
Api use:
when to add or delete server, you need to open the transaction;
after the transaction is committed, haproxy automatically reload, modify configuration files automatically
dataplaneapi service starts
/root/dataplaneapi/dataplaneapi-master/build/dataplaneapi --host 10.40.20.203 --port 5555 -b /usr/local/haproxy/sbin/haproxy -c /usr/local/haproxy/conf/haproxy.cfg -d 5 -r "/usr/local/haproxy/haproxy_mgr.sh restart" -s "/usr/local/haproxy/haproxy_mgr.sh reload" -u api -t /tmp/haproxyThe main steps:
查询信息,获取当前version
# curl -X GET -u admin:admin \
> -H "Content-Type: application/json" \
> "http://10.40.20.203:5555/v2/services/haproxy/configuration/servers?backend=test-proxy-srv"
{
"_version":1,
"data":[
{"address":"10.40.20.208","check":"enabled","name":"10.40.20.208","port":222},
{"address":"10.45.0.10","check":"enabled","name":"10.45.0.10","port":22,"weight":80},
{"address":"10.45.0.11","check":"enabled","name":"10.45.0.11","port":22,"weight":80}
]
}
开启事务，获取事务id
参数version根据上述步骤结果递增
# curl -X POST -u admin:admin \
> -H "Content-Type: application/json" \
> http://10.40.20.203:5555/v2/services/haproxy/transactions?version=2
{"_version":1,"id":"c69fa5fe-8dc7-4c85-8912-0ac86b3ad59d","status":"in_progress"}
删除server
curl -X DELETE -u admin:admin \
-H "Content-Type: application/json" \
"http://10.40.20.203:5555/v2/services/haproxy/configuration/servers/10.40.20.207?backend=test-proxy-srv&transaction_id=c69fa5fe-8dc7-4c85-8912-0ac86b3ad59d"
增加server
curl -X POST -u admin:admin \
-H "Content-Type: application/json" \
--data '{"address": "10.45.0.11", "check": "enabled", "max-connections": 500, "name": "10.45.0.11", "port": 22, "weight": 80}' \
"http://10.40.20.203:5555/v2/services/haproxy/configuration/servers?backend=test-proxy-srv&transaction_id=c69fa5fe-8dc7-4c85-8912-0ac86b3ad59d "
提交事务
curl -X PUT -u admin:admin \
-H "Content-Type: application/json" \
http://10.40.20.203:5555/v2/services/haproxy/transactions/c69fa5fe-8dc7-4c85-8912-0ac86b3ad59d3. Compare program
|
|
Nginx + lua |
Haproxy + dataplaneapi |
| stability |
Write your own lua, through a wide range of functional testing and performance testing, stability needs to be verified online |
Own characteristics, stability is more assured |
| Monitoring program requirements |
Redis can adjust a simple program for |
Need to know api, the need for separately processing nodes N |
| Operation and maintenance requirements |
Lua introducing complex programming (in particular failure or performance problems in high concurrency), subsequent to further optimize |
The newly introduced haproxy, operation and maintenance to further study
|
