Preparation before installation
Nginx1.9.9
LuaJIT-2.0.4
lua-nginx-module-0.10.2
ngx_devel_kit-0.3.0rc1
lua-resty-kafka
kafka_2.11-0.9.0.1
The referenced articles form this plan
Nginx Log real-time monitoring system based on Lua+Kafka+Heka
How Lua reads nginx built-in objects and methods
nginx+lua+kafka realizes unified collection and summary of logs
Provide a network disk download address for related software and detailed configuration instructions http://pan.baidu.com/s/1c2LmX36
1 Install LuaJIT
Download http://luajit.org/download.html
http://luajit.org/install.html The
command is as follows:
tar zxf LuaJIT-2.0.4.tar.gz
cd LuaJIT-2.0.4
make PREFIX=/usr/local /LuaJIT
make install PREFIX=/usr/local/LuaJIT
echo "/usr/local/LuaJIT/lib" > /etc/ld.so.conf.d/usr_local_luajit_lib.conf
ldconfig #Pay
attention to environment variables!
export LUAJIT_LIB=/usr/ local/LuaJIT/lib
export LUAJIT_INC=/usr/local/LuaJIT/include/luajit-2.0
2 Install lua-nginx-module
https://github.com/openresty/lua-nginx-module/tags
cd /usr/local
tar zxvf lua-nginx-module-0.10.2.tar.gz
3 Install ngx_devel_kit
https://github.com/simpl/ngx_devel_kit/tags
http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
cd /usr/local
tar zxvf ngx_devel_kit-0.3.0rc1.tar.gz
4 Install and compile Nginx
http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
给Nginx添加下面的参数，如果已经安装了Nginx则用
nginx -V
--add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1
如：
cd /usr/local/nginx-1.9.9
./configure --prefix=/usr/local/nginxnew/ --with-http_stub_status_module --with-http_ssl_module --with-http_realip_module --add-module=/root/install/ngx_log_if-master --add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1
5 lua plugin lua-resty-kafka
https://github.com/doujiang24/lua-resty-kafka
mkdir /usr/local/lua
Upload lua-cjson-2.1.0.3.tar.gz to /usr/local/lua
Upload lua-resty-kafka to /usr /local/lua
6 lua plugin cjson
http://www.kyne.com.au/~mark/software/lua-cjson-manual.html
cd /usr/local/lua
tar zxvf lua-cjson-2.1.0.3.tar.gz
7, nginx configuration
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
log_format ht-video '$request|$msec|$http_x_forwarded_for|$http_user_agent|$request_body';
lua_package_path "/usr/local/lua/lua-resty-kafka/lib/?.lua;;";
server{
listen 8080;
server_name 192.168.50.42;
location / {
root /usr/local/nginx/htm;
index index.html;
}
location /ht{
#root html;
#index index.html index.htm;
if ( $request ~ "GET" ) {
#access_log logs/ht-video.log ht-video;
access_log /htlog/ht-video.log ht-video;
}
# proxy_set_header Host $http_host;
# proxy_pass http://localhost:6099;
client_max_body_size 8000M;
#error_page 405 =200 $1;
}
}
server {
listen 6099;
server_name localhost;
location /ht {
root /usr/local/nginx;
index index.html index.htm;
client_max_body_size 8000M;
error_page 405 =200 $1;
}
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main; #To
facilitate debugging, close lua_code_cache, if it is a production environment, it should be turned on.
lua_code_cache off;
location / {
root html;
index index.html index.htm;
}
location /lua {
default_type 'text/plain';
content_by_lua 'ngx.say("hello, lua")';
}
location = /lua-v {
content_by_lua '
ngx.header.content_type = "text/plain";
if jit then
ngx.say(jit.version)
else
ngx.say(_VERSION)
end
';
}
location /ht3/video/p {
content_by_lua '
ngx.header.content_type = "text/plain";
--local cjson = require "cjson"
--local client = require "resty.kafka.client"
local producer = require "resty.kafka.producer"
local broker_list = {
{ host = "192.168.50.42", port = 9092 }
}
local broker_list_host_ip = {
{ host = "kafka1.livecourse.com", ip = "192.168.50.42" },
}
local key = "key"
--local message = $request|$msec|$remote_addrx|$http_user_agent|$request_body
local myIP = ngx.req.get_headers()["X-Real-IP"]
if myIP == nil then
myIP = ngx.req.get_headers()["x_forwarded_for"]
end
if myIP == nil then
myIP = ngx.var.remote_addr
end
local h = ngx.req.get_headers()
local message = ngx.req.get_method() .. " " .. ngx.var.uri
if ngx.var.args ~= nil then
message = message .. "?" .. ngx.var.args .. "|"
end
message = message .. ngx.now() .. "|"
message = message .. myIP .. "|"
message = message .. h["user-agent"] .. "|"
local bodyData = ngx.req.get_body_data()
if bodyData == nil then
bodyData = "-"
end
message = message .. bodyData
-- this is async producer_type and bp will be reused in the whole nginx worker
local bp = producer:new(broker_list, { producer_type = "async" , broker_list_host_ip = broker_list_host_ip,refresh_interval = 3000})
local ok, err = bp:send("test", key, message)
-- Define kafka asynchronous producers
-- send log messages, send the second parameter key, which is used for kafka routing control:
-- when the key is nill (null), write data to the same partition for a period of time
-- specify the key, according to The hash of the key is written to the corresponding partition
-- Specify the key and write it to the corresponding partition according to the hash of the key
if not ok then
--ngx.say("send err:", err)
ngx.say("void(1);");
return
end
ngx.say("void(0);");
--ngx.say("send success, ok:", ok)
';
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
Note that if the configuration file is marked in red, the first one marked in red is the address and port of the kafka installation server. If there are more than one, configure multiple ones, and the second marked red test is the topic name of kafka.
8 complete
Request Url: http://localhost/ht3/video/p?a=bbbb to send log content to kafka