kafka中的原始数据格式(1条数据)
{
"body": {
"cwd": "/home/test/",
"monitor": {
"proc_num": 2,
"procs": [{
"cmd": "",
"cpu_usage_rate": 2.0,
"mem_usage_rate": 3.0,
"pid": 4976,
"procname": "test-name"
}, {
"cmd": "/home/test2",
"cpu_usage_rate": 5.0,
"mem_usage_rate": 6.0,
"pid": 4977,
"procname": "test-name2"
}],
"timestamp": 1547124214814
},
"os_tag": "Linux",
"system": {
"connection": {
"haddr": "00:50:56:B3:7E:7A",
"ip": "192.168.21.80",
"name": "ens160"
},
"cpu": ["Intel Xeon", "Intel Xeon", "Intel Xeon", "Intel Xeon"],
"memory": {
"swap_total": "7918841856",
"total": "15600787456"
},
"uname": "Linux Linux 3.10.0-862.el7.x86_64 x86_64 x86_64",
"vendor": "CentOS 7.5.1804"
}
},
"meta": {
"request_id": "3-14865"
}
}
logstash处理后的数据格式(2条数据)
{
"hostname": "test",
"procs": {
"mem_usage_rate": 2.0,
"cpu_usage_rate": 3.0,
"pid": 4976,
"cmd": "",
"procname": "test-name"
},
"@timestamp": "2019-01-11T02:08:57.225Z",
"memory": {
"total": "3975188480",
"swap_total": "4177522688"
},
"connection": {
"ip": "192.168.31.182",
"name": "ens160",
"haddr": "00:50:56:B3:7E:35"
},
"proc_num": 4
}
{
"hostname": "test",
"procs": {
"mem_usage_rate": 5.0,
"cpu_usage_rate": 6.0,
"pid": 4976,
"cmd": "test",
"procname": "test-name"
},
"connection": {
"ip": "192.168.31.182",
"name": "ens160",
"haddr": "00:50:56:B3:7E:35"
},
"proc_num": 4
}
logstash的配置
input {
kafka {
bootstrap_servers=> "192.168.31.92:9092,192.168.31.93:9092,192.168.31.94:9092"
group_id => "test_group"
topics =>"test_topic"
auto_offset_reset => "earliest"
type => "test_type"
consumer_threads => 1
codec => "json"
}
}
filter{
if !([body][monitor][procs]) {
drop { }
}
mutate {
remove_field => ["body[cwd]","body[os_tag]","body[system][filesystem]","body[system][cpu]","body[system][disk]",
"body[system][has_docker]","body[system][if]","body[system][uname]","body[system][vendor]","meta","url","body[configuration]"]
}
date {
match => ["body[monitor][timestamp]","UNIX_MS"]
remove_field => ["body[monitor][timestamp]"]
}
mutate {
add_field => {
"client_id" => "%{params[client_id]}"
"system" => "%{body[system]}"
"monitor" => "%{body[monitor]}"
}
remove_field => ["body","params"]
}
json {
source => "system"
remove_field => ["system"]
}
json {
source => "monitor"
remove_field => ["monitor"]
}
if ([procs]) {
split {
field => "procs"
}
}
}
output {
elasticsearch {
hosts => ["192.168.21.80:9200"]
index => "test_index"
codec => "json"
}
}