文章目录
联级agent
采集资产漏洞扫描日志/var/log/lynis.log，agent端agent.properties，大数据端hbase.properties。
启动两个采集服务，客户端start-agent.sh和大数据端start-h.sh
创建hbase表
create 'T_WARNING_LYNIS', {NAME => 'BASIC_INFO', VERSIONS => 3, BLOCKCACHE => false, COMPRESSION => 'SNAPPY'}, SPLITS => ['1','2','3','4','5','6','7','8','9','10']
创建目录
- 客户端create.sh
mkdir -p /home/dataFlume/data_lynis/checkpoint
mkdir -p /home/dataFlume/data_lynis/datadir
mkdir -p /home/dataFlume/data_lynis/backup_checkpoint
- 大数据端create-h.sh
mkdir -p /home/hadoop/dataFlume/data_lynis/checkpoint
mkdir -p /home/hadoop/dataFlume/data_lynis/datadir
mkdir -p /home/hadoop/dataFlume/data_lynis/backup_checkpoint
客户端flume配置
- agent.properties
#list all the compents of this agent
type_agent.sources = r1
type_agent.sinks = k1
type_agent.channels = c1
#10.lynis
# list the properties of source
type_agent.sources.r1.type = exec
type_agent.sources.r1.command = tail -F /var/log/lynis.log | sed -u "s/^/$HOSTNAME /g"
type_agent.sources.r1.shell = /bin/bash -c
# list the properties of channels
type_agent.channels.c1.type = file
type_agent.channels.c1.checkpointDir = /home/dataFlume/data_lynis/checkpoint
# Using multiple directories on separate disks can improve file channel peformance
type_agent.channels.c1.dataDirs = /home/dataFlume/data_lynis/datadir
type_agent.channels.c1.useDualCheckpoints = true
type_agent.channels.c1.backupCheckpointDir = /home/dataFlume/data_lynis/backup_checkpoint
# list the properties of sinks
type_agent.sinks.k1.type = avro
type_agent.sinks.k1.hostname = master
type_agent.sinks.k1.port = 16080
type_agent.sinks.k1.compression-type = deflate
type_agent.sinks.k1.batch-size = 16
# link the flow
type_agent.sources.r1.channels = c1
type_agent.sinks.k1.channel = c1
其中tail -F /var/log/lynis.log | sed -u "s/^/$HOSTNAME /g"后面得sed用于在每行行首添加HOSTNAME
- start-agent.sh
nohup flume-ng agent -n type_agent -c $FLUME_HOME/conf -f $FLUME_HOME/conf/Inventory/Lynis/agent.properties &
- 记得服务sh文件执行权限
chmod -R 744 start-agent.sh
- 运行命令
nohup ./start-agent.sh 1>out.log 2>&1
大数据端
- hbase.properties
#list all the compents of this agent
type_hbase.sources = r1
type_hbase.sinks = k1
type_hbase.channels = c1
#10.lynis
# list the properties of source
type_hbase.sources.r1.type = avro
type_hbase.sources.r1.bind = 0.0.0.0
type_hbase.sources.r1.port = 16080
type_hbase.sources.r1.threads = 10
type_hbase.sources.r1.compression-type = deflate
# list the properties of channels
type_hbase.channels.c1.type = file
type_hbase.channels.c1.checkpointDir = /home/hadoop/dataFlume/data_lynis/checkpoint
# Using multiple directories on separate disks can improve file channel peformance
type_hbase.channels.c1.dataDirs = /home/hadoop/dataFlume/data_lynis/datadir
type_hbase.channels.c1.useDualCheckpoints = true
type_hbase.channels.c1.backupCheckpointDir = /home/hadoop/dataFlume/data_lynis/backup_checkpoint
# list the properties of sinks
type_hbase.sinks.k1.type = hbase2
type_hbase.sinks.k1.serializer = org.apache.flume.sink.hbase2.RegexHBase2EventSerializer
type_hbase.sinks.k1.columnFamily = BASIC_INFO
# first regex expression
#type_hbase.sinks.k1.serializer.regex = ([^ ]*) ([0-9]*-[0-9]*-[0-9]*) ([0-9]*:[0-9]*:[0-9]*) (Warning|Suggestion): (.+)
#type_hbase.sinks.k1.serializer.colNames = DST_IP,YMD,TIME,TYPE,INFO
type_hbase.sinks.k10.serializer.regex = ([^ ]*) ([0-9]*-[0-9]*-[0-9]*) ([0-9]*:[0-9]*:[0-9]*) (Warning|Suggestion): (.*?) \\[test:(.*?)-(.*?)\\] \\[details:(.*?)\\] \\[solution:(.*?)\\]
type_hbase.sinks.k10.serializer.colNames = DST_IP,YMD,TIME,TYPE,INFO,TEST,TEST_ID,DETAIL,SOLUTION
type_hbase.sinks.k1.table = T_WARNING_LYNIS
# link the flow
type_hbase.sources.r1.channels = c1
type_hbase.sinks.k10.channel = c1
- start-h.sh
nohup flume-ng agent -n type_hbase -c $FLUME_HOME/conf -f $FLUME_HOME/conf/Inventory/Lynis/hbase.properties &
- 记得服务sh文件执行权限
chmod -R 744 start-h.sh
- 运行命令
nohup ./start-h.sh 1>out.log 2>&1