下载编译flume-ng-sql-source,下载地址并按照网站里的说明添加相应的包
安装说明文档编译和拷贝jar包
flume配置文件
a1.channels = c1 c2
a1.sources = s1 s2
a1.sinks = k1 k2
###########sources#################
# s1
a1.sources.s1.type = org.keedio.flume.source.SQLSource
a1.sources.s1.hibernate.connection.url = jdbc:mysql://192.168.1.177:3306/bigdata
a1.sources.s1.hibernate.connection.user = root
a1.sources.s1.hibernate.connection.password = chineseall
a1.sources.s1.hibernate.connection.autocommit = true
a1.sources.s1.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect
a1.sources.s1.hibernate.connection.driver_class = com.mysql.jdbc.Driver
a1.sources.s1.run.query.delay=10000
a1.sources.s1.status.file.path = /root/data/flume/
a1.sources.s1.status.file.name = sqlSource.status
a1.sources.s1.start.from = 0
a1.sources.s1.custom.query = select username,devicetype,logintime from device_log
a1.sources.s1.batch.size = 1000
a1.sources.s1.max.rows = 1000
a1.sources.s1.hibernate.connection.provider_class = org.hibernate.connection.C3P0ConnectionProvider
a1.sources.s1.hibernate.c3p0.min_size=1
a1.sources.s1.hibernate.c3p0.max_size=10
#s2
a1.sources.s2.type = org.keedio.flume.source.SQLSource
a1.sources.s2.hibernate.connection.url = jdbc:mysql://192.168.1.177:3306/bigdata
a1.sources.s2.hibernate.connection.user = root
a1.sources.s2.hibernate.connection.password = chineseall
a1.sources.s2.hibernate.connection.autocommit = true
a1.sources.s2.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect
a1.sources.s2.hibernate.connection.driver_class = com.mysql.jdbc.Driver
a1.sources.s2.run.query.delay=10000
a1.sources.s2.status.file.path = /root/data/flume/
a1.sources.s2.status.file.name = sqlSource.status
a1.sources.s2.start.from = 0
a1.sources.s2.custom.query = select realname,school,source,create_time from report_table
a1.sources.s2.batch.size = 1000
a1.sources.s2.max.rows = 1000
a1.sources.s2.hibernate.connection.provider_class = org.hibernate.connection.C3P0ConnectionProvider
a1.sources.s2.hibernate.c3p0.min_size=1
a1.sources.s2.hibernate.c3p0.max_size=10
############channels###############
a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 10000
a1.channels.c1.byteCapacityBufferPercentage = 20
a1.channels.c1.byteCapacity = 800000
a1.channels.c2.type = memory
a1.channels.c2.capacity = 10000
a1.channels.c2.transactionCapacity = 10000
a1.channels.c2.byteCapacityBufferPercentage = 20
a1.channels.c2.byteCapacity = 800000
############sinks##################
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = Device
a1.sinks.k1.brokerList = 192.168.1.233:9092
a1.sinks.k1.requiredAcks = 1
a1.sinks.k1.batchSize = 20
a1.sinks.k1.channel = c1
a1.sinks.k2.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k2.topic = Report
a1.sinks.k2.brokerList = 192.168.1.233:9092
a1.sinks.k2.requiredAcks = 1
a1.sinks.k2.batchSize = 20
a1.sinks.k2.channel = c1
# Bind the source and sink to the channel
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2
a1.sources.s1.channels=c1
a1.sources.s2.channels=c2