# Define the name of the three components
######## collection of static files
# startup command
bin / flume-ng agent -c conf / -f dir-hdfs.conf -n ag1 -Dflume.root.logger = INFO, the print log console console, in the situation placed Black Hole
Configuration is as follows:
ag1.sources = source1
ag1.sinks = sink1
ag1.channels = channel1
# Set the source component
ag1.sources.source1.type = spooldir #spooldir is flume comes with a read source directory, its behavior is characterized as long as there is a new file will help you go read the directory
ag1.sources. source1.spoolDir = / root / log / # specific directory read
file as a suffix following ag1.sources.source1.fileSuffix = .FINISHED # file is read is completed
ag1.sources.source1.deserializer.maxLineLength = 5120
# Configure sink assembly
ag1.sinks.sink1.type = hdfs # hdfs taken to the
ag1.sinks.sink1.hdfs.path = hdfs: // hdp-1 : 9000 / access_log /% y-% m-% d /% H-% M
prefix such ag1.sinks.sink1.hdfs.filePrefix = app_log # plus the cut-out files app_log
ag1.sinks.sink1.hdfs.fileSuffix cut # = .log file suffix added app_log such as: app_log .1563695834065.log
ag1.sinks.sink1.hdfs.batchSize = 100 # 100 data refresh hdfs
ag1.sinks.sink1.hdfs.fileType # = DataStream type of file to write hdfs for DataStream
ag1.sinks.sink1.hdfs. writeFormat = Text # hdfs written as Text, fileType with the use of the writeFormat
## roll: Scroll Switch: control switching rules to write files generated files. The three meet that will be executed which, if you specify which one, then with the other two can be equal to 0! ! ! Can not remember the comment to the other two, or allowed to scroll through the file
## by the file size (bytes) to cut, equivalent to 512000 500K
ag1.sinks.sink1.hdfs.rollSize = 512000
## by cutting the number of event, 1000000 switching data corresponding to a file 1000000
ag1.sinks.sink1.hdfs.rollCount = 1000000
## file switching time intervals, the equivalent of 60 60 seconds
ag1.sinks.sink1.hdfs.rollInterval = 60
## control rule generating directory
ag1.sinks.sink1.hdfs.round = true # do not want to cut, to
ag1.sinks.sink1.hdfs.roundValue = 10 # How often cut once
ag1.sinks.sink1.hdfs.roundUnit = minute # ten minutes cut once
ag1.sinks.sink1.hdfs.useLocalTimeStamp = true # used to get the local time stamp, can put /% y-% m-% d /% H-% M time to replace
# Channel assembly is configured
ag1.channels.channel1.type = Memory
## Event Article Number
ag1.channels.channel1.capacity = 500000
## Flume transaction control required buffer capacity 600 Event
ag1.channels.channel1.transactionCapacity = 600
# Binding link between the source, channel and sink
ag1.sources.source1.channels = CHANnel1
ag1.sinks.sink1.channel = CHANnel1
Start command:
[] # Bin / flume-ng agent -c conf / -f dir-hdfs.conf -n ag1 -Dflume.root.logger = INFO, console
log printed in the foreground, the actual process on the Black Hole