flume starts code analysis record
1.Application.java-->main method: First load the corresponding file category according to the command line content:
try {
boolean isZkConfigured = false;
Options options = new Options();
Option option = new Option("n", "name", true, "the name of this agent");
option.setRequired(true);
options.addOption(option);
option = new Option("f", "conf-file", true,
"specify a config file (required if -z missing)");
option.setRequired(false);
options.addOption(option);
option = new Option(null, "no-reload-conf", false,
"do not reload config file if changed");
options.addOption(option);
// Options for Zookeeper
option = new Option("z", "zkConnString", true,
"specify the ZooKeeper connection to use (required if -f missing)");
option.setRequired(false);
options.addOption(option);
option = new Option("p", "zkBasePath", true,
"specify the base path in ZooKeeper for agent configs");
option.setRequired(false);
options.addOption(option);
option = new Option("h", "help", false, "display help text");
options.addOption(option);
CommandLineParser parser = new GnuParser();
CommandLine commandLine = parser.parse(options, args);
...
I won't discuss it in depth here, and the next part is the key part:
if (reload) {
EventBus eventBus = new EventBus(agentName + "-event-bus");
PollingPropertiesFileConfigurationProvider configurationProvider =
new PollingPropertiesFileConfigurationProvider(
agentName, configurationFile, eventBus, 30);
components.add(configurationProvider);
application = new Application(components);
eventBus.register(application);
} else {
//2222
PropertiesFileConfigurationProvider configurationProvider =
new PropertiesFileConfigurationProvider(agentName, configurationFile);
application = new Application();
application.handleConfigurationEvent(configurationProvider.getConfiguration());
}
}
application.start();
final Application appReference = application;
Runtime.getRuntime().addShutdownHook(new Thread("agent-shutdown-hook") {
@Override
public void run() {
appReference.stop();
}
});
} catch (Exception e) {
logger.error("A fatal error occurred while running. Exception follows.", e);
}
The reload here is a boolean type, which is used to load the configuration file every 30s. If it is set to true, the content of the EventBus block is designed. For details, see the blog post. . . Next, we mainly explain the type of configuration file that is not loaded cyclically, that is, the code at the else code is used here
PropertiesFileConfigurationProvider configurationProvider =
new PropertiesFileConfigurationProvider(agentName, configurationFile);
The configurationProvider object is generated, mainly to generate the following objects
public AbstractConfigurationProvider(String agentName) {
super();
this.agentName = agentName;
this.sourceFactory = new DefaultSourceFactory();
this.sinkFactory = new DefaultSinkFactory();
this.channelFactory = new DefaultChannelFactory();
channelCache = new HashMap<Class<? extends Channel>, Map<String, Channel>>();
}
Then mainly use:
application.handleConfigurationEvent(configurationProvider.getConfiguration());
Load the configuration file and form components, and then mainly focus on this line of code to explain
public MaterializedConfiguration getConfiguration() {
MaterializedConfiguration conf = new SimpleMaterializedConfiguration();
// getFlumeConfiguration()方法，是关键核心，负责整个配置加载，下边代码说明
FlumeConfiguration fconfig = getFlumeConfiguration();
AgentConfiguration agentConf = fconfig.getConfigurationFor(getAgentName());
if (agentConf != null) {
Map<String, ChannelComponent> channelComponentMap = Maps.newHashMap();
Map<String, SourceRunner> sourceRunnerMap = Maps.newHashMap();
Map<String, SinkRunner> sinkRunnerMap = Maps.newHashMap();
try {
loadChannels(agentConf, channelComponentMap);
loadSources(agentConf, channelComponentMap, sourceRunnerMap);
loadSinks(agentConf, channelComponentMap, sinkRunnerMap);
Set<String> channelNames = new HashSet<String>(channelComponentMap.keySet());
for (String channelName : channelNames) {
ChannelComponent channelComponent = channelComponentMap.get(channelName);
if (channelComponent.components.isEmpty()) {
LOGGER.warn(String.format("Channel %s has no components connected" +
" and has been removed.", channelName));
channelComponentMap.remove(channelName);
Map<String, Channel> nameChannelMap =
channelCache.get(channelComponent.channel.getClass());
if (nameChannelMap != null) {
nameChannelMap.remove(channelName);
}
} else {
LOGGER.info(String.format("Channel %s connected to %s",
channelName, channelComponent.components.toString()));
conf.addChannel(channelName, channelComponent.channel);
}
}
for (Map.Entry<String, SourceRunner> entry : sourceRunnerMap.entrySet()) {
conf.addSourceRunner(entry.getKey(), entry.getValue());
}
for (Map.Entry<String, SinkRunner> entry : sinkRunnerMap.entrySet()) {
conf.addSinkRunner(entry.getKey(), entry.getValue());
}
} catch (InstantiationException ex) {
LOGGER.error("Failed to instantiate component", ex);
} finally {
channelComponentMap.clear();
sourceRunnerMap.clear();
sinkRunnerMap.clear();
}
} else {
LOGGER.warn("No configuration found for this host:{}", getAgentName());
}
return conf;
}
new FlumeConfiguration(toMap(properties)), the code is below:
@Override
public FlumeConfiguration getFlumeConfiguration() {
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(file));
String resolverClassName = System.getProperty("propertiesImplementation",
DEFAULT_PROPERTIES_IMPLEMENTATION);
Class<? extends Properties> propsclass = Class.forName(resolverClassName)
.asSubclass(Properties.class);
Properties properties = propsclass.newInstance();
properties.load(reader);
return new FlumeConfiguration(toMap(properties));
} catch (IOException ex) {
LOGGER.error("Unable to load file:" + file
+ " (I/O failure) - Exception follows.", ex);
} catch (ClassNotFoundException e) {
LOGGER.error("Configuration resolver class not found", e);
} catch (InstantiationException e) {
LOGGER.error("Instantiation exception", e);
} catch (IllegalAccessException e) {
LOGGER.error("Illegal access exception", e);
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException ex) {
LOGGER.warn(
"Unable to close file reader for file: " + file, ex);
}
}
}
return new FlumeConfiguration(new HashMap<String, String>());
}
FlumeConfiguration(Map<String, String> properties)
/**
* Creates a populated Flume Configuration object.
*/
public FlumeConfiguration(Map<String, String> properties) {
agentConfigMap = new HashMap<String, AgentConfiguration>();
errors = new LinkedList<FlumeConfigurationError>();
// Construct the in-memory component hierarchy
for (String name : properties.keySet()) {
String value = properties.get(name);
// addRawProperty里对agentConfigMap初始化,
// 1：这里插入的是agentConfiguration对象里的contextMap
if (!addRawProperty(name, value)) {
logger.warn("Configuration property ignored: " + name + " = " + value);
}
}
// Now iterate thru the agentContext and create agent configs and add them
// to agentConfigMap
//这里插入的是agentConfiguration对象里的configMap
// validate and remove improperly configured components
validateConfiguration();
}
Then, enter FlumeConfiguration.addRawProperty(name,value):
private boolean addRawProperty(String name, String value) {
// Null names and values not supported
if (name == null || value == null) {
errors
.add(new FlumeConfigurationError("", "",
FlumeConfigurationErrorType.AGENT_NAME_MISSING,
ErrorOrWarning.ERROR));
return false;
}
// Empty values are not supported
if (value.trim().length() == 0) {
errors
.add(new FlumeConfigurationError(name, "",
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
}
// Remove leading and trailing spaces
name = name.trim();
value = value.trim();
int index = name.indexOf('.');
// All configuration keys must have a prefix defined as agent name
if (index == -1) {
errors
.add(new FlumeConfigurationError(name, "",
FlumeConfigurationErrorType.AGENT_NAME_MISSING,
ErrorOrWarning.ERROR));
return false;
}
String agentName = name.substring(0, index);
// Agent name must be specified for all properties
if (agentName.length() == 0) {
errors
.add(new FlumeConfigurationError(name, "",
FlumeConfigurationErrorType.AGENT_NAME_MISSING,
ErrorOrWarning.ERROR));
return false;
}
String configKey = name.substring(index + 1);
// Configuration key must be specified for every property
if (configKey.length() == 0) {
errors
.add(new FlumeConfigurationError(name, "",
FlumeConfigurationErrorType.PROPERTY_NAME_NULL,
ErrorOrWarning.ERROR));
return false;
}
AgentConfiguration aconf = agentConfigMap.get(agentName);
// 这里创建AgentConfiguration，并插入到FlumeConfiguration的Map<String, AgentConfiguration> agentConfigMap中
if (aconf == null) {
aconf = new AgentConfiguration(agentName, errors);
agentConfigMap.put(agentName, aconf);
}
// Each configuration key must begin with one of the three prefixes:
// sources, sinks, or channels.
// 最终，键值对被加载到agentConfiguration中
return aconf.addProperty(configKey, value);
}
Finally loaded into: agentConfiguration.addProperty(String key, String value)
ComponentNameAndConfigKey cnck = parseConfigKey(key,
BasicConfigurationConstants.CONFIG_SOURCES_PREFIX);
if (cnck != null) {
// it is a source
String name = cnck.getComponentName();
Context srcConf = sourceContextMap.get(name);
if (srcConf == null) {
srcConf = new Context();
sourceContextMap.put(name, srcConf);
}
// sourceContextMap中存放new Context().put(cnck.getConifyKey(),value)
srcConf.put(cnck.getConfigKey(), value);
return true;
}
So finally it is loaded into the corresponding *ContextMap in AgentConfiguration;
Finally there is another place in the code
// validate and remove improperly configured components
validateConfiguration();
The corresponding detailed codes are:
private void validateConfiguration() {
Iterator<String> it = agentConfigMap.keySet().iterator();
while (it.hasNext()) {
String agentName = it.next();
AgentConfiguration aconf = agentConfigMap.get(agentName);
if (!aconf.isValid()) {
logger.warn("Agent configuration invalid for agent '" + agentName
+ "'. It will be removed.");
errors.add(new FlumeConfigurationError(agentName, "",
FlumeConfigurationErrorType.AGENT_CONFIGURATION_INVALID,
ErrorOrWarning.ERROR));
it.remove();
}
logger.debug("Channels:" + aconf.channels + "\n");
logger.debug("Sinks " + aconf.sinks + "\n");
logger.debug("Sources " + aconf.sources + "\n");
}
logger.info("Post-validation flume configuration contains configuration"
+ " for agents: " + agentConfigMap.keySet());
}
aconf.isValid()
private boolean isValid() {
logger.debug("Starting validation of configuration for agent: {}", agentName);
if (logger.isDebugEnabled() && LogPrivacyUtil.allowLogPrintConfig()) {
logger.debug("Initial configuration: {}", this.getPrevalidationConfig());
}
// Make sure that at least one channel is specified
if (channels == null || channels.trim().length() == 0) {
logger.warn("Agent configuration for '" + agentName
+ "' does not contain any channels. Marking it as invalid.");
errorList.add(new FlumeConfigurationError(agentName,
BasicConfigurationConstants.CONFIG_CHANNELS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
}
channelSet =
new HashSet<String>(Arrays
.asList(channels.split("\\s+")));
// validateComponent(channelSet, channelConfigMap, CLASS_CHANNEL,
// ATTR_TYPE);
channelSet = validateChannels(channelSet);
if (channelSet.size() == 0) {
logger.warn("Agent configuration for '" + agentName
+ "' does not contain any valid channels. Marking it as invalid.");
errorList.add(new FlumeConfigurationError(agentName,
BasicConfigurationConstants.CONFIG_CHANNELS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
}
sourceSet = validateSources(channelSet);
sinkSet = validateSinks(channelSet);
sinkgroupSet = validateGroups(sinkSet);
// If no sources or sinks are present, then this is invalid
if (sourceSet.size() == 0 && sinkSet.size() == 0) {
logger.warn("Agent configuration for '" + agentName
+ "' has no sources or sinks. Will be marked invalid.");
errorList.add(new FlumeConfigurationError(agentName,
BasicConfigurationConstants.CONFIG_SOURCES,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
errorList.add(new FlumeConfigurationError(agentName,
BasicConfigurationConstants.CONFIG_SINKS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
}
// Now rewrite the sources/sinks/channels
this.sources = getSpaceDelimitedList(sourceSet);
this.channels = getSpaceDelimitedList(channelSet);
this.sinks = getSpaceDelimitedList(sinkSet);
this.sinkgroups = getSpaceDelimitedList(sinkgroupSet);
if (logger.isDebugEnabled() && LogPrivacyUtil.allowLogPrintConfig()) {
logger.debug("Post validation configuration for {}", agentName);
logger.debug(this.getPostvalidationConfig());
}
return true;
}
validateChannels
/**
* If it is a known component it will do the full validation required for
* that component, else it will do the validation required for that class.
*/
private Set<String> validateChannels(Set<String> channelSet) {
Iterator<String> iter = channelSet.iterator();
Map<String, Context> newContextMap = new HashMap<String, Context>();
ChannelConfiguration conf = null;
/*
* The logic for the following code:
*
* Is it a known component?
* -Yes: Get the ChannelType and set the string name of that to
* config and set configSpecified to true.
* -No.Look for config type for the given component:
* -Config Found:
* Set config to the type mentioned, set configSpecified to true
* -No Config found:
* Set config to OTHER, configSpecified to false,
* do basic validation. Leave the context in the
* contextMap to process later. Setting it to other returns
* a vanilla configuration(Source/Sink/Channel Configuration),
* which does basic syntactic validation. This object is not
* put into the map, so the context is retained which can be
* picked up - this is meant for older classes which don't
* implement ConfigurableComponent.
*/
while (iter.hasNext()) {
String channelName = iter.next();
Context channelContext = channelContextMap.get(channelName);
// Context exists in map.
if (channelContext != null) {
// Get the configuration object for the channel:
ChannelType chType = getKnownChannel(channelContext.getString(
BasicConfigurationConstants.CONFIG_TYPE));
boolean configSpecified = false;
String config = null;
// Not a known channel - cannot do specific validation to this channel
if (chType == null) {
config = channelContext.getString(BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
} else {
configSpecified = true;
}
} else {
config = chType.toString().toUpperCase(Locale.ENGLISH);
configSpecified = true;
}
try {
//建立channel组件
conf =
(ChannelConfiguration) ComponentConfigurationFactory.create(
channelName, config, ComponentType.CHANNEL);
logger.debug("Created channel " + channelName);
if (conf != null) {
//配置组件信息
conf.configure(channelContext);
}
if ((configSpecified && conf.isNotFoundConfigClass()) ||
!configSpecified) {
newContextMap.put(channelName, channelContext);
} else if (configSpecified) {
channelConfigMap.put(channelName, conf);
}
if (conf != null) {
errorList.addAll(conf.getErrors());
}
} catch (ConfigurationException e) {
// Could not configure channel - skip it.
// No need to add to error list - already added before exception is
// thrown
if (conf != null) errorList.addAll(conf.getErrors());
iter.remove();
logger.warn("Could not configure channel " + channelName
+ " due to: " + e.getMessage(), e);
}
} else {
iter.remove();
errorList.add(new FlumeConfigurationError(agentName, channelName,
FlumeConfigurationErrorType.CONFIG_ERROR, ErrorOrWarning.ERROR));
}
}
channelContextMap = newContextMap;
Set<String> tempchannelSet = new HashSet<String>();
tempchannelSet.addAll(channelConfigMap.keySet());
tempchannelSet.addAll(channelContextMap.keySet());
channelSet.retainAll(tempchannelSet);
return channelSet;
}
At this point, a complete FlumeConfiguration object has been fully loaded; continue below; Next, create a channel
AgentConfiguration agentConf = fconfig.getConfigurationFor(getAgentName());
if (agentConf != null) {
Map<String, ChannelComponent> channelComponentMap = Maps.newHashMap();
Map<String, SourceRunner> sourceRunnerMap = Maps.newHashMap();
Map<String, SinkRunner> sinkRunnerMap = Maps.newHashMap();
try {
loadChannels(agentConf, channelComponentMap);