Kafka+Strom+mongoDB 整合

一、实现模型

    数据流程

    a、Kafka Producter生成topic1主题的消息　

    b、 Storm中有个Topology，包含了KafkaSpout、SenqueceBolt、KafkaBolt三个组件。其中KafkaSpout订阅了topic1主题消息，然后发送给SenqueceBolt加工处理，最后数据由KafkaBolt生成topic2主题消息发送给Kafka。

    c、Kafka Consumer负责消费topic2主题的消息

   实例模型

  

代码实现

一、设置日志生成信息，并配置log4j.xml

      a、在数据拦截层，配置，配置拦截信息

扫描二维码关注公众号，回复： 2429110 查看本文章

 logger.warn("处理请求|"+((null==userId || "".equals(userId.toString()))?"0":userId.toString())+"|"+system+"|"+version+"|" + request.getRequestURI() +"|"+GbdDateUtils.format(Calendar.getInstance().getTime(),"yyyy-MM-dd HH:mm:ss")+ "|" + (System.currentTimeMillis() - accessTime) + "|毫秒！");

      b、配置log4j.xml拦截信息

<?xml version="1.0" encoding="GBK" ?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
	<appender name="console" class="org.apache.log4j.ConsoleAppender">
		<layout class="org.apache.log4j.PatternLayout">
			<param name="ConversionPattern" value="[%-5p] [%d{yyyy-MM-dd HH:mm:ss SSS}] %c - %m%n" />
		</layout>
		<!--过滤器设置输出的级别 -->
		<filter class="org.apache.log4j.varia.LevelRangeFilter">
			<param name="levelMin" value="debug" />
			<param name="levelMax" value="error" />
			<param name="AcceptOnMatch" value="true" />
		</filter>
	</appender>
	<appender name="kafka" class="kafka.producer.KafkaLog4jAppender">
		<param name="topic" value="gmap" />
		<param name="brokerList" value="192.168.3.17:9092" />
		<param name="compressionType" value="none" />
		<param name="requiredNumAcks" value="0" />
		<param name="syncSend" value="false" />
		<layout class="org.apache.log4j.PatternLayout">
			<param name="ConversionPattern" value="[%-5p] [%d{yyyy-MM-dd HH:mm:ss SSS}] %c - %m%n" />
		</layout>

		<!--过滤器设置输出的级别 -->
		<filter class="org.apache.log4j.varia.LevelRangeFilter">
			<param name="levelMin" value="warn" />
			<param name="levelMax" value="warn" />
			<param name="AcceptOnMatch" value="true" />
		</filter>
	</appender>
	<!-- 根logger的设置 -->
	<root>
		<priority value="info" />
        <appender-ref ref="kafka" />
	</root>
</log4j:configuration>

二、设置kafka  并用mongoDB保存数据

      a、配置kafka  

import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.TopologyBuilder;
import storm.kafka.BrokerHosts;
import storm.kafka.KafkaSpout;
import storm.kafka.SpoutConfig;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;

public class Start {
    private static Logger logger = Logger.getLogger(Start.class);

    @SuppressWarnings("resource")
    public static void main(String[] args) {
        logger.info("初始化storm-kafka......");
        //配置kafka topic
        String topic = "gmap"; 
        //配置kafka zookeeper
        BrokerHosts hosts = new ZkHosts("192.168.17:2181");
        //配置storm spout
        SpoutConfig spoutConfig = new SpoutConfig(hosts, topic, "/kafkastorm", "gmapKafka");
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        //配置storm zookeeper
        List<String> zkServers = new ArrayList<>();
        zkServers.add("192.168.3.18");
        spoutConfig.zkServers = zkServers;
        spoutConfig.zkPort = Integer.valueOf(2182);

        spoutConfig.forceFromStart = false;
        spoutConfig.socketTimeoutMs = 60 * 1000;
        //配置topology
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", new KafkaSpout(spoutConfig),2);
        //配置storm bolt
        builder.setBolt("bolt",new StormKafkaBolt(), 2).shuffleGrouping("spout");
        //开启topology
        Config config = new Config();
        config.setDebug(false);
        new LocalCluster().submitTopology("topology", config, builder.createTopology());
        logger.info("日志处理中心启动成功！");
    }
}

     b.配置mongoDB并存储到临时文件中

import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.icloudmoo.common.util.GbdDateUtils;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.IBasicBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;

public class StormKafkaBolt implements IBasicBolt {

    private static Logger logger = Logger.getLogger(StormKafkaBolt.class);
    private static final long serialVersionUID = 1L;
    private static final MongoTemplate mongoTemplate = SpringBeanUtil.getBean("mongoTemplate");

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("logs"));
    }

    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        try {
            String kafkaMsg = input.getString(0);
            if (StringUtils.isNotEmpty(kafkaMsg) && kafkaMsg.contains("|")) {
                int start = kafkaMsg.indexOf("|");
                int end = kafkaMsg.lastIndexOf("|");
                kafkaMsg = kafkaMsg.substring(start + 1, end);
                System.err.println(kafkaMsg);
                String[] logs = kafkaMsg.split("\\|");
                if (6 == logs.length) {
                    GmapLog gmapLog = new GmapLog(Integer.valueOf(logs[0]), logs[1], logs[2], logs[3],
                            GbdDateUtils.parseDate(logs[4]), Integer.valueOf(logs[5]));
                    System.err.println(null == mongoTemplate);
                    mongoTemplate.insert(gmapLog, "logs");
                }
            }
        }catch (Exception e) {
            logger.error(e);
        }
    }

    @Override
    public void prepare(Map map, TopologyContext context) {
    }

    @Override
    public Map<String, Object> getComponentConfiguration() {
        return null;
    }

    @Override
    public void cleanup() {
    }
}

这样当每次数据达到六条的时候，mongoDB就保存数据。