<!-- KafKa 连接 --> <dependency> <groupId>com.youku.data</groupId> <artifactId>data-mq-common</artifactId> <version>0.9.3</version><!--如不使用新版本特性可以不升级成最新版本 --> </dependency>
// topicName为消息通道名称,管理员创建 private static String TOPIC = "log_xm_uploadfr"; // token 用户访问通道的token,一个token可以读或写多个消息 private static String TOKEN = "ZgSSYUxqvH"; // groupId 消费者组名(一个或多个消费者在同一个组里) private static String GROUP = "lf_ct_uploadfr"; // 从Kafka中获取数据 try { readChannel = MqSystem.getMqSystem().open(MqSystem.CHANNEL_MODE_READ, TOPIC, TOKEN, GROUP); } catch (MqException e) { LOG.error(e.getMessage()); }
2、连接Hbase
POM配置
<dependency> <groupId>com.youku.data</groupId> <artifactId>data-storm-common</artifactId> <version>1.0.0</version> <exclusions> <exclusion> <artifactId>hbase</artifactId> <groupId>org.apache.hbase</groupId> </exclusion> </exclusions> </dependency> <!-- Hbase --> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> <version>4.0.0-incubating-client</version> </dependency> <dependency> <groupId>com.youku.data</groupId> <artifactId>data-hbase-phoenix-common</artifactId> <version>1.0.0</version> </dependency>
host配置:
#hbase 10.106.25.11 a001.master.hbase.qingdao.youku 10.106.25.12 a001.region.hbase.qingdao.youku 10.106.25.13 a002.region.hbase.qingdao.youku 10.106.25.14 a003.region.hbase.qingdao.youku
package com.laifeng.control.bolts; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import com.youku.data.store.SQLWriter; import com.youku.data.store.token.TokenPhoenixWriter; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Tuple; import backtype.storm.utils.Time; /** * @author wangqiao * @date 2014-9-24 下午7:06:38 */ public class UploadFlowrateHbaseBolt extends BaseRichBolt { private static final long serialVersionUID = 3218981547130543201L; private static final Logger LOG = Logger.getLogger(UploadFlowrateHbaseBolt.class); private static SQLWriter writer = null; private int timestamp = 0; private List<String> sqlList = new ArrayList<String>(); public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { // 连接Hbase try { this.writer = TokenPhoenixWriter.getWriter("ADcjaS"); timestamp = Time.currentTimeSecs(); } catch (Exception e) { LOG.error(e.getMessage()); } } public void execute(Tuple input) { try { int timenow = Time.currentTimeSecs(); // 初始化语句 sqlList.add(String .format("upsert into T_test(minutecol,roomid,streamid,provinceid,asn,cdnid,city,ip,userid,blockcount,appid,type,duration,receiverip,logversion,ext,visittime) values('%s','%s','%s','%s','%s','%s','%s','%s',%s,%s,%s,'%s','%s','%s','%s','%s','%s')", input.getStringByField("minutecol"), input.getStringByField("roomid"), input.getStringByField("streamid"), input.getStringByField("provinceid"), input.getStringByField("asn"), input.getStringByField("cdnid"), input.getStringByField("city"), input.getStringByField("ip"), input.getIntegerByField("userid"), input.getIntegerByField("blockcount"), input.getIntegerByField("appid"), input.getStringByField("type"), input.getStringByField("duration"), input.getStringByField("receiverip"), input.getStringByField("logversion"), input.getStringByField("ext"), input.getStringByField("visittime"))); if (sqlList.size() >= 500 || (timenow - timestamp >= 60 && sqlList.size() > 0)) { writer.writeBatchSQL(sqlList); writer.close(); sqlList.clear(); timestamp = timenow; } } catch (Exception e) { LOG.error(e.getMessage()); } } public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub } }
maven install 然后在target找到含有dependjar的jar包放在storm环境下运行即可
运行命令如下:
storm jar /home/xm_wangqiao/stormtest-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.laifeng.control.toplogy.UploadFlowrateToplogy 'lf-upload-flowrate' storm kill 'lf-upload-flowrate'