Storm Trident 项目

企业开发 2018-05-09 19:44:11 阅读次数: 1 
事务：运用IPartitionedTridentSpout
DRPC：Web端通过DRPC获得结果数据
topN：按销售额排行
基于Hbase存储的Trident state：支持Topo重启时不丢数据，也可支撑Web端读数据

数据获得实现 
通过开源Storm-kafka项目提供  
采用Storm-kafka 项目中提供的TransactionalTridentKafkaSpout
  
开发思路
1、业务逻辑处理，bolt中实现；
2、数据落地格式可更加前台HighCharts的需要而定，有时需要特别为HighCharts的数据格式来存储落地数据，如项目三中就会这样。
采用内存+磁盘方式避免断电、重启等会造成数据丢失的问题。
3、必须前后台分离，有利于稳定性；
4、Web端开发思路相对简单，就是长链接读数据，推数据到HighCharts
5、HighCharts开发，很多实例及代码，可以现学现用。
 
优点：1、前台、后台分离，重启维护互不影响；
           2、Storm重启不会影响结果数据，不影响HighCharts图表展示；
           3、Tomcat重启不会影响数据处理；
           4、前台后台数据传输通过Hbase 或 DRPC，注意DRPC是通过服务访问，稳定性不如DB。



package cn.wh.trident;
import storm.kafka.BrokerHosts;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;
import storm.kafka.trident.TransactionalTridentKafkaSpout;
import storm.kafka.trident.TridentKafkaConfig;
import storm.trident.TridentState;
import storm.trident.TridentTopology;
import storm.trident.operation.builtin.Count;
importstorm.trident.operation.builtin.FirstN;
import storm.trident.operation.builtin.MapGet;
import storm.trident.operation.builtin.Sum;
import storm.trident.testing.MemoryMapState;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.LocalDRPC;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.tuple.Fields;
import backtype.storm.utils.Utils;
import cn.wh.storm.KafkaProperties;
 
publicclass TridentTopo {
    /**
     * @param args
     */
    publicstaticvoid main(String[] args) {
        BrokerHosts zkHosts = new ZkHosts(KafkaProperties.zkConnect);
        String topic = "track";
        TridentKafkaConfig config = new TridentKafkaConfig(zkHosts, topic);
       
        config.forceFromStart = false; //测试时用true，上线时必须改为false
        
        //输入格式
        config.scheme = new SchemeAsMultiScheme(new StringScheme());  
        config.fetchSizeBytes = 100 ;     //batch size
        LocalDRPC drpc = new LocalDRPC();
        TransactionalTridentKafkaSpout spout  = newTransactionalTridentKafkaSpout(config) ;
       
        TridentTopology topology = new TridentTopology() ;
        //销售额
        TridentState amtState = topology.newStream("spout", spout)
        .parallelismHint(3)
        .each(new Fields(StringScheme.STRING_SCHEME_KEY),new OrderSplit("\\t"), new Fields("order_id","order_amt","create_date","province_id"))
        .shuffle()
        .groupBy(new Fields("create_date","province_id"))
        .persistentAggregate(new MemoryMapState.Factory(), new Fields("order_amt"), new Sum(), new Fields("sum_amt"));
       
        topology.newDRPCStream("getOrderAmt", drpc)
        .each(new Fields("args"), new Split(" "), new Fields("arg"))
        .each(new Fields("arg"), new SplitBy("\\:"), new Fields("create_date","province_id"))
        .groupBy(new Fields("create_date","province_id"))
        .stateQuery(amtState, new Fields("create_date","province_id"), new MapGet(), new Fields("sum_amt"))
//      .applyAssembly(new FirstN(5, "sum_amt", true))
        ; 
        //订单数
        TridentState orderState = topology.newStream("orderSpout", spout)
        .parallelismHint(3)
        .each(new Fields(StringScheme.STRING_SCHEME_KEY),new OrderSplit("\\t"), new Fields("order_id","order_amt","create_date","province_id"))
        .shuffle()
        .groupBy(new Fields("create_date","province_id"))
        .persistentAggregate(new MemoryMapState.Factory(), new Fields("order_id"), new Count(), new Fields("order_num"));
       
        topology.newDRPCStream("getOrderNum", drpc)
        .each(new Fields("args"), new Split(" "), new Fields("arg"))
        .each(new Fields("arg"), new SplitBy("\\:"), new Fields("create_date","province_id"))
        .groupBy(new Fields("create_date","province_id"))
        .stateQuery(orderState, new Fields("create_date","province_id"), new MapGet(), new Fields("order_num"))
//      .applyAssembly(new FirstN(5, "order_num", true))
        ; 
        Config conf = new Config() ;
        conf.setDebug(false);
        LocalCluster cluster = new LocalCluster() ;
        cluster.submitTopology("myTopo", conf, topology.build());

        while(true){
//          System.err.println("销售额："+drpc.execute("getOrderAmt", "2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5")) ;
            System.err.println("订单数："+drpc.execute("getOrderNum", "2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5")) ;
            Utils.sleep(5000); 
        }
    }
}



DRPC方式基于hbase state方式
package cn.wh.stormtest;
import hbase.state.HBaseAggregateState;
import hbase.state.TridentConfig;
import kafka.productor.KafkaProperties;
import storm.kafka.BrokerHosts;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;
import storm.kafka.trident.TransactionalTridentKafkaSpout;
import storm.kafka.trident.TridentKafkaConfig;
import storm.trident.TridentState;
import storm.trident.TridentTopology;
import storm.trident.operation.builtin.Count;
import storm.trident.operation.builtin.FilterNull;
import storm.trident.operation.builtin.FirstN;
import storm.trident.operation.builtin.MapGet;
import storm.trident.operation.builtin.Sum;
import storm.trident.state.StateFactory;
import storm.trident.testing.MemoryMapState;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.LocalDRPC;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.generated.StormTopology;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.tuple.Fields;
import backtype.storm.utils.Utils;
 
public class TridentTopo {
 
    public static StormTopology builder(LocalDRPC drpc)
    {
        TridentConfig tridentConfig = new TridentConfig("state");
        StateFactory state = HBaseAggregateState.transactional(tridentConfig);
       
        BrokerHosts zkHosts = new ZkHosts(KafkaProperties.zkConnect);
        String topic = "track";
        TridentKafkaConfig config = new TridentKafkaConfig(zkHosts, topic);
        config.forceFromStart = false; //测试时用true，上线时必须改为false
        config.scheme = new SchemeAsMultiScheme(new StringScheme());
        config.fetchSizeBytes = 100 ;//batch size
       
        TransactionalTridentKafkaSpout spout  = new TransactionalTridentKafkaSpout(config) ;
       
        TridentTopology topology = new TridentTopology() ;
        //销售额
        TridentState amtState = topology.newStream("spout", spout)
        .parallelismHint(3)
        .each(new Fields(StringScheme.STRING_SCHEME_KEY),new OrderAmtSplit("\\t"), new Fields("order_id","order_amt","create_date","province_id","cf"))
        .shuffle()
        .groupBy(new Fields("create_date","cf","province_id"))
        .persistentAggregate(state, new Fields("order_amt"), new Sum(), new Fields("sum_amt"));
//      .persistentAggregate(new MemoryMapState.Factory(), new Fields("order_amt"), new Sum(), new Fields("sum_amt"));
       
        topology.newDRPCStream("getOrderAmt", drpc)
        .each(new Fields("args"), new Split(" "), new Fields("arg"))
        .each(new Fields("arg"), new SplitBy("\\:"), new Fields("create_date","cf","province_id"))
        .groupBy(new Fields("create_date","cf","province_id"))
        .stateQuery(amtState, new Fields("create_date","cf","province_id"), new MapGet(), new Fields("sum_amt"))
        .each(new Fields("sum_amt"),new FilterNull())
        .applyAssembly(new FirstN(5, "sum_amt", true))
        ;
       
        //订单数
        TridentState orderState = topology.newStream("orderSpout", spout)
        .parallelismHint(3)
        .each(new Fields(StringScheme.STRING_SCHEME_KEY),new OrderNumSplit("\\t"), new Fields("order_id","order_amt","create_date","province_id","cf"))
        .shuffle()
        .groupBy(new Fields("create_date","cf","province_id"))
        .persistentAggregate(state, new Fields("order_id"), new Count(), new Fields("order_num"));
//      .persistentAggregate(new MemoryMapState.Factory(), new Fields("order_id"), new Count(), new Fields("order_num"));
       
        topology.newDRPCStream("getOrderNum", drpc)
        .each(new Fields("args"), new Split(" "), new Fields("arg"))
        .each(new Fields("arg"), new SplitBy("\\:"), new Fields("create_date","cf","province_id"))
        .groupBy(new Fields("create_date","cf","province_id"))
        .stateQuery(orderState, new Fields("create_date","cf","province_id"), new MapGet(), new Fields("order_num"))
        .each(new Fields("order_num"),new FilterNull())
//      .applyAssembly(new FirstN(5, "order_num", true))
        ;
        return topology.build() ;
    }
    /**
     * @param args
     */
    public static void main(String[] args) {
        // TODO Auto-generated method stub
        LocalDRPC drpc = new LocalDRPC();
        Config conf = new Config() ;
        conf.setNumWorkers(5);
        conf.setDebug(false);
        if (args.length > 0) {
            try {
                StormSubmitter.submitTopology(args[0], conf, builder(null));
            } catch (AlreadyAliveException e) {
                e.printStackTrace();
            } catch (InvalidTopologyException e) {
                e.printStackTrace();
            }
        }else{
            LocalCluster localCluster = new LocalCluster();
            localCluster.submitTopology("mytopology", conf, builder(drpc));
        }
//      Utils.sleep(60000);
//      while (true) {
//          System.err.println("销售额："+drpc.execute("getOrderAmt", "2014-09-13:cf:amt_3 2014-09-13:cf:amt_2 2014-09-13:cf:amt_1 2014-09-13:cf:amt_7 2014-09-13:cf:amt_6 2014-09-13:cf:amt_5 2014-09-13:cf:amt_4 2014-09-13:cf:amt_8")) ;
//          Utils.sleep(5000);
//      }
        /**
         * [["2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5","2014-08-19:1","2014-08-19","1",821.9000000000001],
         * ["2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5","2014-08-19:2","2014-08-19","2",631.3000000000001],
         * ["2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5","2014-08-19:3","2014-08-19","3",240.7],
         * ["2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5","2014-08-19:4","2014-08-19","4",340.4],
         * ["2014-08-19:1 2014-08-19:2 2014-08-19:3 2014-08-19:4 2014-08-19:5","2014-08-19:5","2014-08-19","5",460.8]]
         */
       
       
    }
 
}
 


----------------------------DRPC Client-------------
import backtype.storm.LocalDRPC;
import backtype.storm.utils.DRPCClient;
import backtype.storm.utils.Utils;
 
public class TridentDRPCclient {
    public static void main(String[] args) {
       
        DRPCClient client = new DRPCClient("192.168.1.107", 3772);
//      LocalDRPC client = new LocalDRPC(); 
        try {
                while (true)
                {
                    System.err.println("销售额："+client.execute("getOrderAmt", "2014-09-13:cf:amt_5 2014-09-13:cf:amt_8")) ;
                    System.err.println("订单数："+client.execute("getOrderNum", "2014-09-13:cf:amt_1 2014-09-13:cf:amt_2")) ;
                    Utils.sleep(5000);
                }
            } catch (Exception e) {
                e.printStackTrace() ;
            }
       
    }
}

猜你喜欢

转载自houston123.iteye.com/blog/2317501
今日推荐
TIOBE 5 月榜单：Fortran “复活”进入 Top 10
GCC 14.1 发布
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
Java自定义时间格式
同步整形电路
在开发中最最最常用的字符串的属性大集合
Linux 查看端口占用并杀掉
Java基础四：ArrayList
多线程之死锁就是这么简单
mysql 基础命令集
awk 命令详解
Centos6.3编译安装nginx+php步骤
OCR （Optical Character Recognition，光学字符识别）
每日归档
更多
2024-05-08(42)
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022