8.2.0Storm集成Redis

Storm集成Redis

一、       新建项目

二、       导入Pom依赖

<dependencies>
  <dependency>
    <groupId>junit</groupId>
   <artifactId>junit</artifactId>
    <version>3.8.1</version>
    <scope>test</scope>
  </dependency>
  <dependency>
   <groupId>org.apache.storm</groupId>
    <artifactId>storm-core</artifactId>
    <version>1.1.0</version>
    <scope>provided</scope>
   <!--<scope>compile</scope>-->
  </dependency>
  <dependency>
   <groupId>org.apache.storm</groupId>
   <artifactId>storm-redis</artifactId>
    <version>1.1.0</version>
  </dependency>
  <dependency>
   <groupId>org.apache.storm</groupId>
   <artifactId>storm-kafka</artifactId>
    <version>1.1.0</version>
  </dependency>
</dependencies>
<build>
  <plugins>
    <plugin>
     <artifactId>maven-assembly-plugin</artifactId>
      <configuration>
        <descriptorRefs>
         <descriptorRef>jar-with-dependencies</descriptorRef>
        </descriptorRefs>
        <archive>
          <manifest>
           <mainClass>com.simon.storm.URLCountTopology</mainClass>
          </manifest>
        </archive>
      </configuration>
      <executions>
        <execution>
         <id>make-assembly</id>
         <phase>package</phase>
          <goals>
           <goal>single</goal>
          </goals>
        </execution>
      </executions>
    </plugin>
    <plugin>
     <groupId>org.apache.maven.plugins</groupId>
     <artifactId>maven-compiler-plugin</artifactId>
      <configuration>
        <source>1.8</source>
        <target>1.8</target>
      </configuration>
    </plugin>
  </plugins>
</build>

三、       定义Spout

            继承BaseRishSpout，重写方法。

public static class URLCountSpout extends BaseRichSpout {
   //模拟数据
   String[] phones = { "xiaomi", "apple", "huawei", "moto", "nokia", "sansung", "LG" ,"htc","Iphone", "dell", "huasuo", "hongqi", "lianxiang", "lenovo","shunfeng","meituan", "alibaba", "wangyi", "qq", "luoji", "leise","nongfu", "yibao","rongyao", "oppo", "vivo"};
   private SpoutOutputCollector collector;
   // 从外部数据源（比如kafka消息缓存中）获取url数据，源源不断获取
   @Override
   public void nextTuple() {
      //休眠一下
      Utils.sleep(500);
      //模拟产生数据
      int index = new Random().nextInt(20);
      String phone = phones[index];
      // 将拿到的数据封装为tuple发送出去
      //collector在open方法中，需要设置为全局
      // 一个tuple中可以封装多个数据（类似list的元素）
      // collector.emit(new Values("value1","value2","value3"));
      String url =  phone + UUID.randomUUID();
      collector.emit(new Values(url));
   }
   @Override
   public void open(Map config, TopologyContext context, SpoutOutputCollector collector) {
      this.collector = collector; 
   }
   @Override
  public void declareOutputFields(OutputFieldsDeclarer declarer) {
      // 为发出去的tuple中的每一个字段定义一个名字
      // declarer.declare(new Fields("field1","field2","field3"));
      declarer.declare(new Fields("url"));
   }
}

四、定义Bolt

public static class URLCountRedisBolt extends BaseRichBolt{
   private Jedis jedis = null;
   @Override
   public void prepare(Map conifg, TopologyContext context, OutputCollector collector) {
      jedis = new Jedis("10.10.56.138",6379);
   }
   @Override
   public void execute(Tuple tuple) {
      String word = tuple.getString(0);
      //截取url中的网站名字
      if(word != null){
         //将url写入到redis的一个zset集合中
         // 参数1： zset的主键 参数2：要增加的分数 参数3：url中的网站部分——site
         jedis.zincrby("kafkawords", 1, word);
      }
   }
   /**
    * 没有后续组件了，所以不需要定义输出tuple的schema
    */
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
   }
}

五、定义Topology主函数

public static void main(String[] args) throws AlreadyAliveException, 
      InvalidTopologyException, AuthorizationException {
   TopologyBuilder builder =  new TopologyBuilder();
   //2、添加spout和bolt
   builder.setSpout("urlspout", new URLCountSpout());
   builder.setBolt("rediscountbolt", new URLCountRedisBolt()).shuffleGrouping("urlspout");
   StormTopology topology = builder.createTopology();
   Config config = new Config();
   config.setNumWorkers(8);
   //集群模式
   StormSubmitter.submitTopology("topwords",config,topology);
}

六、打包

七、上传

八、运行jar

./storm jar /usr/apps/testsource/storm-1.0-SNAPSHOT-jar-with-dependencies.jar  com.simon.storm.URLCountTopology myTest

九、运行成功

十、查看Redis