storm学习篇（三）——集成MySQL

版权声明： https://blog.csdn.net/typ1805/article/details/81211251

以统计单词代码为例子，用jdbcClient直接执行SQL

添加依赖pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    
    <modelVersion>4.0.0</modelVersion>
    <artifactId>storm-jdbc</artifactId>
    <packaging>jar</packaging>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>1.1.1</version>
            <!--<scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>

        <!--storm整合jdbc-->
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-jdbc</artifactId>
            <version>1.0.1</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.31</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                            <mainClass>cn.ljh.storm.helloworld.ExclamationTopology</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
        </plugins>
    </build>



</project>

创建spout：WordCountSpout.java

package com.storm.demo;

import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

import java.util.Map;
import java.util.Random;

/**
 * 路径：com.storm.demo
 * 类名：
 * 功能：《用一句话描述一下》
 * 备注：
 * 创建人：typ
 * 创建时间：2018/7/25 15:51
 * 修改人：
 * 修改备注：
 * 修改时间：
 */
public class WordCountSpout extends BaseRichSpout {

    private SpoutOutputCollector collector;

    private static final String[] words = new String[]{"aaa","bbb","cad","adv","ed","eaf"};

    public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        this.collector = spoutOutputCollector;
    }
    
    /**
     * 方法名：
     * 功能：把每一行数据发射出去
     * 描述：
     * 创建人：typ
     * 创建时间：2018/7/25 15:58
     * 修改人：
     * 修改描述：
     * 修改时间：
     */
    public void nextTuple() {
        Random random = new Random();
        String word = words[random.nextInt(words.length)];
        //发射出去
        this.collector.emit(new Values(word));
        System.out.println("emit==="+word);
//        Utils.sleep(1000);
    }

    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("word"));
    }
}

创建bolt：CountBolt.java

package com.storm.demo;

import com.google.common.collect.Maps;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.HikariCPConnectionProvider;
import org.apache.storm.jdbc.common.JdbcClient;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;

import java.sql.Types;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * 路径：com.storm.demo
 * 类名：
 * 功能：词频汇总Bolt
 * 备注：
 * 创建人：typ
 * 创建时间：2018/7/25 16:09
 * 修改人：
 * 修改备注：
 * 修改时间：
 */
public class CountBolt extends BaseRichBolt {

    private OutputCollector collector;
    private JdbcClient jdbcClient;
    private ConnectionProvider connectionProvider;

    Map<String,Integer> map = new HashMap<String,Integer>();

    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        this.collector = outputCollector;
        Map hikariConfigMap = Maps.newHashMap();
        hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
        hikariConfigMap.put("dataSource.url", "jdbc:mysql://localhost/mc_config");
        hikariConfigMap.put("dataSource.user","root");
        hikariConfigMap.put("dataSource.password","admin");
        connectionProvider = new HikariCPConnectionProvider(hikariConfigMap);
        //对数据库连接进行初始化
        connectionProvider.prepare();
        jdbcClient = new JdbcClient(connectionProvider,30);
    }

    /**
     * 方法名：
     * 功能：业务逻辑
     *     1.获取每个单词
     *     2.对所有单词进行汇总
     *     3.输出
     * 描述：
     * 创建人：typ
     * 创建时间：2018/7/25 16:17
     * 修改人：
     * 修改描述：
     * 修改时间：
     */
    public void execute(Tuple tuple) {
        String word = tuple.getStringByField("word");
        Integer count = map.get(word);
        if(count == null){
            count = 0;
        }
        count++;
        map.put(word,count);
        //查询该word是否存在
        List<Column> list = new ArrayList<Column>();
        //创建一列将值传入   列名  值  值的类型
        list.add(new Column("word",word, Types.VARCHAR));
        List<List<Column>> select = jdbcClient.select("select word from wordcount where word = ?",list);
        //计算出查询的条数
        Long n = select.stream().count();
        if(n>=1){
            //update
            jdbcClient.executeSql("update wordcount set word_count = "+map.get(word)+" where word = '"+word+"'");
        }else{
            //insert
            jdbcClient.executeSql("insert into wordcount values( '"+word+"',"+map.get(word)+")");
        }

    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        //后面jdbc insert bolt直接把这里的输出写Mysql里去了，所以这里的fileds的名字要跟mysql表的字段名字对应
        declarer.declare(new Fields("word","word_count"));
    }
}

创建topology ：WordCountStormJdbcTopology.java

package com.storm.demo;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.topology.TopologyBuilder;

/**
 * 路径：com.storm.demo
 * 类名：
 * 功能：《用一句话描述一下》
 * 备注：
 * 创建人：typ
 * 创建时间：2018/7/25 16:36
 * 修改人：
 * 修改备注：
 * 修改时间：
 */
public class WordCountStormJdbcTopology {

    public static void main(String[] args) {

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", new WordCountSpout());
        builder.setBolt("bolt", new CountBolt()).shuffleGrouping("spout");

        Config config = new Config();
        config.setNumWorkers(3);

        //本地模式
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("stormJdbcTopology", config, builder.createTopology());



    }

}

运行结果：