Flink 二

主要也是代码的一些操作，API的一些使用

1.从文件中读取数据流然后存入Mysql

package com.yjp.flink.demo6;


import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * 从文件中读取数据
 *
 * @Author : WenBao
 * Date : 17:01 2018/3/5
 */
public class Demo2 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().disableSysoutLogging();
        env.registerCachedFile("F://1.txt", "WordCount");
        DataStream<Tuple2<String, Integer>> sourceStream =
                env.readTextFile("F://1.txt")
                .flatMap(new Splitter())
                .keyBy(0)
                .sum(1);
        sourceStream.print();
        sourceStream.writeAsText("/w");
        sourceStream.addSink(new MysqlSink());

        sourceStream.print();
        env.execute("Window WordCount");

    }

    public static class Splitter implements FlatMapFunction<String, Tuple2<String, Integer>> {

        public void flatMap(String sentence, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (String word : sentence.split(" ")) {
                if (null != word && !word.isEmpty()) {
                    out.collect(new Tuple2<String, Integer>(word, 1));
                }
            }
        }

    }
}

package com.yjp.flink.demo6;

import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;

/**
 * 自定义JDBC sink
 * 向mysql数据库中插入数据
 *
 * @Author : WenBao
 * Date : 17:20 2018/3/1
 */

public class MysqlSink extends RichSinkFunction<Tuple2<String, Integer>> {
    private static final long serialVersionUID = 1L;

    private Connection connection;

    private PreparedStatement preparedStatement;

    /**
     * open方法是初始化方法，会在invoke方法之前执行，执行一次。
     *
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws Exception {
        //JDBC连接信息
        String USERNAME = "root";
        String PASSWORD = "root";
        String driverClass = "com.mysql.jdbc.Driver";
        //String driverClass = "com.mysql.cj.jdbc.Driver";
        String URL = "jdbc:mysql://localhost:3306/login?characterEncoding=UTF-8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&useUnicode=true";
        //加载jdbc的驱动
        Class.forName(driverClass);
        //获取数据库的连接
        connection = DriverManager.getConnection(URL, USERNAME, PASSWORD);
        String sql = "insert into word_num(word, num) values (?,?)";
        preparedStatement = connection.prepareStatement(sql);
        super.open(parameters);
    }

    /**
     * invoke()方法解析一个元组数据，并插入到数据库中。
     *
     * @param value 输入的数据
     * @throws Exception
     */
    @Override
    public void invoke(Tuple2<String, Integer> value, Context context) throws Exception {
        try {
            String word = value.f0;
            Integer num = value.f1;
            preparedStatement.setString(1, word);
            preparedStatement.setInt(2, num);
            preparedStatement.executeUpdate();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * close()是tear down的方法，在销毁时执行，关闭连接。
     */
    @Override
    public void close() throws Exception {
        if (preparedStatement != null) {
            preparedStatement.close();
        }
        if (connection != null) {
            connection.close();
        }
        super.close();
    }

}

2.从mysql中读取数据流将值转换为对象流

package com.yjp.flink.demo6;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

/**
 * 从mysql中读取数据流
 *
 * @Author : WenBao
 * Date : 10:35 2018/3/5
 */

public class WordSourceFromMysql extends RichSourceFunction<Word> {


    private static final long serialVersionUID = 1L;

    private Connection connection;

    private PreparedStatement preparedStatement;

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        String USERNAME = "root";
        String PASSWORD = "root";
        String driverClass = "com.mysql.jdbc.Driver";
        //String driverClass = "com.mysql.cj.jdbc.Driver";
        String URL = "jdbc:mysql://localhost:3306/login?characterEncoding=UTF-8&autoReconnect=true&zeroDateTimeBehavior=convertToNull&useUnicode=true";
        //加载jdbc的驱动
        Class.forName(driverClass);
        connection = DriverManager.getConnection(URL, USERNAME, PASSWORD);
        String sql = " SELECT * FROM  word_num ";
        preparedStatement = connection.prepareStatement(sql);
    }

    @Override
    public void run(SourceContext<Word> sourceContext) throws Exception {
        try {
            ResultSet resultSet = preparedStatement.executeQuery();
            while (resultSet.next()) {
                Word word = new Word();
                word.setId(resultSet.getInt("id"));
                word.setWord(resultSet.getString("word"));
                word.setNum(resultSet.getInt("num"));
                sourceContext.collect(word);
            }

        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    @Override
    public void cancel() {

    }

    @Override
    public void close() throws Exception {
        super.close();
        if (connection != null) {
            connection.close();
        }
        if (preparedStatement != null) {
            preparedStatement.close();
        }
    }
}

package com.yjp.flink.demo6;

public class Word {

    private int id;
    private String word;
    private int num;


    public Word(int id, String word, int num) {
        this.id = id;
        this.word = word;
        this.num = num;
    }

    public Word() {
    }

    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public int getNum() {
        return num;
    }

    public void setNum(int num) {
        this.num = num;
    }

    @Override
    public String toString() {
        return "Word{" +
                "id=" + id +
                ", word='" + word + '\'' +
                ", num=" + num +
                '}';
    }
}

package com.yjp.flink.demo6;

import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class WordSourceFromMysqlTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().disableSysoutLogging();
        DataStream<Word> stream = env.addSource(new WordSourceFromMysql());
        stream.print();
        env.execute();
    }
}

努力吧，皮卡丘

猜你喜欢