Flume实时读取数据到MySQL

准备工作:

在MySQL的db_test数据库下创建表tb_user:

CREATE TABLE `tb_user` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `username` varchar(255) DEFAULT NULL,
  `password` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;

具体实现

第一步:自定义Sink:

public class MySQLSink extends AbstractSink implements Configurable {
    private Connection conn;
    private Statement stmt;
    private String columnNames;
    private String url;
    private String user;
    private String password;
    private String tableName;
    // 在整个sink结束时执行一遍
    @Override
    public synchronized void stop() {
    }
    // 在整个sink开始时执行一遍
    @Override
    public synchronized void start() {
        try {
            Class.forName("com.mysql.jdbc.Driver");
            conn = DriverManager.getConnection(url, user, password);
            stmt = conn.createStatement();
        } catch (ClassNotFoundException | SQLException e) {
            e.printStackTrace();
        }
    }
    // 不断循环调用
    @Override
    public Status process() throws EventDeliveryException {
        Status res = Status.READY;
        Channel channel = getChannel();
        Transaction tx = channel.getTransaction();
        Event event = null;
        tx.begin();
        while (true) {
            event = channel.take();
            if (event != null) {
                break;
            }
        }
        try {
            String body = new String(event.getBody());
            if (body.split(",").length == columnNames.split(",").length) {
                String sql = "insert into " + tableName + "(" + columnNames + ") values(" + body + ")";
                System.out.println("========sql======= " + sql);
                stmt.executeUpdate(sql);
                tx.commit();
            } else {
                System.out.println("input error!");
                res = Status.BACKOFF;
                tx.rollback();
            }
        } catch (Throwable th) {
            tx.rollback();
        } finally {
            tx.close();
        }
        return res;
    }

    @Override
    public void configure(Context context) {
        columnNames = context.getString("columnNames");
        url = context.getString("url");
        user = context.getString("user");
        password = context.getString("password");
        tableName = context.getString("tableName");
    }

}

第二步:在flume的job文件夹下自定义配置文件flume-to-mysql.conf:

agent.sources = s1   
agent.channels = c1 
agent.sinks = sk1 
 
agent.sources.s1.type = netcat 
agent.sources.s1.bind = localhost 
agent.sources.s1.port = 5678 
agent.sources.s1.channels = c1 
 
agent.sinks.sk1.type = com.hc.tomysql.MySQLSink
agent.sinks.sk1.url=jdbc:mysql://hcmaster:3306/db_test?useSSL=false
agent.sinks.sk1.tableName= tb_user
agent.sinks.sk1.user=root
agent.sinks.sk1.password=root
agent.sinks.sk1.columnNames=id, username, password
agent.sinks.sk1.channel = c1 
 
agent.channels.c1.type = memory
agent.channels.c1.capacity = 1000    
agent.channels.c1.transactionCapacity = 100

第三步:项目打包并上传到flume的lib目录

第四步:执行代码:

  • 启动flume:
    在这里插入图片描述
  • 启动telnet:
    在这里插入图片描述
  • 在telnet中输入数据:
    在这里插入图片描述
  • Flume中查看结果
    在这里插入图片描述
  • 数据库中结果:
    在这里插入图片描述

发布了407 篇原创文章 · 获赞 798 · 访问量 7万+

猜你喜欢

转载自blog.csdn.net/lianghecai52171314/article/details/104931272