Springboot integrates TDengine to realize data subscription - multi-threaded fast consumption

1. TDengine data subscription service

In order to help applications obtain data written to TDengine in real time, or process data in the order of event arrival, TDengine provides data subscription and consumption interfaces similar to message queue products. In this way, in many scenarios, the time series data processing system using TDengine no longer needs to integrate message queue products, such as Kafka, thereby simplifying the complexity of system design and reducing operation and maintenance costs.

For detailed documentation on the TDengine data subscription service, please refer to the official website: TDengine—Data Subscription

No more description here, the main content of this article: In the context of the Internet of Things, real-time data monitoring is a very common business function. How can we efficiently obtain real-time data in the face of PB-level data volumes? TDengine is an excellent timing The database integrates services like Kafka message queues. The message queue can play the role of asynchronous decoupling and peak elimination, but generally speaking, the sending speed of data is much higher than the speed of data consumption (because there is consumption logic for business), so the possibility of data accumulation is very high. So increasing the speed of consumption is naturally the top priority.

2. Multi-threaded batch consumption

2.0 preparation

Create database: tmqdb

Create a super table:

CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT)
TAGS (groupid INT, location BINARY(16))

创建子表d0和d1：INSERT INTO d1 USING meters TAGS(1, ‘San Francisco’) values(now - 9s, 10.1, 119)
INSERT INTO d0 values(now - 8s, NULL, NULL)

Create topic: create topic topic_name as select * from meters

rely:

    <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <!--  TDengine Java Connector       -->
        <dependency>
            <groupId>com.taosdata.jdbc</groupId>
            <artifactId>taos-jdbcdriver</artifactId>
            <version>3.0.0</version>
     </dependency>

2.1 Super table entity class

@Data
@AllArgsConstructor
@NoArgsConstructor
public class Meters {
    
    

    //动态属性
    private Timestamp ts;
    private float current;
    private int voltage;
    //静态属性
    private int groupid;
    private String location;
}

2.2 Simulated data insertion

/**
 * 模拟写数据
 *
 * @author zhmsky
 * @date 2022/9/12 17:18
 */
public class WriteData {
    
    

    private static int count = 0;

    public static void main(String[] args) {
    
    
        TaosCrudUtils taosCrudUtils = new TaosCrudUtils();
        //一次性插入两万数据
        while (count < 20000) {
    
    
            Random random = new Random();
            int i = random.nextInt(235);
            String sql = "INSERT INTO tmqdb.d1 VALUES(now, " + (float) i + ", " + i + ");";
            taosCrudUtils.insert(sql);
            count++;
        }
    }

}

2.3 Configuration file

#  服务器主机
taos.hostName=localdomain.com:6030
# 消费组
taos.groupId=test
# 主题名
taos.topicName=topic_name

2.4 Consumer multi-threaded batch consumption

package com.zhmsky.springboottdengine.数据订阅.消费者多线程消费;

import com.taosdata.jdbc.tmq.ConsumerRecords;
import com.taosdata.jdbc.tmq.TMQConstants;
import com.taosdata.jdbc.tmq.TaosConsumer;
import com.zhmsky.springboottdengine.数据订阅.pojo.Meters;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * @author zhmsky
 * @date 2022/9/12 19:58
 */
@Component
public class ConsumerHandler {
    
    

    private static String HOST_NAME;
    private static String GROUP_ID;
    private static String TOPICNAME;

    private TaosConsumer<Meters> consumer;
    private ExecutorService executors;

    //消息队列消息拉取是否开启
    private boolean active = true;

    public static Properties initConfig() {
    
    
        //获取配置文件
        InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("application.properties");
        Properties fileProperties = new Properties();
        try {
    
    
            //读取配置文件
            fileProperties.load(is);
            HOST_NAME = fileProperties.getProperty("taos.hostName");
            GROUP_ID = fileProperties.getProperty("taos.groupId");
            TOPICNAME = fileProperties.getProperty("taos.topicName");
        } catch (IOException e) {
    
    
            throw new RuntimeException(e);
        }

        //消费者配置
        Properties properties = new Properties();
        //连接地址
        properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, HOST_NAME);
        //允许从消息中解析表名
        properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
        //开启自动提交
        properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
        properties.setProperty(TMQConstants.GROUP_ID, GROUP_ID);
        //值解析方法，需要实现com.taosdata.jdbc.tmq.Deserializer 接口或继承 com.taosdata.jdbc.tmq.ReferenceDeserializer 类
        properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
                "com.zhmsky.springboottdengine.数据订阅.MetersDeserializer");
        return properties;
    }

    /**
     * 项目启动时完成初始化配置
     */
    @PostConstruct
    public void initTaosConfig() {
    
    
        Properties properties = initConfig();
        try {
    
    
            //创建消费者实例
            consumer = new TaosConsumer<>(properties);
            //订阅主题
            consumer.subscribe(Collections.singletonList(TOPICNAME));
        } catch (SQLException e) {
    
    
            throw new RuntimeException(e);
        }

    }

    /**
     * 多线程批量消费(执行这个方法即可循环拉取消息)
     *
     * @param workerNum
     */
    public void execute(int workerNum) {
    
    
        executors = new ThreadPoolExecutor(workerNum, 20, 10,
                TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), new ThreadPoolExecutor.CallerRunsPolicy());
        while (active) {
    
    
            try {
    
    
                ConsumerRecords<Meters> records = consumer.poll(Duration.ofMillis(100));
                if (!records.isEmpty()) {
    
    
                    //将消息交给线程池认领
                    executors.submit(new TaskWorker(records));
                }
            } catch (SQLException e) {
    
    
                throw new RuntimeException(e);
            }
        }
    }

    /**
     * 停止拉取消息
     */
    public void stopTaosPoll() {
    
    
        this.active = false;
    }
}

2.5 Custom thread class (handling real consumption logic)

/**
 * @author zhmsky
 * @date 2022/9/12 20:29
 */
@Slf4j
public class TaskWorker implements Runnable {
    
    

    private ConsumerRecords<Meters> consumerRecords;

    public TaskWorker(ConsumerRecords<Meters> records) {
    
    
        this.consumerRecords = records;
    }

    /**
     * 线程处理逻辑(正真的消息消费逻辑)
     */
    @Override
    public void run() {
    
    
        //TODO 真实的消费处理逻辑
        for (Meters consumerRecord : consumerRecords) {
    
    
            log.info(Thread.currentThread().getName() + "::" + consumerRecord.getTs() + " " + consumerRecord.getCurrent() + " "
                    + consumerRecord.getVoltage() + " " + consumerRecord.getLocation() + " " + consumerRecord.getGroupid());
        }
    }
}

2.6 Value Parsing

/**
 * 值解析方法
 *
 * @author zhmsky
 * @date 2022/9/12 16:43
 */
public class MetersDeserializer extends ReferenceDeserializer<Meters> {
    
    

}

2.7 Testing

@SpringBootApplication
@EnableScheduling
public class SpringbootTDengineApplication {
    
    
    @Autowired
    private ConsumerHandler consumers;

    public static void main(String[] args) {
    
    
        SpringApplication.run(SpringbootTDengineApplication.class, args);
    }

    //定时任务启动
    @Scheduled(cron = "* 3 21 * * ? ")
    public void test() {
    
    
        consumers.execute(10);
    }

}