SpringBoot与Kafka整合案例

一,搭建好Zookeeper集群与Kafka集群

二.构建Maven项目

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.1.RELEASE</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.jy.data.yh.bigdata.drools.hadoop.kafka</groupId>
    <artifactId>SSO-Hadoop-Kafka</artifactId>
    <version>1.0-SNAPSHOT</version>
    <name>SSO-Hadoop-Kafka</name>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
            <version>2.1.1.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.mybatis.spring.boot</groupId>
            <artifactId>mybatis-spring-boot-starter</artifactId>
            <version>1.3.2</version>
        </dependency>
        <dependency>
            <groupId>org.mybatis</groupId>
            <artifactId>mybatis</artifactId>
            <version>3.5.2</version>
        </dependency>
        <!--alibaba druid连接池  -->
        <!--Druid是Java语言中最好的数据库连接池。Druid能够提供强大的监控和扩展功能。-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>1.1.10</version>
        </dependency>


        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.13</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <version>2.1.1.RELEASE</version>
            <scope>test</scope>
        </dependency>

        <!-- 通用 UTIL 包 -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.6</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.9</version>
        </dependency>

        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>20.0</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.54</version>
        </dependency>

        <!--<dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
        </dependency>-->

        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.2</version>
        </dependency>



        <!-- spring boot 集成 redis -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
            <version>2.1.1.RELEASE</version>
            <exclusions>
                <exclusion><!-- 排除lettuce依赖,使用jedis来代替 -->
                    <groupId>io.lettuce</groupId>
                    <artifactId>lettuce-core</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>2.9.0</version>
        </dependency>
        <!-- session共享需要的Jar -->
        <dependency>
            <groupId>org.springframework.session</groupId>
            <artifactId>spring-session-data-redis</artifactId>
            <version>2.1.2.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>2.1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-streams</artifactId>
            <version>2.1.1</version>
        </dependency>
        <!--springboot整合-->
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>2.2.4.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.0</version>
            <optional>true</optional>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-surefire-plugin</artifactId>
                <version>2.22.1</version>
                <configuration>
                    <skipTests>true</skipTests>
                    <testFailureIgnore>true</testFailureIgnore>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <version>2.1.1.RELEASE</version>
                <!-- 支持调试 热部署 -->
                <configuration>
                    <fork>true</fork>
                </configuration>
                <!-- 支持调试 热部署 end -->
            </plugin>
        </plugins>
    </build>

</project>

application.properties文件

server.port=8855
server.session-timeout=60
server.tomcat.max-threads=1000
server.tomcat.uri-encoding=UTF-8
logging.path=../logs
logging.file=kafka.log
logging.level.root=info

spring.thymeleaf.mode=HTML
spring.thymeleaf.encoding=UTF-8
spring.thymeleaf.prefix=classpath:/templates/
spring.thymeleaf.suffix=.html
spring.thymeleaf.basename=messages


spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3306/security?characterEncoding=utf-8&serverTimezone=GMT%2B8
spring.datasource.username=root
spring.datasource.password=123456
spring.datasource.initialSize=200
spring.datasource.minIdle=100
spring.datasource.maxActive=500
spring.datasource.maxWait=60000
spring.datasource.timeBetweenEvictionRunsMillis=60000
spring.datasource.minEvictableIdleTimeMillis=300000
spring.datasource.validationQuery=SELECT 1 FROM DUAL
spring.datasource.testWhileIdle=true
spring.datasource.testOnBorrow=false
spring.datasource.testOnReturn=false
spring.datasource.poolPreparedStatements=true
spring.datasource.maxPoolPreparedStatementPerConnectionSize=2000
spring.datasource.filters=stat,wall,logback
spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000

mybatis.mapperLocations=classpath:mapper/*.xml
mybatis.typeAliasesPackage=org.jy.data.elasticsearch.search.model
mybatis.configuration.log-impl=org.apache.ibatis.logging.slf4j.Slf4jImpl
mybatis.configuration.log-prefix=es_

# redis集群配置
spring.redis.cluster.nodes=192.168.227.128:6379,192.168.227.128:6380,192.168.227.129:6381,192.168.227.129:6382,192.168.227.130:6383,192.168.227.130:6384
spring.redis.cluster.timeout=2000
spring.redis.cluster.max-redirects=100
spring.redis.cluster.maxIdle=200
spring.redis.cluster.maxTotal=1000
spring.redis.cluster.maxWaitMillis=2000
spring.session.store-type = redis

#============== kafka ===================
# 指定kafka 代理地址，可以多个
spring.kafka.bootstrap-servers=centosnode01:9092,centosnode02:9092,centosnode03:9092
#=============== provider  =======================
spring.kafka.producer.retries=0
# 每次批量发送消息的数量
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432
# 指定消息key和消息体的编解码方式
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
#=============== consumer  =======================
# 指定默认消费者group id
spring.kafka.consumer.group-id=kafka-consumer-group
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=100

# 指定消息key和消息体的编解码方式
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

三.完整代码实例

(1)消息生成者

package org.jy.data.yh.bigdata.drools.hadoop.kafka.producer;

import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.jy.data.yh.bigdata.drools.hadoop.kafka.model.KafkaMessage;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.Date;
import java.util.UUID;

@Component
@Slf4j
public class KafkaMessageProducer {

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;

    //发送消息方法
    @Scheduled(fixedDelay=2000)  // 2秒调度一次
    public void send() {
        KafkaMessage message = new KafkaMessage();
        message.setId(System.currentTimeMillis());
        message.setMsg(UUID.randomUUID().toString());
        message.setSendTime(new Date());
        String jsonContent = JSON.toJSONString(message);
        kafkaTemplate.send("topic_template",jsonContent);
    }
}

(2)消费者

package org.jy.data.yh.bigdata.drools.hadoop.kafka.consumer;

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

import java.util.Optional;

@Component
@Slf4j
public class KafkaMessageReceive {

    private static final Logger log = LoggerFactory.getLogger(KafkaMessageReceive.class);

    /**
     * @TopicPartition  有必要研究一些这个注解
     * @param record
     */
    @KafkaListener(topics = {"topic_template"})  // 订阅的主题，可以订阅多个主题
    public void listen(ConsumerRecord<?, ?> record) {
        Optional<?> kafkaMessage = Optional.ofNullable(record.value());
        if (kafkaMessage.isPresent()) {
            Object message = kafkaMessage.get();
            System.out.println("这是订阅topic_template主题的消息内容: "+message);
            log.info("----------------- record =" + record);
            log.info("------------------ message =" + message);
        }

    }

}

（3）消息类

package org.jy.data.yh.bigdata.drools.hadoop.kafka.model;

import java.io.Serializable;
import java.util.Date;
/**
 * 消息实体类
 */
public class KafkaMessage implements Serializable {
    private Long id;    //id
    private String msg; //消息
    private Date sendTime;  //时间戳

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getMsg() {
        return msg;
    }

    public void setMsg(String msg) {
        this.msg = msg;
    }

    public Date getSendTime() {
        return sendTime;
    }

    public void setSendTime(Date sendTime) {
        this.sendTime = sendTime;
    }
}

四，运行项目打印日志如下:

2020-04-20 15:04:39.323  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366279318,"msg":"f5da04ec-a4f7-4bc3-b909-8a044839fbaf","sendTime":1587366279318}
这是订阅topic_template主题的消息内容: {"id":1587366281319,"msg":"c3177a1b-9645-415e-b376-c98119159339","sendTime":1587366281319}
2020-04-20 15:04:41.323  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 1, leaderEpoch = 0, offset = 820, CreateTime = 1587366281319, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366281319,"msg":"c3177a1b-9645-415e-b376-c98119159339","sendTime":1587366281319})
2020-04-20 15:04:41.324  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366281319,"msg":"c3177a1b-9645-415e-b376-c98119159339","sendTime":1587366281319}
这是订阅topic_template主题的消息内容: {"id":1587366283320,"msg":"8314d756-48a1-4f96-bf17-c6d2560145d0","sendTime":1587366283320}
2020-04-20 15:04:43.324  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 0, leaderEpoch = 0, offset = 820, CreateTime = 1587366283320, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366283320,"msg":"8314d756-48a1-4f96-bf17-c6d2560145d0","sendTime":1587366283320})
2020-04-20 15:04:43.324  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366283320,"msg":"8314d756-48a1-4f96-bf17-c6d2560145d0","sendTime":1587366283320}
这是订阅topic_template主题的消息内容: {"id":1587366285320,"msg":"bff540ac-e925-4aad-959f-a0c8672e9dbe","sendTime":1587366285320}
2020-04-20 15:04:45.324  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 1, leaderEpoch = 0, offset = 821, CreateTime = 1587366285320, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366285320,"msg":"bff540ac-e925-4aad-959f-a0c8672e9dbe","sendTime":1587366285320})
2020-04-20 15:04:45.324  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366285320,"msg":"bff540ac-e925-4aad-959f-a0c8672e9dbe","sendTime":1587366285320}
这是订阅topic_template主题的消息内容: {"id":1587366287321,"msg":"d5a9ec88-15ae-45d5-a17f-a2483b76f6dd","sendTime":1587366287321}
2020-04-20 15:04:47.325  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 0, leaderEpoch = 0, offset = 821, CreateTime = 1587366287321, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366287321,"msg":"d5a9ec88-15ae-45d5-a17f-a2483b76f6dd","sendTime":1587366287321})
2020-04-20 15:04:47.325  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366287321,"msg":"d5a9ec88-15ae-45d5-a17f-a2483b76f6dd","sendTime":1587366287321}
这是订阅topic_template主题的消息内容: {"id":1587366289321,"msg":"5688b0f5-5862-4b84-b3e1-132f11f3a568","sendTime":1587366289321}
2020-04-20 15:04:49.325  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 1, leaderEpoch = 0, offset = 822, CreateTime = 1587366289321, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366289321,"msg":"5688b0f5-5862-4b84-b3e1-132f11f3a568","sendTime":1587366289321})
2020-04-20 15:04:49.325  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366289321,"msg":"5688b0f5-5862-4b84-b3e1-132f11f3a568","sendTime":1587366289321}
这是订阅topic_template主题的消息内容: {"id":1587366291322,"msg":"0174a72b-a5df-4805-b23f-3faac8a32e77","sendTime":1587366291322}
2020-04-20 15:04:51.326  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ----------------- record =ConsumerRecord(topic = topic_template, partition = 0, leaderEpoch = 0, offset = 822, CreateTime = 1587366291322, serialized key size = -1, serialized value size = 90, headers = RecordHeaders(headers = [], isReadOnly = false), key = null, value = {"id":1587366291322,"msg":"0174a72b-a5df-4805-b23f-3faac8a32e77","sendTime":1587366291322})
2020-04-20 15:04:51.326  INFO 23764 --- [ntainer#0-0-C-1] o.j.d.y.b.d.h.k.c.KafkaMessageReceive    : ------------------ message ={"id":1587366291322,"msg":"0174a72b-a5df-4805-b23f-3faac8a32e77","sendTime":1587366291322}