Kafka The production data （partition）

pom

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-streams</artifactId>
            <version>1.0.0</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <!-- java编译插件 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
        </plugins>
    </build>

Producer： 

package Producer;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

/*
用于生产数据到kafka集群
 */
public class Producer1 {

    /*
    程序的入口
     */
    public static void main(String[] args){

    //编写生产数据的程序

        //1、配置kafka集群环境（设置）
        Properties props = new Properties();
        //kafka服务器地址
        props.put("bootstrap.servers", "node001:9092,node002:9092,node003:9092");
        //消息确认机制
        props.put("acks", "all");
        //重试机制
        props.put("retries", 0);
        //批量发送的大小
        props.put("batch.size", 16384);
        //消息延迟
        props.put("linger.ms", 1);
        //批量的缓冲区大小
        props.put("buffer.memory", 33554432);
        // kafka   key 和value的序列化
        props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");

      //  props.put("partitioner.class", "Producer.ProducerPartition");
        //2、实例一个生产者对象
        KafkaProducer<String, String> kafkaProducer = new KafkaProducer<String, String>(props);

        for (int i = 0; i < 9; i++) {

            //3、通过生产者对象将数据发送到kafka集群

            //ProducerRecord<K, V> record
            //1、没有指定分区编号，没有指定key,时采用轮询方式存户数据
            ProducerRecord producerRecord = new ProducerRecord("18BD12","bbbb___"+i);
            //2、没有指定分区编号，指定key时，数据分发策略为对key求取hash值，这个值与分区数量取余，于数就是分区编号。
           //ProducerRecord producerRecord = new ProducerRecord("18BD12","test","aaaa___"+i);
            //3、指定分区编号，所有数据输入到指定的分区内
            //ProducerRecord producerRecord = new ProducerRecord("18BD12",1,"test","aaaa___"+i);

            //4、定义分区策略
            //ProducerRecord producerRecord = new ProducerRecord("18BD12","test","aaaa___"+i);


            kafkaProducer.send(producerRecord);


        }
        //4、关闭成产者
        kafkaProducer.close();

    }
}

Partition ：

package Producer;

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;

import java.util.Map;

/*
编写自定义分区逻辑
 */
public class ProducerPartition implements Partitioner {
    @Override
    public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {

   /*
   编写自定义分区代码

    */

        //数据目标的分区
        return 0;
    }















    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> configs) {

    }
}

 

Consumer ： 

package Consumer;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;

import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.Set;

/*
以分区为单位获取数据
 */
public class ConsumerPartition {

    //入口
    public static void main(String[] args){
        //1配置文件
        Properties props = new Properties();
        //指定kafka服务器
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        //消费组
        props.put("group.id", "test1");
        //以下两行代码 ---消费者自动提交offset值
        props.put("enable.auto.commit", "false");
        //自动提交的周期
        //props.put("auto.commit.interval.ms",  "1000");
        //kafka   key 和value的反序列化
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        //2消费者
        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(props);
        //3、设置topic
        kafkaConsumer.subscribe(Arrays.asList("18BD12"));

        while (true){

            //4、拉取数据，并输出
            ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);
            //通过数据获取到多有的分区
            Set<TopicPartition> partitions = consumerRecords.partitions();
            //遍历所有分区，或得到一个分区
            for (TopicPartition partition : partitions) {
                //获取每个分区的数据,多条数据
                List<ConsumerRecord<String, String>> records = consumerRecords.records(partition);
                //遍历分区内的所有数据，或得到一条
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(record.value()+"      "+record.partition());
                }


                //手动提交offset
                kafkaConsumer.commitSync();
            }




        }
    }


}

二 ：

package Consumer;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;

import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.Set;

/*
消费指定分区的数据
 */
public class ConsumerMyPartition {
    public static void main(String[] args){

        //1配置文件
        Properties props = new Properties();
        //指定kafka服务器
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        //消费组
        props.put("group.id", "test1");
        //以下两行代码 ---消费者自动提交offset值
        props.put("enable.auto.commit", "false");
        //自动提交的周期
        //props.put("auto.commit.interval.ms",  "1000");
        //kafka   key 和value的反序列化
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        //2消费者
        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(props);
        //3、设置topic
        //kafkaConsumer.subscribe(Arrays.asList("18BD12"));



             //Collection<TopicPartition>
        TopicPartition topicPartition0 = new TopicPartition("18BD12",0);
        TopicPartition topicPartition2 = new TopicPartition("18BD12",1);

        kafkaConsumer.assign(Arrays.asList(topicPartition0,topicPartition2));








        while (true){

            //4、拉取数据，并输出
            ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);
            //通过数据获取到多有的分区  0   2
            Set<TopicPartition> partitions = consumerRecords.partitions();
            //遍历所有分区，或得到一个分区
            for (TopicPartition partition : partitions) {
                //获取每个分区的数据,多条数据
                List<ConsumerRecord<String, String>> records = consumerRecords.records(partition);
                //遍历分区内的所有数据，或得到一条
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(record.value()+"      "+record.partition());
                }


                //手动提交offset
                kafkaConsumer.commitSync();
            }




        }

    } 
}