默认内网访问,要在外网访问的话,需要在修改config/server.properties中的配置
将listeners和advertised.listeners的值用主机名进行替换,在外用使用java进行生产者或消费者连接的时候,不填写具体的IP,填写安装kafka的主机名,然后,在hosts目录中,配置该主机名对应的真是IP地址即可;
以下命令都是摘抄与官网http://kafka.apache.org/quickstart
先启动zookeeper,默认自带的
bin/zookeeper-server-start.sh -daemon config/zookeeper.properties &
然后启动kafka服务
bin/kafka-server-start.sh -daemon config/server.properties &
列举拥有哪些topics
bin/kafka-topics.sh --list --bootstrap-server localhost:9092
在服务器上打开一个生产者,然后把输入的每行数据发送到kafka中的命令
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
#后面光标提示数据数据,然后回车就会发送到kafka中了
打开一个消费者
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
当有数据往kafka的test主题发送消息,这边就会进行消费。
java调用作为生产者和消费者代码:
项目需要引入的依赖pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.theorydance</groupId>
<artifactId>kafkademo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>kafkademo</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.1.1</version>
</dependency>
</dependencies>
</project>
生产者代码ProducerDemo.java
package com.theorydance.kafkademo;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
public class ProducerDemo {
public static void main(String[] args){
Properties properties = new Properties();
properties.put("bootstrap.servers", "node125:9092");
properties.put("acks", "all");
properties.put("retries", 0);
properties.put("batch.size", 16384);
properties.put("linger.ms", 1);
properties.put("buffer.memory", 33554432);
properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
Producer<String, String> producer = null;
try {
producer = new KafkaProducer<String, String>(properties);
for (int i = 0; i < 100; i++) {
String msg = "This is Message " + i;
producer.send(new ProducerRecord<String, String>("HelloWorld", msg));
System.out.println("Sent:" + msg);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
producer.close();
}
}
}
消费者代码ConsumerDemo.java
package com.theorydance.kafkademo;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
public class ConsumerDemo {
public static void main(String[] args) throws InterruptedException {
Properties properties = new Properties();
properties.put("bootstrap.servers", "node125:9092");
properties.put("group.id", "group-1");
properties.put("enable.auto.commit", "true");
properties.put("auto.commit.interval.ms", "1000");
properties .put("auto.offset.reset", "earliest");
properties.put("session.timeout.ms", "30000");
properties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
properties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties);
while(true){
Map<String, List<PartitionInfo>> maps = kafkaConsumer.listTopics();
System.out.println("监听topics="+maps.keySet());
Set<String> sets = new HashSet<>();
for (String topic : maps.keySet()) {
if(topic.startsWith("Hello")){ // 制定规则,监听哪一些的topic
sets.add(topic);
}
}
kafkaConsumer.subscribe(sets);
long startTime = System.currentTimeMillis();
while (true) {
ConsumerRecords<String, String> records = kafkaConsumer.poll(100);
for (ConsumerRecord<String, String> record : records) {
System.out.printf("offset = %d, value = %s, topic = %s", record.offset(), record.value(), record.topic());
System.out.println("=====================>");
}
long endTime = System.currentTimeMillis();
if(endTime - startTime > 30000){
System.out.println("------------------------------------------------------------------");
break;
}
}
}
}
}