Table of contents
Get started with Kafka quickly
Kafka relies on the Java runtime environment, so before starting Kafka or ZooKeeper, make sure Java is installed on your system.
Install OpenJDK by following these steps:
- Update package information:
sudo apt update - Install OpenJDK:
sudo apt install default-jdk
This will install the default version of OpenJDK. If you need a specific version of OpenJDK, you can use the corresponding package name. - After the installation is complete, verify that the Java installation was successful:
java -version - Rerun the ZooKeeper or Kafka command.
Step 1: Install Kafka
-
Download Kafka: Go to Apache Kafka official website to download the latest Kafka version.
-
Unzip the file: Unzip the downloaded Kafka zip package to a directory of your choice.
tar -xzf kafka_2.12-3.6.0.tgz- -x: means decompression.
- -z: indicates using gzip to decompress.
- -f: followed by the file name of the compressed package.
-
Start ZooKeeper: Kafka depends on ZooKeeper, so before starting Kafka, you need to start ZooKeeper. Enter the bin directory in the Kafka decompression directory and execute the following command to start ZooKeeper:
./zookeeper-server-start.sh ../config/zookeeper.properties -
Start the Kafka server: Open a new terminal, enter the bin directory in the Kafka decompression directory, and execute the following command to start the Kafka server:
./kafka-server-start.sh ../config/server.properties
Step 2: Create a topic
In Kafka, data is published into topics, and consumers subscribe to these topics to receive data.
- Create a topic: Open a new terminal, enter the bin directory in the Kafka decompression directory, and execute the following command to create a topic named my-topic:
./kafka-topics.sh --create --topic my-topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1--topic my-topicSpecify the name of the created topic as "my-topic".--bootstrap-server localhost:9092Specifies the bootstrap server address of the Kafka cluster used to create the topic.--partitions 1Specifies the number of partitions that the created topic will contain. A partition is the unit of parallel processing of messages in Kafka. Here, the specified number of partitions is 1, indicating that the topic has only one partition. The choice of the number of partitions usually depends on parallelism requirements and the distribution of message load.--replication-factor 1Specify the number of replicas per partition. Here, the specified number of replicas is 1, which means there is only one replica for each partition. Replicas are used to improve data reliability and fault tolerance. If there are multiple replicas, the data will be replicated on multiple brokers. Typically, you will want to set the number of replicas greater than 1 to ensure data reliability.
Step 3: Producer sends message
In Kafka, producers are responsible for publishing messages to topics.
-
Start a producer: Open a new terminal, enter the bin directory in the Kafka decompression directory, and execute the following command to start a producer:
./kafka-console-producer.sh --topic my-topic --bootstrap-server localhost:9092 -
Send a message: In the producer terminal, type some message and press enter to send.
Step 4: Consumer receives message
In Kafka, consumers subscribe to topics to receive messages sent by producers.
-
Start a consumer: Open a new terminal, enter the bin directory in the Kafka decompression directory, and execute the following command to start a consumer:
./kafka-console-consumer.sh --topic my-topic --bootstrap-server localhost:9092 --from-beginning--from-beginning
-
Receiving messages: In the consumer terminal, you will see the messages sent by the producer.
Implement producer and consumer sample code using kafka-go
producer
package main
// ./kafka-console-producer.sh --topic my-topic --bootstrap-server localhost:9092
import (
"context"
"fmt"
"strconv"
"time"
"github.com/segmentio/kafka-go"
)
var (
reader *kafka.Reader
topic = "user_click"
)
func Write(ctx context.Context) {
writer := &kafka.Writer{
Addr: kafka.TCP("localhost:9092"),
Topic: topic,
Balancer: &kafka.Hash{
},
WriteTimeout: 1 * time.Second,
RequiredAcks: kafka.RequireNone,
AllowAutoTopicCreation: true,
}
defer writer.Close()
tm := time.Now().Unix()
for i := 0; i < 3; i++ {
if err := writer.WriteMessages(
ctx,
kafka.Message{
Key: []byte("1"), Value: []byte("小" + strconv.Itoa(int(tm)))},
kafka.Message{
Key: []byte("2"), Value: []byte("白" + strconv.Itoa(int(tm)))},
kafka.Message{
Key: []byte("3"), Value: []byte("小" + strconv.Itoa(int(tm)))},
kafka.Message{
Key: []byte("1"), Value: []byte("熊" + strconv.Itoa(int(tm)))},
kafka.Message{
Key: []byte("1"), Value: []byte("猫" + strconv.Itoa(int(tm)))},
); err != nil {
if err == kafka.LeaderNotAvailable {
time.Sleep(500 * time.Millisecond)
continue
} else {
fmt.Println("批量写kafka失败：%v \n", err)
}
} else {
break
}
}
}
func main() {
ctx := context.Background()
Write(ctx)
}
consumer
package main
// ./kafka-console-consumer.sh --topic user_click --bootstrap-server localhost:9092 --from-beginning --group a
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/segmentio/kafka-go"
)
var (
reader *kafka.Reader
topic = "user_click"
)
func read(ctx context.Context) {
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{
"localhost:9092"},
Topic: topic,
CommitInterval: 1 * time.Second,
GroupID: "rec_team",
StartOffset: kafka.FirstOffset,
})
for {
if message, err := reader.ReadMessage(ctx); err != nil {
fmt.Println("读kafka失败: ", err)
break
} else {
fmt.Printf("topic=%s, partition=%d, offset=%d, key=%s, value=%s \n", message.Topic, message.Partition, message.Offset, string(message.Key), string(message.Value))
}
}
}
func listenSignal() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
sig := <-c
fmt.Println("接收到信号: ", sig.String())
if reader != nil {
reader.Close()
}
os.Exit(0)
}
func main() {
ctx := context.Background()
go listenSignal()
read(ctx)
}