Kafka cluster containerized deployment

Kafka cluster containerized deployment

1 Cluster construction

1.1 Cluster Planning

host address	Role	ZOO_MY_ID	KAFKA_BROKER_ID
10.247.3.33	zookeeper1,kafka1	1	1
10.247.3.127	zookeeper2,kafka2	2	2
10.247.3.135	zookeeper3,kafka3	3	3

Create directories on three hosts:

mkdir -p /data/kafka/zookeeper/{
    
    data,datalog}
mkdir -p /data/kafka/kafka/logs

environmentSetting tips: In docker-compose, the environment name server.propertiesis improved from the options in the kafka configuration file. For example, server.propertiesthe wording of advertised.listeners in: KAFKA_ADVERTISED_LISTENERS, the rule is to add KAFKA_a prefix, and the parameter name is changed to a larger one, .replaced by _.

Host one:

version: '3'
services:
  zookeeper1:
    image: zookeeper:3.8.1
    container_name: zookeeper
    volumes:
      #- /data/kafka/zookeeper/conf:/conf
      - /data/kafka/zookeeper/data:/data
      - /data/kafka/zookeeper/datalog:/datalog
    environment:
      TZ: "Asia/Shanghai"
      ZOO_CFG_EXTRA: "quorumListenOnAllIPs=true"
      ZOO_MY_ID: 1
      ZOO_SERVERS: "server.1=10.247.3.33:2888:3888;2181 server.2=10.247.3.127:2888:3888;2181 server.3=10.247.3.135:2888:3888;2181"
    ports: 
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    restart: always

  kafka1:
    image: wurstmeister/kafka:2.13-2.8.1
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      # client 要访问的 broker 地址
      KAFKA_ADVERTISED_HOST_NAME: 10.247.3.33
      # 通过端口连接 zookeeper
      KAFKA_ZOOKEEPER_CONNECT: "10.247.3.33:2181,10.247.3.127:2181,10.247.3.135:2181"
      # 每个容器就是一个 broker，设置其对应的 ID
      KAFKA_BROKER_ID: 1
      # 外部网络只能获取到容器名称，在内外网络隔离情况下
      # 通过名称是无法成功访问 kafka 的
      # kafka 监听器，告诉外部连接者要通过什么协议访问指定主机名和端口开放的 Kafka 服务。
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      # 因此需要通过绑定这个监听器能够让外部获取到的是 IP 
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.247.3.33:9092
      # Kafka默认使用-Xmx1G -Xms1G的JVM内存配置，由于服务器小，调整下启动配置
      # 这个看自己的现状做调整，如果资源充足，可以不用配置这个
      KAFKA_HEAP_OPTS: "-Xms1024M -Xmx1024M"
      # 设置 kafka 日志位置
      KAFKA_LOG_DIRS: "/kafka/logs"
      KAFKA_SOCKET_REQUEST_MAX_BYTES: 1347375956
    volumes:
      # 挂载 kafka 日志
      - /data/kafka/kafka/logs:/kafka/logs
    restart: always
    depends_on:
      - zookeeper1

Host two:

version: '3'
services:
  zookeeper2:
    image: zookeeper:3.8.1
    container_name: zookeeper
    volumes:
      #- /data/kafka/zookeeper/conf:/conf
      - /data/kafka/zookeeper/data:/data
      - /data/kafka/zookeeper/datalog:/datalog
    environment:
      TZ: "Asia/Shanghai"
      ZOO_CFG_EXTRA: "quorumListenOnAllIPs=true"
      ZOO_MY_ID: 2
      ZOO_SERVERS: "server.1=10.247.3.33:2888:3888;2181 server.2=10.247.3.127:2888:3888;2181 server.3=10.247.3.135:2888:3888;2181"
    ports: 
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    restart: always

  kafka2:
    image: wurstmeister/kafka:2.13-2.8.1
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      # client 要访问的 broker 地址
      KAFKA_ADVERTISED_HOST_NAME: 10.247.3.127
      # 通过端口连接 zookeeper
      KAFKA_ZOOKEEPER_CONNECT: "10.247.3.33:2181,10.247.3.127:2181,10.247.3.135:2181"
      # 每个容器就是一个 broker，设置其对应的 ID
      KAFKA_BROKER_ID: 2
      # 外部网络只能获取到容器名称，在内外网络隔离情况下
      # 通过名称是无法成功访问 kafka 的
      # kafka 监听器，告诉外部连接者要通过什么协议访问指定主机名和端口开放的 Kafka 服务。
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      # 因此需要通过绑定这个监听器能够让外部获取到的是 IP 
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.247.3.127:9092
      # Kafka默认使用-Xmx1G -Xms1G的JVM内存配置，由于服务器小，调整下启动配置
      # 这个看自己的现状做调整，如果资源充足，可以不用配置这个
      KAFKA_HEAP_OPTS: "-Xms1024M -Xmx1024M"
      # 设置 kafka 日志位置
      KAFKA_LOG_DIRS: "/kafka/logs"
      KAFKA_SOCKET_REQUEST_MAX_BYTES: 1347375956
    volumes:
      # 挂载 kafka 日志
      - /data/kafka/kafka/logs:/kafka/logs
    restart: always
    depends_on:
      - zookeeper2

Host three:

version: '3'
services:
  zookeeper3:
    image: zookeeper:3.8.1
    container_name: zookeeper
    volumes:
      #- /data/kafka/zookeeper/conf:/conf
      - /data/kafka/zookeeper/data:/data
      - /data/kafka/zookeeper/datalog:/datalog
    environment:
      TZ: "Asia/Shanghai"
      ZOO_CFG_EXTRA: "quorumListenOnAllIPs=true"
      ZOO_MY_ID: 3
      ZOO_SERVERS: "server.1=10.247.3.33:2888:3888;2181 server.2=10.247.3.127:2888:3888;2181 server.3=10.247.3.135:2888:3888;2181" 
    ports: 
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    restart: always

  kafka3:
    image: wurstmeister/kafka:2.13-2.8.1
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      # client 要访问的 broker 地址
      KAFKA_ADVERTISED_HOST_NAME: 10.247.3.135
      # 通过端口连接 zookeeper
      KAFKA_ZOOKEEPER_CONNECT: "10.247.3.33:2181,10.247.3.127:2181,10.247.3.135:2181"
      # 每个容器就是一个 broker，设置其对应的 ID
      KAFKA_BROKER_ID: 3
      # 外部网络只能获取到容器名称，在内外网络隔离情况下
      # 通过名称是无法成功访问 kafka 的
      # kafka 监听器，告诉外部连接者要通过什么协议访问指定主机名和端口开放的 Kafka 服务。
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      # 因此需要通过绑定这个监听器能够让外部获取到的是 IP 
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.247.3.135:9092
      # Kafka默认使用-Xmx1G -Xms1G的JVM内存配置，由于服务器小，调整下启动配置
      # 这个看自己的现状做调整，如果资源充足，可以不用配置这个
      KAFKA_HEAP_OPTS: "-Xms1024M -Xmx1024M"
      # 设置 kafka 日志位置
      KAFKA_LOG_DIRS: "/kafka/logs"
      KAFKA_SOCKET_REQUEST_MAX_BYTES: 1347375956
    volumes:
      # 挂载 kafka 日志
      - /data/kafka/kafka/logs:/kafka/logs
    restart: always
    depends_on:
      - zookeeper3

2 kafka-related knowledge

• Create Topics

Create a topic named test with only one copy and one partition:

./kafka-topics.sh --create --zookeeper 10.247.3.33:2181,10.247.3.127:2181,10.247.3.135:2181 --replication-factor 3 --partitions 1 --topic testT

Check out the created topics:

./kafka-topics.sh -list -zookeeper 10.247.3.33:2181,10.247.3.127:2181,10.247.3.135:2181

• Start Producer to send messages

Open a window, in the kafka decompression directory:

./kafka-console-producer.sh --broker-list 10.247.3.33:9092,10.247.3.127:9092,10.247.3.135:9092 --topic testT

*注意：不要localhost:9092这样写，也不要127.0.0.1:9092这样写，正确的写法是：10.221.19.217:9092。

• Start Consumer

In a new remote window:

./kafka-console-consumer.sh --bootstrap-server 10.247.3.33:9092,10.247.3.127:9092,10.247.3.135:9092 --topic testT --from-beginning

At this point, send a message in the Producer window, and print out the consumed message in the Consumer window.