docker-compose搭建kafka集群，支持程序外部访问

创建网络

docker network create \
  --driver=bridge \
  --subnet=172.28.0.0/16 \
  --gateway=172.28.0.1 \
  dev

docker-compose 搭建

KAFKA_ADVERTISED_LISTENERS的配置是关键,OUTSIDE://localhost:9093的填写是因为在本地搭建的docker集群, 可以允许本机的程序访问, 如果要支持外部程序访问, 可以更换ip

version: '3.3'

services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - 2181:2181
    container_name: zookeeper
    networks:
      default:
        ipv4_address: 172.28.0.99

  kafka0:
    image: wurstmeister/kafka:2.13-2.6.0
    depends_on:
      - zookeeper
    container_name: kafka0
    hostname: kafka0
    ports:
      - 9093:9093
    environment:
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9093
      KAFKA_ADVERTISED_LISTENERS: INSIDE://172.28.0.100:9092,OUTSIDE://localhost:9093
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 0
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
    networks:
      default:
        ipv4_address: 172.28.0.100

  kafka1:
    image: wurstmeister/kafka:2.13-2.6.0
    depends_on:
      - zookeeper
    container_name: kafka1
    hostname: kafka1
    ports:
      - 9094:9094
    environment:
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://172.28.0.101:9092,OUTSIDE://localhost:9094
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
    networks:
      default:
        ipv4_address: 172.28.0.101

  kafka2:
    image: wurstmeister/kafka:2.13-2.6.0
    depends_on:
      - zookeeper
    container_name: kafka2
    hostname: kafka2
    ports:
      - 9095:9095
    environment:
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9095
      KAFKA_ADVERTISED_LISTENERS: INSIDE://172.28.0.102:9092,OUTSIDE://localhost:9095
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
    networks:
      default:
        ipv4_address: 172.28.0.102

networks:
  default:
    external:
      name: dev

docker 内部访问验证

• 创建topic

docker exec -it kafka0 /bin/bash

kafka-topics.sh \
--create --topic kafka_demo \
--partitions 3 \
--zookeeper zookeeper:2181 \
--replication-factor 2

• 消费者

docker exec -it kafka0 /bin/bash

kafka-console-consumer.sh \
--topic kafka_demo \
--bootstrap-server 172.28.0.100:9092,172.28.0.101:9092,172.28.0.102:9092

• 生产者

kafka-cluster docker exec -it kafka2 /bin/bash

kafka-console-producer.sh \
--topic kafka_demo \
--broker-list 172.28.0.100:9092,172.28.0.101:9092,172.28.0.102:9092

python 外部访问验证

• 消费者代码

import json

from kafka import KafkaConsumer


def consumer_demo():
    consumer = KafkaConsumer(
        'kafka_demo',
        security_protocol="PLAINTEXT",
        bootstrap_servers=[
            'localhost:9095',
            'localhost:9093',
            'localhost:9094'
        ],
    )
    for message in consumer:
        print("receive, key: {}, value: {}".format(
            json.loads(message.key.decode()),
            json.loads(message.value.decode())
        )
        )


if __name__ == '__main__':
    consumer_demo()

• 生产者代码

import json
import traceback

from kafka import KafkaProducer
from kafka.errors import kafka_errors


def producer_demo():
    producer = KafkaProducer(
        security_protocol="PLAINTEXT",
        bootstrap_servers=[
            'localhost:9095',
            'localhost:9093',
            'localhost:9094'
        ],
        key_serializer=lambda k: json.dumps(k).encode(),
        value_serializer=lambda v: json.dumps(v).encode())
    # 发送三条消息
    for i in range(0, 3):
        future = producer.send(
            'kafka_demo',
            key='count_num', 
            value=str(i),
            partition=0)  
        print("send {}".format(str(i)))
        try:
            future.get(timeout=10)  # 监控是否发送成功
        except kafka_errors:  # 发送失败抛出kafka_errors
            traceback.format_exc()


if __name__ == '__main__':
    producer_demo()