kafka使用备忘docker安装配置本地host新建topic新建消费者组写入数据查询topictopic详细信息安装

docker安装

version: '3'

services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"
  kafka-1:
    image: wurstmeister/kafka:2.12-2.4.0
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LOG_DIRS: /kafka/logs
      KAFKA_LISTENERS: PLAINTEXT://kafka-1:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /Users/firaga/Code/docker/compose/kafka/data/1:/kafka
  kafka-2:
    image: wurstmeister/kafka:2.12-2.4.0
    ports:
      - "9093:9093"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LOG_DIRS: /kafka/logs
      KAFKA_LISTENERS: PLAINTEXT://kafka-2:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9093
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /Users/firaga/Code/docker/compose/kafka/data/2:/kafka
  kafka-3:
    image: wurstmeister/kafka:2.12-2.4.0
    ports:
      - "9094:9094"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LOG_DIRS: /kafka/logs
      KAFKA_LISTENERS: PLAINTEXT://kafka-3:9094 
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9094
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /Users/firaga/Code/docker/compose/kafka/data/3:/kafka

配置本地host

127.0.0.1 kafka-1
127.0.0.1 kafka-2
127.0.0.1 kafka-3

新建topic

bin/kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --config retention.ms=604800000 --config min.insync.replicas=2 --create --partitions 3 --replication-factor 3 --topic user-topic

新建消费者组

bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --from-beginning --group user-topic-to-es --topic user-topic

写入数据

bin/kafka-console-producer.sh --broker-list 127.0.0.1:9092 --topic user-topic

查询topic

bin/kafka-topics.sh --bootstrap-server 127.0.0.1:9092  --list

topic 详细信息

bin/kafka-topics.sh --bootstrap-server 127.0.0.1:9092  --topic user-log --describe

安装kafka tool查看kafka状态

链接

查看偏移量

kafka_2.12-2.4.0/bin/kafka-console-consumer.sh --topic __consumer_offsets --bootstrap-server kafka-1:9092,kafka-2:9093,kafka-3:9094 --from-beginning --formatter "kafka.coordinator.group.GroupMetadataManager\$OffsetsMessageFormatter"

kafka_2.12-2.4.0/bin/kafka-consumer-groups.sh --bootstrap-server kafka-1:9092,kafka-2:9093,kafka-3:9094 --group user-log-to-es  --describe